diff --git a/README.md b/README.md index f37405e9..bc3d4eaa 100644 --- a/README.md +++ b/README.md @@ -219,11 +219,19 @@ curl http://YOUR_MITHRIL_IP:8899 -X POST -H "Content-Type: application/json" -d **Currently supported RPC methods:** - `getAccountInfo` - Get account data and lamports +- `getBalance` - Get an account's lamport balance - `getBankHash` - Get bankhash for a slot (Mithril extension, not standard Solana RPC) - `getBlockHeight` - Get current block height +- `getBlockProduction` - Get produced and assigned block counts by validator - `getEpochInfo` - Get current epoch info - `getInflationReward` - Get retained inflation rewards for an epoch +- `getGenesisHash` - Get the cluster genesis hash - `getLatestBlockhash` - Get recent blockhash +- `getLeaderSchedule` - Get the leader schedule for an epoch +- `getSlot` - Get the slot at the requested commitment (defaults to finalized) +- `getVoteAccounts` - Get current and delinquent vote accounts +- `sendTransaction` - Submit a signed transaction +- `simulateTransaction` - Simulate a transaction On Alpenglow, `getInflationReward` can publish verified canonical rewards before the account checkpoint reaches disk. `confirmed` and `finalized` share that view; @@ -236,7 +244,10 @@ Credited rewards are readable while other staking partitions are still pending. Requests for pending partitions return `-32017` with verified distribution progress; pruned history returns `-32001`, and unknown history returns `-32004`. -We're actively expanding RPC method coverage. Upcoming methods include transaction simulation, send transaction, and get leader schedule. +`getSlot` accepts an optional config object with `commitment` and `minContextSlot`. +On Alpenglow, `processed` uses the live replay slot; `confirmed` and `finalized` +use the published rooted slot. If that slot is below `minContextSlot`, the request +returns error `-32016` with the available slot in `data.contextSlot`. ### Current Limitations diff --git a/cmd/mithril/node/checkpoint_recovery_test.go b/cmd/mithril/node/checkpoint_recovery_test.go index d6ccc1f2..ff5a1588 100644 --- a/cmd/mithril/node/checkpoint_recovery_test.go +++ b/cmd/mithril/node/checkpoint_recovery_test.go @@ -134,6 +134,34 @@ func TestRecoveredManifestContextAdvancesStaleStateBeforeIntegrity(t *testing.T) assert.Equal(t, manifestCtx, s.LastRootedContext) } +type recordingRPCBankState struct { + slot uint64 + blockHeight uint64 + transactionCount uint64 +} + +func (state *recordingRPCBankState) SetRootedBankState(slot, blockHeight, transactionCount uint64) { + state.slot = slot + state.blockHeight = blockHeight + state.transactionCount = transactionCount +} + +func TestPublishRPCBankStateUsesRecoveredRoot(t *testing.T) { + transactionCount := uint64(789) + server := new(recordingRPCBankState) + publishRPCBankState(server, &state.MithrilState{ + ManifestParentSlot: 10, + ManifestBlockHeight: 20, + ManifestTransactionCount: 30, + LastRootedContext: &state.ResumeContext{ + Slot: 123, + BlockHeight: 456, + TransactionCount: &transactionCount, + }, + }) + assert.Equal(t, recordingRPCBankState{slot: 123, blockHeight: 456, transactionCount: 789}, *server) +} + func writeCheckpointManifest(t *testing.T, accountsDbPath, manifestsDir string, seq, root, fileID uint64, suffix string) *state.TransactionStatusCheckpointRef { t.Helper() payload := []byte(fmt.Sprintf("checkpoint-payload-%d", root)) diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index 574e3041..8850ba47 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -2600,6 +2600,7 @@ postBootstrap: klog.Fatalf("unable to load RPC epoch rewards: %v", err) } } + publishRPCBankState(rpcServer, mithrilState) rpcServer.Start() mlog.Log.Infof("Started RPC server on port %d", rpcPort) } @@ -4648,6 +4649,7 @@ func runReplayWithRecovery( mlog.Log.Errorf("fork switch: no rooted checkpoint context to re-replay from; halting") break } + publishRPCBankState(rpcServer, mithrilState) if mithrilState.LastRootedSlot > prevRooted { // progress since last attempt -> fresh budget attempt = 0 prevRooted = mithrilState.LastRootedSlot @@ -4687,3 +4689,24 @@ func runReplayWithRecovery( } return result } + +type rpcBankStateSetter interface { + SetRootedBankState(slot, blockHeight, transactionCount uint64) +} + +func publishRPCBankState(rpcServer rpcBankStateSetter, mithrilState *state.MithrilState) { + if rpcServer == nil || mithrilState == nil { + return + } + rootedSlot := mithrilState.ManifestParentSlot + rootedBlockHeight := mithrilState.ManifestBlockHeight + rootedTransactionCount := mithrilState.ManifestTransactionCount + if rootedCtx := mithrilState.LastRootedContext; rootedCtx != nil { + rootedSlot = rootedCtx.Slot + rootedBlockHeight = rootedCtx.BlockHeight + if rootedCtx.TransactionCount != nil { + rootedTransactionCount = *rootedCtx.TransactionCount + } + } + rpcServer.SetRootedBankState(rootedSlot, rootedBlockHeight, rootedTransactionCount) +} diff --git a/docs/rooted-stake-rpc-cache.md b/docs/rooted-stake-rpc-cache.md new file mode 100644 index 00000000..e725e679 --- /dev/null +++ b/docs/rooted-stake-rpc-cache.md @@ -0,0 +1,168 @@ +# Rooted stake totals for getVoteAccounts + +`getVoteAccounts` maintains activated stake by vote account in memory. The +initial request loads the existing stake-candidate index and reads accounts in +batches of 512. Subsequent account commits update only changed delegations. +Filtered requests select one vote record and one total; unfiltered requests still +enumerate vote accounts, but neither path rescans stake accounts on each scrape. +Initialized vote-account records are decoded lazily and reused until a committed +write touches that account. The cache owns the RPC fields and up to five credit +entries, not account buffers. Stake totals, epoch membership and delinquency are +applied per request. Returned credit slices are copies owned by the caller. + +The writer only invalidates touched vote records; it performs no additional vote +decoding or I/O. Rewind, recovery and failed writes clear all parsed records. Loads +check the account version under the invalidation lock before entering the cache, +so an old in-flight decode cannot resurrect a record after a concurrent commit. +The final response still checks bank identity and account version even on a fully +cached read. Unknown or invalid filtered keys do not create negative cache entries. + +The retained data is one compact delegation/contribution per live delegated +stake account, plus totals per vote account. Full account buffers are not retained. +Memory is proportional to delegated stake accounts. Bootstrap temporarily tracks +changed keys while loading candidates. Recalculation walks the existing map +without allocating or copying a whole-cache key list. + +## RPC options + +`getVoteAccounts`, `getBalance`, `getLeaderSchedule`, `getBlockProduction`, and +`getAccountInfo` read the published rooted state. They accept `confirmed` and +`finalized`; `processed` returns an unsupported-commitment error rather than a +rooted result mislabeled as processed. This does not provide a separate optimistic +confirmed account view for classic consensus. `getSlot` and `getEpochInfo` keep +their existing processed-counter support. + +`keepUnstakedDelinquents: true` includes unstaked accounts in the delinquent list. +The default is false. It does not change the current-account list. + +`getLeaderSchedule` supports `keyByVoteAccount: true` when the retained schedule +contains the selected vote account for each leader slot. The `identity` filter +still selects a node identity. A node-keyed compatibility schedule without vote +account provenance returns an error for this option; it does not infer a vote +account from the node identity. + +## Publication and recovery contract + +- AccountsDB brackets writes with a process-local version. An odd version, + pending legacy write, or failed write cannot serve a committed cache view. +- Observers receive borrowed changed accounts after successful writes, before the + version becomes readable. They perform no I/O and retain only decoded values. +- A response must match both the account version and the RPC's published bank + identity from start to finish. A durable fold or completed legacy write alone + does not publish its account values under the previous RPC bank context. +- Initialization merges commits while scanning. Placeholders cannot resurrect + closed accounts, and accounts created after the candidate snapshot are supplied + by the observer. No unchanged-root interval spanning the full scan is required. +- Epoch, stake-history, or warmup-feature changes trigger a RAM-only recalculation + in bounded chunks. Per-entry calculation tags count concurrent updates exactly + once; new activation inputs cancel an unfinished calculation. +- Recalculation holds the cache mutex for every map iterator step, read and write, + releasing it after each 512 visited entries, including already-counted entries. + Commits between chunks may delete or insert keys: the observer removes old + contributions and counts updated/new entries with the current calculation tag. + Those tags prevent double counting whether the iterator sees an insertion or + skips it, as permitted by [Go's map range semantics](https://go.dev/ref/spec#For_range). + After each lock reacquisition, generation and calculation checks reject resets + or changed activation inputs before the iterator advances again. This bounds + entries processed per lock hold, not a hard wall-clock latency guarantee. +- Rewind, recovery, and failed writes invalidate the derived view. Recovery also + clears old account-read caches before a rebuild. Reusing the same slot on another + fork cannot reuse its old stake totals. Failed writes require successful repair + or a subsequent successful write before the account version is readable again. + +This cache adds no durable index, disk writes, or manifest format changes. It is +reconstructed after process restart. + +The durable vote-candidate index also keeps the original version-1 manifest +format: `PrevValid` is exactly 0 or 1, as required by older rewind readers. +Vote membership is an in-memory hint during normal commits. Recovery derives it +from account headers in the CRC-checked segment before atomically installing the +primary index, vote candidates, and fold watermark. This adds header reads only +for batches whose index application must be replayed. Metadata/read failures +leave that batch's files intact and do not advance its watermark. + +The reader still accepts the 2/3 flag bytes emitted by earlier revisions of the +RPC branch, preserving their undo pointers. Existing files are not automatically +rewritten: stores that already contain those flags must use the corrected reader +for recovery/rewind, or be rebuilt before downgrading to a pre-RPC binary. Newly +written manifests are compatible with the original reader. A format-version bump +alone would not make downgrade safe, because older recovery treats unknown +manifest headers as orphans. + +## Work and validation + +The first request and recovery still require account reads. Epoch/input changes +require work proportional to delegated stake accounts, but no disk scan. Normal +commits inspect changed accounts and decode only stake candidates; this cost is +paid by the account writer. RPC callers waiting for initialization can cancel. +The cache checks cancellation between read/calculation batches. + +```sh +GOMAXPROCS=2 go test -race ./pkg/accountsdb ./pkg/rpcserver +GOMAXPROCS=2 go vet ./pkg/accountsdb ./pkg/rpcserver +GOMAXPROCS=2 go test ./pkg/rpcserver -run '^$' \ + -bench '^BenchmarkRootedStakeTotals$' -benchtime=500ms -count=3 +``` + +The benchmark uses 20,000 stake accounts and 1,000 vote accounts. It compares the +old per-query scan against warm aggregate reads, and separately measures applying +64 stake updates among 20,000 changed accounts. It reports time and allocations. +These are component measurements: they exclude JSON encoding, network latency, +initial cache loading, and an epoch transition. They do not establish mainnet +capacity or live validator performance. + +`BenchmarkRootedStakeRecalculation` isolates RAM traversal at 20,000 and 2 million +delegations with 1,000 voters. The delegation cache is seeded directly and all +stakes are bootstrap-active; it measures recalculation time and allocations, +not startup, disk, or writer latency. For concurrent commits and mixed activation +history, use the on-disk sizing experiment below. An optional Go mutex profile +can attribute writer/reader lock wait; profile times sum waiting goroutines and +must not be reported as a single request's wall-clock latency. The sizing test's +full-scan correctness oracles also appear in whole-process profiles. + +```sh +GOMAXPROCS=2 GOMEMLIMIT=4GiB go test ./pkg/rpcserver -run '^$' \ + -bench '^BenchmarkRootedStakeRecalculation$' -benchtime=700ms -count=3 +``` + +`BenchmarkGetVoteAccountsRecords` measures the warm handler plus JSON encoding +for 1,000 V4 vote accounts (31 lockouts and 64 credit-history entries each), or one +filtered account. It isolates vote records with initialized empty stake totals. +The changed-record cases invalidate 10% or 100% of decoded records before every +request, including invalidation cost, while account buffers remain warm. They +simulate cache churn without disk commits; they do not measure writer contention. +Reuse benefits depend on how many vote accounts change between requests. A full +refresh still pays decoding plus cache maintenance and response ownership costs. + +```sh +GOMAXPROCS=2 go test ./pkg/rpcserver -run '^$' \ + -bench '^BenchmarkGetVoteAccountsRecords$' -benchtime=700ms -count=3 +``` + +## Larger on-disk sizing experiment + +The opt-in `TestRootedStakeCacheScale` exercises 1,000 initialized vote accounts +and a configurable count of delegated stake accounts (up to 2 million). Its +synthetic mix is 80% active, 10% activating, and 10% deactivating, with stake-history +changes at epoch boundaries. Run each size in a fresh process: + +```sh +MITHRIL_STAKE_SCALE=1000000 GOMAXPROCS=2 GOMEMLIMIT=4GiB \ + go test ./pkg/rpcserver -run '^TestRootedStakeCacheScale$' \ + -count=1 -v -timeout=10m +``` + +JSON log records report initial loading, warm all-voter/single-voter RPC handler +latencies and allocations, epoch recalculation, and concurrent bootstrap and +epoch changes. Four callers pause 50 ms between requests while 100 durable folds +update 64 stake accounts each, with a 20 ms pause between folds. A separate +no-reader stage measures fold latency. Every stage checks totals against a full +account scan; concurrent root advancement must not restart the bootstrap scan. + +Handler timings include JSON response encoding but exclude HTTP. Startup clears +application account caches; fixture creation still warms the OS page cache, so +this is not a cold-storage benchmark. The retained-cache estimate is the change +in live Go heap after dropping only the derived cache and collecting garbage. +Whole-process peak memory includes fixture construction and full-scan oracles; +it must not be described as the cache size or a complete RPC-node memory budget. +Reported tail percentiles describe these short samples, not production SLOs. diff --git a/pkg/accountsdb/account_observer.go b/pkg/accountsdb/account_observer.go new file mode 100644 index 00000000..83987ce8 --- /dev/null +++ b/pkg/accountsdb/account_observer.go @@ -0,0 +1,67 @@ +package accountsdb + +import "github.com/Overclock-Validator/mithril/pkg/accounts" + +// AccountStateObserver maintains disposable derived state, not durable state. +// Calls are serialized after a successful write, before its version becomes +// readable. Reset is true on rewind, recovery, or any failed/partial write. +// Accounts are borrowed for this call only. Observers must not retain mutable +// account data, call back into AccountsDb, or perform I/O. +type AccountStateObserver interface { + AccountsChanged(version uint64, changed []*accounts.Account, reset bool) +} + +// ObserveAccountState installs an observer for subsequent write completions. It is intended for +// node-lifetime derived indexes; the returned function unregisters it. +func (db *AccountsDb) ObserveAccountState(observer AccountStateObserver) func() { + db.accountObserversMu.Lock() + if db.accountObservers == nil { + db.accountObservers = make(map[AccountStateObserver]struct{}) + } + db.accountObservers[observer] = struct{}{} + db.accountObserversMu.Unlock() + return func() { + db.accountObserversMu.Lock() + delete(db.accountObservers, observer) + db.accountObserversMu.Unlock() + } +} + +// CommittedAccountVersion is a process-local coherence fence, not a slot or an +// on-disk format version. Readers must check it before AND after reading derived +// state and accounts, and independently match the RPC's published bank. An odd +// version or queued legacy write cannot serve a committed view. +func (db *AccountsDb) CommittedAccountVersion() (uint64, bool) { + version := db.accountStateVersion.Load() + if version&1 != 0 || db.accountStateUncertain.Load() || db.StoreQueueLen() != 0 { + return version, false + } + return version, db.accountStateVersion.Load() == version +} + +// CommittedAccountSlot identifies the last completed write. Zero is the +// snapshot baseline, whose slot is supplied by the RPC's initial bank state. +// Use only inside a stable CommittedAccountVersion fence. +func (db *AccountsDb) CommittedAccountSlot() uint64 { return db.accountCommittedSlot.Load() } + +// Existing fold/legacy locks are acquired first. This additional serialization +// also covers tests/tools that mix the two write APIs. No RPC read holds this +// mutex while scanning or loading accounts. +func (db *AccountsDb) beginAccountChange() func([]*accounts.Account, uint64, bool, bool) { + db.accountStateMu.Lock() + db.accountStateVersion.Add(1) + return func(changed []*accounts.Account, slot uint64, success, reset bool) { + version := db.accountStateVersion.Load() + 1 + db.accountObserversMu.Lock() + for observer := range db.accountObservers { + observer.AccountsChanged(version, changed, reset || !success) + } + db.accountObserversMu.Unlock() + if success { + db.accountCommittedSlot.Store(slot) + } + db.accountStateUncertain.Store(!success) + db.accountStateVersion.Store(version) + db.accountStateMu.Unlock() + } +} diff --git a/pkg/accountsdb/account_observer_test.go b/pkg/accountsdb/account_observer_test.go new file mode 100644 index 00000000..30605bf8 --- /dev/null +++ b/pkg/accountsdb/account_observer_test.go @@ -0,0 +1,74 @@ +package accountsdb + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/stretchr/testify/require" +) + +type accountObserverFunc func(uint64, []*accounts.Account, bool) + +func (f *accountObserverFunc) AccountsChanged(version uint64, changed []*accounts.Account, reset bool) { + (*f)(version, changed, reset) +} + +func TestAccountObserverCommitFailureRecoveryAndUnregister(t *testing.T) { + db, _ := newFoldTestDb(t) + defer db.CloseDb() + notifications := 0 + resetCount := 0 + var lastVersion uint64 + observer := accountObserverFunc(func(version uint64, changed []*accounts.Account, reset bool) { + _, stable := db.CommittedAccountVersion() + require.False(t, stable, "derived state must be updated before the version is readable") + require.Greater(t, version, lastVersion) + lastVersion = version + notifications++ + if reset { + resetCount++ + } else { + require.Len(t, changed, 1) + } + }) + unregister := db.ObserveAccountState(&observer) + db.foldHooks.afterPublicationStart = func() { _, stable := db.CommittedAccountVersion(); require.False(t, stable) } + _, err := db.CommitBatch(foldDeltas(accounts.SlotDelta{Slot: 10, Delta: []*accounts.Account{foldAcct(1, 100, nil)}}), 10, nil, nil) + require.NoError(t, err) + version, stable := db.CommittedAccountVersion() + require.True(t, stable) + require.Equal(t, lastVersion, version) + require.Equal(t, 1, notifications) + require.Zero(t, resetCount) + cached, err := db.GetAccount(10, foldAcct(1, 0, nil).Key) + require.NoError(t, err) + db.CommonAcctsCache.Set(cached.Key, cached) + // Rejecting invalid arguments does not poison a previously valid view. + _, err = db.CommitBatch(foldDeltas(accounts.SlotDelta{Slot: 12}, accounts.SlotDelta{Slot: 11}), 12, nil, nil) + require.Error(t, err) + got, stable := db.CommittedAccountVersion() + require.True(t, stable) + require.Equal(t, version, got) + + db.foldHooks.afterPublicationStart = func() { panic("crash after publication began") } + require.Panics(t, func() { + _, _ = db.CommitBatch(foldDeltas(accounts.SlotDelta{Slot: 11, Delta: []*accounts.Account{foldAcct(1, 200, nil)}}), 11, nil, nil) + }) + _, stable = db.CommittedAccountVersion() + require.False(t, stable, "partial writes fail closed until repaired") + require.Equal(t, 1, resetCount) + db.foldHooks.afterPublicationStart = nil + _, err = db.RecoverFoldState() + require.NoError(t, err) + _, stable = db.CommittedAccountVersion() + require.True(t, stable) + require.Equal(t, 2, resetCount) + account, err := db.GetAccount(11, foldAcct(1, 0, nil).Key) + require.NoError(t, err) + require.Equal(t, uint64(200), account.Lamports) + unregister() + before := notifications + _, err = db.CommitBatch(foldDeltas(accounts.SlotDelta{Slot: 12}), 12, nil, nil) + require.NoError(t, err) + require.Equal(t, before, notifications) +} diff --git a/pkg/accountsdb/accountsdb.go b/pkg/accountsdb/accountsdb.go index 121eb5ae..c0f6bb3b 100644 --- a/pkg/accountsdb/accountsdb.go +++ b/pkg/accountsdb/accountsdb.go @@ -34,12 +34,21 @@ type AccountsDb struct { AcctsDir string LargestFileId atomic.Uint64 VoteAcctCache otter.Cache[solana.PublicKey, *accounts.Account] + voteIndexMu sync.Mutex CommonAcctsCache otter.Cache[solana.PublicKey, *accounts.Account] ProgramCache otter.Cache[solana.PublicKey, *ProgramCacheEntry] // Otter permits concurrent ordinary operations but not Clear. Rewind takes // the write side; hot-path program cache operations take the read side. programCacheMu sync.RWMutex + // Coherent, process-local derived views of committed account state. + accountStateMu sync.Mutex + accountStateVersion atomic.Uint64 + accountCommittedSlot atomic.Uint64 + accountStateUncertain atomic.Bool + accountObserversMu sync.Mutex + accountObservers map[AccountStateObserver]struct{} + // readCacheEpochMu protects the cache epoch and the pending fold view. // CommitBatch publishes its immutable newest-wins union here before the // Pebble index commit. Readers use that view for changed keys while the @@ -110,6 +119,11 @@ func (accountsDb *AccountsDb) StoreQueueLen() int { return accountsDb.inProgressStoreRequests.Len() } +// DurableThrough returns the highest slot whose fold is fully committed. +func (accountsDb *AccountsDb) DurableThrough() uint64 { + return accountsDb.durableThrough.Load() +} + // silentLogger implements pebble.Logger but discards all messages. // This suppresses verbose WAL recovery messages on startup. type silentLogger struct{} @@ -705,11 +719,15 @@ func (accountsDb *AccountsDb) storeAccountsSync(accts []*accounts.Account, slot accountsDb.refreshReadCaches(accts) accountsDb.appendVecReadMu.Lock() defer accountsDb.appendVecReadMu.Unlock() + finish := accountsDb.beginAccountChange() + success := false + defer func() { finish(accts, slot, success, false) }() if StoreAccountsWorkers == 1 { accountsDb.storeAccountsInternal(accts, slot) } else { accountsDb.parallelStoreAccounts(StoreAccountsWorkers, accts, slot) } + success = true } // refreshReadCaches keeps already-hot common entries coherent after a store, diff --git a/pkg/accountsdb/appendvec_test.go b/pkg/accountsdb/appendvec_test.go index 9314740a..09c6c4a3 100644 --- a/pkg/accountsdb/appendvec_test.go +++ b/pkg/accountsdb/appendvec_test.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "testing" + "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/gagliardetto/solana-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -21,7 +22,7 @@ func TestBuildIndexEntriesDoesNotReadPastSliceLen(t *testing.T) { copy(backing[len(real):], phantom) data := backing[:len(real)] // len excludes phantom; cap and manifest include it. - pubkeys, entries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(cap(data)), 7, 9) + pubkeys, entries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(cap(data)), 7, 9) require.NoError(t, err) require.Equal(t, []solana.PublicKey{realKey}, pubkeys) require.Len(t, entries, 1) @@ -37,7 +38,7 @@ func TestBuildIndexEntriesBoundsOversizedManifestByDataLen(t *testing.T) { var pubkeys []solana.PublicKey require.NotPanics(t, func() { var err error - pubkeys, _, _, err = BuildIndexEntriesFromAppendVecs(data, uint64(len(data)+hdrLen), 8, 10) + pubkeys, _, _, _, err = BuildIndexEntriesFromAppendVecs(data, uint64(len(data)+hdrLen), 8, 10) require.NoError(t, err) }) require.Equal(t, []solana.PublicKey{key}, pubkeys) @@ -54,7 +55,7 @@ func TestBuildIndexEntriesStopsAtDefaultZeroLamportTerminator(t *testing.T) { binary.LittleEndian.PutUint64(terminator[dataLenOffset:dataLenOffset+8], ^uint64(0)) data := append(append(append([]byte{}, first...), terminator...), afterTerminator...) - pubkeys, entries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 12, 13) + pubkeys, entries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 12, 13) require.NoError(t, err) require.Equal(t, []solana.PublicKey{firstKey}, pubkeys) require.Len(t, entries, 1) @@ -67,7 +68,7 @@ func TestBuildIndexEntriesTerminatorRequiresDefaultKeyAndZeroLamports(t *testing zeroLamportAccount := marshalAppendVecTestAccount(t, zeroLamportKey, 0, []byte("tombstone")) data := append(defaultKeyAccount, zeroLamportAccount...) - pubkeys, entries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 18, 19) + pubkeys, entries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 18, 19) require.NoError(t, err) require.Equal(t, []solana.PublicKey{defaultKey, zeroLamportKey}, pubkeys) require.Len(t, entries, 2) @@ -80,7 +81,7 @@ func TestBuildIndexEntriesRejectsTruncatedAccountData(t *testing.T) { copy(data[pubkeyOffset:pubkeyOffset+32], key[:]) binary.LittleEndian.PutUint64(data[lamportsOffset:lamportsOffset+8], 1) - pubkeys, entries, stakeEntries, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 14, 15) + pubkeys, entries, stakeEntries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 14, 15) require.ErrorContains(t, err, "truncated appendvec account data") assert.Nil(t, pubkeys) assert.Nil(t, entries) @@ -92,12 +93,23 @@ func TestBuildIndexEntriesAcceptsFinalAccountWithoutAlignmentPadding(t *testing. encoded := marshalAppendVecTestAccount(t, key, 77, []byte{1, 2, 3}) data := encoded[: hdrLen+3 : hdrLen+3] - pubkeys, entries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(encoded)), 16, 17) + pubkeys, entries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(encoded)), 16, 17) require.NoError(t, err) require.Equal(t, []solana.PublicKey{key}, pubkeys) require.Len(t, entries, 1) } +func TestBuildIndexEntriesCollectsVoteAccounts(t *testing.T) { + vote := appendVecTestPubkey(10) + ordinary := appendVecTestPubkey(11) + voteData := marshalAppendVecTestAccount(t, vote, 1, nil) + copy(voteData[ownerOffset:ownerOffset+32], addresses.VoteProgramAddr[:]) + data := append(voteData, marshalAppendVecTestAccount(t, ordinary, 1, nil)...) + _, _, _, votes, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 20, 21) + require.NoError(t, err) + require.Equal(t, []solana.PublicKey{vote}, votes) +} + func appendVecTestPubkey(firstByte byte) solana.PublicKey { var key solana.PublicKey key[0] = firstByte diff --git a/pkg/accountsdb/compact.go b/pkg/accountsdb/compact.go index a007c9ec..f080fdb4 100644 --- a/pkg/accountsdb/compact.go +++ b/pkg/accountsdb/compact.go @@ -432,7 +432,7 @@ func (db *AccountsDb) compactFile(c compactCandidate, minDeadFraction float64) ( return false, 0, err } - pubkeys, idxEntries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), c.slot, c.fileId) + pubkeys, idxEntries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), c.slot, c.fileId) if err != nil { return false, 0, err } diff --git a/pkg/accountsdb/fold.go b/pkg/accountsdb/fold.go index d4b8a1f9..cb3b8a4a 100644 --- a/pkg/accountsdb/fold.go +++ b/pkg/accountsdb/fold.go @@ -13,6 +13,7 @@ import ( "sort" "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/cockroachdb/pebble" "golang.org/x/sync/errgroup" ) @@ -174,6 +175,11 @@ func (db *AccountsDb) CommitBatch( } sort.Slice(keys, func(i, j int) bool { return bytes.Compare(keys[i][:], keys[j][:]) < 0 }) + finish := db.beginAccountChange() + var committed []*accounts.Account + success := false + defer func() { finish(committed, throughSlot, success, false) }() + // (2) fileId allocation; persist high-water BEFORE the first data byte (I7). fileId := db.LargestFileId.Add(1) if err := db.persistLargestFileId(); err != nil { @@ -197,7 +203,10 @@ func (db *AccountsDb) CommitBatch( segErr := func() error { for _, k := range keys { v := union[k] - records = append(records, ManifestRecord{Pubkey: k, Offset: dataLen, OwnerSlot: v.ownerSlot}) + records = append(records, ManifestRecord{ + Pubkey: k, Offset: dataLen, OwnerSlot: v.ownerSlot, + Vote: v.acct.Lamports > 0 && v.acct.Owner == addresses.VoteProgramAddr, + }) ava := AppendVecAccount{ DataLen: uint64(len(v.acct.Data)), Pubkey: v.acct.Key, @@ -320,6 +329,9 @@ func (db *AccountsDb) CommitBatch( // ordinary concurrent-safe cache operations finish. db.refreshReadCacheEntries(live) + // Advance the watermark before removing pendingFold so rooted readers keep + // waiting until the matching bank state is published. + db.durableThrough.Store(throughSlot) db.readCacheEpochMu.Lock() db.pendingFold = nil db.readCacheEpochMu.Unlock() @@ -327,7 +339,7 @@ func (db *AccountsDb) CommitBatch( // (8) Publish. db.lastBatchSeq = batchSeq - db.durableThrough.Store(throughSlot) + committed, success = live, true return BatchCommitResult{ BatchSeq: batchSeq, @@ -353,6 +365,11 @@ func (db *AccountsDb) applyManifestToIndex(m *SegmentManifest) error { if err := batch.Set(r.Pubkey[:], idxBuf[:], nil); err != nil { return err } + if r.Vote { + if err := batch.Set(voteIndexKey(r.Pubkey), nil, nil); err != nil { + return err + } + } } if err := batch.Set(metaKeyLastBatch, encodeFoldMeta(foldMeta{ BatchSeq: m.BatchSeq, diff --git a/pkg/accountsdb/fold_test.go b/pkg/accountsdb/fold_test.go index fbdff0ad..64f01d0f 100644 --- a/pkg/accountsdb/fold_test.go +++ b/pkg/accountsdb/fold_test.go @@ -116,6 +116,19 @@ func TestCommitBatchReadsBackAndDedupes(t *testing.T) { assert.Equal(t, []byte("ctx-108"), manifest.ResumeCtx) } +func TestCommitBatchPublishesDurableThroughWithIndex(t *testing.T) { + db, _ := newFoldTestDb(t) + defer db.CloseDb() + + db.foldHooks.afterIndexCommit = func() { + require.Equal(t, uint64(42), db.DurableThrough()) + } + _, err := db.CommitBatch(foldDeltas( + accounts.SlotDelta{Slot: 42, Delta: []*accounts.Account{foldAcct(1, 1, nil)}}, + ), 42, nil, nil) + require.NoError(t, err) +} + // Manifest encode/decode round-trip, and CRC detection of a torn manifest. func TestSegmentManifestRoundTripAndTornDetection(t *testing.T) { dir := t.TempDir() diff --git a/pkg/accountsdb/index.go b/pkg/accountsdb/index.go index 7601d73f..0a064517 100644 --- a/pkg/accountsdb/index.go +++ b/pkg/accountsdb/index.go @@ -100,10 +100,12 @@ func WriteStakePubkeyIndex(path string, entries []StakeIndexEntry) error { // - pubkeys: all account pubkeys // - acctIdxEntries: index entries for each account // - stakeEntries: stake account pubkeys with their appendvec location hints -func BuildIndexEntriesFromAppendVecs(data []byte, fileSize uint64, slot uint64, fileId uint64) ([]solana.PublicKey, []AccountIndexEntry, []StakeIndexEntry, error) { +// - votePubkeys: vote-program account candidates, including accounts without active stake +func BuildIndexEntriesFromAppendVecs(data []byte, fileSize uint64, slot uint64, fileId uint64) ([]solana.PublicKey, []AccountIndexEntry, []StakeIndexEntry, []solana.PublicKey, error) { pubkeys := make([]solana.PublicKey, 0, 20000) acctIdxEntries := make([]AccountIndexEntry, 0, 20000) stakeEntries := make([]StakeIndexEntry, 0, 1000) + var votePubkeys []solana.PublicKey parser := &appendVecParser{Buf: data, FileSize: fileSize, FileId: fileId, Slot: slot} var owner solana.PublicKey @@ -117,7 +119,7 @@ func BuildIndexEntriesFromAppendVecs(data []byte, fileSize uint64, slot uint64, if errors.Is(err, io.EOF) { break } - return nil, nil, nil, fmt.Errorf("parse appendvec slot=%d file_id=%d: %w", slot, fileId, err) + return nil, nil, nil, nil, fmt.Errorf("parse appendvec slot=%d file_id=%d: %w", slot, fileId, err) } // Collect stake account entries with appendvec location hints if bytes.Equal(owner[:], addresses.StakeProgramAddr[:]) { @@ -128,7 +130,10 @@ func BuildIndexEntriesFromAppendVecs(data []byte, fileSize uint64, slot uint64, Offset: acctIdxEntries[idx].Offset, }) } + if owner == addresses.VoteProgramAddr { + votePubkeys = append(votePubkeys, pubkeys[len(pubkeys)-1]) + } } - return pubkeys, acctIdxEntries, stakeEntries, nil + return pubkeys, acctIdxEntries, stakeEntries, votePubkeys, nil } diff --git a/pkg/accountsdb/manifest_compat_test.go b/pkg/accountsdb/manifest_compat_test.go new file mode 100644 index 00000000..d99fa345 --- /dev/null +++ b/pkg/accountsdb/manifest_compat_test.go @@ -0,0 +1,170 @@ +package accountsdb + +import ( + "context" + "encoding/binary" + "hash/crc32" + "os" + "path/filepath" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/cockroachdb/pebble" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Pin the original v1 wire layout independently of the current decoder. In +// particular the old reader used equality with 1, not a bit mask, for undo. +func TestManifestV1BooleanCompatibility(t *testing.T) { + for _, prevValid := range []bool{false, true} { + for _, vote := range []bool{false, true} { + m := &SegmentManifest{Kind: ManifestKindFold, ThroughSlot: 20, FileId: 2, + Records: []ManifestRecord{{Pubkey: [32]byte{1}, PrevValid: prevValid, Vote: vote, + Prev: AccountIndexEntry{Slot: 10, FileId: 1, Offset: 136}}}} + raw := m.encode() + require.Equal(t, uint32(1), binary.LittleEndian.Uint32(raw[4:8])) + // Fixed prefix + empty bankhash count + empty context length + record count. + const recordStart = 4 + 4 + 1 + 5*8 + 4 + 4 + 4 + 8 + const booleanOffset = recordStart + 32 + 8 + 8 + require.LessOrEqual(t, raw[booleanOffset], byte(1)) + require.Equal(t, prevValid, raw[booleanOffset] == 1, "legacy undo decision") + var previous AccountIndexEntry + previous.Unmarshal((*[24]byte)(raw[booleanOffset+1 : booleanOffset+25])) + require.Equal(t, m.Records[0].Prev, previous) + require.Equal(t, crc32.ChecksumIEEE(raw[:len(raw)-4]), binary.LittleEndian.Uint32(raw[len(raw)-4:])) + + // Earlier versions of this RPC branch emitted bit 1 as a vote hint. The + // fixed reader accepts those files; an explicit rewrite emits a boolean. + raw[booleanOffset] |= 2 + binary.LittleEndian.PutUint32(raw[len(raw)-4:], crc32.ChecksumIEEE(raw[:len(raw)-4])) + dir := t.TempDir() + path := segmentManifestPath(dir, 20, 2) + require.NoError(t, os.WriteFile(path, raw, 0600)) + got, err := ReadSegmentManifest(path) + require.NoError(t, err) + require.Equal(t, prevValid, got.Records[0].PrevValid) + require.True(t, got.Records[0].Vote) + require.NoError(t, WriteSegmentManifest(dir, got)) + canonical, err := os.ReadFile(path) + require.NoError(t, err) + require.LessOrEqual(t, canonical[booleanOffset], byte(1)) + } + } +} + +func TestManifestRecoveryDerivesVoteCandidates(t *testing.T) { + for name, legacyFlags := range map[string]bool{"original-v1": false, "legacy-rpc-flags": true} { + t.Run(name, func(t *testing.T) { + db, dir := newFoldTestDb(t) + require.NoError(t, db.SeedVoteAccountPubkeys(nil)) // No migration scan may mask recovery bugs. + vote := foldAcct(1, 10, []byte("vote")) + vote.Owner = addresses.VoteProgramAddr + closed := foldAcct(2, 0, nil) + closed.Owner = addresses.VoteProgramAddr + ordinary := foldAcct(3, 10, []byte("ordinary")) + res := commitTestBatch(t, db, 10, vote, closed, ordinary) + manifest, err := ReadSegmentManifest(segmentManifestPath(db.AcctsDir, 10, res.FileId)) + require.NoError(t, err) + for _, r := range manifest.Records { + require.False(t, r.Vote, "vote hints must not be serialized") + } + if legacyFlags { + // Even a stale legacy vote hint must not override the segment's owner + // or zero balance. Mark all three records as votes in this fixture. + path := segmentManifestPath(db.AcctsDir, 10, res.FileId) + raw, err := os.ReadFile(path) + require.NoError(t, err) + recordStart := len(raw) - 4 - len(manifest.Records)*manifestRecordSize + for i := range manifest.Records { + raw[recordStart+i*manifestRecordSize+32+8+8] |= 2 + } + binary.LittleEndian.PutUint32(raw[len(raw)-4:], crc32.ChecksumIEEE(raw[:len(raw)-4])) + require.NoError(t, os.WriteFile(path, raw, 0600)) + } + require.NoError(t, db.Index.Delete(voteIndexKey(vote.Key), pebble.Sync)) + require.NoError(t, db.Index.Delete(metaKeyLastBatch, pebble.Sync)) + for _, a := range []*accounts.Account{vote, closed, ordinary} { + require.NoError(t, db.Index.Delete(a.Key[:], pebble.Sync)) + } + db = reopenFoldTestDb(t, db, dir) + defer db.CloseDb() + recovered, err := db.RecoverFoldState() + require.NoError(t, err) + require.Equal(t, []uint64{1}, recovered.ReplayedBatches) + got, err := db.VoteAccountPubkeys(context.Background()) + require.NoError(t, err) + require.Equal(t, []solana.PublicKey{vote.Key}, got) + require.Equal(t, uint64(10), mustColdRead(t, db, 10, vote.Key).Lamports) + again, err := db.RecoverFoldState() + require.NoError(t, err) + require.Empty(t, again.ReplayedBatches) + }) + } +} + +func TestVoteManifestRewindRestoresExistingAccount(t *testing.T) { + db, _ := newFoldTestDb(t) + defer db.CloseDb() + old := foldAcct(1, 10, []byte("old")) + old.Owner = addresses.VoteProgramAddr + commitTestBatch(t, db, 10, old) + newer := foldAcct(1, 20, []byte("new")) + newer.Owner = addresses.VoteProgramAddr + created := foldAcct(2, 30, nil) + created.Owner = addresses.VoteProgramAddr + commitTestBatch(t, db, 20, newer, created) + _, err := db.RewindToBatchBoundary(10) + require.NoError(t, err) + restored := mustColdRead(t, db, 10, old.Key) + require.Equal(t, old.Lamports, restored.Lamports) + require.Equal(t, old.Data, restored.Data) + _, closer, err := db.Index.Get(created.Key[:]) + if closer != nil { + closer.Close() + } + require.ErrorIs(t, err, pebble.ErrNotFound) +} + +// A structurally valid, CRC-valid manifest with invalid record metadata must +// fail before changing the index, and must not be deleted as an orphan. +func TestManifestVoteRecoveryMetadataErrorPreservesFiles(t *testing.T) { + for _, kind := range []string{"offset", "pubkey", "payload"} { + t.Run(kind, func(t *testing.T) { + db, _ := newFoldTestDb(t) + defer db.CloseDb() + vote := foldAcct(1, 10, nil) + vote.Owner = addresses.VoteProgramAddr + res := commitTestBatch(t, db, 10, vote) + path := segmentManifestPath(db.AcctsDir, 10, res.FileId) + m, err := ReadSegmentManifest(path) + require.NoError(t, err) + dataPath := filepath.Join(db.AcctsDir, SegmentDataName(10, res.FileId)) + switch kind { + case "offset": + m.Records[0].Offset = ^uint64(0) + case "pubkey": + m.Records[0].Pubkey[0]++ + case "payload": + data, err := os.ReadFile(dataPath) + require.NoError(t, err) + binary.LittleEndian.PutUint64(data[dataLenOffset:], ^uint64(0)) + require.NoError(t, os.WriteFile(dataPath, data, 0600)) + m.DataCRC = crc32.ChecksumIEEE(data) + } + require.NoError(t, WriteSegmentManifest(db.AcctsDir, m)) + require.NoError(t, db.Index.Delete(metaKeyLastBatch, pebble.Sync)) + require.NoError(t, db.Index.Delete(voteIndexKey(vote.Key), pebble.Sync)) + before := snapshotIndex(t, db) + _, err = db.RecoverFoldState() + require.ErrorContains(t, err, "restore vote candidates") + require.Equal(t, before, snapshotIndex(t, db)) + require.FileExists(t, path) + require.FileExists(t, dataPath) + _, haveMeta, err := db.readFoldMeta() + require.NoError(t, err) + require.False(t, haveMeta) + }) + } +} diff --git a/pkg/accountsdb/recovery.go b/pkg/accountsdb/recovery.go index cac4a38e..0586fa84 100644 --- a/pkg/accountsdb/recovery.go +++ b/pkg/accountsdb/recovery.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" + "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/Overclock-Validator/mithril/pkg/mlog" ) @@ -53,6 +54,9 @@ type RecoveryResult struct { func (db *AccountsDb) RecoverFoldState() (RecoveryResult, error) { db.foldMu.Lock() defer db.foldMu.Unlock() + finish := db.beginAccountChange() + success := false + defer func() { finish(nil, db.durableThrough.Load(), success, true) }() // Derived indexes must rebuild, including on partial failures. res := RecoveryResult{} @@ -95,6 +99,12 @@ func (db *AccountsDb) RecoverFoldState() (RecoveryResult, error) { mlog.Log.Warnf("accountsdb: fold manifest seq %d failed verification (%v); treating batch and everything above as undecided", seq, verr) break } + // Vote membership is derived from the CRC-checked segment, keeping the + // durable manifest compatible with pre-vote-index binaries. A read or + // metadata error must leave this decided batch intact for diagnosis/retry. + if err := db.restoreManifestVoteFlags(manifest); err != nil { + return res, fmt.Errorf("accountsdb: restore vote candidates for fold seq %d: %w", seq, err) + } if err := db.applyManifestToIndex(manifest); err != nil { return res, fmt.Errorf("accountsdb: replay fold manifest seq %d: %w", seq, err) } @@ -165,8 +175,19 @@ func (db *AccountsDb) RecoverFoldState() (RecoveryResult, error) { // reconciliation instead of condemning the store as data loss. res.RewindInProgress = hasParkedRewindManifests(db.AcctsDir) + // Recovery can also run in-process after a partial fold. Discard cached + // account bytes before rebuilding any derived view against the repaired index. + db.readCacheEpochMu.Lock() + db.readCacheEpoch++ + db.pendingFold = nil + if db.commonAdmission != nil { + db.resetReadCachesLocked() + } + db.readCacheEpochMu.Unlock() + db.lastBatchSeq = res.BatchSeq db.durableThrough.Store(res.DurableThrough) + success = true return res, nil } @@ -300,3 +321,39 @@ func (db *AccountsDb) removeStaleTmpManifests() ([]string, error) { } return removed, nil } + +// restoreManifestVoteFlags reconstructs the transient vote-index hints before +// replaying a durable manifest. Normal commits already have these hints from +// the accounts being written and do not reread the segment. Read fixed headers +// only, without loading account payloads or consulting the possibly stale index. +// The caller must first verify the segment length and CRC. +func (db *AccountsDb) restoreManifestVoteFlags(m *SegmentManifest) error { + if len(m.Records) == 0 { + return nil + } + f, err := os.Open(filepath.Join(db.AcctsDir, SegmentDataName(m.ThroughSlot, m.FileId))) + if err != nil { + return err + } + defer f.Close() + var header [hdrLen]byte + for i := range m.Records { + r := &m.Records[i] + if r.Offset > m.DataLen || m.DataLen-r.Offset < hdrLen || r.Offset > uint64(1<<63-1)-hdrLen { + return fmt.Errorf("account %x: header offset %d outside segment", r.Pubkey, r.Offset) + } + if _, err := f.ReadAt(header[:], int64(r.Offset)); err != nil { + return err + } + if [32]byte(header[pubkeyOffset:pubkeyOffset+32]) != r.Pubkey { + return fmt.Errorf("account %x: segment pubkey mismatch", r.Pubkey) + } + dataLen := binary.LittleEndian.Uint64(header[dataLenOffset : dataLenOffset+8]) + if dataLen > m.DataLen-r.Offset-hdrLen { + return fmt.Errorf("account %x: payload outside segment", r.Pubkey) + } + r.Vote = binary.LittleEndian.Uint64(header[lamportsOffset:lamportsOffset+8]) > 0 && + [32]byte(header[ownerOffset:ownerOffset+32]) == addresses.VoteProgramAddr + } + return nil +} diff --git a/pkg/accountsdb/rewind.go b/pkg/accountsdb/rewind.go index 6ad48b5b..6c202a78 100644 --- a/pkg/accountsdb/rewind.go +++ b/pkg/accountsdb/rewind.go @@ -153,6 +153,10 @@ func (db *AccountsDb) RewindToBatchBoundary(throughSlot uint64) (RewindResult, e return res, fmt.Errorf("accountsdb: rewind: store has no fold meta (nothing folded)") } if meta.ThroughSlot == throughSlot { + finish := db.beginAccountChange() + success := false + defer func() { finish(nil, db.durableThrough.Load(), success, true) }() // Derived indexes must rebuild, including on partial failures. + // This also repairs the in-memory publication state after a prior // WAL-less commit succeeded but its explicit Flush reported an error. db.readCacheEpochMu.Lock() @@ -176,6 +180,7 @@ func (db *AccountsDb) RewindToBatchBoundary(throughSlot uint64) (RewindResult, e // crashed before moving the undone files aside — finalize any leftover // parked manifests now so recovery stops reporting a rewind in progress. db.finalizeParkedRewindLeftoversLocked(throughSlot) + success = true return res, nil // already at the target boundary } @@ -229,6 +234,10 @@ func (db *AccountsDb) RewindToBatchBoundary(throughSlot uint64) (RewindResult, e } } + finish := db.beginAccountChange() + success := false + defer func() { finish(nil, db.durableThrough.Load(), success, true) }() // Derived indexes must rebuild, including on partial failures. + // Step 1: park suffix manifests (ascending) so recovery treats the // batches as undone even if we crash mid-rewind. for seq := seqT + 1; seq <= meta.BatchSeq; seq++ { @@ -322,5 +331,6 @@ func (db *AccountsDb) RewindToBatchBoundary(throughSlot uint64) (RewindResult, e res.NewThrough = target.manifest.ThroughSlot res.ResumeCtx = target.manifest.ResumeCtx mlog.Log.Warnf("accountsdb: REWOUND %d fold batch(es) (%d keys) — durable state restored to slot %d", res.UndoneBatches, res.UndoneKeys, res.NewThrough) + success = true return res, nil } diff --git a/pkg/accountsdb/segment.go b/pkg/accountsdb/segment.go index c3367df0..004fddbf 100644 --- a/pkg/accountsdb/segment.go +++ b/pkg/accountsdb/segment.go @@ -62,6 +62,7 @@ type ManifestRecord struct { Offset uint64 // record offset within this segment's data file OwnerSlot uint64 // slot that produced this version (observability) PrevValid bool // false => the key was absent from the index before this batch + Vote bool // derived in memory; not encoded in the v1 manifest Prev AccountIndexEntry // index entry before this batch overwrote it } @@ -138,11 +139,13 @@ func (m *SegmentManifest) encode() []byte { body.Write(r.Pubkey[:]) put64(r.Offset) put64(r.OwnerSlot) + // Version 1 encodes a boolean, not a bit field. Older rewind readers + // use pv == 1; adding a vote flag here would silently lose undo pointers. + var prevValid byte if r.PrevValid { - body.WriteByte(1) - } else { - body.WriteByte(0) + prevValid = 1 } + body.WriteByte(prevValid) r.Prev.Marshal(&prevBuf) body.Write(prevBuf[:]) } @@ -324,7 +327,14 @@ func ReadSegmentManifest(path string) (*SegmentManifest, error) { if err != nil { return nil, err } - r.PrevValid = pv == 1 + if pv&^3 != 0 { + return nil, ErrTornManifest + } + // Accept the flags emitted by the earlier RPC branch so those stores + // remain readable. New writes use only 0/1; recovery derives Vote from + // the segment even for these older flagged manifests. + r.PrevValid = pv&1 != 0 + r.Vote = pv&2 != 0 prevRaw, err := d.bytes(24) if err != nil { return nil, err diff --git a/pkg/accountsdb/sparse_compact_test.go b/pkg/accountsdb/sparse_compact_test.go index 6c4a2cae..8a1cd7b8 100644 --- a/pkg/accountsdb/sparse_compact_test.go +++ b/pkg/accountsdb/sparse_compact_test.go @@ -69,7 +69,7 @@ func TestSparseCompactionRetainsExactLiveAndUndoRecords(t *testing.T) { // reader; only obsolete payload bytes can differ. data, err := os.ReadFile(source) require.NoError(t, err) - keys, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 110, r1.FileId) + keys, _, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 110, r1.FileId) require.NoError(t, err) require.Len(t, keys, 3) _, err = db.RewindToBatchBoundary(130) diff --git a/pkg/accountsdb/vote_index.go b/pkg/accountsdb/vote_index.go new file mode 100644 index 00000000..982fd8f6 --- /dev/null +++ b/pkg/accountsdb/vote_index.go @@ -0,0 +1,153 @@ +package accountsdb + +import ( + "context" + "fmt" + + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/cockroachdb/pebble" + "github.com/gagliardetto/solana-go" +) + +// Candidate keys are separate from the 32-byte account-index keys. They are +// append-only; callers check the current account state before returning it. +const voteIndexPrefix = "\x00mithril.vote." + +var voteIndexReadyKey = []byte("\x00mithril.meta.vote_index_ready") + +func voteIndexKey(pubkey solana.PublicKey) []byte { + key := make([]byte, len(voteIndexPrefix)+len(pubkey)) + copy(key, voteIndexPrefix) + copy(key[len(voteIndexPrefix):], pubkey[:]) + return key +} + +// SeedVoteAccountPubkeys seeds a freshly built snapshot index with vote-program +// account candidates observed while parsing its appendvecs. +func (db *AccountsDb) SeedVoteAccountPubkeys(pubkeys []solana.PublicKey) error { + batch := db.Index.NewBatch() + defer batch.Close() + for _, pubkey := range pubkeys { + if err := batch.Set(voteIndexKey(pubkey), nil, nil); err != nil { + return err + } + } + if err := batch.Set(voteIndexReadyKey, nil, nil); err != nil { + return err + } + if err := batch.Commit(pebble.Sync); err != nil { + return err + } + if db.IndexWALDisabled { + return db.Index.Flush() + } + return nil +} + +// VoteAccountPubkeys returns every known vote-account candidate. Existing +// stores built before this index was added are migrated once, on first use. +func (db *AccountsDb) VoteAccountPubkeys(ctx context.Context) ([]solana.PublicKey, error) { + db.voteIndexMu.Lock() + defer db.voteIndexMu.Unlock() + if _, closer, err := db.Index.Get(voteIndexReadyKey); err == nil { + if err := closer.Close(); err != nil { + return nil, err + } + } else if err == pebble.ErrNotFound { + if err := db.migrateVoteIndex(ctx); err != nil { + return nil, err + } + } else { + return nil, err + } + + iter, err := db.Index.NewIter(&pebble.IterOptions{ + LowerBound: []byte(voteIndexPrefix), + UpperBound: []byte("\x00mithril.vote/"), + }) + if err != nil { + return nil, err + } + var pubkeys []solana.PublicKey + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + iter.Close() + return nil, err + } + key := iter.Key() + if len(key) != len(voteIndexPrefix)+32 { + iter.Close() + return nil, fmt.Errorf("invalid vote-index key length %d", len(key)) + } + pubkeys = append(pubkeys, solana.PublicKeyFromBytes(key[len(voteIndexPrefix):])) + } + iterErr := iter.Error() + closeErr := iter.Close() + if iterErr != nil { + return nil, iterErr + } + return pubkeys, closeErr +} + +// migrateVoteIndex scans an older store once. Finalized folds can run during +// the scan: they add their new vote candidates to the same append-only index. +func (db *AccountsDb) migrateVoteIndex(ctx context.Context) error { + iter, err := db.Index.NewIter(nil) + if err != nil { + return err + } + defer iter.Close() + batch := db.Index.NewBatch() + defer batch.Close() + keys := make([]solana.PublicKey, 0, 512) + flush := func() error { + if len(keys) == 0 { + return nil + } + accounts, err := db.GetAccountsBatch(ctx, db.DurableThrough(), keys) + if err != nil { + return err + } + for i, account := range accounts { + if account != nil && account.Lamports > 0 && account.Owner == addresses.VoteProgramAddr { + if err := batch.Set(voteIndexKey(keys[i]), nil, nil); err != nil { + return err + } + } + } + keys = keys[:0] + return nil + } + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + if key := iter.Key(); len(key) == 32 { + keys = append(keys, solana.PublicKeyFromBytes(key)) + if len(keys) == cap(keys) { + if err := flush(); err != nil { + return err + } + } + } + } + if err := iter.Error(); err != nil { + return err + } + if err := flush(); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + if err := batch.Set(voteIndexReadyKey, nil, nil); err != nil { + return err + } + if err := batch.Commit(pebble.Sync); err != nil { + return err + } + if db.IndexWALDisabled { + return db.Index.Flush() + } + return nil +} diff --git a/pkg/accountsdb/vote_index_test.go b/pkg/accountsdb/vote_index_test.go new file mode 100644 index 00000000..bdbbc300 --- /dev/null +++ b/pkg/accountsdb/vote_index_test.go @@ -0,0 +1,70 @@ +package accountsdb + +import ( + "context" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/cockroachdb/pebble" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestVoteIndexMigratesExistingStoreAndPersists(t *testing.T) { + db, dir := newFoldTestDb(t) + vote := &accounts.Account{Key: solana.PublicKey{1}, Owner: addresses.VoteProgramAddr, Lamports: 1} + ordinary := &accounts.Account{Key: solana.PublicKey{2}, Lamports: 1} + _, err := db.CommitBatch(foldDeltas(accounts.SlotDelta{Slot: 10, Delta: []*accounts.Account{vote, ordinary}}), 10, nil, nil) + require.NoError(t, err) + // Mimic a database written before the vote index existed. + require.NoError(t, db.Index.Delete(voteIndexKey(vote.Key), pebble.Sync)) + db.VoteAcctCache.Clear() + got, err := db.VoteAccountPubkeys(context.Background()) + require.NoError(t, err) + require.Equal(t, []solana.PublicKey{vote.Key}, got) + + db = reopenFoldTestDb(t, db, dir) + defer db.CloseDb() + _, err = db.RecoverFoldState() + require.NoError(t, err) + got, err = db.VoteAccountPubkeys(context.Background()) + require.NoError(t, err) + require.Equal(t, []solana.PublicKey{vote.Key}, got) +} + +func TestVoteIndexSnapshotSeedAndFoldRecovery(t *testing.T) { + db, dir := newFoldTestDb(t) + seedVote := solana.PublicKey{1} + foldVote := &accounts.Account{Key: solana.PublicKey{2}, Owner: addresses.VoteProgramAddr, Lamports: 1} + require.NoError(t, db.SeedVoteAccountPubkeys([]solana.PublicKey{seedVote})) + _, err := db.CommitBatch(foldDeltas(accounts.SlotDelta{Slot: 10, Delta: []*accounts.Account{foldVote}}), 10, nil, nil) + require.NoError(t, err) + got, err := db.VoteAccountPubkeys(context.Background()) + require.NoError(t, err) + require.Equal(t, []solana.PublicKey{seedVote, foldVote.Key}, got) + + // Simulate a lost index tail: recovery must restore the candidate from the + // durable fold manifest along with the account's primary index entry. + require.NoError(t, db.Index.Delete(voteIndexKey(foldVote.Key), pebble.Sync)) + require.NoError(t, db.Index.Delete(metaKeyLastBatch, pebble.Sync)) + db = reopenFoldTestDb(t, db, dir) + defer db.CloseDb() + _, err = db.RecoverFoldState() + require.NoError(t, err) + got, err = db.VoteAccountPubkeys(context.Background()) + require.NoError(t, err) + require.Equal(t, []solana.PublicKey{seedVote, foldVote.Key}, got) +} + +func TestVoteIndexMigrationCanRetryAfterCancellation(t *testing.T) { + db, _ := newFoldTestDb(t) + defer db.CloseDb() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := db.VoteAccountPubkeys(ctx) + require.ErrorIs(t, err, context.Canceled) + got, err := db.VoteAccountPubkeys(context.Background()) + require.NoError(t, err) + require.Empty(t, got) +} diff --git a/pkg/replay/block.go b/pkg/replay/block.go index 191f1b73..6418ba1e 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -46,9 +46,11 @@ import ( "github.com/panjf2000/ants/v2" ) -// SlotCtxSetter is implemented by types that accept a SlotCtx update (e.g. RpcServer). +// SlotCtxSetter is implemented by the RPC server so replay can publish both +// its live execution bank and the latest bank durably folded into AccountsDB. type SlotCtxSetter interface { SetSlotCtx(slotCtx *sealevel.SlotCtx) + SetRootedBankState(slot, blockHeight, transactionCount uint64) } type epochRewardPublisher interface { @@ -2194,6 +2196,13 @@ func ReplayBlocks( mlog.Log.Errorf("failed to advance rooted epoch rewards through slot %d: %v", promotedThrough, err) } } + if rpcServer != nil { + var transactionCount uint64 + if rootedCtx.TransactionCount != nil { + transactionCount = *rootedCtx.TransactionCount + } + rpcServer.SetRootedBankState(promotedThrough, rootedCtx.BlockHeight, transactionCount) + } if rewardsCompletion.retire(&partitionedRewardsInfo, promotedThrough) { rewardsHoldBelowSlot = 0 mlog.Log.Infof("epoch rewards bookkeeping retired through durable slot %d; later fork switches may unwind in memory", promotedThrough) @@ -3400,6 +3409,9 @@ func ReplayBlocks( if rpcServer != nil { rpcServer.SetSlotCtx(lastSlotCtx) + if unrootedTailState == nil { + rpcServer.SetRootedBankState(block.Slot, global.BlockHeight(), global.TransactionCount()) + } } replayCtx.Capitalization -= lastSlotCtx.LamportsBurnt diff --git a/pkg/rpcserver/get_account_info.go b/pkg/rpcserver/get_account_info.go index 11d01e4c..22886133 100644 --- a/pkg/rpcserver/get_account_info.go +++ b/pkg/rpcserver/get_account_info.go @@ -3,12 +3,12 @@ package rpcserver import ( "context" "encoding/base64" + "errors" "fmt" - "reflect" "github.com/DataDog/zstd" - "github.com/Overclock-Validator/mithril/pkg/global" - "github.com/Overclock-Validator/mithril/pkg/mlog" + "github.com/Overclock-Validator/mithril/pkg/accountsdb" + "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/Overclock-Validator/mithril/pkg/safemath" "github.com/filecoin-project/go-jsonrpc" "github.com/gagliardetto/solana-go" @@ -54,7 +54,7 @@ const ( ) func (rpcServer *RpcServer) GetAccountInfo(ctx context.Context, p jsonrpc.RawParams) (GetAccountInfoResp, error) { - params, err := jsonrpc.DecodeParams[[]interface{}](p) + params, err := decodeOptionalParams(p) if err != nil { return GetAccountInfoResp{}, fmt.Errorf("decoding params: %w", err) } @@ -73,18 +73,31 @@ func (rpcServer *RpcServer) GetAccountInfo(ctx context.Context, p jsonrpc.RawPar return GetAccountInfoResp{}, fmt.Errorf("invalid base58 encoding") } - acct, err := rpcServer.acctsDb.GetAccount(0, pk) - if err != nil { - return GetAccountInfoResp{}, nil - } - conf, err := parseGetAccountInfoConfMap(params) if err != nil { - return GetAccountInfoResp{}, err + return GetAccountInfoResp{}, &InvalidParamsError{Message: err.Error()} + } + rooted, acct, err := rpcServer.readRootedAccount(ctx, pk) + if err != nil && !errors.Is(err, accountsdb.ErrNoAccount) { + return GetAccountInfoResp{}, fmt.Errorf("read account at rooted slot %d: %w", rooted.Slot, err) + } + if conf.MinContextSlot > rooted.Slot { + return GetAccountInfoResp{}, &MinContextSlotNotReachedError{ContextSlot: rooted.Slot} + } + if errors.Is(err, accountsdb.ErrNoAccount) { + return GetAccountInfoResp{Context: GetAccountInfoRespContext{ApiVersion: "mithril 0.1", Slot: rooted.Slot}}, nil } var acctData interface{} - acctData, err = encodeAcctDataWithConfig(acct.Data, conf) + if conf.EncodingType != nil && *conf.EncodingType == GetAccountEncodingJson && conf.DataSlice != nil { + return GetAccountInfoResp{}, fmt.Errorf("cannot use jsonParsed with dataSlice") + } + if conf.EncodingType != nil && *conf.EncodingType == GetAccountEncodingJson && acct.Owner == addresses.VoteProgramAddr { + acctData, err = parsedVoteAccountData(acct.Data, pk) + } + if acctData == nil || err != nil { + acctData, err = encodeAcctDataWithConfig(acct.Data, conf) + } if err != nil { return GetAccountInfoResp{}, err } @@ -97,11 +110,11 @@ func (rpcServer *RpcServer) GetAccountInfo(ctx context.Context, p jsonrpc.RawPar RentEpoch: acct.RentEpoch, Space: uint64(len(acct.Data))} - return GetAccountInfoResp{Context: GetAccountInfoRespContext{ApiVersion: "mithril 0.1", Slot: global.Slot()}, Value: val}, nil + return GetAccountInfoResp{Context: GetAccountInfoRespContext{ApiVersion: "mithril 0.1", Slot: rooted.Slot}, Value: val}, nil } func parseGetAccountInfoConfMap(params []interface{}) (*GetAccountInfoConfig, error) { - if len(params) < 2 { + if len(params) < 2 || params[1] == nil { return &GetAccountInfoConfig{}, nil } @@ -113,19 +126,23 @@ func parseGetAccountInfoConfMap(params []interface{}) (*GetAccountInfoConfig, er conf := &GetAccountInfoConfig{} - // commitment is optional - commitmentObj, ok := confMap["commitment"] - if ok { - commitmentStr, ok := commitmentObj.(string) - if !ok { - return nil, fmt.Errorf("invalid commitment") + if err := validateRootedCommitment(confMap["commitment"]); err != nil { + return nil, err + } + if commitment, ok := confMap["commitment"].(string); ok { + conf.Commitment = commitment + } + + if minContextSlot, ok := confMap["minContextSlot"]; ok && minContextSlot != nil { + conf.MinContextSlot, err = rpcUint64(minContextSlot, "minContextSlot") + if err != nil { + return nil, err } - conf.Commitment = commitmentStr } // encoding type is optional. defaults to base58. encodingObj, ok := confMap["encoding"] - if ok { + if ok && encodingObj != nil { encodingStr, ok := encodingObj.(string) if ok { var encodingType int @@ -141,8 +158,7 @@ func parseGetAccountInfoConfMap(params []interface{}) (*GetAccountInfoConfig, er // dataSlice is optional. if present, it must contain both 'length' and 'offset' fields. dataSliceObj, ok := confMap["dataSlice"] - if ok { - mlog.Log.Infof("dataSliceObj: %+v, type: %s", dataSliceObj, reflect.TypeOf(dataSliceObj)) + if ok && dataSliceObj != nil { dsMap, ok := dataSliceObj.(map[string]interface{}) if ok { conf.DataSlice, err = parseDataSlice(dsMap) @@ -161,30 +177,22 @@ func parseDataSlice(dataSlice map[string]interface{}) (*GetAccountInfoDataSlice, lengthObj, hasLen := dataSlice["length"] offsetObj, hasOffset := dataSlice["offset"] - if hasLen && !hasOffset { + if !hasOffset { return nil, fmt.Errorf("missing dataSlice field offset") - } else if !hasLen && hasOffset { + } else if !hasLen { return nil, fmt.Errorf("missing dataSlice field length") } - ds := &GetAccountInfoDataSlice{} - if hasLen == hasOffset && hasLen { - length, ok := lengthObj.(float64) - if !ok { - return nil, fmt.Errorf("invalid dataSlice length, expected usize, got %s", reflect.TypeOf(lengthObj)) - } - offset, ok := offsetObj.(float64) - if !ok { - return nil, fmt.Errorf("invalid dataSlice offset, expected usize, got %s", reflect.TypeOf(offsetObj)) - } - - lengthUint := uint64(length) - offsetUint := uint64(offset) - ds.Len = &lengthUint - ds.Offset = &offsetUint + length, err := rpcUint64(lengthObj, "dataSlice length") + if err != nil { + return nil, err + } + offset, err := rpcUint64(offsetObj, "dataSlice offset") + if err != nil { + return nil, err } - return ds, nil + return &GetAccountInfoDataSlice{Len: &length, Offset: &offset}, nil } func parseGetAcctDataEncodingType(encodingStr string) (int, error) { @@ -199,7 +207,7 @@ func parseGetAcctDataEncodingType(encodingStr string) (int, error) { return GetAccountEncodingBase64Zstd, nil case "jsonParsed": - return GetAccountEncodingBase64, nil + return GetAccountEncodingJson, nil default: return 0, fmt.Errorf("invalid data encoding %s", encodingStr) diff --git a/pkg/rpcserver/get_account_info_parsed_test.go b/pkg/rpcserver/get_account_info_parsed_test.go new file mode 100644 index 00000000..9f9e7f08 --- /dev/null +++ b/pkg/rpcserver/get_account_info_parsed_test.go @@ -0,0 +1,128 @@ +package rpcserver + +import ( + "encoding/json" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetAccountInfoParsesVoteAccountAtRoot(t *testing.T) { + const rootedSlot = 500 + votePubkey := solana.PublicKey{1} + nodePubkey := solana.PublicKey{2} + withdrawer := solana.PublicKey{3} + collector := solana.PublicKey{4} + blockCollector := solana.PublicKey{5} + authorized := solana.PublicKey{6} + rootSlot := uint64(490) + bls := [48]byte{7} + state := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4} + state.V4.NodePubkey = nodePubkey + state.V4.AuthorizedWithdrawer = withdrawer + state.V4.InflationRewardsCollector = collector + state.V4.BlockRevenueCollector = blockCollector + state.V4.InflationRewardsCommissionBps = 755 + state.V4.BlockRevenueCommissionBps = 321 + state.V4.PendingDelegatorRewards = 42 + state.V4.BlsPubkeyCompressed = &bls + state.V4.RootSlot = &rootSlot + state.V4.AuthorizedVoters.AuthorizedVoters.Set(3, authorized) + state.V4.Votes.PushBack(sealevel.LandedVote{Latency: 2, Lockout: sealevel.VoteLockout{Slot: 499, ConfirmationCount: 4}}) + state.V4.EpochCredits = []sealevel.EpochCredits{{Epoch: 3, Credits: 9, PrevCredits: 5}} + state.V4.LastTimestamp = sealevel.BlockTimestamp{Slot: 499, Timestamp: 1_700_000_000} + + db := newRPCAccountsDB(t) + account := voteAccountForRPC(t, votePubkey, &state) + account.Lamports = 27_074_400 + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: rootedSlot, Delta: []*accounts.Account{account}}}, rootedSlot, nil, nil) + require.NoError(t, err) + + server := &RpcServer{acctsDb: db} + server.SetRootedBankState(rootedSlot, 495, 100) + got, err := server.GetAccountInfo(t.Context(), mustRawParams(t, []interface{}{ + votePubkey.String(), + map[string]interface{}{"commitment": "finalized", "encoding": "jsonParsed", "minContextSlot": float64(rootedSlot)}, + })) + require.NoError(t, err) + require.Equal(t, uint64(rootedSlot), got.Context.Slot) + require.NotNil(t, got.Value) + require.Equal(t, uint64(27_074_400), got.Value.Lamports) + + raw, err := json.Marshal(got.Value.Data) + require.NoError(t, err) + require.Contains(t, string(raw), `"lastTimestamp":{"slot":499,"timestamp":1700000000}`) + var decoded struct { + Program string `json:"program"` + Parsed struct { + Type string `json:"type"` + Info parsedVoteAccountInfo `json:"info"` + } `json:"parsed"` + Space uint64 `json:"space"` + } + require.NoError(t, json.Unmarshal(raw, &decoded)) + require.Equal(t, "vote", decoded.Program) + require.Equal(t, "vote", decoded.Parsed.Type) + require.Equal(t, nodePubkey.String(), decoded.Parsed.Info.NodePubkey) + require.Equal(t, withdrawer.String(), decoded.Parsed.Info.AuthorizedWithdrawer) + require.Equal(t, uint8(8), decoded.Parsed.Info.Commission) + require.Equal(t, uint16(755), decoded.Parsed.Info.InflationRewardsCommissionBPS) + require.Equal(t, "42", decoded.Parsed.Info.PendingDelegatorRewards) + require.Equal(t, parsedBlockTimestamp{Slot: 499, Timestamp: 1_700_000_000}, decoded.Parsed.Info.LastTimestamp) + require.Equal(t, []parsedAuthorizedVoter{{Epoch: 3, AuthorizedVoter: authorized.String()}}, decoded.Parsed.Info.AuthorizedVoters) + require.Equal(t, []parsedVote{{Latency: 2, Slot: 499, ConfirmationCount: 4}}, decoded.Parsed.Info.Votes) + require.Equal(t, []parsedEpochCredits{{Epoch: 3, Credits: "9", PreviousCredits: "5"}}, decoded.Parsed.Info.EpochCredits) +} + +func TestGetAccountInfoJsonParsedFallsBackToBase64(t *testing.T) { + db := newRPCAccountsDB(t) + address := solana.PublicKey{9} + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 10, Delta: []*accounts.Account{{Key: address, Lamports: 1, Data: []byte{1, 2, 3}}}}}, 10, nil, nil) + require.NoError(t, err) + server := &RpcServer{acctsDb: db} + server.SetRootedBankState(10, 9, 0) + + got, err := server.GetAccountInfo(t.Context(), mustRawParams(t, []interface{}{ + address.String(), map[string]interface{}{"encoding": "jsonParsed"}, + })) + require.NoError(t, err) + require.Equal(t, []string{"AQID", "base64"}, got.Value.Data) +} + +func TestGetAccountInfoParsesLegacyVoteAccountWithV4Defaults(t *testing.T) { + votePubkey := solana.PublicKey{1} + nodePubkey := solana.PublicKey{2} + state := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionCurrent} + state.Current.NodePubkey = nodePubkey + state.Current.Commission = 7 + + db := newRPCAccountsDB(t) + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 10, Delta: []*accounts.Account{ + voteAccountForRPC(t, votePubkey, &state), + }}}, 10, nil, nil) + require.NoError(t, err) + server := &RpcServer{acctsDb: db} + server.SetRootedBankState(10, 9, 0) + + got, err := server.GetAccountInfo(t.Context(), mustRawParams(t, []interface{}{ + votePubkey.String(), map[string]interface{}{"encoding": "jsonParsed"}, + })) + require.NoError(t, err) + raw, err := json.Marshal(got.Value.Data) + require.NoError(t, err) + var decoded struct { + Parsed struct { + Info parsedVoteAccountInfo `json:"info"` + } `json:"parsed"` + } + require.NoError(t, json.Unmarshal(raw, &decoded)) + require.Equal(t, uint8(7), decoded.Parsed.Info.Commission) + require.Equal(t, uint16(700), decoded.Parsed.Info.InflationRewardsCommissionBPS) + require.Equal(t, votePubkey.String(), decoded.Parsed.Info.InflationRewardsCollector) + require.Equal(t, nodePubkey.String(), decoded.Parsed.Info.BlockRevenueCollector) + require.Equal(t, uint16(10000), decoded.Parsed.Info.BlockRevenueCommissionBPS) + require.Equal(t, "0", decoded.Parsed.Info.PendingDelegatorRewards) +} diff --git a/pkg/rpcserver/get_account_info_test.go b/pkg/rpcserver/get_account_info_test.go new file mode 100644 index 00000000..48ad909d --- /dev/null +++ b/pkg/rpcserver/get_account_info_test.go @@ -0,0 +1,129 @@ +package rpcserver + +import ( + "encoding/json" + "fmt" + "net/http/httptest" + "strings" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetAccountInfoParameterHTTP(t *testing.T) { + db := newRPCAccountsDB(t) + key := solana.PublicKey{87} + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 10, Delta: []*accounts.Account{{Key: key, Lamports: 42, Data: []byte("abcd")}}}}, 10, nil, nil) + require.NoError(t, err) + server := NewRpcServer(db, 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 432000}, solana.Hash{1}) + require.NoError(t, server.listener.Close()) + server.SetRootedBankState(10, 10, 0) + endpoint := httptest.NewServer(server) + t.Cleanup(endpoint.Close) + for _, test := range []struct { + config string + code int + data string + }{ + {`{"encoding":"base64","minContextSlot":10}`, 0, "YWJjZA=="}, + {`{"encoding":"base64","minContextSlot":10,"dataSlice":{"offset":1,"length":2}}`, 0, "YmM="}, + {`{"encoding":"base64","dataSlice":{"offset":18446744073709551615,"length":1}}`, 0, ""}, + {`{"encoding":"base64","commitment":null,"minContextSlot":null,"dataSlice":null}`, 0, "YWJjZA=="}, + {`{"minContextSlot":9007199254740993}`, -32016, ""}, + {`{"minContextSlot":1.0000000000000001}`, -32602, ""}, + {`{"dataSlice":{"offset":-1,"length":1}}`, -32602, ""}, + {`{"dataSlice":{"offset":0,"length":1.5}}`, -32602, ""}, + {`{"dataSlice":{}}`, -32602, ""}, + {`{"dataSlice":{"offset":0,"length":null}}`, -32602, ""}, + } { + t.Run(test.config, func(t *testing.T) { + body := fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":[%q,%s]}`, key.String(), test.config) + response, err := endpoint.Client().Post(endpoint.URL, "application/json", strings.NewReader(body)) + require.NoError(t, err) + defer response.Body.Close() + var got struct { + Result GetAccountInfoResp `json:"result"` + Error *struct { + Code int `json:"code"` + } `json:"error"` + } + require.NoError(t, json.NewDecoder(response.Body).Decode(&got)) + if test.code != 0 { + require.NotNil(t, got.Error) + require.Equal(t, test.code, got.Error.Code) + return + } + require.Nil(t, got.Error) + require.Equal(t, uint64(10), got.Result.Context.Slot) + require.NotNil(t, got.Result.Value) + require.Equal(t, []interface{}{test.data, "base64"}, got.Result.Value.Data) + }) + } +} + +func TestGetAccountInfoAccountExistence(t *testing.T) { + for _, state := range []string{"never-created", "funded-empty-data", "deleted"} { + t.Run(state, func(t *testing.T) { + db := newRPCAccountsDB(t) + server := &RpcServer{acctsDb: db} + address := solana.PublicKey{87} + slot := uint64(10) + lamports := uint64(0) + server.SetRootedBankState(slot, slot, 0) + if state != "never-created" { + lamports = 42 + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: []*accounts.Account{{Key: address, Lamports: lamports}}}}, slot, nil, nil) + require.NoError(t, err) + // Prime the account-read cache before deleting, and confirm that a live + // account with empty data is not mistaken for a missing placeholder. + funded, err := server.GetAccountInfo(t.Context(), mustRawParams(t, []interface{}{address.String()})) + require.NoError(t, err) + require.NotNil(t, funded.Value) + require.Equal(t, lamports, funded.Value.Lamports) + } + if state == "deleted" { + slot++ + lamports = 0 + // A zero-lamport tombstone is absent even if it retains payload bytes. + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: []*accounts.Account{{Key: address, Data: []byte{1, 2, 3}}}}}, slot, nil, nil) + require.NoError(t, err) + server.SetRootedBankState(slot, slot, 0) + } + for _, encoding := range []string{"", "base58", "base64", "base64+zstd", "jsonParsed"} { + params := []interface{}{address.String()} + if encoding != "" { + params = append(params, map[string]interface{}{"encoding": encoding}) + } + got, err := server.GetAccountInfo(t.Context(), mustRawParams(t, params)) + require.NoError(t, err) + require.Equal(t, slot, got.Context.Slot) + if lamports == 0 { + require.Nil(t, got.Value, "state=%s encoding=%s", state, encoding) + wire, err := json.Marshal(got) + require.NoError(t, err) + var decoded map[string]json.RawMessage + require.NoError(t, json.Unmarshal(wire, &decoded)) + require.Equal(t, "null", string(decoded["value"]), "value must be present as JSON null") + } else { + require.NotNil(t, got.Value) + require.Equal(t, lamports, got.Value.Lamports) + } + } + balance, err := server.GetBalance(t.Context(), mustRawParams(t, []interface{}{address.String()})) + require.NoError(t, err) + require.Equal(t, slot, balance.Context.Slot) + require.Equal(t, lamports, balance.Value) + + params := mustRawParams(t, []interface{}{address.String(), map[string]interface{}{"minContextSlot": slot + 1}}) + _, err = server.GetAccountInfo(t.Context(), params) + var minSlotErr *MinContextSlotNotReachedError + require.ErrorAs(t, err, &minSlotErr, "missing accounts must still enforce minContextSlot") + require.Equal(t, slot, minSlotErr.ContextSlot) + _, err = server.GetBalance(t.Context(), params) + require.ErrorAs(t, err, &minSlotErr) + }) + } +} diff --git a/pkg/rpcserver/get_balance.go b/pkg/rpcserver/get_balance.go new file mode 100644 index 00000000..23ec3050 --- /dev/null +++ b/pkg/rpcserver/get_balance.go @@ -0,0 +1,87 @@ +package rpcserver + +import ( + "context" + "errors" + "fmt" + + "github.com/Overclock-Validator/mithril/pkg/accountsdb" + "github.com/filecoin-project/go-jsonrpc" + "github.com/gagliardetto/solana-go" +) + +type GetBalanceResp struct { + Context GetBalanceRespContext `json:"context"` + Value uint64 `json:"value"` +} + +type GetBalanceRespContext struct { + APIVersion string `json:"apiVersion"` + Slot uint64 `json:"slot"` +} + +func (rpcServer *RpcServer) GetBalance(ctx context.Context, p jsonrpc.RawParams) (GetBalanceResp, error) { + if err := ctx.Err(); err != nil { + return GetBalanceResp{}, err + } + + params, err := decodeOptionalParams(p) + if err != nil { + return GetBalanceResp{}, &InvalidParamsError{Message: fmt.Sprintf("decoding params: %v", err)} + } + if len(params) < 1 || len(params) > 2 { + return GetBalanceResp{}, &InvalidParamsError{Message: "getBalance requires an address and optional config"} + } + address, ok := params[0].(string) + if !ok { + return GetBalanceResp{}, &InvalidParamsError{Message: "getBalance requires an address as first parameter"} + } + pubkey, err := solana.PublicKeyFromBase58(address) + if err != nil { + return GetBalanceResp{}, &InvalidParamsError{Message: "Invalid param: Invalid"} + } + + minContextSlot, err := parseBalanceConfig(params) + if err != nil { + return GetBalanceResp{}, err + } + rooted, account, err := rpcServer.readRootedAccount(ctx, pubkey) + if err != nil && !errors.Is(err, accountsdb.ErrNoAccount) { + return GetBalanceResp{}, fmt.Errorf("read balance at rooted slot %d: %w", rooted.Slot, err) + } + if minContextSlot != nil && rooted.Slot < *minContextSlot { + return GetBalanceResp{}, &MinContextSlotNotReachedError{ContextSlot: rooted.Slot} + } + + var lamports uint64 + if account != nil { + lamports = account.Lamports + } + + return GetBalanceResp{ + Context: GetBalanceRespContext{APIVersion: "mithril 0.1", Slot: rooted.Slot}, + Value: lamports, + }, nil +} + +func parseBalanceConfig(params []interface{}) (*uint64, error) { + if len(params) == 1 || params[1] == nil { + return nil, nil + } + config, ok := params[1].(map[string]interface{}) + if !ok { + return nil, &InvalidParamsError{Message: "invalid getBalance config"} + } + if err := validateRootedCommitment(config["commitment"]); err != nil { + return nil, err + } + value, exists := config["minContextSlot"] + if !exists || value == nil { + return nil, nil + } + minContextSlot, err := rpcUint64(value, "minContextSlot") + if err != nil { + return nil, err + } + return &minContextSlot, nil +} diff --git a/pkg/rpcserver/get_balance_test.go b/pkg/rpcserver/get_balance_test.go new file mode 100644 index 00000000..8ce0831f --- /dev/null +++ b/pkg/rpcserver/get_balance_test.go @@ -0,0 +1,112 @@ +package rpcserver + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/accountsdb" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetBalanceUsesPublishedRootedBank(t *testing.T) { + db := newRPCAccountsDB(t) + address := solana.PublicKey{7} + _, err := db.CommitBatch([]accounts.SlotDelta{{ + Slot: 42, + Delta: []*accounts.Account{{ + Key: address, + Lamports: 123_456_789, + }}, + }}, 42, nil, nil) + require.NoError(t, err) + + server := &RpcServer{acctsDb: db} + server.SetRootedBankState(42, 40, 10) + got, err := server.GetBalance(t.Context(), mustRawParams(t, []interface{}{ + address.String(), + map[string]interface{}{"commitment": "confirmed", "minContextSlot": float64(42)}, + })) + require.NoError(t, err) + require.Equal(t, uint64(42), got.Context.Slot) + require.Equal(t, uint64(123_456_789), got.Value) + + _, err = server.GetBalance(t.Context(), mustRawParams(t, []interface{}{ + address.String(), + map[string]interface{}{"minContextSlot": float64(43)}, + })) + var minSlotErr *MinContextSlotNotReachedError + require.True(t, errors.As(err, &minSlotErr)) + require.Equal(t, uint64(42), minSlotErr.ContextSlot) +} + +func TestGetBalanceMissingAccountIsZero(t *testing.T) { + server := &RpcServer{acctsDb: newRPCAccountsDB(t)} + server.SetRootedBankState(9, 8, 0) + + got, err := server.GetBalance(t.Context(), mustRawParams(t, []interface{}{solana.PublicKey{8}.String()})) + require.NoError(t, err) + require.Equal(t, uint64(9), got.Context.Slot) + require.Zero(t, got.Value) +} + +func TestGetBalanceDoesNotMixNewAccountsWithOldRoot(t *testing.T) { + db := newRPCAccountsDB(t) + address := solana.PublicKey{9} + _, err := db.CommitBatch([]accounts.SlotDelta{{ + Slot: 43, + Delta: []*accounts.Account{{ + Key: address, + Lamports: 43, + }}, + }}, 43, nil, nil) + require.NoError(t, err) + + server := &RpcServer{acctsDb: db} + server.SetRootedBankState(42, 40, 10) + deadline, cancel := context.WithTimeout(t.Context(), 5*time.Millisecond) + defer cancel() + _, err = server.GetBalance(deadline, mustRawParams(t, []interface{}{address.String()})) + require.ErrorIs(t, err, context.DeadlineExceeded) + + server.SetRootedBankState(43, 41, 11) + got, err := server.GetBalance(t.Context(), mustRawParams(t, []interface{}{address.String()})) + require.NoError(t, err) + require.Equal(t, uint64(43), got.Context.Slot) + require.Equal(t, uint64(43), got.Value) +} + +func TestGetBalanceRejectsInvalidParams(t *testing.T) { + server := &RpcServer{} + for _, params := range [][]interface{}{ + {}, + {true}, + {"not-a-pubkey"}, + {solana.PublicKey{1}.String(), true}, + {solana.PublicKey{1}.String(), map[string]interface{}{"commitment": "unknown"}}, + {solana.PublicKey{1}.String(), map[string]interface{}{"minContextSlot": 1.5}}, + } { + _, err := server.GetBalance(t.Context(), mustRawParams(t, params)) + var invalid *InvalidParamsError + require.True(t, errors.As(err, &invalid), "params: %#v; error: %v", params, err) + } +} + +func newRPCAccountsDB(t testing.TB) *accountsdb.AccountsDb { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "accounts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "largest_file_id"), make([]byte, 8), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "bootstrap_high_file_id"), make([]byte, 8), 0o644)) + db, err := accountsdb.OpenDb(dir) + require.NoError(t, err) + db.RootedDurable = true + db.InitCaches() + t.Cleanup(db.CloseDb) + return db +} diff --git a/pkg/rpcserver/get_block_production.go b/pkg/rpcserver/get_block_production.go new file mode 100644 index 00000000..55096307 --- /dev/null +++ b/pkg/rpcserver/get_block_production.go @@ -0,0 +1,185 @@ +package rpcserver + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/filecoin-project/go-jsonrpc" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" +) + +type GetBlockProductionResp struct { + Context GetBalanceRespContext `json:"context"` + Value BlockProductionResults `json:"value"` +} + +type BlockProductionResults struct { + ByIdentity map[string][2]uint64 `json:"byIdentity"` + Range BlockProductionRange `json:"range"` +} + +type BlockProductionRange struct { + FirstSlot uint64 `json:"firstSlot"` + LastSlot uint64 `json:"lastSlot"` +} + +type getBlockProductionConfig struct { + identity *solana.PublicKey + firstSlot *uint64 + lastSlot *uint64 +} + +func (rpcServer *RpcServer) GetBlockProduction(ctx context.Context, p jsonrpc.RawParams) (GetBlockProductionResp, error) { + if err := ctx.Err(); err != nil { + return GetBlockProductionResp{}, err + } + params, err := decodeOptionalParams(p) + if err != nil { + return GetBlockProductionResp{}, &InvalidParamsError{Message: fmt.Sprintf("decoding params: %v", err)} + } + config, err := parseBlockProductionConfig(params) + if err != nil { + return GetBlockProductionResp{}, err + } + if rpcServer.epochSchedule == nil { + return GetBlockProductionResp{}, fmt.Errorf("node has no epoch schedule available") + } + rooted, historyAccount, err := rpcServer.readRootedAccount(ctx, sealevel.SysvarSlotHistoryAddr) + if err != nil { + return GetBlockProductionResp{}, fmt.Errorf("read slot history at rooted slot %d: %w", rooted.Slot, err) + } + + firstSlot := rpcServer.epochSchedule.FirstSlotInEpoch(rpcServer.epochSchedule.GetEpoch(rooted.Slot)) + if config.firstSlot != nil { + firstSlot = *config.firstSlot + } + lastSlot := rooted.Slot + if config.lastSlot != nil { + lastSlot = *config.lastSlot + } + if lastSlot < firstSlot { + return GetBlockProductionResp{}, &InvalidParamsError{Message: fmt.Sprintf("lastSlot, %d, cannot be less than firstSlot, %d", lastSlot, firstSlot)} + } + + var history sealevel.SysvarSlotHistory + if err := history.UnmarshalWithDecoder(bin.NewBinDecoder(historyAccount.Data)); err != nil { + return GetBlockProductionResp{}, fmt.Errorf("decode slot history at rooted slot %d: %w", rooted.Slot, err) + } + oldest, newest, err := slotHistoryBounds(&history) + if err != nil { + return GetBlockProductionResp{}, err + } + if firstSlot < oldest { + return GetBlockProductionResp{}, &InvalidParamsError{Message: fmt.Sprintf("firstSlot, %d, is too small; min %d", firstSlot, oldest)} + } + if lastSlot > newest { + return GetBlockProductionResp{}, &InvalidParamsError{Message: fmt.Sprintf("lastSlot, %d, is too large; max %d", lastSlot, newest)} + } + + byIdentity := make(map[string][2]uint64) + for slot := firstSlot; ; slot++ { + leader, ok := global.LeaderForSlot(slot) + if !ok { + return GetBlockProductionResp{}, fmt.Errorf("leader schedule unavailable for slot %d", slot) + } + if config.identity == nil || leader == *config.identity { + production := byIdentity[leader.String()] + production[0]++ + if slotHistoryContains(&history, slot) { + production[1]++ + } + byIdentity[leader.String()] = production + } + if slot == lastSlot { + break + } + } + + return GetBlockProductionResp{ + Context: GetBalanceRespContext{APIVersion: "mithril 0.1", Slot: rooted.Slot}, + Value: BlockProductionResults{ + ByIdentity: byIdentity, + Range: BlockProductionRange{FirstSlot: firstSlot, LastSlot: lastSlot}, + }, + }, nil +} + +func parseBlockProductionConfig(params []interface{}) (getBlockProductionConfig, error) { + var config getBlockProductionConfig + if len(params) > 1 { + return config, &InvalidParamsError{Message: "getBlockProduction accepts at most one config object"} + } + if len(params) == 0 || params[0] == nil { + return config, nil + } + values, ok := params[0].(map[string]interface{}) + if !ok { + return config, &InvalidParamsError{Message: "invalid getBlockProduction config"} + } + if err := validateRootedCommitment(values["commitment"]); err != nil { + return config, err + } + if raw, exists := values["identity"]; exists && raw != nil { + value, ok := raw.(string) + if !ok { + return config, &InvalidParamsError{Message: "invalid identity"} + } + identity, err := solana.PublicKeyFromBase58(value) + if err != nil { + return config, &InvalidParamsError{Message: "invalid identity"} + } + config.identity = &identity + } + if raw, exists := values["range"]; exists && raw != nil { + rangeValues, ok := raw.(map[string]interface{}) + if !ok { + return config, &InvalidParamsError{Message: "invalid range"} + } + first, exists := rangeValues["firstSlot"] + if !exists { + return config, &InvalidParamsError{Message: "range requires firstSlot"} + } + firstSlot, err := rpcUint64(first, "firstSlot") + if err != nil { + return config, err + } + config.firstSlot = &firstSlot + if last, exists := rangeValues["lastSlot"]; exists && last != nil { + lastSlot, err := rpcUint64(last, "lastSlot") + if err != nil { + return config, err + } + config.lastSlot = &lastSlot + } + } + return config, nil +} + +func rpcUint64(raw interface{}, name string) (uint64, error) { + value, ok := raw.(json.Number) + if !ok { + return 0, &InvalidParamsError{Message: fmt.Sprintf("invalid %s", name)} + } + parsed, err := strconv.ParseUint(string(value), 10, 64) + if err != nil { + return 0, &InvalidParamsError{Message: fmt.Sprintf("invalid %s", name)} + } + return parsed, nil +} + +func slotHistoryBounds(history *sealevel.SysvarSlotHistory) (uint64, uint64, error) { + if history == nil || history.Bits.Len == 0 || history.Bits.Bits.BlocksLen == 0 || len(history.Bits.Bits.Blocks) < int(history.Bits.Bits.BlocksLen) || history.NextSlot == 0 { + return 0, 0, fmt.Errorf("node has no usable slot history") + } + return history.NextSlot - min(history.NextSlot, history.Bits.Len), history.NextSlot - 1, nil +} + +func slotHistoryContains(history *sealevel.SysvarSlotHistory, slot uint64) bool { + block := (slot / 64) % history.Bits.Bits.BlocksLen + return history.Bits.Bits.Blocks[block]&(uint64(1)<<(slot%64)) != 0 +} diff --git a/pkg/rpcserver/get_block_production_test.go b/pkg/rpcserver/get_block_production_test.go new file mode 100644 index 00000000..19e7e396 --- /dev/null +++ b/pkg/rpcserver/get_block_production_test.go @@ -0,0 +1,86 @@ +package rpcserver + +import ( + "errors" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/leaderschedule" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetBlockProductionUsesRootedSlotHistory(t *testing.T) { + const rootedSlot = uint64(13) + firstLeader := solana.PublicKey{1} + secondLeader := solana.PublicKey{2} + global.SetLeaderScheduleForEpoch(0, leaderschedule.NewLeaderScheduleFromKeyedSlots( + map[solana.PublicKey][]uint64{ + firstLeader: {10, 11}, + secondLeader: {12, 13}, + }, + 0, + )) + t.Cleanup(func() { global.SetLeaderScheduleForEpoch(0, nil) }) + + history := sealevel.SysvarSlotHistory{ + Bits: sealevel.SlotHistoryBitvec{ + Bits: sealevel.SlotHistoryInner{BlocksLen: 1, Blocks: []uint64{(1 << 10) | (1 << 12) | (1 << 13)}}, + Len: 64, + }, + NextSlot: rootedSlot + 1, + } + db := newRPCAccountsDB(t) + _, err := db.CommitBatch([]accounts.SlotDelta{{ + Slot: rootedSlot, + Delta: []*accounts.Account{{ + Key: sealevel.SysvarSlotHistoryAddr, + Lamports: 1, + Data: history.MustMarshal(), + }}, + }}, rootedSlot, nil, nil) + require.NoError(t, err) + + server := &RpcServer{ + acctsDb: db, + epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 32}, + } + server.SetRootedBankState(rootedSlot, 12, 0) + got, err := server.GetBlockProduction(t.Context(), mustRawParams(t, []interface{}{ + map[string]interface{}{ + "commitment": "finalized", + "range": map[string]interface{}{ + "firstSlot": float64(10), + "lastSlot": float64(rootedSlot), + }, + }, + })) + require.NoError(t, err) + require.Equal(t, rootedSlot, got.Context.Slot) + require.Equal(t, BlockProductionRange{FirstSlot: 10, LastSlot: 13}, got.Value.Range) + require.Equal(t, [2]uint64{2, 1}, got.Value.ByIdentity[firstLeader.String()]) + require.Equal(t, [2]uint64{2, 2}, got.Value.ByIdentity[secondLeader.String()]) +} + +func TestGetBlockProductionRejectsInvalidRange(t *testing.T) { + server := &RpcServer{ + acctsDb: newRPCAccountsDB(t), + epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 32}, + } + server.SetRootedBankState(13, 12, 0) + for _, params := range [][]interface{}{ + {true}, + {map[string]interface{}{"commitment": "unknown"}}, + {map[string]interface{}{"identity": "invalid"}}, + {map[string]interface{}{"range": true}}, + {map[string]interface{}{"range": map[string]interface{}{}}}, + {map[string]interface{}{"range": map[string]interface{}{"firstSlot": 1.5}}}, + {map[string]interface{}{}, map[string]interface{}{}}, + } { + _, err := server.GetBlockProduction(t.Context(), mustRawParams(t, params)) + var invalid *InvalidParamsError + require.True(t, errors.As(err, &invalid), "params: %#v; error: %v", params, err) + } +} diff --git a/pkg/rpcserver/get_epoch_info.go b/pkg/rpcserver/get_epoch_info.go index 2ee4a9cd..f60c0a84 100644 --- a/pkg/rpcserver/get_epoch_info.go +++ b/pkg/rpcserver/get_epoch_info.go @@ -18,25 +18,61 @@ type GetEpochInfoResp struct { } func (rpcServer *RpcServer) GetEpochInfo(ctx context.Context, p jsonrpc.RawParams) (GetEpochInfoResp, error) { - params, err := jsonrpc.DecodeParams[[]interface{}](p) + params, err := decodeOptionalParams(p) if err != nil { - return GetEpochInfoResp{}, fmt.Errorf("decoding params: %w", err) + return GetEpochInfoResp{}, &InvalidParamsError{Message: fmt.Sprintf("decoding params: %v", err)} + } + if len(params) > 1 { + return GetEpochInfoResp{}, &InvalidParamsError{Message: "getEpochInfo accepts at most one config object"} + } + commitment := "finalized" + var minContextSlot *uint64 + if len(params) == 1 && params[0] != nil { + config, ok := params[0].(map[string]interface{}) + if !ok { + return GetEpochInfoResp{}, &InvalidParamsError{Message: "invalid getEpochInfo config"} + } + if raw, exists := config["commitment"]; exists && raw != nil { + value, ok := raw.(string) + if !ok || (value != "processed" && value != "confirmed" && value != "finalized") { + return GetEpochInfoResp{}, &InvalidParamsError{Message: "invalid commitment"} + } + commitment = value + } + if raw, exists := config["minContextSlot"]; exists && raw != nil { + value, err := rpcUint64(raw, "minContextSlot") + if err != nil { + return GetEpochInfoResp{}, err + } + minContextSlot = &value + } + } + if rpcServer.epochSchedule == nil { + return GetEpochInfoResp{}, fmt.Errorf("node has no epoch schedule available") } - - _ = params - - epoch := global.Epoch() slot := global.Slot() - firstSlotInEpoch := rpcServer.epochSchedule.FirstSlotInEpoch(epoch) - slotIndex := slot - firstSlotInEpoch + blockHeight := global.BlockHeight() + transactionCount := global.TransactionCount() + if commitment != "processed" { + rooted, ok := rpcServer.getRootedBankState() + if !ok { + return GetEpochInfoResp{}, fmt.Errorf("node has no rooted bank available") + } + slot, blockHeight, transactionCount = rooted.Slot, rooted.BlockHeight, rooted.TransactionCount + } + if minContextSlot != nil && slot < *minContextSlot { + return GetEpochInfoResp{}, &MinContextSlotNotReachedError{ContextSlot: slot} + } + epoch := rpcServer.epochSchedule.GetEpoch(slot) + slotIndex := slot - rpcServer.epochSchedule.FirstSlotInEpoch(epoch) resp := GetEpochInfoResp{ - AbsoluteSlot: global.Slot(), - BlockHeight: global.BlockHeight(), - Epoch: global.Epoch(), + AbsoluteSlot: slot, + BlockHeight: blockHeight, + Epoch: epoch, SlotIndex: slotIndex, SlotsInEpoch: 432000, - TransactionCount: global.TransactionCount(), + TransactionCount: transactionCount, } return resp, nil diff --git a/pkg/rpcserver/get_epoch_info_rooted_test.go b/pkg/rpcserver/get_epoch_info_rooted_test.go new file mode 100644 index 00000000..0d133641 --- /dev/null +++ b/pkg/rpcserver/get_epoch_info_rooted_test.go @@ -0,0 +1,33 @@ +package rpcserver + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/filecoin-project/go-jsonrpc" + "github.com/stretchr/testify/require" +) + +func TestGetEpochInfoUsesRootedBankForFinalized(t *testing.T) { + oldSlot, oldEpoch := global.Slot(), global.Epoch() + oldHeight, oldTransactions := global.BlockHeight(), global.TransactionCount() + t.Cleanup(func() { + global.SetSlot(oldSlot) + global.SetEpoch(oldEpoch) + global.SetBlockHeight(oldHeight) + global.SetTransactionCount(oldTransactions) + }) + global.SetSlot(432250) + global.SetEpoch(1) + global.SetBlockHeight(432000) + global.SetTransactionCount(2000) + + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 432000, LeaderScheduleSlotOffset: 432000}} + server.SetRootedBankState(432123, 431900, 1900) + got, err := server.GetEpochInfo(t.Context(), jsonrpc.RawParams(`[{"commitment":"finalized"}]`)) + require.NoError(t, err) + require.Equal(t, uint64(432123), got.AbsoluteSlot) + require.Equal(t, uint64(431900), got.BlockHeight) + require.Equal(t, uint64(1900), got.TransactionCount) +} diff --git a/pkg/rpcserver/get_leader_schedule.go b/pkg/rpcserver/get_leader_schedule.go new file mode 100644 index 00000000..aeaebc79 --- /dev/null +++ b/pkg/rpcserver/get_leader_schedule.go @@ -0,0 +1,123 @@ +package rpcserver + +import ( + "context" + "fmt" + + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/filecoin-project/go-jsonrpc" + "github.com/gagliardetto/solana-go" +) + +type getLeaderScheduleConfig struct { + identity *solana.PublicKey + keyByVoteAccount bool +} + +func (rpcServer *RpcServer) GetLeaderSchedule(ctx context.Context, p jsonrpc.RawParams) (map[string][]uint64, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + params, err := decodeOptionalParams(p) + if err != nil { + return nil, &InvalidParamsError{Message: fmt.Sprintf("decoding params: %v", err)} + } + if len(params) > 2 { + return nil, &InvalidParamsError{Message: "getLeaderSchedule accepts an optional slot and config"} + } + rooted, ok := rpcServer.getRootedBankState() + if !ok { + return nil, fmt.Errorf("node has no rooted bank available") + } + if rpcServer.epochSchedule == nil { + return nil, fmt.Errorf("node has no epoch schedule available") + } + + slot := rooted.Slot + var rawConfig interface{} + if len(params) > 0 && params[0] != nil { + if _, ok := params[0].(map[string]interface{}); ok { + if len(params) != 1 { + return nil, &InvalidParamsError{Message: "leader schedule config cannot be provided twice"} + } + rawConfig = params[0] + } else { + value, err := rpcUint64(params[0], "slot") + if err != nil { + return nil, err + } + slot = value + } + } + if len(params) == 2 { + rawConfig = params[1] + } + config, err := parseLeaderScheduleConfig(rawConfig) + if err != nil { + return nil, err + } + + epoch := rpcServer.epochSchedule.GetEpoch(slot) + firstSlot := rpcServer.epochSchedule.FirstSlotInEpoch(epoch) + slotsInEpoch := rpcServer.epochSchedule.SlotsInEpoch(epoch) + schedule := make(map[string][]uint64) + for index := uint64(0); index < slotsInEpoch; index++ { + scheduledSlot := firstSlot + index + var leader, key solana.PublicKey + var ok bool + if config.keyByVoteAccount { + leader, key, ok = global.LeaderForSlotWithVoteAccount(scheduledSlot) + if !ok { + if _, exists := global.LeaderForSlot(scheduledSlot); exists { + return nil, fmt.Errorf("vote-account leader schedule unavailable for slot %d", scheduledSlot) + } + return nil, nil + } + } else { + leader, ok = global.LeaderForSlot(scheduledSlot) + if !ok { + return nil, nil + } + key = leader + } + if config.identity != nil && leader != *config.identity { + continue + } + keyString := key.String() + schedule[keyString] = append(schedule[keyString], index) + } + return schedule, nil +} + +func parseLeaderScheduleConfig(raw interface{}) (getLeaderScheduleConfig, error) { + var config getLeaderScheduleConfig + if raw == nil { + return config, nil + } + values, ok := raw.(map[string]interface{}) + if !ok { + return config, &InvalidParamsError{Message: "invalid getLeaderSchedule config"} + } + if err := validateRootedCommitment(values["commitment"]); err != nil { + return config, err + } + if raw, exists := values["identity"]; exists && raw != nil { + value, ok := raw.(string) + if !ok { + return config, &InvalidParamsError{Message: "invalid identity"} + } + identity, err := solana.PublicKeyFromBase58(value) + if err != nil { + return config, &InvalidParamsError{Message: "invalid identity"} + } + config.identity = &identity + } + if raw, exists := values["keyByVoteAccount"]; exists && raw != nil { + value, ok := raw.(bool) + if !ok { + return config, &InvalidParamsError{Message: "invalid keyByVoteAccount"} + } + config.keyByVoteAccount = value + } + return config, nil +} diff --git a/pkg/rpcserver/get_leader_schedule_test.go b/pkg/rpcserver/get_leader_schedule_test.go new file mode 100644 index 00000000..43ee1dec --- /dev/null +++ b/pkg/rpcserver/get_leader_schedule_test.go @@ -0,0 +1,130 @@ +package rpcserver + +import ( + "errors" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/leaderschedule" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetLeaderScheduleReturnsRelativeEpochSlots(t *testing.T) { + epochSchedule := &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 4} + firstSlot := epochSchedule.FirstSlotInEpoch(3) + firstLeader := solana.PublicKey{1} + secondLeader := solana.PublicKey{2} + global.SetLeaderScheduleForEpoch(3, leaderschedule.NewLeaderScheduleFromKeyedSlots( + map[solana.PublicKey][]uint64{ + firstLeader: {0, 2}, + secondLeader: {1, 3}, + }, + firstSlot, + )) + t.Cleanup(func() { global.SetLeaderScheduleForEpoch(3, nil) }) + + server := &RpcServer{epochSchedule: epochSchedule} + server.SetRootedBankState(firstSlot+2, firstSlot+2, 0) + got, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{ + float64(firstSlot + 1), + map[string]interface{}{"commitment": "confirmed"}, + })) + require.NoError(t, err) + require.Equal(t, []uint64{0, 2}, got[firstLeader.String()]) + require.Equal(t, []uint64{1, 3}, got[secondLeader.String()]) + + filtered, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{ + float64(firstSlot), + map[string]interface{}{"identity": secondLeader.String()}, + })) + require.NoError(t, err) + require.Equal(t, map[string][]uint64{secondLeader.String(): {1, 3}}, filtered) + + filtered, err = server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{ + map[string]interface{}{"commitment": "finalized", "identity": secondLeader.String()}, + })) + require.NoError(t, err) + require.Equal(t, map[string][]uint64{secondLeader.String(): {1, 3}}, filtered) + + missing, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{ + float64(firstSlot), map[string]interface{}{"identity": solana.PublicKey{9}.String()}, + })) + require.NoError(t, err) + require.Empty(t, missing) +} + +func TestGetLeaderScheduleKeysBySelectedVoteAccount(t *testing.T) { + epochSchedule := &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 32} + const epoch = 13 + firstSlot := epochSchedule.FirstSlotInEpoch(epoch) + node := solana.PublicKey{31} + other := solana.PublicKey{32} + votes := map[solana.PublicKey]*epochstakes.VoteAccount{ + {41}: {NodePubkey: node}, + {42}: {NodePubkey: node}, + {43}: {NodePubkey: other}, + } + schedule := leaderschedule.New(votes, map[solana.PublicKey]uint64{{41}: 100, {42}: 100, {43}: 100}, epochSchedule, epoch, 32, 1) + global.SetLeaderScheduleForEpoch(epoch, schedule) + t.Cleanup(func() { global.SetLeaderScheduleForEpoch(epoch, nil) }) + server := &RpcServer{epochSchedule: epochSchedule} + server.SetRootedBankState(firstSlot, firstSlot, 0) + for _, identity := range []string{"", node.String(), other.String()} { + config := map[string]interface{}{"keyByVoteAccount": true} + if identity != "" { + config["identity"] = identity + } + want := make(map[string][]uint64) + for index := uint64(0); index < 32; index++ { + leader, vote, ok := schedule.LeaderForSlotWithVoteAccount(firstSlot + index) + require.True(t, ok) + if identity == "" || leader.String() == identity { + want[vote.String()] = append(want[vote.String()], index) + } + } + got, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{nil, config})) + require.NoError(t, err) + require.Equal(t, want, got) + if identity == node.String() { + require.Len(t, got, 2, "two vote accounts sharing an identity must stay distinct") + } + } + global.SetLeaderScheduleForEpoch(epoch, leaderschedule.NewLeaderScheduleFromKeyedSlots(map[solana.PublicKey][]uint64{node: {0}}, firstSlot)) + got, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{nil, map[string]interface{}{"keyByVoteAccount": true}})) + require.ErrorContains(t, err, "vote-account leader schedule unavailable") + require.Nil(t, got, "node-keyed compatibility schedules must not invent vote accounts") + global.SetLeaderScheduleForEpoch(epoch, nil) + got, err = server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{nil, map[string]interface{}{"keyByVoteAccount": true}})) + require.NoError(t, err) + require.Nil(t, got, "missing epochs keep the standard null result") +} + +func TestGetLeaderScheduleReturnsNullWhenEpochScheduleIsUnavailable(t *testing.T) { + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 4}} + server.SetRootedBankState(40_002, 1, 0) + + got, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{float64(40_002)})) + require.NoError(t, err) + require.Nil(t, got) +} + +func TestGetLeaderScheduleRejectsInvalidParams(t *testing.T) { + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 4}} + server.SetRootedBankState(1, 1, 0) + for _, params := range [][]interface{}{ + {-1.0}, + {1.5}, + {float64(1), true}, + {float64(1), map[string]interface{}{"commitment": "unknown"}}, + {float64(1), map[string]interface{}{"identity": "invalid"}}, + {float64(1), map[string]interface{}{"keyByVoteAccount": "true"}}, + {float64(1), nil, nil}, + } { + _, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, params)) + var invalid *InvalidParamsError + require.True(t, errors.As(err, &invalid), "params: %#v; error: %v", params, err) + } +} diff --git a/pkg/rpcserver/get_slot.go b/pkg/rpcserver/get_slot.go new file mode 100644 index 00000000..7dbb52fb --- /dev/null +++ b/pkg/rpcserver/get_slot.go @@ -0,0 +1,56 @@ +package rpcserver + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/filecoin-project/go-jsonrpc" +) + +// GetSlot returns the slot at the requested commitment, defaulting to finalized. +func (rpcServer *RpcServer) GetSlot(ctx context.Context, p jsonrpc.RawParams) (uint64, error) { + if err := ctx.Err(); err != nil { + return 0, err + } + var params []json.RawMessage + if len(p) != 0 { + if err := json.Unmarshal(p, ¶ms); err != nil { + return 0, &InvalidParamsError{Message: fmt.Sprintf("decoding params: %v", err)} + } + } + if len(params) > 1 { + return 0, &InvalidParamsError{Message: "getSlot accepts at most one config object"} + } + var config struct { + Commitment *string `json:"commitment"` + MinContextSlot *uint64 `json:"minContextSlot"` + } + if len(params) == 1 { + if err := json.Unmarshal(params[0], &config); err != nil { + return 0, &InvalidParamsError{Message: fmt.Sprintf("invalid getSlot config: %v", err)} + } + } + commitment := "finalized" + if config.Commitment != nil { + commitment = *config.Commitment + } + if commitment != "processed" && commitment != "confirmed" && commitment != "finalized" { + return 0, &InvalidParamsError{Message: "invalid commitment"} + } + + slot := global.Slot() + if commitment != "processed" { + // Match getEpochInfo's published rooted view for confirmed and finalized. + rooted, ok := rpcServer.getRootedBankState() + if !ok { + return 0, fmt.Errorf("node has no rooted bank available") + } + slot = rooted.Slot + } + if config.MinContextSlot != nil && slot < *config.MinContextSlot { + return 0, &MinContextSlotNotReachedError{ContextSlot: slot} + } + return slot, nil +} diff --git a/pkg/rpcserver/get_slot_test.go b/pkg/rpcserver/get_slot_test.go new file mode 100644 index 00000000..a61e7780 --- /dev/null +++ b/pkg/rpcserver/get_slot_test.go @@ -0,0 +1,121 @@ +package rpcserver + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetSlotHTTP(t *testing.T) { + oldSlot := global.Slot() + t.Cleanup(func() { global.SetSlot(oldSlot) }) + global.SetSlot(150) + server := NewRpcServer(nil, 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 512}, solana.Hash{}) + t.Cleanup(func() { require.NoError(t, server.listener.Close()) }) + server.SetRootedBankState(100, 90, 0) + httpServer := httptest.NewServer(server) + t.Cleanup(httpServer.Close) + + check := func(t *testing.T, params string, want uint64, code int) { + t.Helper() + body := `{"jsonrpc":"2.0","id":7,"method":"getSlot"` + if params != "" { + body += `,"params":` + params + } + resp, err := httpServer.Client().Post(httpServer.URL, "application/json", strings.NewReader(body+`}`)) + require.NoError(t, err) + defer resp.Body.Close() + var got struct { + JSONRPC string `json:"jsonrpc"` + ID int `json:"id"` + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Data struct { + ContextSlot uint64 `json:"contextSlot"` + } `json:"data"` + } `json:"error"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + require.Equal(t, "2.0", got.JSONRPC) + require.Equal(t, 7, got.ID) + if code != 0 { + require.NotNil(t, got.Error) + require.Equal(t, code, got.Error.Code) + require.Empty(t, got.Result) + if code == -32016 { + require.Equal(t, want, got.Error.Data.ContextSlot) + } + return + } + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Nil(t, got.Error) + require.Equal(t, fmt.Sprint(want), string(got.Result)) + } + + for _, tc := range []struct { + name string + params string + want uint64 + code int + }{ + {"omitted", "", 100, 0}, + {"empty", `[]`, 100, 0}, + {"null params", `null`, 100, 0}, + {"null config", `[null]`, 100, 0}, + {"empty config", `[{}]`, 100, 0}, + {"null fields", `[{"commitment":null,"minContextSlot":null}]`, 100, 0}, + {"finalized", `[{"commitment":"finalized"}]`, 100, 0}, + {"confirmed", `[{"commitment":"confirmed"}]`, 100, 0}, + {"processed", `[{"commitment":"processed"}]`, 150, 0}, + {"root boundary", `[{"minContextSlot":100}]`, 100, 0}, + {"root too old", `[{"minContextSlot":101}]`, 100, -32016}, + {"confirmed too old", `[{"commitment":"confirmed","minContextSlot":101}]`, 100, -32016}, + {"processed boundary", `[{"commitment":"processed","minContextSlot":150}]`, 150, 0}, + {"processed too old", `[{"commitment":"processed","minContextSlot":151}]`, 150, -32016}, + {"unknown field", `[{"futureOption":true}]`, 100, 0}, + {"extra argument", `[{},{}]`, 0, -32602}, + {"scalar config", `[true]`, 0, -32602}, + {"array config", `[[]]`, 0, -32602}, + {"unknown commitment", `[{"commitment":"invalid"}]`, 0, -32602}, + {"empty commitment", `[{"commitment":""}]`, 0, -32602}, + {"typed commitment", `[{"commitment":1}]`, 0, -32602}, + {"negative slot", `[{"minContextSlot":-1}]`, 0, -32602}, + {"fractional slot", `[{"minContextSlot":1.5}]`, 0, -32602}, + {"string slot", `[{"minContextSlot":"100"}]`, 0, -32602}, + {"overflow slot", `[{"minContextSlot":18446744073709551616}]`, 0, -32602}, + } { + t.Run(tc.name, func(t *testing.T) { check(t, tc.params, tc.want, tc.code) }) + } + + t.Run("exact uint64", func(t *testing.T) { + server.SetRootedBankState(1<<53, 0, 0) + check(t, `[{"minContextSlot":9007199254740993}]`, 1<<53, -32016) + server.SetRootedBankState(^uint64(0), 0, 0) + check(t, `[{"minContextSlot":18446744073709551615}]`, ^uint64(0), 0) + }) + t.Run("no rooted bank", func(t *testing.T) { + server.rootedBank.Store(nil) + check(t, `[]`, 0, 1) + check(t, `[{"commitment":"processed"}]`, 150, 0) + }) + t.Run("zero slot", func(t *testing.T) { + server.SetRootedBankState(0, 0, 0) + check(t, `[]`, 0, 0) + }) + t.Run("canceled request", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err := server.GetSlot(ctx, nil) + require.ErrorIs(t, err, context.Canceled) + }) +} diff --git a/pkg/rpcserver/get_vote_accounts.go b/pkg/rpcserver/get_vote_accounts.go new file mode 100644 index 00000000..78ca737f --- /dev/null +++ b/pkg/rpcserver/get_vote_accounts.go @@ -0,0 +1,201 @@ +package rpcserver + +import ( + "context" + "errors" + "fmt" + "slices" + + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/filecoin-project/go-jsonrpc" + "github.com/gagliardetto/solana-go" +) + +const ( + delinquentValidatorSlotDistance = 128 + maxRPCEpochCreditsHistory = 5 +) + +type GetVoteAccountsResp struct { + Current []VoteAccountInfo `json:"current"` + Delinquent []VoteAccountInfo `json:"delinquent"` +} + +type VoteAccountInfo struct { + VotePubkey string `json:"votePubkey"` + NodePubkey string `json:"nodePubkey"` + ActivatedStake uint64 `json:"activatedStake"` + Commission uint8 `json:"commission"` + InflationRewardsCommissionBPS uint16 `json:"inflationRewardsCommissionBps"` + EpochCredits [][3]uint64 `json:"epochCredits"` + EpochVoteAccount bool `json:"epochVoteAccount"` + LastVote uint64 `json:"lastVote"` + RootSlot uint64 `json:"rootSlot"` +} + +type getVoteAccountsConfig struct { + votePubkey *solana.PublicKey + delinquentSlotDistance uint64 + keepUnstakedDelinquents bool +} + +func (rpcServer *RpcServer) GetVoteAccounts(ctx context.Context, p jsonrpc.RawParams) (GetVoteAccountsResp, error) { + if err := ctx.Err(); err != nil { + return GetVoteAccountsResp{}, err + } + params, err := decodeOptionalParams(p) + if err != nil { + return GetVoteAccountsResp{}, &InvalidParamsError{Message: fmt.Sprintf("decoding params: %v", err)} + } + config, err := parseGetVoteAccountsConfig(params) + if err != nil { + return GetVoteAccountsResp{}, err + } + if rpcServer.epochSchedule == nil { + return GetVoteAccountsResp{}, fmt.Errorf("node has no epoch schedule available") + } + if rpcServer.acctsDb == nil { + return GetVoteAccountsResp{}, fmt.Errorf("node has no accounts database available") + } + + for { + if err := ctx.Err(); err != nil { + return GetVoteAccountsResp{}, err + } + bank := rpcServer.rootedBank.Load() + version, stable := rpcServer.acctsDb.CommittedAccountVersion() + if !stable { + if err := waitForRootedPublication(ctx); err != nil { + return GetVoteAccountsResp{}, err + } + continue + } + rooted, ok := rpcServer.getRootedBankState() + if !ok { + return GetVoteAccountsResp{}, fmt.Errorf("node has no rooted bank available") + } + epoch := rpcServer.epochSchedule.GetEpoch(rooted.Slot) + stakes, ok := global.EpochStakesSnapshot(epoch) + if !ok { + return GetVoteAccountsResp{}, fmt.Errorf("epoch stakes unavailable for rooted epoch %d", epoch) + } + activeStakes, err := rpcServer.rootedActivatedStakes(ctx, rooted.Slot, epoch, config.votePubkey) + if errors.Is(err, errStakeViewChanged) { + if err := waitForRootedPublication(ctx); err != nil { + return GetVoteAccountsResp{}, err + } + continue + } + if err != nil { + return GetVoteAccountsResp{}, err + } + + var votePubkeys []solana.PublicKey + if config.votePubkey != nil { + votePubkeys = []solana.PublicKey{*config.votePubkey} + } else { + votePubkeys, err = rpcServer.acctsDb.VoteAccountPubkeys(ctx) + if err != nil { + return GetVoteAccountsResp{}, fmt.Errorf("list vote accounts: %w", err) + } + // Epoch stakes also cover stores created before the durable index was built. + for key := range stakes.Stakes { + votePubkeys = append(votePubkeys, key) + } + slices.SortFunc(votePubkeys, func(a, b solana.PublicKey) int { return slices.Compare(a[:], b[:]) }) + votePubkeys = slices.Compact(votePubkeys) + } + records, err := rpcServer.rootedVoteRecords(ctx, rooted.Slot, version, votePubkeys) + if errors.Is(err, errStakeViewChanged) { + continue + } + if err != nil { + return GetVoteAccountsResp{}, fmt.Errorf("read vote accounts at rooted slot %d: %w", rooted.Slot, err) + } + + response := GetVoteAccountsResp{ + Current: make([]VoteAccountInfo, 0, len(votePubkeys)), + Delinquent: make([]VoteAccountInfo, 0), + } + for index, votePubkey := range votePubkeys { + record := records[index] + if record == nil { + continue + } + info := *record + // Response slices are owned by the caller, never by the retained cache. + info.EpochCredits = make([][3]uint64, len(record.EpochCredits)) + copy(info.EpochCredits, record.EpochCredits) + info.ActivatedStake = activeStakes[votePubkey] + _, info.EpochVoteAccount = stakes.Stakes[votePubkey] + lastVote := info.LastVote + current := lastVote > 0 + if rooted.Slot >= config.delinquentSlotDistance { + current = lastVote > rooted.Slot-config.delinquentSlotDistance + } + if current { + response.Current = append(response.Current, info) + } else if info.ActivatedStake > 0 || config.keepUnstakedDelinquents { + response.Delinquent = append(response.Delinquent, info) + } + } + latest, stable := rpcServer.acctsDb.CommittedAccountVersion() + if !stable || latest != version || rpcServer.rootedBank.Load() != bank || rpcServer.rootedPublicationPending(rooted.Slot) { + continue + } + return response, nil + } +} + +func parseGetVoteAccountsConfig(params []interface{}) (getVoteAccountsConfig, error) { + config := getVoteAccountsConfig{delinquentSlotDistance: delinquentValidatorSlotDistance} + if len(params) > 1 { + return config, &InvalidParamsError{Message: "getVoteAccounts accepts at most one config object"} + } + if len(params) == 0 || params[0] == nil { + return config, nil + } + values, ok := params[0].(map[string]interface{}) + if !ok { + return config, &InvalidParamsError{Message: "invalid getVoteAccounts config"} + } + if err := validateRootedCommitment(values["commitment"]); err != nil { + return config, err + } + if raw, exists := values["votePubkey"]; exists && raw != nil { + value, ok := raw.(string) + if !ok { + return config, &InvalidParamsError{Message: "invalid votePubkey"} + } + pubkey, err := solana.PublicKeyFromBase58(value) + if err != nil { + return config, &InvalidParamsError{Message: "invalid votePubkey"} + } + config.votePubkey = &pubkey + } + if raw, exists := values["keepUnstakedDelinquents"]; exists && raw != nil { + value, ok := raw.(bool) + if !ok { + return config, &InvalidParamsError{Message: "invalid keepUnstakedDelinquents"} + } + config.keepUnstakedDelinquents = value + } + if raw, exists := values["delinquentSlotDistance"]; exists && raw != nil { + value, err := rpcUint64(raw, "delinquentSlotDistance") + if err != nil { + return config, err + } + config.delinquentSlotDistance = value + } + return config, nil +} + +func voteCommission(versioned *sealevel.VoteStateVersions, current *sealevel.VoteState) (uint8, uint16) { + if versioned.Type == sealevel.VoteStateVersionV4 { + basisPoints := versioned.V4.InflationRewardsCommissionBps + percent := (uint32(basisPoints) + 99) / 100 + return uint8(min(uint32(255), percent)), basisPoints + } + return current.Commission, uint16(current.Commission) * 100 +} diff --git a/pkg/rpcserver/get_vote_accounts_test.go b/pkg/rpcserver/get_vote_accounts_test.go new file mode 100644 index 00000000..db5608c7 --- /dev/null +++ b/pkg/rpcserver/get_vote_accounts_test.go @@ -0,0 +1,250 @@ +package rpcserver + +import ( + "bytes" + "errors" + "math" + "path/filepath" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetVoteAccountsUsesRootedVoteState(t *testing.T) { + const ( + rootedSlot = 500 + epoch = 0 + ) + currentVote := solana.PublicKey{1} + delinquentVote := solana.PublicKey{2} + newVote := solana.PublicKey{3} + invalidVote := solana.PublicKey{5} + currentNode := solana.PublicKey{11} + delinquentNode := solana.PublicKey{12} + newNode := solana.PublicKey{13} + currentRoot := uint64(498) + delinquentRoot := uint64(299) + + currentState := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4} + currentState.V4.NodePubkey = currentNode + currentState.V4.AuthorizedVoters.AuthorizedVoters.Set(0, currentNode) + currentState.V4.InflationRewardsCommissionBps = 755 + currentState.V4.RootSlot = ¤tRoot + currentState.V4.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: 499}}) + currentState.V4.EpochCredits = []sealevel.EpochCredits{ + {Epoch: 0, Credits: 1}, + {Epoch: 1, Credits: 2, PrevCredits: 1}, + {Epoch: 2, Credits: 3, PrevCredits: 2}, + {Epoch: 3, Credits: 4, PrevCredits: 3}, + {Epoch: 4, Credits: 5, PrevCredits: 4}, + {Epoch: 5, Credits: 9, PrevCredits: 5}, + } + + delinquentState := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionCurrent} + delinquentState.Current.NodePubkey = delinquentNode + delinquentState.Current.AuthorizedVoters.AuthorizedVoters.Set(0, delinquentNode) + delinquentState.Current.Commission = 5 + delinquentState.Current.RootSlot = &delinquentRoot + delinquentState.Current.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: 300}}) + + newState := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionCurrent} + newState.Current.NodePubkey = newNode + newState.Current.AuthorizedVoters.AuthorizedVoters.Set(0, newNode) + newState.Current.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: 498}}) + uninitializedVote := solana.PublicKey{4} + uninitializedState := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionCurrent} + + db := newRPCAccountsDB(t) + global.ClearPendingStakePubkeys() + t.Cleanup(func() { + global.EnqueuePendingStakePubkey(rootedSlot, solana.PublicKey{21}) + _, err := global.FlushPendingStakePubkeys(filepath.Join(db.AcctsDir, "..")) + require.NoError(t, err) + global.ClearPendingStakePubkeys() + }) + stakeHistory := sealevel.SysvarStakeHistory{} + var historyData bytes.Buffer + require.NoError(t, stakeHistory.MarshalWithEncoder(bin.NewBinEncoder(&historyData))) + stakeAccount := func(key, vote solana.PublicKey, lamports uint64) *accounts.Account { + state := &sealevel.StakeStateV2{Status: sealevel.StakeStateV2StatusStake} + state.Stake.Stake.Delegation = sealevel.Delegation{ + VoterPubkey: vote, StakeLamports: lamports, + ActivationEpoch: math.MaxUint64, DeactivationEpoch: math.MaxUint64, + } + data, err := sealevel.MarshalStakeStake(state) + require.NoError(t, err) + global.EnqueuePendingStakePubkey(rootedSlot, key) + return &accounts.Account{Key: key, Owner: addresses.StakeProgramAddr, Lamports: lamports, Data: data} + } + _, err := db.CommitBatch([]accounts.SlotDelta{{ + Slot: rootedSlot, + Delta: []*accounts.Account{ + voteAccountForRPC(t, currentVote, ¤tState), + voteAccountForRPC(t, delinquentVote, &delinquentState), + voteAccountForRPC(t, newVote, &newState), + voteAccountForRPC(t, uninitializedVote, &uninitializedState), + {Key: invalidVote, Owner: addresses.VoteProgramAddr, Lamports: 1}, + stakeAccount(solana.PublicKey{21}, currentVote, 125), + stakeAccount(solana.PublicKey{22}, delinquentVote, 200), + stakeAccount(solana.PublicKey{23}, uninitializedVote, 50), + {Key: sealevel.SysvarStakeHistoryAddr, Owner: addresses.SysvarOwnerAddr, Lamports: 1, Data: historyData.Bytes()}, + }, + }}, rootedSlot, nil, nil) + require.NoError(t, err) + _, err = global.FlushPendingStakePubkeys(filepath.Join(db.AcctsDir, "..")) + require.NoError(t, err) + + previous, hadPrevious := global.EpochStakesSnapshot(epoch) + global.PutEpochStakes(epoch, + map[solana.PublicKey]uint64{currentVote: 100, delinquentVote: 200}, + map[solana.PublicKey]*epochstakes.VoteAccount{ + currentVote: {NodePubkey: currentNode}, + delinquentVote: {NodePubkey: delinquentNode}, + }, + 300, + ) + t.Cleanup(func() { + if hadPrevious { + global.PutEpochStakes(epoch, previous.Stakes, previous.VoteAccounts, previous.TotalStake) + } else { + global.ClearEpochStakes(epoch) + } + }) + + server := &RpcServer{ + acctsDb: db, + epochSchedule: &sealevel.SysvarEpochSchedule{ + SlotsPerEpoch: 1_000, + }, + } + server.SetRootedBankState(rootedSlot, 490, 1_000) + got, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{ + map[string]interface{}{"commitment": "confirmed"}, + })) + require.NoError(t, err) + require.Len(t, got.Current, 2) + require.Len(t, got.Delinquent, 1) + require.Equal(t, currentVote.String(), got.Current[0].VotePubkey) + require.Equal(t, currentNode.String(), got.Current[0].NodePubkey) + require.Equal(t, uint64(125), got.Current[0].ActivatedStake, "live stake differs from frozen epoch stake") + require.Equal(t, uint8(8), got.Current[0].Commission) + require.Equal(t, uint16(755), got.Current[0].InflationRewardsCommissionBPS) + require.Equal(t, uint64(499), got.Current[0].LastVote) + require.Equal(t, [][3]uint64{{1, 2, 1}, {2, 3, 2}, {3, 4, 3}, {4, 5, 4}, {5, 9, 5}}, got.Current[0].EpochCredits) + require.True(t, got.Current[0].EpochVoteAccount) + require.Equal(t, newVote.String(), got.Current[1].VotePubkey) + require.Equal(t, uint64(0), got.Current[1].ActivatedStake) + require.False(t, got.Current[1].EpochVoteAccount) + require.Equal(t, delinquentVote.String(), got.Delinquent[0].VotePubkey) + require.Equal(t, uint8(5), got.Delinquent[0].Commission) + + filtered, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{ + map[string]interface{}{"votePubkey": currentVote.String()}, + })) + require.NoError(t, err) + require.Len(t, filtered.Current, 1) + require.Empty(t, filtered.Delinquent) + + db.VoteAcctCache.Clear() + cold, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{})) + require.NoError(t, err) + require.Len(t, cold.Current, 2) + require.Equal(t, currentVote.String(), cold.Current[0].VotePubkey) + require.Equal(t, newVote.String(), cold.Current[1].VotePubkey) + require.Len(t, cold.Delinquent, 1) + require.Equal(t, delinquentVote.String(), cold.Delinquent[0].VotePubkey) + + // A zero distance makes every fixture vote delinquent, including the + // initialized vote account with no stake or epoch membership. + for _, keep := range []interface{}{true, false, nil} { + config := map[string]interface{}{"delinquentSlotDistance": 0, "keepUnstakedDelinquents": keep} + all, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{config})) + require.NoError(t, err) + require.Empty(t, all.Current) + if keep == true { + require.Len(t, all.Delinquent, 3) + require.Equal(t, newVote.String(), all.Delinquent[2].VotePubkey) + require.Zero(t, all.Delinquent[2].ActivatedStake) + require.False(t, all.Delinquent[2].EpochVoteAccount) + } else { + require.Len(t, all.Delinquent, 2) + } + config["votePubkey"] = newVote.String() + filtered, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{config})) + require.NoError(t, err) + require.Empty(t, filtered.Current) + if keep == true { + require.Len(t, filtered.Delinquent, 1) + } else { + require.Empty(t, filtered.Delinquent) + } + } + + // Candidate keys are append-only, but an account that changes owner must + // no longer be returned as a vote account. + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 501, Delta: []*accounts.Account{{ + Key: newVote, Lamports: 1, Owner: addresses.SystemProgramAddr, + }}}}, 501, nil, nil) + require.NoError(t, err) + server.SetRootedBankState(501, 490, 1_000) + changed, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{})) + require.NoError(t, err) + require.Len(t, changed.Current, 1) + require.Equal(t, currentVote.String(), changed.Current[0].VotePubkey) + + activationSlot := uint64(0) + featureData, err := features.MarshalFeatureAcct(&features.FeatureAcct{ActivatedAt: &activationSlot}) + require.NoError(t, err) + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 502, Delta: []*accounts.Account{{ + Key: features.ReduceStakeWarmupCooldown.Address, Owner: addresses.FeatureAddr, Lamports: 1, Data: featureData, + }}}}, 502, nil, nil) + require.NoError(t, err) + server.SetRootedBankState(502, 490, 1_000) + withFeature, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{})) + require.NoError(t, err) + require.Equal(t, uint64(125), withFeature.Current[0].ActivatedStake) + + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 503, Delta: []*accounts.Account{{ + Key: features.ReduceStakeWarmupCooldown.Address, Owner: addresses.FeatureAddr, Lamports: 1, Data: []byte{1}, + }}}}, 503, nil, nil) + require.NoError(t, err) + server.SetRootedBankState(503, 490, 1_000) + _, err = server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{})) + require.ErrorContains(t, err, "decode rooted stake warmup feature") +} + +func TestGetVoteAccountsRejectsInvalidConfig(t *testing.T) { + server := &RpcServer{} + for _, params := range [][]interface{}{ + {true}, + {map[string]interface{}{"commitment": "unknown"}}, + {map[string]interface{}{"votePubkey": "invalid"}}, + {map[string]interface{}{"keepUnstakedDelinquents": "yes"}}, + {map[string]interface{}{"delinquentSlotDistance": 1.5}}, + {map[string]interface{}{}, map[string]interface{}{}}, + } { + _, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, params)) + var invalid *InvalidParamsError + require.True(t, errors.As(err, &invalid), "params: %#v; error: %v", params, err) + } +} + +func voteAccountForRPC(t testing.TB, key solana.PublicKey, state *sealevel.VoteStateVersions) *accounts.Account { + t.Helper() + data, err := sealevel.MarshalVersionedVoteState(state) + require.NoError(t, err) + return &accounts.Account{ + Key: key, + Lamports: 1, + Owner: addresses.VoteProgramAddr, + Data: data, + } +} diff --git a/pkg/rpcserver/optional_params.go b/pkg/rpcserver/optional_params.go new file mode 100644 index 00000000..f6a609b1 --- /dev/null +++ b/pkg/rpcserver/optional_params.go @@ -0,0 +1,45 @@ +package rpcserver + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + "github.com/filecoin-project/go-jsonrpc" +) + +func decodeOptionalParams(p jsonrpc.RawParams) ([]interface{}, error) { + if len(p) == 0 { + return nil, nil + } + decoder := json.NewDecoder(bytes.NewReader(p)) + decoder.UseNumber() + var params []interface{} + if err := decoder.Decode(¶ms); err != nil { + return nil, err + } + if err := decoder.Decode(new(interface{})); err != io.EOF { + return nil, fmt.Errorf("parameters must contain one JSON value") + } + return params, nil +} + +// validateRootedCommitment rejects requests that need an unpublished account view. +func validateRootedCommitment(raw interface{}) error { + if raw == nil { + return nil + } + value, ok := raw.(string) + if !ok { + return &InvalidParamsError{Message: "invalid commitment"} + } + switch value { + case "confirmed", "finalized": + return nil + case "processed": + return &InvalidParamsError{Message: "processed commitment is not supported by this method; use confirmed or finalized"} + default: + return &InvalidParamsError{Message: "invalid commitment"} + } +} diff --git a/pkg/rpcserver/optional_params_test.go b/pkg/rpcserver/optional_params_test.go new file mode 100644 index 00000000..b0cfa31a --- /dev/null +++ b/pkg/rpcserver/optional_params_test.go @@ -0,0 +1,45 @@ +package rpcserver + +import ( + "context" + "testing" + + "github.com/filecoin-project/go-jsonrpc" + "github.com/stretchr/testify/require" +) + +func TestOptionalRPCParamsCanBeOmitted(t *testing.T) { + server := &RpcServer{} + methods := map[string]func(context.Context, jsonrpc.RawParams) error{ + "getEpochInfo": func(ctx context.Context, p jsonrpc.RawParams) error { + _, err := server.GetEpochInfo(ctx, p) + return err + }, + "getVoteAccounts": func(ctx context.Context, p jsonrpc.RawParams) error { + _, err := server.GetVoteAccounts(ctx, p) + return err + }, + "getLeaderSchedule": func(ctx context.Context, p jsonrpc.RawParams) error { + _, err := server.GetLeaderSchedule(ctx, p) + return err + }, + "getBlockProduction": func(ctx context.Context, p jsonrpc.RawParams) error { + _, err := server.GetBlockProduction(ctx, p) + return err + }, + } + for name, call := range methods { + t.Run(name, func(t *testing.T) { + // Missing node state should fail identically for each empty parameter form. + want := call(t.Context(), jsonrpc.RawParams(`[]`)) + require.Error(t, want) + for _, params := range []jsonrpc.RawParams{nil, jsonrpc.RawParams(`null`)} { + require.EqualError(t, call(t.Context(), params), want.Error()) + } + for _, params := range []jsonrpc.RawParams{jsonrpc.RawParams(`{}`), jsonrpc.RawParams(`true`)} { + var invalid *InvalidParamsError + require.ErrorAs(t, call(t.Context(), params), &invalid) + } + }) + } +} diff --git a/pkg/rpcserver/parse_vote_account.go b/pkg/rpcserver/parse_vote_account.go new file mode 100644 index 00000000..6dd41553 --- /dev/null +++ b/pkg/rpcserver/parse_vote_account.go @@ -0,0 +1,127 @@ +package rpcserver + +import ( + "strconv" + + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/mr-tron/base58" +) + +type parsedVoteAccount struct { + Program string `json:"program"` + Parsed parsedVoteAccountBody `json:"parsed"` + Space uint64 `json:"space"` +} + +type parsedVoteAccountBody struct { + Type string `json:"type"` + Info parsedVoteAccountInfo `json:"info"` +} + +type parsedVoteAccountInfo struct { + NodePubkey string `json:"nodePubkey"` + AuthorizedWithdrawer string `json:"authorizedWithdrawer"` + Commission uint8 `json:"commission"` + Votes []parsedVote `json:"votes"` + RootSlot *uint64 `json:"rootSlot"` + AuthorizedVoters []parsedAuthorizedVoter `json:"authorizedVoters"` + PriorVoters []parsedPriorVoter `json:"priorVoters"` + EpochCredits []parsedEpochCredits `json:"epochCredits"` + LastTimestamp parsedBlockTimestamp `json:"lastTimestamp"` + InflationRewardsCommissionBPS uint16 `json:"inflationRewardsCommissionBps"` + InflationRewardsCollector string `json:"inflationRewardsCollector"` + BlockRevenueCollector string `json:"blockRevenueCollector"` + BlockRevenueCommissionBPS uint16 `json:"blockRevenueCommissionBps"` + PendingDelegatorRewards string `json:"pendingDelegatorRewards"` + BLSPubkeyCompressed *string `json:"blsPubkeyCompressed"` +} + +type parsedVote struct { + Latency uint8 `json:"latency"` + Slot uint64 `json:"slot"` + ConfirmationCount uint32 `json:"confirmationCount"` +} + +type parsedAuthorizedVoter struct { + Epoch uint64 `json:"epoch"` + AuthorizedVoter string `json:"authorizedVoter"` +} + +type parsedPriorVoter struct { + AuthorizedPubkey string `json:"authorizedPubkey"` + EpochOfLastAuthorizedSwitch uint64 `json:"epochOfLastAuthorizedSwitch"` + TargetEpoch uint64 `json:"targetEpoch"` +} + +type parsedEpochCredits struct { + Epoch uint64 `json:"epoch"` + Credits string `json:"credits"` + PreviousCredits string `json:"previousCredits"` +} + +type parsedBlockTimestamp struct { + Slot uint64 `json:"slot"` + Timestamp int64 `json:"timestamp"` +} + +func parsedVoteAccountData(data []byte, votePubkey solana.PublicKey) (parsedVoteAccount, error) { + versioned, err := sealevel.UnmarshalVersionedVoteState(data) + if err != nil { + return parsedVoteAccount{}, err + } + state := versioned.ConvertToCurrent() + info := parsedVoteAccountInfo{ + NodePubkey: state.NodePubkey.String(), + AuthorizedWithdrawer: state.AuthorizedWithdrawer.String(), + Commission: state.Commission, + Votes: make([]parsedVote, 0, state.Votes.Len()), + RootSlot: state.RootSlot, + AuthorizedVoters: make([]parsedAuthorizedVoter, 0, state.AuthorizedVoters.AuthorizedVoters.Len()), + PriorVoters: make([]parsedPriorVoter, 0), + EpochCredits: make([]parsedEpochCredits, 0, len(state.EpochCredits)), + LastTimestamp: parsedBlockTimestamp{ + Slot: state.LastTimestamp.Slot, Timestamp: state.LastTimestamp.Timestamp, + }, + InflationRewardsCommissionBPS: uint16(state.Commission) * 100, + InflationRewardsCollector: votePubkey.String(), + BlockRevenueCollector: state.NodePubkey.String(), + BlockRevenueCommissionBPS: 10000, + PendingDelegatorRewards: "0", + } + for i := 0; i < state.Votes.Len(); i++ { + vote := state.Votes.At(i) + info.Votes = append(info.Votes, parsedVote{ + Latency: vote.Latency, + Slot: vote.Lockout.Slot, + ConfirmationCount: vote.Lockout.ConfirmationCount, + }) + } + epochs, voters := state.AuthorizedVoters.AuthorizedVoters.KeyValues() + for i := range epochs { + info.AuthorizedVoters = append(info.AuthorizedVoters, parsedAuthorizedVoter{ + Epoch: epochs[i], AuthorizedVoter: voters[i].String(), + }) + } + for _, credits := range state.EpochCredits { + info.EpochCredits = append(info.EpochCredits, parsedEpochCredits{ + Epoch: credits.Epoch, Credits: strconv.FormatUint(credits.Credits, 10), PreviousCredits: strconv.FormatUint(credits.PrevCredits, 10), + }) + } + if versioned.Type == sealevel.VoteStateVersionV4 { + info.Commission, info.InflationRewardsCommissionBPS = voteCommission(versioned, state) + info.InflationRewardsCollector = versioned.V4.InflationRewardsCollector.String() + info.BlockRevenueCollector = versioned.V4.BlockRevenueCollector.String() + info.BlockRevenueCommissionBPS = versioned.V4.BlockRevenueCommissionBps + info.PendingDelegatorRewards = strconv.FormatUint(versioned.V4.PendingDelegatorRewards, 10) + if versioned.V4.BlsPubkeyCompressed != nil { + encoded := base58.Encode(versioned.V4.BlsPubkeyCompressed[:]) + info.BLSPubkeyCompressed = &encoded + } + } + return parsedVoteAccount{ + Program: "vote", + Parsed: parsedVoteAccountBody{Type: "vote", Info: info}, + Space: uint64(len(data)), + }, nil +} diff --git a/pkg/rpcserver/rooted_account_publication_test.go b/pkg/rpcserver/rooted_account_publication_test.go new file mode 100644 index 00000000..8baad986 --- /dev/null +++ b/pkg/rpcserver/rooted_account_publication_test.go @@ -0,0 +1,125 @@ +package rpcserver + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetBalanceHTTPWaitsForFailedRewindRepair(t *testing.T) { + db := newRPCAccountsDB(t) + key := solana.PublicKey{204} + for _, slot := range []uint64{10, 20} { + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: []*accounts.Account{{Key: key, Lamports: slot}}}}, slot, nil, nil) + require.NoError(t, err) + } + server := NewRpcServer(db, 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 100}, solana.Hash{1}) + t.Cleanup(func() { _ = server.listener.Close() }) + server.SetRootedBankState(20, 20, 0) + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 100*time.Millisecond) + defer cancel() + server.ServeHTTP(w, r.WithContext(ctx)) + })) + t.Cleanup(endpoint.Close) + endpoint.Client().Timeout = time.Second + request, err := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": "getBalance", "params": []any{key.String()}}) + require.NoError(t, err) + call := func() (*GetBalanceResp, *rpcProbeError) { + t.Helper() + response, err := endpoint.Client().Post(endpoint.URL, "application/json", bytes.NewReader(request)) + require.NoError(t, err) + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + require.NoError(t, err) + t.Logf("HTTP status=%d body=%s", response.StatusCode, body) + var reply struct { + Result *GetBalanceResp `json:"result"` + Error *rpcProbeError `json:"error"` + } + require.NoError(t, json.Unmarshal(body, &reply)) + return reply.Result, reply.Error + } + rewoundPath := filepath.Join(db.AcctsDir, "rewound") + require.NoError(t, os.WriteFile(rewoundPath, nil, 0o644)) + _, err = db.RewindToBatchBoundary(10) + require.Error(t, err) + result, rpcError := call() + require.Nil(t, result, "partial rewind must not publish an account under the previous bank") + require.NotNil(t, rpcError) + require.Contains(t, rpcError.Message, context.DeadlineExceeded.Error()) + + require.NoError(t, os.Remove(rewoundPath)) + _, err = db.RewindToBatchBoundary(10) + require.NoError(t, err) + server.SetRootedBankState(10, 10, 0) + result, rpcError = call() + require.Nil(t, rpcError) + require.NotNil(t, result) + require.Equal(t, uint64(10), result.Context.Slot) + require.Equal(t, uint64(10), result.Value) +} + +func TestRootedAccountRPCsWaitForFailedRewindRepair(t *testing.T) { + key := solana.PublicKey{203} + for _, test := range []struct { + name string + read func(context.Context, *RpcServer) error + }{ + {"balance", func(ctx context.Context, server *RpcServer) error { + _, err := server.GetBalance(ctx, mustRawParams(t, []any{key.String()})) + return err + }}, + {"account info", func(ctx context.Context, server *RpcServer) error { + _, err := server.GetAccountInfo(ctx, mustRawParams(t, []any{key.String(), map[string]any{"encoding": "base64"}})) + return err + }}, + {"block production", func(ctx context.Context, server *RpcServer) error { + _, err := server.GetBlockProduction(ctx, nil) + return err + }}, + } { + t.Run(test.name, func(t *testing.T) { + db := newRPCAccountsDB(t) + for _, slot := range []uint64{10, 20} { + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: []*accounts.Account{{Key: key, Lamports: slot}}}}, slot, nil, nil) + require.NoError(t, err) + } + server := &RpcServer{acctsDb: db, epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 100}} + server.SetRootedBankState(20, 20, 0) + // Fail forensic-directory creation after rewind's atomic index undo. + rewoundPath := filepath.Join(db.AcctsDir, "rewound") + require.NoError(t, os.WriteFile(rewoundPath, nil, 0o644)) + _, err := db.RewindToBatchBoundary(10) + require.Error(t, err) + _, stable := db.CommittedAccountVersion() + require.False(t, stable) + require.Equal(t, uint64(20), db.DurableThrough()) + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + require.ErrorIs(t, test.read(ctx, server), context.DeadlineExceeded, + "partial rewind must not publish account values under the previous bank") + + require.NoError(t, os.Remove(rewoundPath)) + _, err = db.RewindToBatchBoundary(10) + require.NoError(t, err) + server.SetRootedBankState(10, 10, 0) + rooted, account, err := server.readRootedAccount(t.Context(), key) + require.NoError(t, err) + require.Equal(t, uint64(10), rooted.Slot) + require.Equal(t, uint64(10), account.Lamports) + }) + } +} diff --git a/pkg/rpcserver/rooted_commitment_test.go b/pkg/rpcserver/rooted_commitment_test.go new file mode 100644 index 00000000..4eef9063 --- /dev/null +++ b/pkg/rpcserver/rooted_commitment_test.go @@ -0,0 +1,54 @@ +package rpcserver + +import ( + "encoding/json" + "fmt" + "net/http/httptest" + "strings" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestRootedRPCRejectsProcessedHTTP(t *testing.T) { + server := NewRpcServer(newRPCAccountsDB(t), 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 1000}, solana.Hash{1}) + require.NoError(t, server.listener.Close()) + server.SetRootedBankState(500, 400, 0) + oldSlot := global.Slot() + global.SetSlot(600) + t.Cleanup(func() { global.SetSlot(oldSlot) }) + endpoint := httptest.NewServer(server) + t.Cleanup(endpoint.Close) + key := (solana.PublicKey{8}).String() + for _, test := range []struct{ method, params string }{ + {"getAccountInfo", `["` + key + `",{"commitment":"processed"}]`}, + {"getBalance", `["` + key + `",{"commitment":"processed"}]`}, + {"getVoteAccounts", `[{"commitment":"processed"}]`}, + {"getLeaderSchedule", `[{"commitment":"processed"}]`}, + {"getLeaderSchedule", `[null,{"commitment":"processed"}]`}, + {"getLeaderSchedule", `[500,{"commitment":"processed"}]`}, + {"getBlockProduction", `[{"commitment":"processed"}]`}, + } { + t.Run(test.method+test.params, func(t *testing.T) { + request := fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":%q,"params":%s}`, test.method, test.params) + response, err := endpoint.Client().Post(endpoint.URL, "application/json", strings.NewReader(request)) + require.NoError(t, err) + defer response.Body.Close() + var reply struct { + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + require.NoError(t, json.NewDecoder(response.Body).Decode(&reply)) + require.NotNil(t, reply.Error, "must not substitute rooted state for processed") + require.Equal(t, -32602, reply.Error.Code) + require.Contains(t, reply.Error.Message, "processed commitment is not supported") + require.Empty(t, reply.Result) + }) + } +} diff --git a/pkg/rpcserver/rooted_stake_cache.go b/pkg/rpcserver/rooted_stake_cache.go new file mode 100644 index 00000000..5d9130ab --- /dev/null +++ b/pkg/rpcserver/rooted_stake_cache.go @@ -0,0 +1,446 @@ +package rpcserver + +import ( + "context" + "errors" + "fmt" + "maps" + "path/filepath" + "sync" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/accountsdb" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" +) + +var errStakeViewChanged = errors.New("rooted stake view changed") + +const stakeCacheBatchSize = 512 + +type cachedDelegation struct { + delegation sealevel.Delegation + active uint64 + calculation uint64 + loaded bool +} + +// rootedStakeCache is disposable process-local derived state. It owns compact +// delegations and totals, never account buffers. Committed changes replace each +// delegation's contribution; speculative replay does not. Epoch/history/feature +// changes require RAM-only recalculation. Rewind, recovery and failed writes +// discard the view. No on-disk index or manifest format is changed here. +// +// RPC publication is separate: both bank identity and account version must +// match over the entire response. A commit alone cannot expose an unannounced +// bank. Initialization/recalculation merge concurrent commits in bounded chunks; +// they do not need replay to stop or one root to last for an entire stake scan. +type rootedStakeCache struct { + mu sync.Mutex + building chan struct{} + version uint64 + generation uint64 + calculation uint64 + dataReady bool + totalsReady bool + byStake map[solana.PublicKey]cachedDelegation + totals map[solana.PublicKey]uint64 + // While candidate keys are being loaded/seeded, remember touched keys so an + // old candidate cannot resurrect a concurrently closed account. Released as + // soon as placeholders are seeded, before the account scan starts. + bootstrapChanged map[solana.PublicKey]struct{} + epoch uint64 + history sealevel.SysvarStakeHistory + activationEpoch *uint64 + // Work counters used by tests/benchmarks; protected by mu. + scans uint64 + recalculations uint64 + updates uint64 + // Deterministic interleaving hooks; nil outside tests. + afterBootstrapSeed func() + beforeRecalculationChunk func(int) +} + +func stakeDelegation(account *accounts.Account) (sealevel.Delegation, bool) { + if account == nil || account.Lamports == 0 || account.Owner != addresses.StakeProgramAddr { + return sealevel.Delegation{}, false + } + state, err := sealevel.UnmarshalStakeState(account.Data) + if err != nil || state.Status != sealevel.StakeStateV2StatusStake { + return sealevel.Delegation{}, false + } + return state.Stake.Stake.Delegation, true +} + +// AccountsChanged runs on the account writer: no I/O or whole-cache scan. +// Invalidate activation inputs before applying this batch's delegations, so +// correctness does not depend on the order of accounts in the batch. +func (c *rootedStakeCache) AccountsChanged(version uint64, changed []*accounts.Account, reset bool) { + c.mu.Lock() + defer c.mu.Unlock() + if reset || c.byStake == nil || c.version+2 != version { + c.byStake, c.totals, c.bootstrapChanged = nil, nil, nil + c.dataReady, c.totalsReady = false, false + c.generation++ + c.version = version + return + } + for _, account := range changed { + if account != nil && (account.Key == sealevel.SysvarStakeHistoryAddr || account.Key == features.ReduceStakeWarmupCooldown.Address) { + c.totals, c.totalsReady = nil, false + c.calculation++ + break + } + } + for _, account := range changed { + if account == nil { + continue + } + if c.bootstrapChanged != nil { + c.bootstrapChanged[account.Key] = struct{}{} + } + old, known := c.byStake[account.Key] + if !known && account.Owner != addresses.StakeProgramAddr { + continue + } + if known && old.loaded && old.calculation == c.calculation && c.totals != nil { + c.subtract(old) + } + delete(c.byStake, account.Key) + if delegation, ok := stakeDelegation(account); ok { + entry := cachedDelegation{delegation: delegation, loaded: true} + if c.totals != nil { + entry.active = delegation.Stake(c.epoch, &c.history, c.activationEpoch) + entry.calculation = c.calculation + if entry.active != 0 { + c.totals[delegation.VoterPubkey] += entry.active + } + } + c.byStake[account.Key] = entry + } + c.updates++ + } + c.version = version +} + +func (c *rootedStakeCache) subtract(entry cachedDelegation) { + voter := entry.delegation.VoterPubkey + total := c.totals[voter] - entry.active + if total == 0 { + delete(c.totals, voter) + } else { + c.totals[voter] = total + } +} + +// selectTotals copies only voter totals, or one total for a filtered request. +// Callers never receive mutable cache maps. +func (c *rootedStakeCache) selectTotals(vote *solana.PublicKey) map[solana.PublicKey]uint64 { + if vote != nil { + return map[solana.PublicKey]uint64{*vote: c.totals[*vote]} + } + return maps.Clone(c.totals) +} + +func (s *RpcServer) rootedActivatedStakes(ctx context.Context, slot, epoch uint64, vote *solana.PublicKey) (map[solana.PublicKey]uint64, error) { + s.stakeCacheOnce.Do(func() { s.acctsDb.ObserveAccountState(&s.stakeCache) }) + c := &s.stakeCache + for { + if err := ctx.Err(); err != nil { + return nil, err + } + bank := s.rootedBank.Load() + version, stable := s.acctsDb.CommittedAccountVersion() + if bank == nil || bank.Slot != slot || !stable || s.rootedPublicationPending(slot) { + return nil, errStakeViewChanged + } + c.mu.Lock() + // Do not clear a newer cache if a write completed between the initial + // version read and acquiring the cache lock. + current, valid := s.acctsDb.CommittedAccountVersion() + if !valid || current != version || s.rootedBank.Load() != bank { + c.mu.Unlock() + return nil, errStakeViewChanged + } + if c.dataReady && c.totalsReady && c.version == version && c.epoch == epoch { + totals := c.selectTotals(vote) + c.mu.Unlock() + return totals, nil + } + if done := c.building; done != nil { + c.mu.Unlock() + select { + case <-done: + continue + case <-ctx.Done(): + return nil, ctx.Err() + } + } + c.building = make(chan struct{}) + cold := !c.dataReady || c.version != version + if cold { + c.generation++ + c.byStake = make(map[solana.PublicKey]cachedDelegation) + c.bootstrapChanged = make(map[solana.PublicKey]struct{}) + c.dataReady, c.totalsReady = false, false + c.totals = nil + c.version = version + } + generation := c.generation + c.mu.Unlock() + + var err error + if cold { + err = s.bootstrapStakeCache(ctx, generation) + } + if err == nil { + err = s.recalculateStakeTotals(ctx, slot, epoch, generation) + } + c.mu.Lock() + latest, valid := s.acctsDb.CommittedAccountVersion() + if err == nil && (!valid || c.version != latest || s.rootedBank.Load() != bank || s.rootedPublicationPending(slot)) { + err = errStakeViewChanged + } + var totals map[solana.PublicKey]uint64 + if err == nil { + totals = c.selectTotals(vote) + } + close(c.building) + c.building = nil + c.mu.Unlock() + return totals, err + } +} +func (s *RpcServer) rootedStakeInputs(ctx context.Context, slot uint64) (sealevel.SysvarStakeHistory, *uint64, error) { + var history sealevel.SysvarStakeHistory + rooted, values, err := s.readRootedAccounts(ctx, []solana.PublicKey{sealevel.SysvarStakeHistoryAddr, features.ReduceStakeWarmupCooldown.Address}) + if err != nil { + return history, nil, fmt.Errorf("read rooted stake history: %w", err) + } + if rooted.Slot != slot { + return history, nil, errStakeViewChanged + } + if values[0] == nil || values[0].Lamports == 0 { + return history, nil, fmt.Errorf("stake history unavailable at rooted slot %d", slot) + } + if err := history.UnmarshalWithDecoder(bin.NewBinDecoder(values[0].Data)); err != nil { + return history, nil, fmt.Errorf("decode stake history at rooted slot %d: %w", slot, err) + } + var activation *uint64 + account := values[1] + if account != nil && account.Lamports > 0 && account.Owner == addresses.FeatureAddr { + var feature features.FeatureAcct + if err := feature.UnmarshalWithDecoder(bin.NewBinDecoder(account.Data)); err != nil { + return history, nil, fmt.Errorf("decode rooted stake warmup feature: %w", err) + } + if feature.ActivatedAt != nil && *feature.ActivatedAt <= slot { + epoch := s.epochSchedule.GetEpoch(*feature.ActivatedAt) + activation = &epoch + } + } + return history, activation, nil +} + +// bootstrapStakeCache seeds placeholders from the existing candidate index. +// Incoming commits win over placeholders, including deletion/owner changes. +// Each disk batch is coherent, but batches may come from different roots: the +// observer reconciles touched accounts to the latest committed state. Only a +// reset (rewind/recovery/failure), not ordinary root advancement, cancels this. +func (s *RpcServer) bootstrapStakeCache(ctx context.Context, generation uint64) (err error) { + c := &s.stakeCache + defer func() { + if err != nil { + c.mu.Lock() + if c.generation == generation { + c.byStake, c.bootstrapChanged = nil, nil + c.dataReady = false + } + c.mu.Unlock() + } + }() + // Read pending FIRST: if a flush moves entries to disk between the two + // reads, either the pending snapshot or the subsequent file load contains them. + pending := global.PendingStakeEntriesSnapshot() + entries, err := global.LoadStakePubkeyIndex(filepath.Join(s.acctsDb.AcctsDir, "..")) + if err != nil { + return fmt.Errorf("load stake account index: %w", err) + } + for _, list := range [][]accountsdb.StakeIndexEntry{entries, pending} { + for start := 0; start < len(list); start += stakeCacheBatchSize { + if err = ctx.Err(); err != nil { + return err + } + c.mu.Lock() + if c.generation != generation { + c.mu.Unlock() + return errStakeViewChanged + } + for _, entry := range list[start:min(start+stakeCacheBatchSize, len(list))] { + if _, changed := c.bootstrapChanged[entry.Pubkey]; !changed { + if _, exists := c.byStake[entry.Pubkey]; !exists { + c.byStake[entry.Pubkey] = cachedDelegation{} + } + } + } + c.mu.Unlock() + } + } + c.mu.Lock() + if c.generation != generation { + c.mu.Unlock() + return errStakeViewChanged + } + c.bootstrapChanged = nil + c.mu.Unlock() + if c.afterBootstrapSeed != nil { + c.afterBootstrapSeed() + } + for _, list := range [][]accountsdb.StakeIndexEntry{entries, pending} { + for start := 0; start < len(list); start += stakeCacheBatchSize { + keys := make([]solana.PublicKey, 0, stakeCacheBatchSize) + c.mu.Lock() + if c.generation != generation { + c.mu.Unlock() + return errStakeViewChanged + } + for _, candidate := range list[start:min(start+stakeCacheBatchSize, len(list))] { + if entry, exists := c.byStake[candidate.Pubkey]; exists && !entry.loaded { + keys = append(keys, candidate.Pubkey) + } + } + c.mu.Unlock() + if len(keys) == 0 { + continue + } + for { + if err = ctx.Err(); err != nil { + return err + } + version, stable := s.acctsDb.CommittedAccountVersion() + if !stable { + if err = waitForRootedPublication(ctx); err != nil { + return err + } + continue + } + _, values, readErr := s.readRootedAccounts(ctx, keys) + if readErr != nil { + return readErr + } + c.mu.Lock() + if c.generation != generation { + c.mu.Unlock() + return errStakeViewChanged + } + latest, stable := s.acctsDb.CommittedAccountVersion() + if !stable || latest != version { + c.mu.Unlock() + continue + } + for i, account := range values { + key := keys[i] + // A commit already installed the latest value (or deleted it). + if entry, exists := c.byStake[key]; !exists || entry.loaded { + continue + } + delete(c.byStake, key) + if delegation, ok := stakeDelegation(account); ok { + c.byStake[key] = cachedDelegation{delegation: delegation, loaded: true} + } + } + c.mu.Unlock() + break + } + } + } + c.mu.Lock() + defer c.mu.Unlock() + if c.generation != generation { + return errStakeViewChanged + } + c.dataReady = true + c.scans++ + return nil +} + +// Recalculate in bounded chunks. Every map iterator step and entry access holds +// c.mu; commits may modify the map only between chunks. Go's map range semantics +// permit those modifications: deleted entries disappear, and new entries may or +// may not be visited. The observer already includes new/updated entries in this +// calculation, so their tags prevent double counting either way. A reset or new +// activation input cancels the iterator before it can advance again. +func (s *RpcServer) recalculateStakeTotals(ctx context.Context, slot, epoch, generation uint64) (err error) { + c := &s.stakeCache + bank := s.rootedBank.Load() + version, stable := s.acctsDb.CommittedAccountVersion() + if bank == nil || bank.Slot != slot || !stable { + return errStakeViewChanged + } + history, activation, err := s.rootedStakeInputs(ctx, slot) + if err != nil { + return err + } + c.mu.Lock() + latest, stable := s.acctsDb.CommittedAccountVersion() + if !stable || latest != version || c.version != version || c.generation != generation || s.rootedBank.Load() != bank { + c.mu.Unlock() + return errStakeViewChanged + } + c.calculation++ + calculation := c.calculation + c.epoch, c.history, c.activationEpoch = epoch, history, activation + c.totals = make(map[solana.PublicKey]uint64) + c.totalsReady = false + // From here every return owns c.mu, including errors from yield. Invalidate + // only this calculation; a reset may already have installed a different view. + defer func() { + if err != nil && c.generation == generation && c.calculation == calculation { + c.totals = nil + c.totalsReady = false + } + c.mu.Unlock() + }() + // Enter and return locked. Never advance the map iterator while unlocked. + yield := func(visited int) error { + c.mu.Unlock() + if c.beforeRecalculationChunk != nil { + c.beforeRecalculationChunk(visited) + } + canceled := ctx.Err() + c.mu.Lock() + if canceled != nil { + return canceled + } + if c.generation != generation || c.calculation != calculation { + return errStakeViewChanged + } + return nil + } + if err = yield(0); err != nil { + return err + } + visited := 0 + for key, entry := range c.byStake { + if entry.calculation != calculation { + entry.active = entry.delegation.Stake(epoch, &history, activation) + entry.calculation = calculation + c.byStake[key] = entry + if entry.active != 0 { + c.totals[entry.delegation.VoterPubkey] += entry.active + } + } + visited++ + if visited%stakeCacheBatchSize == 0 { + if err = yield(visited); err != nil { + return err + } + } + } + c.totalsReady = true + c.recalculations++ + return nil +} diff --git a/pkg/rpcserver/rooted_stake_cache_bench_test.go b/pkg/rpcserver/rooted_stake_cache_bench_test.go new file mode 100644 index 00000000..7fc6d7ff --- /dev/null +++ b/pkg/rpcserver/rooted_stake_cache_bench_test.go @@ -0,0 +1,104 @@ +package rpcserver + +import ( + "encoding/binary" + "math" + "path/filepath" + "sync" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// BenchmarkRootedStakeTotals compares the old query-time scan with the new +// warm cache on the same database. It measures stake aggregation, not the full +// JSON-RPC response. Cold initialization and epoch recalculation are excluded +// from warm timings. No live validator or mainnet-scale performance is implied. +func BenchmarkRootedStakeTotals(b *testing.B) { + const stakeCount = 20_000 + const voterCount = 1_000 + db := newRPCAccountsDB(b) + global.ClearPendingStakePubkeys() + b.Cleanup(global.ClearPendingStakePubkeys) + payloads := make([][]byte, voterCount) + for i := range payloads { + voter := solana.PublicKey{0xb0} + binary.LittleEndian.PutUint64(voter[1:], uint64(i)) + state := &sealevel.StakeStateV2{Status: sealevel.StakeStateV2StatusStake} + state.Stake.Stake.Delegation = sealevel.Delegation{VoterPubkey: voter, StakeLamports: 1_000_000, ActivationEpoch: math.MaxUint64, DeactivationEpoch: math.MaxUint64} + data, err := sealevel.MarshalStakeStake(state) + require.NoError(b, err) + payloads[i] = data + } + delta := make([]*accounts.Account, 0, stakeCount+1) + for i := range stakeCount { + key := solana.PublicKey{0xa0} + binary.LittleEndian.PutUint64(key[1:], uint64(i)) + delta = append(delta, &accounts.Account{Key: key, Owner: addresses.StakeProgramAddr, Lamports: 1_000_001, Data: payloads[i%voterCount]}) + global.EnqueuePendingStakePubkey(10, key) + } + delta = append(delta, cacheHistoryAccount(b, nil)) + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 10, Delta: delta}}, 10, nil, nil) + require.NoError(b, err) + _, err = global.FlushPendingStakePubkeys(filepath.Dir(db.AcctsDir)) + require.NoError(b, err) + s := &RpcServer{acctsDb: db, epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 100}} + s.SetRootedBankState(10, 10, 0) + expected, err := s.rootedActivatedStakes(b.Context(), 10, 0, nil) + require.NoError(b, err) + require.Len(b, expected, voterCount) + voter := solana.PublicKey{0xb0} + b.Run("LegacyScanAll", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + totals := make(map[solana.PublicKey]uint64) + history := sealevel.SysvarStakeHistory{} + var mu sync.Mutex + _, err := global.StreamStakeAccounts(db, 10, func(_ solana.PublicKey, d *sealevel.Delegation, _ uint64) { + active := d.Stake(0, &history, nil) + mu.Lock() + totals[d.VoterPubkey] += active + mu.Unlock() + }) + if err != nil || totals[voter] != 20_000_000 { + b.Fatalf("scan: %v, total: %d", err, totals[voter]) + } + } + }) + for _, tc := range []struct { + name string + filter *solana.PublicKey + }{{"CachedAll", nil}, {"CachedOne", &voter}} { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + totals, err := s.rootedActivatedStakes(b.Context(), 10, 0, tc.filter) + if err != nil || totals[voter] != 20_000_000 { + b.Fatalf("cache: %v, total: %d", err, totals[voter]) + } + } + }) + } + // Include writer-side maintenance: 20k changed accounts, only 64 stakes. + mixed := make([]*accounts.Account, stakeCount) + copy(mixed, delta[:64]) + for i := 64; i < len(mixed); i++ { + key := solana.PublicKey{0xc0} + binary.LittleEndian.PutUint64(key[1:], uint64(i)) + mixed[i] = &accounts.Account{Key: key, Owner: addresses.SystemProgramAddr, Lamports: 1} + } + b.Run("Apply64StakesAmong20000Changes", func(b *testing.B) { + b.ReportAllocs() + version := s.stakeCache.version + for b.Loop() { + version += 2 + s.stakeCache.AccountsChanged(version, mixed, false) + } + }) + +} diff --git a/pkg/rpcserver/rooted_stake_cache_scale_test.go b/pkg/rpcserver/rooted_stake_cache_scale_test.go new file mode 100644 index 00000000..8d97ba4b --- /dev/null +++ b/pkg/rpcserver/rooted_stake_cache_scale_test.go @@ -0,0 +1,304 @@ +package rpcserver + +import ( + "context" + "encoding/binary" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "sync" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Opt-in resource experiment; run each size in a fresh process. This measures +// local fixture scaling, not mainnet capacity. Fixture writes warm the OS page +// cache. RPC timings include response construction/JSON, but exclude HTTP. +// +// MITHRIL_STAKE_SCALE=1000000 GOMAXPROCS=2 go test ./pkg/rpcserver \ +// -run '^TestRootedStakeCacheScale$' -count=1 -v -timeout=10m +func TestRootedStakeCacheScale(t *testing.T) { + size := os.Getenv("MITHRIL_STAKE_SCALE") + if size == "" { + t.Skip("set MITHRIL_STAKE_SCALE to run the on-disk sizing experiment") + } + count, err := strconv.Atoi(size) + require.NoError(t, err) + require.GreaterOrEqual(t, count, 1000) + require.LessOrEqual(t, count, 2_000_000) + const voters = 1000 + const epochSlots = uint64(1_000_000) + key := func(prefix byte, i int) solana.PublicKey { + k := solana.PublicKey{prefix} + binary.LittleEndian.PutUint64(k[1:], uint64(i)) + return k + } + emit := func(stage string, fields map[string]any) { + fields["stage"], fields["stakes"], fields["voters"] = stage, count, voters + raw, err := json.Marshal(fields) + require.NoError(t, err) + t.Log(string(raw)) + } + heap := func() runtime.MemStats { runtime.GC(); var m runtime.MemStats; runtime.ReadMemStats(&m); return m } + quantiles := func(samples []time.Duration) map[string]any { + sort.Slice(samples, func(i, j int) bool { return samples[i] < samples[j] }) + out := map[string]any{"samples": len(samples)} + if len(samples) > 0 { + for name, q := range map[string]float64{"p50_ms": .5, "p95_ms": .95, "p99_ms": .99, "max_ms": 1} { + out[name] = float64(samples[max(0, int(math.Ceil(float64(len(samples))*q))-1)]) / float64(time.Millisecond) + } + } + return out + } + db := newRPCAccountsDB(t) + global.ClearPendingStakePubkeys() + t.Cleanup(global.ClearPendingStakePubkeys) + // 80% bootstrap-active, 10% activating, 10% deactivating at epoch 100. + makeStake := func(i int, amount uint64) *accounts.Account { + activation, deactivation := uint64(math.MaxUint64), uint64(math.MaxUint64) + switch (i / voters) % 10 { + case 0: + activation = 98 + case 1: + deactivation = 100 + } + state := &sealevel.StakeStateV2{Status: sealevel.StakeStateV2StatusStake} + state.Stake.Stake.Delegation = sealevel.Delegation{VoterPubkey: key(0xb0, i%voters), StakeLamports: amount, ActivationEpoch: activation, DeactivationEpoch: deactivation, WarmupCooldownRate: .25} + data, err := sealevel.MarshalStakeStake(state) + require.NoError(t, err) + return &accounts.Account{Key: key(0xa0, i), Owner: addresses.StakeProgramAddr, Lamports: amount + 1, Data: data} + } + fixtureStart := time.Now() + for first := 0; first < count; first += 10_000 { + delta := make([]*accounts.Account, 0, min(10_000, count-first)) + slot := uint64(first/10_000 + 1) + for i := first; i < min(first+10_000, count); i++ { + a := makeStake(i, 1_000_000) + delta = append(delta, a) + global.EnqueuePendingStakePubkey(slot, a.Key) + } + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: delta}}, slot, nil, nil) + require.NoError(t, err) + } + _, err = global.FlushPendingStakePubkeys(filepath.Dir(db.AcctsDir)) + require.NoError(t, err) + var history sealevel.SysvarStakeHistory + entry := sealevel.StakeHistoryEntry{Effective: 10_000_000_000, Activating: 2_000_000_000, Deactivating: 1_000_000_000} + for e := uint64(100); e > 68; e-- { + history = append(history, sealevel.StakeHistoryPair{Epoch: e - 1, Entry: entry}) + } + delta := []*accounts.Account{cacheHistoryAccount(t, history)} + stakes := make(map[solana.PublicKey]uint64, voters) + for i := 0; i < voters; i++ { + voter := key(0xb0, i) + stakes[voter] = 1 + state := &sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4} + state.V4.NodePubkey = key(0xc0, i) + state.V4.AuthorizedVoters.AuthorizedVoters.Set(0, state.V4.NodePubkey) + state.V4.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: 100 * epochSlots}}) + delta = append(delta, voteAccountForRPC(t, voter, state)) + } + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 100 * epochSlots, Delta: delta}}, 100*epochSlots, nil, nil) + require.NoError(t, err) + require.NoError(t, db.SeedVoteAccountPubkeys(nil)) + for epoch := uint64(100); epoch <= 102; epoch++ { + previous, hadPrevious := global.EpochStakesSnapshot(epoch) + global.PutEpochStakes(epoch, stakes, map[solana.PublicKey]*epochstakes.VoteAccount{}, voters) + t.Cleanup(func() { + if hadPrevious { + global.PutEpochStakes(epoch, previous.Stakes, previous.VoteAccounts, previous.TotalStake) + } else { + global.ClearEpochStakes(epoch) + } + }) + } + s := &RpcServer{acctsDb: db, epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: epochSlots}} + s.SetRootedBankState(100*epochSlots, 100*epochSlots, 0) + db.CommonAcctsCache.Clear() + db.VoteAcctCache.Clear() + delta = nil + before := heap() + emit("fixture", map[string]any{"seconds": time.Since(fixtureStart).Seconds(), "heap_bytes": before.HeapAlloc}) + allParams := mustRawParams(t, []interface{}{}) + oneParams := mustRawParams(t, []interface{}{map[string]interface{}{"votePubkey": key(0xb0, 0).String()}}) + query := func(ctx context.Context, filtered bool) error { + params := allParams + if filtered { + params = oneParams + } + response, err := s.GetVoteAccounts(ctx, params) + if err != nil { + return err + } + want := voters + if filtered { + want = 1 + } + if len(response.Current)+len(response.Delinquent) != want { + return fmt.Errorf("got %d votes, want %d", len(response.Current)+len(response.Delinquent), want) + } + _, err = json.Marshal(response) + return err + } + start := time.Now() + require.NoError(t, query(t.Context(), false)) + coldDuration := time.Since(start) + after := heap() + emit("cold", map[string]any{"seconds": coldDuration.Seconds(), "heap_bytes": after.HeapAlloc, "heap_growth_bytes": int64(after.HeapAlloc) - int64(before.HeapAlloc), "allocated_bytes": after.TotalAlloc - before.TotalAlloc, "allocations": after.Mallocs - before.Mallocs, "gc_cycles": after.NumGC - before.NumGC}) + for _, filtered := range []bool{false, true} { + samples := make([]time.Duration, 0, 100) + var a, b runtime.MemStats + runtime.ReadMemStats(&a) + for i := 0; i < 100; i++ { + start := time.Now() + require.NoError(t, query(t.Context(), filtered)) + samples = append(samples, time.Since(start)) + } + runtime.ReadMemStats(&b) + stats := quantiles(samples) + stats["bytes_per_query"] = (b.TotalAlloc - a.TotalAlloc) / 100 + stats["allocs_per_query"] = (b.Mallocs - a.Mallocs) / 100 + name := "warm_all" + if filtered { + name = "warm_one" + } + emit(name, stats) + } + // Keep a direct, full-scan aggregation as the independent update oracle. + verify := func(stage string) { + bank := s.rootedBank.Load() + epoch := s.epochSchedule.GetEpoch(bank.Slot) + got, err := s.rootedActivatedStakes(t.Context(), bank.Slot, epoch, nil) + require.NoError(t, err) + expected := make(map[solana.PublicKey]uint64) + var mu sync.Mutex + start := time.Now() + _, err = global.StreamStakeAccounts(db, bank.Slot, func(_ solana.PublicKey, d *sealevel.Delegation, _ uint64) { + active := d.Stake(epoch, &history, nil) + if active > 0 { + mu.Lock() + expected[d.VoterPubkey] += active + mu.Unlock() + } + }) + require.NoError(t, err) + require.Equal(t, expected, got) + emit(stage+"_oracle", map[string]any{"seconds": time.Since(start).Seconds()}) + } + verify("cold") + // Measure a quiet epoch recalculation before adding concurrent traffic. + history = append(sealevel.SysvarStakeHistory{{Epoch: 100, Entry: entry}}, history...) + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 101 * epochSlots, Delta: []*accounts.Account{cacheHistoryAccount(t, history)}}}, 101*epochSlots, nil, nil) + require.NoError(t, err) + s.SetRootedBankState(101*epochSlots, 101*epochSlots, 0) + start = time.Now() + require.NoError(t, query(t.Context(), false)) + emit("epoch_quiet", map[string]any{"seconds": time.Since(start).Seconds()}) + verify("epoch_quiet") + // Four paced callers mix whole-cluster and single-voter requests while 100 + // durable folds update 64 stakes apiece. Include wall time waiting for roots. + concurrent := func(stage string, readers int) { + ctx, cancel := context.WithTimeout(t.Context(), 90*time.Second) + var wg sync.WaitGroup + defer func() { cancel(); wg.Wait() }() + var mu sync.Mutex + var all, one []time.Duration + failures := make(chan error, 4) + for worker := 0; worker < readers; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + if ctx.Err() != nil { + return + } + start := time.Now() + err := query(ctx, worker%2 == 1) + elapsed := time.Since(start) + if err != nil { + if ctx.Err() == nil { + failures <- err + } + return + } + mu.Lock() + if worker%2 == 1 { + one = append(one, elapsed) + } else { + all = append(all, elapsed) + } + mu.Unlock() + select { + case <-ctx.Done(): + return + case <-time.After(50 * time.Millisecond): + } + } + }() + } + var commits []time.Duration + startSlot := s.rootedBank.Load().Slot + for n := 0; n < 100; n++ { + changed := make([]*accounts.Account, 64) + for i := range changed { + changed[i] = makeStake((n*64+i)%count, uint64(1_000_000+n+1)) + } + slot := startSlot + uint64(n+1) + start := time.Now() + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: changed}}, slot, nil, nil) + if err != nil { + cancel() + wg.Wait() + t.Fatal(err) + } + s.SetRootedBankState(slot, slot, 0) + commits = append(commits, time.Since(start)) + time.Sleep(20 * time.Millisecond) + } + // Let in-flight initial requests finish before stopping the readers. + require.NoError(t, query(ctx, false)) + time.Sleep(200 * time.Millisecond) + cancel() + wg.Wait() + close(failures) + for err := range failures { + require.NoError(t, err) + } + emit(stage+"_all", quantiles(all)) + emit(stage+"_one", quantiles(one)) + emit(stage+"_commit", quantiles(commits)) + verify(stage) + } + concurrent("quiet_commits", 0) + concurrent("warm_concurrent", 4) + history = append(sealevel.SysvarStakeHistory{{Epoch: 101, Entry: entry}}, history...) + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 102 * epochSlots, Delta: []*accounts.Account{cacheHistoryAccount(t, history)}}}, 102*epochSlots, nil, nil) + require.NoError(t, err) + s.SetRootedBankState(102*epochSlots, 102*epochSlots, 0) + concurrent("epoch_concurrent", 4) + version, _ := db.CommittedAccountVersion() + s.stakeCache.AccountsChanged(version, nil, true) + db.CommonAcctsCache.Clear() + db.VoteAcctCache.Clear() + concurrent("bootstrap_concurrent", 4) + require.Equal(t, uint64(2), s.stakeCache.scans, "concurrent root changes must not restart the account scan") + retained := heap() + version, _ = db.CommittedAccountVersion() + s.stakeCache.AccountsChanged(version, nil, true) + released := heap() + emit("memory", map[string]any{"heap_before_drop_bytes": retained.HeapAlloc, "heap_after_drop_bytes": released.HeapAlloc, "derived_cache_retained_estimate_bytes": int64(retained.HeapAlloc) - int64(released.HeapAlloc), "scans": s.stakeCache.scans, "recalculations": s.stakeCache.recalculations}) + runtime.KeepAlive(s) +} diff --git a/pkg/rpcserver/rooted_stake_cache_test.go b/pkg/rpcserver/rooted_stake_cache_test.go new file mode 100644 index 00000000..1d795b18 --- /dev/null +++ b/pkg/rpcserver/rooted_stake_cache_test.go @@ -0,0 +1,332 @@ +package rpcserver + +import ( + "bytes" + "context" + "math" + "math/rand" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func cacheStakeAccount(t testing.TB, key, voter byte, amount uint64, activation, deactivation uint64) *accounts.Account { + t.Helper() + state := &sealevel.StakeStateV2{Status: sealevel.StakeStateV2StatusStake} + state.Stake.Stake.Delegation = sealevel.Delegation{VoterPubkey: solana.PublicKey{voter}, StakeLamports: amount, ActivationEpoch: activation, DeactivationEpoch: deactivation, WarmupCooldownRate: 0.25} + data, err := sealevel.MarshalStakeStake(state) + require.NoError(t, err) + return &accounts.Account{Key: solana.PublicKey{key}, Owner: addresses.StakeProgramAddr, Lamports: amount + 1, Data: data} +} + +func cacheHistoryAccount(t testing.TB, history sealevel.SysvarStakeHistory) *accounts.Account { + t.Helper() + var buf bytes.Buffer + require.NoError(t, history.MarshalWithEncoder(bin.NewBinEncoder(&buf))) + return &accounts.Account{Key: sealevel.SysvarStakeHistoryAddr, Owner: addresses.SysvarOwnerAddr, Lamports: 1, Data: buf.Bytes()} +} + +func seedStakeCacheServer(t *testing.T, initial ...*accounts.Account) *RpcServer { + t.Helper() + db := newRPCAccountsDB(t) + global.ClearPendingStakePubkeys() + for _, account := range initial { + global.EnqueuePendingStakePubkey(10, account.Key) + } + // Use existing candidate-index publication, including its cache invalidation. + _, err := global.FlushPendingStakePubkeys(filepath.Dir(db.AcctsDir)) + require.NoError(t, err) + initial = append(initial, cacheHistoryAccount(t, nil)) + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 10, Delta: initial}}, 10, nil, nil) + require.NoError(t, err) + s := &RpcServer{acctsDb: db, epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 100}} + s.SetRootedBankState(10, 10, 0) + t.Cleanup(global.ClearPendingStakePubkeys) + return s +} + +func commitCacheAccounts(t *testing.T, s *RpcServer, slot uint64, changed ...*accounts.Account) { + t.Helper() + for _, account := range changed { + if account.Owner == addresses.StakeProgramAddr { + global.EnqueuePendingStakePubkey(slot, account.Key) + } + } + _, err := s.acctsDb.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: changed}}, slot, nil, nil) + require.NoError(t, err) + s.SetRootedBankState(slot, slot, 0) +} + +func cacheTotals(t *testing.T, s *RpcServer) map[solana.PublicKey]uint64 { + t.Helper() + bank, _ := s.getRootedBankState() + totals, err := s.rootedActivatedStakes(t.Context(), bank.Slot, s.epochSchedule.GetEpoch(bank.Slot), nil) + require.NoError(t, err) + return totals +} + +func TestRootedStakeCacheIncrementalChangesAndRewind(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64), cacheStakeAccount(t, 22, 2, 200, math.MaxUint64, math.MaxUint64)) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 100, {2}: 200}, cacheTotals(t, s)) + require.Equal(t, uint64(1), s.stakeCache.scans) + // RPC callers cannot mutate the retained totals. + got := cacheTotals(t, s) + got[solana.PublicKey{1}] = 999 + voter := solana.PublicKey{1} + filtered, err := s.rootedActivatedStakes(t.Context(), 10, 0, &voter) + require.NoError(t, err) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 100}, filtered) + + commitCacheAccounts(t, s, 11, cacheStakeAccount(t, 21, 2, 150, math.MaxUint64, math.MaxUint64), cacheStakeAccount(t, 23, 1, 300, math.MaxUint64, math.MaxUint64)) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 300, {2}: 350}, cacheTotals(t, s)) + commitCacheAccounts(t, s, 12, &accounts.Account{Key: solana.PublicKey{22}, Owner: addresses.SystemProgramAddr, Lamports: 5}, &accounts.Account{Key: solana.PublicKey{23}}) + require.Equal(t, map[solana.PublicKey]uint64{{2}: 150}, cacheTotals(t, s)) + commitCacheAccounts(t, s, 13, &accounts.Account{Key: solana.PublicKey{21}, Owner: addresses.StakeProgramAddr, Lamports: 1, Data: []byte{0}}) + require.Empty(t, cacheTotals(t, s)) + require.Equal(t, uint64(1), s.stakeCache.scans, "all updates must be incremental") + require.Equal(t, uint64(1), s.stakeCache.recalculations) + + _, err = s.acctsDb.RewindToBatchBoundary(10) + require.NoError(t, err) + s.SetRootedBankState(10, 10, 0) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 100, {2}: 200}, cacheTotals(t, s)) + require.Equal(t, uint64(2), s.stakeCache.scans) + // A different fork can reuse exactly the same slots: version fencing must not. + commitCacheAccounts(t, s, 11, cacheStakeAccount(t, 21, 1, 500, math.MaxUint64, math.MaxUint64)) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 500, {2}: 200}, cacheTotals(t, s)) + _, err = s.acctsDb.RecoverFoldState() + require.NoError(t, err) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 500, {2}: 200}, cacheTotals(t, s)) + require.Equal(t, uint64(3), s.stakeCache.scans) +} + +func TestRootedStakeCacheEpochHistoryAndFeatureChanges(t *testing.T) { + stake := cacheStakeAccount(t, 21, 1, 100, 0, 2) + s := seedStakeCacheServer(t, stake) + require.Empty(t, cacheTotals(t, s), "activation epoch has no effective stake") + history := sealevel.SysvarStakeHistory{{Epoch: 0, Entry: sealevel.StakeHistoryEntry{Effective: 1000, Activating: 1000}}} + commitCacheAccounts(t, s, 100, cacheHistoryAccount(t, history)) + delegation, _ := stakeDelegation(stake) + require.Equal(t, delegation.Stake(1, &history, nil), cacheTotals(t, s)[solana.PublicKey{1}]) + require.Equal(t, uint64(25), cacheTotals(t, s)[solana.PublicKey{1}]) + activation := uint64(0) + data, err := features.MarshalFeatureAcct(&features.FeatureAcct{ActivatedAt: &activation}) + require.NoError(t, err) + // Include a stake update in the same batch BEFORE the feature account: input + // invalidation cannot depend on the serialized account order. + changed := cacheStakeAccount(t, 21, 1, 200, 0, 2) + commitCacheAccounts(t, s, 101, changed, &accounts.Account{Key: features.ReduceStakeWarmupCooldown.Address, Owner: addresses.FeatureAddr, Lamports: 1, Data: data}) + delegation, _ = stakeDelegation(changed) + require.Equal(t, delegation.Stake(1, &history, &activation), cacheTotals(t, s)[solana.PublicKey{1}]) + require.Equal(t, uint64(18), cacheTotals(t, s)[solana.PublicKey{1}]) + commitCacheAccounts(t, s, 300) // Epoch changes even without a history account write. + require.Zero(t, cacheTotals(t, s)[solana.PublicKey{1}]) + require.Equal(t, uint64(1), s.stakeCache.scans, "activation changes must not reread all accounts") + require.Equal(t, uint64(4), s.stakeCache.recalculations) +} + +func TestRootedStakeCacheWaitersCancelAndDoNotPublishEarly(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64)) + cacheTotals(t, s) + _, err := s.acctsDb.CommitBatch([]accounts.SlotDelta{{Slot: 11, Delta: []*accounts.Account{cacheStakeAccount(t, 21, 1, 200, math.MaxUint64, math.MaxUint64)}}}, 11, nil, nil) + require.NoError(t, err) + _, err = s.rootedActivatedStakes(t.Context(), 10, 0, nil) + require.ErrorIs(t, err, errStakeViewChanged, "DB commit alone must not publish a new RPC bank") + s.SetRootedBankState(11, 11, 0) + require.Equal(t, uint64(200), cacheTotals(t, s)[solana.PublicKey{1}]) + c := &s.stakeCache + c.mu.Lock() + c.byStake = nil + c.dataReady = false + c.building = make(chan struct{}) + c.mu.Unlock() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err = s.rootedActivatedStakes(ctx, 11, 0, nil) + require.ErrorIs(t, err, context.Canceled) + ctx, cancel = context.WithTimeout(t.Context(), 20*time.Millisecond) + defer cancel() + _, err = s.rootedActivatedStakes(ctx, 11, 0, nil) + require.ErrorIs(t, err, context.DeadlineExceeded) + c.mu.Lock() + close(c.building) + c.building = nil + c.mu.Unlock() + require.Equal(t, uint64(200), cacheTotals(t, s)[solana.PublicKey{1}]) +} + +func TestRootedStakeCacheConcurrentBootstrap(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64)) + var wg sync.WaitGroup + for range 12 { + wg.Go(func() { + _, err := s.rootedActivatedStakes(t.Context(), 10, 0, nil) + if err != nil { + t.Error(err) + } + }) + } + wg.Wait() + require.Equal(t, uint64(1), s.stakeCache.scans) +} + +func TestRootedStakeCacheLegacyStore(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64)) + cacheTotals(t, s) + s.acctsDb.RootedDurable = false + done := make(chan struct{}) + require.NoError(t, s.acctsDb.StoreAccounts([]*accounts.Account{cacheStakeAccount(t, 21, 2, 500, math.MaxUint64, math.MaxUint64)}, 11, func() { close(done) })) + <-done + require.Eventually(t, func() bool { _, ok := s.acctsDb.CommittedAccountVersion(); return ok }, time.Second, time.Millisecond) + _, err := s.rootedActivatedStakes(t.Context(), 10, 0, nil) + require.ErrorIs(t, err, errStakeViewChanged, "a completed legacy write still needs RPC bank publication") + s.SetRootedBankState(11, 11, 0) + require.Equal(t, map[solana.PublicKey]uint64{{2}: 500}, cacheTotals(t, s)) + require.Equal(t, uint64(1), s.stakeCache.scans) +} + +// Compare many independent delegation mutations with complete recomputation, +// including redelegation, closure, invalid state, and ownership changes. +func TestRootedStakeCacheDifferentialUpdates(t *testing.T) { + c := &rootedStakeCache{version: 0, epoch: 3, byStake: make(map[solana.PublicKey]cachedDelegation), totals: make(map[solana.PublicKey]uint64)} + live := make(map[solana.PublicKey]*accounts.Account) + rng := rand.New(rand.NewSource(904)) + for step := uint64(1); step <= 500; step++ { + key := byte(1 + rng.Intn(80)) + voter := byte(100 + rng.Intn(10)) + account := cacheStakeAccount(t, key, voter, uint64(1+rng.Intn(10000)), math.MaxUint64, math.MaxUint64) + switch rng.Intn(5) { + case 0: + account.Lamports = 0 + case 1: + account.Owner = addresses.SystemProgramAddr + case 2: + account.Data = []byte{255} + } + live[account.Key] = account + c.AccountsChanged(step*2, []*accounts.Account{account}, false) + expected := make(map[solana.PublicKey]uint64) + for _, account := range live { + if delegation, ok := stakeDelegation(account); ok { + expected[delegation.VoterPubkey] += delegation.Stake(3, &c.history, nil) + } + } + require.Equal(t, expected, c.totals, "step %d", step) + } +} + +func TestGetVoteAccountsConcurrentCommitConsistency(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 1000, math.MaxUint64, math.MaxUint64)) + previous, hadPrevious := global.EpochStakesSnapshot(0) + global.PutEpochStakes(0, map[solana.PublicKey]uint64{{1}: 1000}, map[solana.PublicKey]*epochstakes.VoteAccount{{1}: {NodePubkey: solana.PublicKey{2}}}, 1000) + t.Cleanup(func() { + if hadPrevious { + global.PutEpochStakes(0, previous.Stakes, previous.VoteAccounts, previous.TotalStake) + } else { + global.ClearEpochStakes(0) + } + }) + voteAccount := func(slot uint64) *accounts.Account { + state := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionCurrent} + state.Current.NodePubkey = solana.PublicKey{2} + state.Current.AuthorizedVoters.AuthorizedVoters.Set(0, solana.PublicKey{2}) + state.Current.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: slot}}) + return voteAccountForRPC(t, solana.PublicKey{1}, &state) + } + commitCacheAccounts(t, s, 11, voteAccount(11), cacheStakeAccount(t, 21, 1, 1100, math.MaxUint64, math.MaxUint64)) + cacheTotals(t, s) + var wg sync.WaitGroup + wg.Go(func() { + for slot := uint64(12); slot < 50; slot++ { + changed := []*accounts.Account{voteAccount(slot), cacheStakeAccount(t, 21, 1, slot*100, math.MaxUint64, math.MaxUint64)} + _, err := s.acctsDb.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: changed}}, slot, nil, nil) + if err != nil { + t.Error(err) + return + } + s.SetRootedBankState(slot, slot, 0) + } + }) + for worker := range 4 { + wg.Go(func() { + params := mustRawParams(t, []interface{}{}) + if worker%2 == 0 { + params = mustRawParams(t, []interface{}{map[string]interface{}{"votePubkey": solana.PublicKey{1}.String()}}) + } + for range 30 { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + got, err := s.GetVoteAccounts(ctx, params) + cancel() + if err != nil { + t.Error(err) + return + } + if len(got.Current) != 1 { + t.Errorf("unexpected response: %+v", got) + return + } + if got.Current[0].ActivatedStake != got.Current[0].LastVote*100 { + t.Errorf("mixed committed banks: %+v", got.Current[0]) + return + } + } + }) + } + wg.Wait() + require.Equal(t, uint64(1), s.stakeCache.scans) +} + +func TestRootedStakeCacheBootstrapMergesConcurrentChanges(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64), cacheStakeAccount(t, 22, 2, 200, math.MaxUint64, math.MaxUint64)) + s.stakeCache.afterBootstrapSeed = func() { + commitCacheAccounts(t, s, 11, &accounts.Account{Key: solana.PublicKey{21}, Owner: addresses.SystemProgramAddr, Lamports: 1}, cacheStakeAccount(t, 22, 1, 250, math.MaxUint64, math.MaxUint64), cacheStakeAccount(t, 23, 3, 300, math.MaxUint64, math.MaxUint64)) + } + _, err := s.rootedActivatedStakes(t.Context(), 10, 0, nil) + require.ErrorIs(t, err, errStakeViewChanged, "request must retry its bank, not its completed bootstrap") + require.True(t, s.stakeCache.dataReady) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 250, {3}: 300}, cacheTotals(t, s)) + require.Equal(t, uint64(1), s.stakeCache.scans) +} + +func TestRootedStakeCacheRecalculationMergesConcurrentChanges(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64), cacheStakeAccount(t, 22, 2, 200, math.MaxUint64, math.MaxUint64)) + s.stakeCache.beforeRecalculationChunk = func(start int) { + if start == 0 { + commitCacheAccounts(t, s, 11, cacheStakeAccount(t, 21, 1, 150, math.MaxUint64, math.MaxUint64), &accounts.Account{Key: solana.PublicKey{22}}, cacheStakeAccount(t, 23, 3, 300, math.MaxUint64, math.MaxUint64)) + } + } + _, err := s.rootedActivatedStakes(t.Context(), 10, 0, nil) + require.ErrorIs(t, err, errStakeViewChanged) + require.True(t, s.stakeCache.totalsReady) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 150, {3}: 300}, cacheTotals(t, s)) + require.Equal(t, uint64(1), s.stakeCache.scans) + require.Equal(t, uint64(1), s.stakeCache.recalculations, "root advancement must not restart the full RAM calculation") +} + +func TestRootedStakeCacheRewindCancelsBootstrap(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64)) + commitCacheAccounts(t, s, 11, cacheStakeAccount(t, 21, 1, 200, math.MaxUint64, math.MaxUint64)) + var once sync.Once + s.stakeCache.afterBootstrapSeed = func() { + once.Do(func() { + _, err := s.acctsDb.RewindToBatchBoundary(10) + require.NoError(t, err) + s.SetRootedBankState(10, 10, 0) + }) + } + _, err := s.rootedActivatedStakes(t.Context(), 11, 0, nil) + require.ErrorIs(t, err, errStakeViewChanged) + require.False(t, s.stakeCache.dataReady) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 100}, cacheTotals(t, s)) +} diff --git a/pkg/rpcserver/rooted_stake_recalculation_bench_test.go b/pkg/rpcserver/rooted_stake_recalculation_bench_test.go new file mode 100644 index 00000000..63b1fdf2 --- /dev/null +++ b/pkg/rpcserver/rooted_stake_recalculation_bench_test.go @@ -0,0 +1,57 @@ +package rpcserver + +import ( + "encoding/binary" + "fmt" + "math" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Isolates RAM-only recalculation and its allocations. The delegation cache is +// seeded directly; this is not a startup, disk, RPC or writer-latency benchmark. +// All stakes are bootstrap-active, so the comparison isolates traversal rather +// than the depth of activation-history calculations. Use the opt-in on-disk +// TestRootedStakeCacheScale for mixed activation states and concurrent commits. +func BenchmarkRootedStakeRecalculation(b *testing.B) { + for _, count := range []int{20_000, 2_000_000} { + b.Run(fmt.Sprint(count), func(b *testing.B) { + db := newRPCAccountsDB(b) + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 10, Delta: []*accounts.Account{cacheHistoryAccount(b, nil)}}}, 10, nil, nil) + require.NoError(b, err) + s := &RpcServer{acctsDb: db, epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 100}} + s.SetRootedBankState(10, 10, 0) + version, stable := db.CommittedAccountVersion() + require.True(b, stable) + c := &s.stakeCache + c.version, c.dataReady = version, true + c.byStake = make(map[solana.PublicKey]cachedDelegation, count) + for i := range count { + key, voter := solana.PublicKey{0xa0}, solana.PublicKey{0xb0} + binary.LittleEndian.PutUint64(key[1:], uint64(i)) + binary.LittleEndian.PutUint64(voter[1:], uint64(i%1000)) + c.byStake[key] = cachedDelegation{loaded: true, delegation: sealevel.Delegation{ + VoterPubkey: voter, StakeLamports: 1_000_000, + ActivationEpoch: math.MaxUint64, DeactivationEpoch: math.MaxUint64, + }} + } + require.NoError(b, s.recalculateStakeTotals(b.Context(), 10, 0, c.generation)) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if err := s.recalculateStakeTotals(b.Context(), 10, 0, c.generation); err != nil { + b.Fatal(err) + } + } + b.StopTimer() + require.Len(b, c.totals, 1000) + for _, total := range c.totals { + require.Equal(b, uint64(count/1000)*1_000_000, total) + } + }) + } +} diff --git a/pkg/rpcserver/rooted_stake_recalculation_test.go b/pkg/rpcserver/rooted_stake_recalculation_test.go new file mode 100644 index 00000000..b7f62cfa --- /dev/null +++ b/pkg/rpcserver/rooted_stake_recalculation_test.go @@ -0,0 +1,124 @@ +package rpcserver + +import ( + "context" + "encoding/binary" + "math" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func recalculationStake(t testing.TB, index int, voter byte, amount uint64) *accounts.Account { + account := cacheStakeAccount(t, 0xa0, voter, amount, math.MaxUint64, math.MaxUint64) + binary.LittleEndian.PutUint64(account.Key[1:], uint64(index)) + return account +} + +func recalculationServer(t *testing.T) *RpcServer { + delta := make([]*accounts.Account, 2048) + for i := range delta { + delta[i] = recalculationStake(t, i, 1, 100) + } + s := seedStakeCacheServer(t, delta...) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 204800}, cacheTotals(t, s)) + commitCacheAccounts(t, s, 100, cacheHistoryAccount(t, nil)) + return s +} + +func TestRootedStakeCacheRecalculationMapChurn(t *testing.T) { + s := recalculationServer(t) + expected := map[solana.PublicKey]uint64{{1}: 204800} + mutated := false + s.stakeCache.beforeRecalculationChunk = func(start int) { + if start != stakeCacheBatchSize || mutated { + return + } + mutated = true + // Select both already-counted and not-yet-counted entries without + // depending on randomized map iteration order. + var counted, pending []solana.PublicKey + c := &s.stakeCache + c.mu.Lock() + for key, entry := range c.byStake { + if entry.calculation == c.calculation && len(counted) < 2 { + counted = append(counted, key) + } else if entry.calculation != c.calculation && len(pending) < 2 { + pending = append(pending, key) + } + } + c.mu.Unlock() + require.Len(t, counted, 2) + require.Len(t, pending, 2) + var changed []*accounts.Account + for _, keys := range [][]solana.PublicKey{counted, pending} { + updated := recalculationStake(t, 0, 2, 250) + updated.Key = keys[0] + changed = append(changed, updated, &accounts.Account{Key: keys[1]}) + expected[solana.PublicKey{1}] -= 200 + expected[solana.PublicKey{2}] += 250 + } + // Force map growth while its iterator is suspended. New entries must + // contribute once whether the iterator later sees them or skips them. + for i := 2048; i < 8192; i++ { + changed = append(changed, recalculationStake(t, i, 3, 75)) + expected[solana.PublicKey{3}] += 75 + } + commitCacheAccounts(t, s, 101, changed...) + // Reinsert keys removed from both sides of the cursor, before resuming. + var recreated []*accounts.Account + for _, key := range []solana.PublicKey{counted[1], pending[1]} { + a := recalculationStake(t, 0, 4, 30) + a.Key = key + recreated = append(recreated, a) + expected[solana.PublicKey{4}] += 30 + } + commitCacheAccounts(t, s, 102, recreated...) + } + _, err := s.rootedActivatedStakes(t.Context(), 100, 1, nil) + require.ErrorIs(t, err, errStakeViewChanged, "the response bank changed, but the calculation may be reused") + require.True(t, mutated) + require.True(t, s.stakeCache.totalsReady) + require.Equal(t, expected, cacheTotals(t, s)) + require.Equal(t, uint64(2), s.stakeCache.recalculations) + require.Equal(t, uint64(1), s.stakeCache.scans) +} + +func TestRootedStakeCacheRecalculationInterruptedMidMap(t *testing.T) { + for _, action := range []string{"cancel", "history", "rewind"} { + t.Run(action, func(t *testing.T) { + s := recalculationServer(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + interrupted := false + s.stakeCache.beforeRecalculationChunk = func(start int) { + if start != stakeCacheBatchSize || interrupted { + return + } + interrupted = true + switch action { + case "cancel": + cancel() + case "history": + commitCacheAccounts(t, s, 101, cacheHistoryAccount(t, nil)) + case "rewind": + _, err := s.acctsDb.RewindToBatchBoundary(10) + require.NoError(t, err) + s.SetRootedBankState(10, 10, 0) + } + } + _, err := s.rootedActivatedStakes(ctx, 100, 1, nil) + if action == "cancel" { + require.ErrorIs(t, err, context.Canceled) + } else { + require.ErrorIs(t, err, errStakeViewChanged) + } + require.True(t, interrupted) + require.False(t, s.stakeCache.totalsReady) + s.stakeCache.beforeRecalculationChunk = nil + require.Equal(t, map[solana.PublicKey]uint64{{1}: 204800}, cacheTotals(t, s)) + }) + } +} diff --git a/pkg/rpcserver/rpc_parameter_contract_test.go b/pkg/rpcserver/rpc_parameter_contract_test.go new file mode 100644 index 00000000..353b85a8 --- /dev/null +++ b/pkg/rpcserver/rpc_parameter_contract_test.go @@ -0,0 +1,175 @@ +package rpcserver + +import ( + "encoding/json" + "errors" + "fmt" + "net/http/httptest" + "strings" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/filecoin-project/go-jsonrpc" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestSpecExactRPCParameters(t *testing.T) { + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 432000}} + server.SetRootedBankState(9007199254740992, 4, 5) + t.Run("getSlot_exact_control", func(t *testing.T) { + _, err := server.GetSlot(t.Context(), jsonrpc.RawParams(`[{"minContextSlot":9007199254740993}]`)) + var behind *MinContextSlotNotReachedError + if !errors.As(err, &behind) { + t.Fatalf("expected minimum-context error, got %v", err) + } + }) + t.Run("epoch_minimum_must_not_round_down", func(t *testing.T) { + got, err := server.GetEpochInfo(t.Context(), jsonrpc.RawParams(`[{"minContextSlot":9007199254740993}]`)) + var behind *MinContextSlotNotReachedError + if !errors.As(err, &behind) { + t.Fatalf("requested minimum 9007199254740993; returned slot %d with error %v", got.AbsoluteSlot, err) + } + }) + t.Run("distance_accepts_full_u64", func(t *testing.T) { + params, err := decodeOptionalParams(jsonrpc.RawParams(`[{"delinquentSlotDistance":18446744073709551615}]`)) + if err != nil { + t.Fatal(err) + } + got, err := parseGetVoteAccountsConfig(params) + if err != nil || got.delinquentSlotDistance != ^uint64(0) { + t.Fatalf("valid maximum u64: value=%d error=%v", got.delinquentSlotDistance, err) + } + }) + t.Run("fraction_must_not_round_to_integer", func(t *testing.T) { + _, err := server.GetEpochInfo(t.Context(), jsonrpc.RawParams(`[{"minContextSlot":1.0000000000000001}]`)) + var invalid *InvalidParamsError + if !errors.As(err, &invalid) { + t.Fatalf("fractional u64 accepted: error=%v", err) + } + }) + t.Run("null_optional_minimum", func(t *testing.T) { + _, err := server.GetEpochInfo(t.Context(), jsonrpc.RawParams(`[{"minContextSlot":null}]`)) + if err != nil { + t.Fatalf("optional minimum null should use default: %v", err) + } + }) + t.Run("balance_minimum_must_not_round_down", func(t *testing.T) { + balanceServer := &RpcServer{acctsDb: newRPCAccountsDB(t)} + balanceServer.SetRootedBankState(9007199254740992, 4, 5) + params := jsonrpc.RawParams(`["` + (solana.PublicKey{8}).String() + `",{"minContextSlot":9007199254740993}]`) + got, err := balanceServer.GetBalance(t.Context(), params) + var behind *MinContextSlotNotReachedError + if !errors.As(err, &behind) { + t.Fatalf("requested minimum 9007199254740993; returned slot %d with error %v", got.Context.Slot, err) + } + }) +} + +func TestRPCParameterHTTPContract(t *testing.T) { + server := NewRpcServer(newRPCAccountsDB(t), 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 432000}, solana.Hash{1}) + require.NoError(t, server.listener.Close()) + server.SetRootedBankState(9007199254740992, 4, 5) + endpoint := httptest.NewServer(server) + t.Cleanup(endpoint.Close) + key := (solana.PublicKey{8}).String() + for _, test := range []struct { + method, params string + code int + }{ + {"getEpochInfo", `[{"minContextSlot":9007199254740993}]`, -32016}, + {"getBalance", `["` + key + `",{"minContextSlot":9007199254740993}]`, -32016}, + {"getEpochInfo", `[{"commitment":null,"minContextSlot":null}]`, 0}, + {"getBalance", `["` + key + `",{"commitment":null,"minContextSlot":null}]`, 0}, + {"getEpochInfo", `[{"minContextSlot":1.0000000000000001}]`, -32602}, + {"getBalance", `["` + key + `",{"minContextSlot":1.0000000000000001}]`, -32602}, + {"getLeaderSchedule", `[1.0000000000000001]`, -32602}, + {"getVoteAccounts", `[{"delinquentSlotDistance":1.0000000000000001}]`, -32602}, + {"getBlockProduction", `[{"range":{"firstSlot":1.0000000000000001}}]`, -32602}, + {"getBlockProduction", `[{"range":{"firstSlot":0,"lastSlot":1.0000000000000001}}]`, -32602}, + {"getBlockProduction", `[{"range":{"firstSlot":null}}]`, -32602}, + {"getBlockProduction", `[{"range":{}}]`, -32602}, + {"getBalance", `[null]`, -32602}, + {"getEpochInfo", `[{"minContextSlot":18446744073709551616}]`, -32602}, + {"getEpochInfo", `[{"minContextSlot":-1}]`, -32602}, + {"getEpochInfo", `[{"minContextSlot":"42"}]`, -32602}, + {"getEpochInfo", `[{"minContextSlot":true}]`, -32602}, + {"getEpochInfo", `[{"minContextSlot":1e0}]`, -32602}, + {"getVoteAccounts", `[{"keepUnstakedDelinquents":"true"}]`, -32602}, + {"getLeaderSchedule", `[null,{"keyByVoteAccount":"true"}]`, -32602}, + {"getLeaderSchedule", `[null,{"keyByVoteAccount":true}]`, 0}, + {"getLeaderSchedule", `[null,{"keyByVoteAccount":null}]`, 0}, + } { + t.Run(test.method+"/"+test.params, func(t *testing.T) { + body := fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":%q,"params":%s}`, test.method, test.params) + response, err := endpoint.Client().Post(endpoint.URL, "application/json", strings.NewReader(body)) + require.NoError(t, err) + defer response.Body.Close() + var got struct { + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Data struct { + ContextSlot uint64 `json:"contextSlot"` + } `json:"data"` + } `json:"error"` + } + require.NoError(t, json.NewDecoder(response.Body).Decode(&got)) + if test.code == 0 { + require.Nil(t, got.Error) + require.NotEmpty(t, got.Result) + if test.method == "getEpochInfo" { + var result GetEpochInfoResp + require.NoError(t, json.Unmarshal(got.Result, &result)) + require.Equal(t, uint64(9007199254740992), result.AbsoluteSlot) + } else if test.method == "getBalance" { + var result GetBalanceResp + require.NoError(t, json.Unmarshal(got.Result, &result)) + require.Equal(t, uint64(9007199254740992), result.Context.Slot) + require.Zero(t, result.Value) + } + } else { + require.NotNil(t, got.Error) + require.Equal(t, test.code, got.Error.Code) + if test.code == -32016 { + require.Equal(t, uint64(9007199254740992), got.Error.Data.ContextSlot) + } + } + }) + } +} + +func TestRPCOptionalConfigAndDecoderContract(t *testing.T) { + params, err := decodeOptionalParams(jsonrpc.RawParams(`[{"commitment":null,"identity":null,"votePubkey":null,"range":null,"keyByVoteAccount":null,"keepUnstakedDelinquents":null,"delinquentSlotDistance":null}]`)) + require.NoError(t, err) + vote, err := parseGetVoteAccountsConfig(params) + require.NoError(t, err) + require.Nil(t, vote.votePubkey) + require.Equal(t, uint64(128), vote.delinquentSlotDistance) + leaders, err := parseLeaderScheduleConfig(params[0]) + require.NoError(t, err) + require.Nil(t, leaders.identity) + production, err := parseBlockProductionConfig(params) + require.NoError(t, err) + require.Nil(t, production.identity) + require.Nil(t, production.firstSlot) + require.Nil(t, production.lastSlot) + + params, err = decodeOptionalParams(jsonrpc.RawParams(`[{"range":{"firstSlot":9007199254740993,"lastSlot":18446744073709551615}}]`)) + require.NoError(t, err) + production, err = parseBlockProductionConfig(params) + require.NoError(t, err) + require.Equal(t, uint64(9007199254740993), *production.firstSlot) + require.Equal(t, ^uint64(0), *production.lastSlot) + params, err = decodeOptionalParams(jsonrpc.RawParams(`[{"range":{"firstSlot":0,"lastSlot":null}}]`)) + require.NoError(t, err) + production, err = parseBlockProductionConfig(params) + require.NoError(t, err) + require.Nil(t, production.lastSlot) + for _, raw := range []string{`[] []`, `[] trailing`, `[`, `{}`, `true`} { + _, err := decodeOptionalParams(jsonrpc.RawParams(raw)) + require.Error(t, err, raw) + } + _, err = decodeOptionalParams(jsonrpc.RawParams("[] \n\t")) + require.NoError(t, err) +} diff --git a/pkg/rpcserver/rpc_registration_test.go b/pkg/rpcserver/rpc_registration_test.go index 2d781021..638f7a52 100644 --- a/pkg/rpcserver/rpc_registration_test.go +++ b/pkg/rpcserver/rpc_registration_test.go @@ -31,12 +31,25 @@ func TestRPCHandlerMethodSet(t *testing.T) { } func TestInternalMethodsAreNotRPC(t *testing.T) { + for _, method := range []string{"enableEpochRewards", "setRootedBankState"} { + t.Run(method, func(t *testing.T) { + testInternalMethodNotRPC(t, method) + }) + } +} + +func testInternalMethodNotRPC(t *testing.T, method string) { + t.Helper() for _, mode := range []string{"single", "mixed-batch", "oversized-single", "direct-dispatch"} { t.Run(mode, func(t *testing.T) { rpc := NewRpcServer(nil, 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 32}, solana.Hash{1}) t.Cleanup(func() { _ = rpc.listener.Close() }) directory := filepath.Join(t.TempDir(), "history") - call := map[string]any{"jsonrpc": "2.0", "id": 1, "method": "enableEpochRewards", "params": []any{directory, 100, 0}} + rpc.SetRootedBankState(100, 90, 80) + call := map[string]any{"jsonrpc": "2.0", "id": 1, "method": method, "params": []any{777, 666, 555}} + if method == "enableEpochRewards" { + call["params"] = []any{directory, 100, 0} + } if mode == "oversized-single" { call["padding"] = strings.Repeat("x", maxQuietMethodProbeBody+1) } @@ -86,6 +99,10 @@ func TestInternalMethodsAreNotRPC(t *testing.T) { if _, err := os.Stat(directory); !os.IsNotExist(err) { t.Fatalf("internal request created history directory: %v", err) } + root, _ := rpc.getRootedBankState() + if root.Slot != 100 { + t.Fatalf("internal request changed rooted bank: %d", root.Slot) + } }) } } diff --git a/pkg/rpcserver/rpcserver.go b/pkg/rpcserver/rpcserver.go index b8459d61..520b1d84 100644 --- a/pkg/rpcserver/rpcserver.go +++ b/pkg/rpcserver/rpcserver.go @@ -11,8 +11,10 @@ import ( "net/http/httptest" "strings" "sync" + "sync/atomic" "time" + "github.com/Overclock-Validator/mithril/pkg/accounts" "github.com/Overclock-Validator/mithril/pkg/accountsdb" b "github.com/Overclock-Validator/mithril/pkg/block" "github.com/Overclock-Validator/mithril/pkg/epochrewards" @@ -31,12 +33,17 @@ type rpcHandler struct { type rpcMethods interface { GetAccountInfo(context.Context, jsonrpc.RawParams) (GetAccountInfoResp, error) + GetBalance(context.Context, jsonrpc.RawParams) (GetBalanceResp, error) GetBankHash(context.Context, jsonrpc.RawParams) (string, error) GetBlockHeight(context.Context, jsonrpc.RawParams) (uint64, error) + GetBlockProduction(context.Context, jsonrpc.RawParams) (GetBlockProductionResp, error) GetEpochInfo(context.Context, jsonrpc.RawParams) (GetEpochInfoResp, error) GetGenesisHash(context.Context, jsonrpc.RawParams) (string, error) GetInflationReward(context.Context, jsonrpc.RawParams) ([]*InflationRewardResp, error) GetLatestBlockhash(context.Context, jsonrpc.RawParams) (GetLatestBlockhashResp, error) + GetLeaderSchedule(context.Context, jsonrpc.RawParams) (map[string][]uint64, error) + GetSlot(context.Context, jsonrpc.RawParams) (uint64, error) + GetVoteAccounts(context.Context, jsonrpc.RawParams) (GetVoteAccountsResp, error) SendTransaction(context.Context, jsonrpc.RawParams) (string, error) SimulateTransaction(context.Context, jsonrpc.RawParams) (SimulateTransactionResp, error) } @@ -55,6 +62,12 @@ type RpcServer struct { slotCtxMu sync.RWMutex genesisHash string epochRewards *epochrewards.Store + rootedBank atomic.Pointer[rootedBankState] + + stakeCacheOnce sync.Once + stakeCache rootedStakeCache + voteRecordsOnce sync.Once + voteRecords rootedVoteRecordCache leaderTPUCacheMu sync.RWMutex leaderTPUByIdentity map[solana.PublicKey]tpuEndpoint @@ -69,16 +82,30 @@ type RpcServer struct { sendTransactionLeaderForwardCount uint64 } +// rootedBankState identifies the durable AccountsDB view served by state RPCs. +// Publishing it only after a successful fold keeps response context and account +// values on the same finalized bank. +type rootedBankState struct { + Slot uint64 + BlockHeight uint64 + TransactionCount uint64 +} + const maxQuietMethodProbeBody = 64 << 10 var supportedRPCMethods = map[string]struct{}{ "getAccountInfo": {}, + "getBalance": {}, "getBankHash": {}, + "getBlockProduction": {}, "getBlockHeight": {}, "getEpochInfo": {}, "getGenesisHash": {}, "getInflationReward": {}, "getLatestBlockhash": {}, + "getLeaderSchedule": {}, + "getSlot": {}, + "getVoteAccounts": {}, "sendTransaction": {}, "simulateTransaction": {}, } @@ -200,6 +227,86 @@ func (rpcServer *RpcServer) SetHistoryRecoveryPending(pending bool) { } } +func (rpcServer *RpcServer) SetRootedBankState(slot, blockHeight, transactionCount uint64) { + rpcServer.rootedBank.Store(&rootedBankState{ + Slot: slot, + BlockHeight: blockHeight, + TransactionCount: transactionCount, + }) +} + +func (rpcServer *RpcServer) getRootedBankState() (rootedBankState, bool) { + rooted := rpcServer.rootedBank.Load() + if rooted == nil { + return rootedBankState{}, false + } + return *rooted, true +} + +func (rpcServer *RpcServer) readRootedAccount(ctx context.Context, pubkey solana.PublicKey) (rootedBankState, *accounts.Account, error) { + rooted, accountSet, err := rpcServer.readRootedAccounts(ctx, []solana.PublicKey{pubkey}) + if err != nil { + return rooted, nil, err + } + // Batch reads use zero-lamport placeholders for absent/deleted accounts. + // Translate that loader representation into absence at the RPC boundary. + if len(accountSet) != 1 || accountSet[0] == nil || accountSet[0].Lamports == 0 { + return rooted, nil, accountsdb.ErrNoAccount + } + return rooted, accountSet[0], nil +} + +func (rpcServer *RpcServer) readRootedAccounts(ctx context.Context, pubkeys []solana.PublicKey) (rootedBankState, []*accounts.Account, error) { + for { + bank := rpcServer.rootedBank.Load() + if bank == nil { + return rootedBankState{}, nil, fmt.Errorf("node has no rooted bank available") + } + rooted := *bank + if rpcServer.acctsDb == nil { + return rootedBankState{}, nil, fmt.Errorf("node has no accounts database available") + } + version, stable := rpcServer.acctsDb.CommittedAccountVersion() + if !stable || rpcServer.rootedPublicationPending(rooted.Slot) { + if err := waitForRootedPublication(ctx); err != nil { + return rooted, nil, err + } + continue + } + accounts, stats, err := rpcServer.acctsDb.GetAccountsBatchSharedWithStats(ctx, rooted.Slot, pubkeys) + latest, stable := rpcServer.acctsDb.CommittedAccountVersion() + if !stable || latest != version || rpcServer.rootedBank.Load() != bank || stats.PendingFoldHits > 0 || + rpcServer.rootedPublicationPending(rooted.Slot) { + if err := waitForRootedPublication(ctx); err != nil { + return rooted, nil, err + } + continue + } + return rooted, accounts, err + } +} + +func (rpcServer *RpcServer) rootedPublicationPending(rootedSlot uint64) bool { + if !rpcServer.acctsDb.RootedDurable { + committed := rpcServer.acctsDb.CommittedAccountSlot() + return committed != 0 && committed != rootedSlot + } + durableThrough := rpcServer.acctsDb.DurableThrough() + // Zero means the snapshot baseline is active and no fold has committed yet. + return durableThrough != 0 && durableThrough != rootedSlot +} + +func waitForRootedPublication(ctx context.Context) error { + timer := time.NewTimer(time.Millisecond) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + func NewRpcServer(acctsDb *accountsdb.AccountsDb, port uint16, epochSchedule *sealevel.SysvarEpochSchedule, genesisHash solana.Hash) *RpcServer { var err error rpcServer := &RpcServer{genesisHash: genesisHash.String()} diff --git a/pkg/rpcserver/rpcserver_test.go b/pkg/rpcserver/rpcserver_test.go index 8ec2c4de..7720037f 100644 --- a/pkg/rpcserver/rpcserver_test.go +++ b/pkg/rpcserver/rpcserver_test.go @@ -10,7 +10,7 @@ import ( func TestServeHTTPQuietlyHandlesUnsupportedMethod(t *testing.T) { rpcServer := &RpcServer{} - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"jsonrpc":"2.0","method":"getSlot","id":7}`)) + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"jsonrpc":"2.0","method":"unsupportedMethod","id":7}`)) rec := httptest.NewRecorder() rpcServer.ServeHTTP(rec, req) @@ -33,7 +33,7 @@ func TestServeHTTPQuietlyHandlesUnsupportedMethod(t *testing.T) { if resp.JSONRPC != "2.0" || resp.ID != 7 { t.Fatalf("unexpected response identity: %+v", resp) } - if resp.Error.Code != -32601 || resp.Error.Message != "method 'getSlot' not found" { + if resp.Error.Code != -32601 || resp.Error.Message != "method 'unsupportedMethod' not found" { t.Fatalf("unexpected error response: %+v", resp.Error) } } diff --git a/pkg/rpcserver/vote_record_cache.go b/pkg/rpcserver/vote_record_cache.go new file mode 100644 index 00000000..1fc8a114 --- /dev/null +++ b/pkg/rpcserver/vote_record_cache.go @@ -0,0 +1,145 @@ +package rpcserver + +import ( + "context" + "sync" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" +) + +// rootedVoteRecordCache retains only immutable, account-derived RPC fields, not +// borrowed account buffers, stake totals, epoch membership or delinquency. Those +// bank/query-dependent values are still calculated for each response. +// +// Writes invalidate touched records before the account version becomes readable. +// Decoding is lazy on the RPC caller, never on the account writer. Recovery, +// rewind and failed writes discard all records, including same-slot fork reuse. +// The enclosing RPC must still validate its account version and bank identity. +type rootedVoteRecordCache struct { + mu sync.Mutex + records map[solana.PublicKey]*VoteAccountInfo +} + +func (c *rootedVoteRecordCache) AccountsChanged(_ uint64, changed []*accounts.Account, reset bool) { + c.mu.Lock() + defer c.mu.Unlock() + if reset { + c.records = nil + return + } + for _, account := range changed { + if account != nil { + delete(c.records, account.Key) + } + } +} + +func decodeVoteRecord(key solana.PublicKey, account *accounts.Account) *VoteAccountInfo { + if account == nil || account.Lamports == 0 || account.Owner != addresses.VoteProgramAddr { + return nil + } + versioned, err := sealevel.UnmarshalVersionedVoteState(account.Data) + if err != nil || !versioned.IsInitialized() { + return nil + } + var voteState *sealevel.VoteState + var v4View sealevel.VoteState + if versioned.Type == sealevel.VoteStateVersionV4 { + // Only these read-only fields are needed by RPC. Avoid allocating the + // full legacy write-back conversion for every refreshed V4 account. + state := &versioned.V4 + v4View.NodePubkey, v4View.RootSlot = state.NodePubkey, state.RootSlot + v4View.Votes, v4View.EpochCredits = state.Votes, state.EpochCredits + voteState = &v4View + } else { + voteState = versioned.ConvertToCurrent() + } + lastVote, _ := voteState.LastVotedSlot() + var rootSlot uint64 + if voteState.RootSlot != nil { + rootSlot = *voteState.RootSlot + } + creditsStart := max(0, len(voteState.EpochCredits)-maxRPCEpochCreditsHistory) + creditsHistory := voteState.EpochCredits[creditsStart:] + // Keep the bounded credit history in the same allocation as the record. + // The returned pointer retains this storage; responses still copy the slice. + record := new(struct { + info VoteAccountInfo + credits [maxRPCEpochCreditsHistory][3]uint64 + }) + epochCredits := record.credits[:len(creditsHistory):len(creditsHistory)] + for i, credits := range creditsHistory { + epochCredits[i] = [3]uint64{credits.Epoch, credits.Credits, credits.PrevCredits} + } + commission, commissionBPS := voteCommission(versioned, voteState) + record.info = VoteAccountInfo{ + VotePubkey: key.String(), NodePubkey: voteState.NodePubkey.String(), + Commission: commission, InflationRewardsCommissionBPS: commissionBPS, + EpochCredits: epochCredits, LastVote: lastVote, RootSlot: rootSlot, + } + return &record.info +} + +func (s *RpcServer) rootedVoteRecords(ctx context.Context, slot, version uint64, keys []solana.PublicKey) ([]*VoteAccountInfo, error) { + s.voteRecordsOnce.Do(func() { s.acctsDb.ObserveAccountState(&s.voteRecords) }) + c := &s.voteRecords + records := make([]*VoteAccountInfo, len(keys)) + var missing []solana.PublicKey + var positions []int + c.mu.Lock() + for i, key := range keys { + records[i] = c.records[key] + if records[i] == nil { + missing = append(missing, key) + positions = append(positions, i) + } + } + c.mu.Unlock() + if len(missing) == 0 { + return records, nil // The enclosing RPC checks the final version/bank fence. + } + rooted, accountSet, err := s.readRootedAccounts(ctx, missing) + if err != nil { + return nil, err + } + if rooted.Slot != slot { + return nil, errStakeViewChanged + } + decoded := make([]*VoteAccountInfo, len(missing)) + for i, account := range accountSet { + if err := ctx.Err(); err != nil { + return nil, err + } + decoded[i] = decodeVoteRecord(missing[i], account) + records[positions[i]] = decoded[i] + } + if !s.storeVoteRecords(version, missing, decoded) { + return nil, errStakeViewChanged + } + return records, nil +} + +func (s *RpcServer) storeVoteRecords(version uint64, keys []solana.PublicKey, records []*VoteAccountInfo) bool { + c := &s.voteRecords + c.mu.Lock() + defer c.mu.Unlock() + // Check under the invalidation lock: a concurrent commit must either make + // this load stale or subsequently remove its entries before readers see it. + latest, stable := s.acctsDb.CommittedAccountVersion() + if !stable || latest != version { + return false + } + if c.records == nil { + c.records = make(map[solana.PublicKey]*VoteAccountInfo) + } + for i, record := range records { + // Do not accumulate negative entries for arbitrary filtered RPC keys. + if record != nil { + c.records[keys[i]] = record + } + } + return true +} diff --git a/pkg/rpcserver/vote_record_cache_bench_test.go b/pkg/rpcserver/vote_record_cache_bench_test.go new file mode 100644 index 00000000..3f075a67 --- /dev/null +++ b/pkg/rpcserver/vote_record_cache_bench_test.go @@ -0,0 +1,96 @@ +package rpcserver + +import ( + "encoding/binary" + "encoding/json" + "path/filepath" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/filecoin-project/go-jsonrpc" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Measures the warm RPC handler plus JSON encoding, not HTTP transport, startup, +// stake aggregation or commits. Vote accounts include lockouts and credit history +// so decoding costs are represented rather than using empty vote-state fixtures. +// Changed cases invalidate decoded records before each query (including that cost), +// simulating account updates with warm account buffers; they do not perform disk +// commits and must not be presented as end-to-end writer/RPC contention results. +func BenchmarkGetVoteAccountsRecords(b *testing.B) { + db := newRPCAccountsDB(b) + global.ClearPendingStakePubkeys() + b.Cleanup(global.ClearPendingStakePubkeys) + global.EnqueuePendingStakePubkey(500, solana.PublicKey{0xee}) + _, err := global.FlushPendingStakePubkeys(filepath.Dir(db.AcctsDir)) + require.NoError(b, err) + delta := []*accounts.Account{cacheHistoryAccount(b, nil)} + stakes := make(map[solana.PublicKey]uint64) + for i := range 1000 { + key := solana.PublicKey{0xb0} + binary.LittleEndian.PutUint64(key[1:], uint64(i)) + v := &sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4} + v.V4.NodePubkey = key + v.V4.AuthorizedVoters.AuthorizedVoters.Set(0, key) + for j := range 31 { + v.V4.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: uint64(470 + j)}}) + } + for j := range 64 { + v.V4.EpochCredits = append(v.V4.EpochCredits, sealevel.EpochCredits{Epoch: uint64(j), Credits: uint64(j + 1), PrevCredits: uint64(j)}) + } + delta = append(delta, voteAccountForRPC(b, key, v)) + stakes[key] = 1 + } + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 500, Delta: delta}}, 500, nil, nil) + require.NoError(b, err) + require.NoError(b, db.SeedVoteAccountPubkeys(nil)) + previous, present := global.EpochStakesSnapshot(0) + global.PutEpochStakes(0, stakes, map[solana.PublicKey]*epochstakes.VoteAccount{}, 1000) + b.Cleanup(func() { + if present { + global.PutEpochStakes(0, previous.Stakes, previous.VoteAccounts, previous.TotalStake) + } else { + global.ClearEpochStakes(0) + } + }) + s := &RpcServer{acctsDb: db, epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 1000}} + s.SetRootedBankState(500, 500, 0) + for _, tc := range []struct { + name string + params []interface{} + count int + changed int + }{ + {"All", []interface{}{}, 1000, 0}, + {"One", []interface{}{map[string]interface{}{"votePubkey": (solana.PublicKey{0xb0}).String()}}, 1, 0}, + {"All10PercentChanged", []interface{}{}, 1000, 100}, + {"AllChanged", []interface{}{}, 1000, 1000}, + } { + b.Run(tc.name, func(b *testing.B) { + raw, err := json.Marshal(tc.params) + require.NoError(b, err) + params := jsonrpc.RawParams(raw) + response, err := s.GetVoteAccounts(b.Context(), params) + require.NoError(b, err) + require.Len(b, response.Current, tc.count) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if tc.changed > 0 { + invalidateVoteRecordsForBenchmark(s, delta[1:1+tc.changed]) + } + response, err := s.GetVoteAccounts(b.Context(), params) + if err != nil { + b.Fatal(err) + } + if _, err := json.Marshal(response); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/rpcserver/vote_record_cache_test.go b/pkg/rpcserver/vote_record_cache_test.go new file mode 100644 index 00000000..ab0646bc --- /dev/null +++ b/pkg/rpcserver/vote_record_cache_test.go @@ -0,0 +1,180 @@ +package rpcserver + +import ( + "context" + "fmt" + "math" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func recordTestAccount(t testing.TB, slot uint64) *accounts.Account { + v := &sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4} + v.V4.NodePubkey = solana.PublicKey{2} + v.V4.AuthorizedVoters.AuthorizedVoters.Set(0, v.V4.NodePubkey) + v.V4.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: slot}}) + v.V4.EpochCredits = []sealevel.EpochCredits{{Epoch: 0, Credits: slot}} + v.V4.RootSlot = &slot + v.V4.InflationRewardsCommissionBps = 755 + return voteAccountForRPC(t, solana.PublicKey{1}, v) +} + +func readTestVoteRecord(t *testing.T, s *RpcServer) *VoteAccountInfo { + t.Helper() + version, stable := s.acctsDb.CommittedAccountVersion() + require.True(t, stable) + bank := s.rootedBank.Load() + records, err := s.rootedVoteRecords(t.Context(), bank.Slot, version, []solana.PublicKey{{1}}) + require.NoError(t, err) + return records[0] +} + +func TestVoteRecordCacheInvalidationAndRecovery(t *testing.T) { + s := seedStakeCacheServer(t, recordTestAccount(t, 10)) + first := readTestVoteRecord(t, s) + require.NotNil(t, first) + require.Same(t, first, readTestVoteRecord(t, s)) + commitCacheAccounts(t, s, 11, &accounts.Account{Key: solana.PublicKey{9}, Lamports: 1, Owner: addresses.SystemProgramAddr}) + require.Same(t, first, readTestVoteRecord(t, s), "unrelated writes must preserve parsed records") + oldVersion, _ := s.acctsDb.CommittedAccountVersion() + commitCacheAccounts(t, s, 12, recordTestAccount(t, 12)) + require.False(t, s.storeVoteRecords(oldVersion, []solana.PublicKey{{1}}, []*VoteAccountInfo{first}), "an in-flight old decode cannot resurrect an invalidated record") + updated := readTestVoteRecord(t, s) + require.Equal(t, uint64(12), updated.LastVote) + require.Equal(t, uint64(10), first.LastVote, "in-flight readers retain immutable old records") + for _, account := range []*accounts.Account{ + {Key: solana.PublicKey{1}, Lamports: 1, Owner: addresses.SystemProgramAddr}, + {Key: solana.PublicKey{1}, Lamports: 0, Owner: addresses.VoteProgramAddr}, + {Key: solana.PublicKey{1}, Lamports: 1, Owner: addresses.VoteProgramAddr, Data: []byte{0}}, + voteAccountForRPC(t, solana.PublicKey{1}, &sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4}), + } { + slot := s.rootedBank.Load().Slot + 1 + commitCacheAccounts(t, s, slot, account) + require.Nil(t, readTestVoteRecord(t, s)) + commitCacheAccounts(t, s, slot+1, recordTestAccount(t, slot+1)) + require.Equal(t, slot+1, readTestVoteRecord(t, s).LastVote) + } + _, err := s.acctsDb.RewindToBatchBoundary(10) + require.NoError(t, err) + s.SetRootedBankState(10, 10, 0) + require.Empty(t, s.voteRecords.records) + require.Equal(t, first, readTestVoteRecord(t, s)) + commitCacheAccounts(t, s, 11, recordTestAccount(t, 9)) // Same slot, different fork. + require.Equal(t, uint64(9), readTestVoteRecord(t, s).LastVote) + _, err = s.acctsDb.RecoverFoldState() + require.NoError(t, err) + require.Empty(t, s.voteRecords.records) + require.Equal(t, uint64(9), readTestVoteRecord(t, s).LastVote) + // Failed writes use the same observer reset contract as recovery. + s.voteRecords.AccountsChanged(0, nil, true) + require.Empty(t, s.voteRecords.records) +} + +func TestVoteRecordCacheResponseOwnershipAndPublication(t *testing.T) { + s := seedStakeCacheServer(t, recordTestAccount(t, 10), cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64)) + previous, present := global.EpochStakesSnapshot(0) + global.PutEpochStakes(0, map[solana.PublicKey]uint64{{1}: 1}, map[solana.PublicKey]*epochstakes.VoteAccount{}, 1) + t.Cleanup(func() { + if present { + global.PutEpochStakes(0, previous.Stakes, previous.VoteAccounts, previous.TotalStake) + } else { + global.ClearEpochStakes(0) + } + }) + params := mustRawParams(t, []interface{}{}) + first, err := s.GetVoteAccounts(t.Context(), params) + require.NoError(t, err) + require.Len(t, first.Current, 1) + first.Current[0].EpochCredits[0][1] = 999 + first.Current[0].NodePubkey = "mutated" + next, err := s.GetVoteAccounts(t.Context(), params) + require.NoError(t, err) + require.Equal(t, uint64(10), next.Current[0].EpochCredits[0][1]) + require.Equal(t, (solana.PublicKey{2}).String(), next.Current[0].NodePubkey) + record := s.voteRecords.records[solana.PublicKey{1}] + // Root-relative classification is not cached with the account record. + _, err = s.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{map[string]interface{}{"delinquentSlotDistance": 0}})) + require.NoError(t, err) + _, err = s.acctsDb.CommitBatch([]accounts.SlotDelta{{Slot: 90, Delta: []*accounts.Account{cacheStakeAccount(t, 21, 1, 200, math.MaxUint64, math.MaxUint64)}}}, 90, nil, nil) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond) + defer cancel() + _, err = s.GetVoteAccounts(ctx, params) + require.ErrorIs(t, err, context.DeadlineExceeded, "cached records must not bypass rooted publication") + s.SetRootedBankState(90, 90, 0) + next, err = s.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{map[string]interface{}{"delinquentSlotDistance": 20}})) + require.NoError(t, err) + require.Empty(t, next.Current) + require.Len(t, next.Delinquent, 1) + require.Equal(t, uint64(200), next.Delinquent[0].ActivatedStake) + require.Same(t, record, s.voteRecords.records[solana.PublicKey{1}]) +} + +func TestVoteRecordCacheLegacyAndUnknownKeys(t *testing.T) { + s := seedStakeCacheServer(t, recordTestAccount(t, 10)) + readTestVoteRecord(t, s) + version, _ := s.acctsDb.CommittedAccountVersion() + keys := make([]solana.PublicKey, 100) + for i := range keys { + keys[i] = solana.PublicKey{0xee, byte(i)} + } + records, err := s.rootedVoteRecords(t.Context(), 10, version, keys) + require.NoError(t, err) + for _, record := range records { + require.Nil(t, record) + } + require.Len(t, s.voteRecords.records, 1, "arbitrary nonexistent filters must not grow retained memory") + s.acctsDb.RootedDurable = false + done := make(chan struct{}) + require.NoError(t, s.acctsDb.StoreAccounts([]*accounts.Account{recordTestAccount(t, 11)}, 11, func() { close(done) })) + <-done + require.Eventually(t, func() bool { _, stable := s.acctsDb.CommittedAccountVersion(); return stable }, time.Second, time.Millisecond) + s.SetRootedBankState(11, 11, 0) + require.Equal(t, uint64(11), readTestVoteRecord(t, s).LastVote) +} + +func invalidateVoteRecordsForBenchmark(s *RpcServer, changed []*accounts.Account) { + s.voteRecords.AccountsChanged(0, changed, false) +} + +func TestDecodeVoteRecordVersions(t *testing.T) { + for version := uint32(0); version <= sealevel.VoteStateVersionV4; version++ { + t.Run(fmt.Sprint(version), func(t *testing.T) { + key, node := solana.PublicKey{1}, solana.PublicKey{2} + root := uint64(9) + credits := []sealevel.EpochCredits{{Epoch: 1, Credits: 12, PrevCredits: 3}} + lockout := sealevel.VoteLockout{Slot: 10} + v := &sealevel.VoteStateVersions{Type: version} + switch version { + case sealevel.VoteStateVersionV0_23_5: + v.V0_23_5 = sealevel.VoteState0_23_5{NodePubkey: node, AuthorizedVoter: node, Commission: 8, RootSlot: &root, EpochCredits: credits} + v.V0_23_5.Votes.PushBack(lockout) + case sealevel.VoteStateVersionV1_14_11: + v.V1_14_11 = sealevel.VoteState1_14_11{NodePubkey: node, Commission: 8, RootSlot: &root, EpochCredits: credits} + v.V1_14_11.AuthorizedVoters.AuthorizedVoters.Set(0, node) + v.V1_14_11.Votes.PushBack(lockout) + case sealevel.VoteStateVersionCurrent: + v.Current = sealevel.VoteState{NodePubkey: node, Commission: 8, RootSlot: &root, EpochCredits: credits} + v.Current.AuthorizedVoters.AuthorizedVoters.Set(0, node) + v.Current.Votes.PushBack(sealevel.LandedVote{Lockout: lockout}) + case sealevel.VoteStateVersionV4: + v.V4.NodePubkey, v.V4.RootSlot, v.V4.EpochCredits = node, &root, credits + v.V4.InflationRewardsCommissionBps = 800 + v.V4.AuthorizedVoters.AuthorizedVoters.Set(0, node) + v.V4.Votes.PushBack(sealevel.LandedVote{Lockout: lockout}) + } + account := voteAccountForRPC(t, key, v) + got := decodeVoteRecord(key, account) + clear(account.Data) // The cache must own everything it retains. + require.Equal(t, &VoteAccountInfo{VotePubkey: key.String(), NodePubkey: node.String(), Commission: 8, InflationRewardsCommissionBPS: 800, RootSlot: 9, LastVote: 10, EpochCredits: [][3]uint64{{1, 12, 3}}}, got) + }) + } +} diff --git a/pkg/snapshot/build_db.go b/pkg/snapshot/build_db.go index dbc9745c..9ae50c8c 100644 --- a/pkg/snapshot/build_db.go +++ b/pkg/snapshot/build_db.go @@ -19,6 +19,7 @@ import ( "github.com/Overclock-Validator/mithril/pkg/statsd" "github.com/Overclock-Validator/mithril/pkg/txstatus" "github.com/cockroachdb/pebble" + "github.com/gagliardetto/solana-go" "github.com/panjf2000/ants/v2" ) @@ -310,12 +311,12 @@ func BuildAccountsDbPaths( numShards := snapshotIndexShards() sl := NewShardLogger(numShards, logsDir) - // Create stake pubkey collector for building stake index during appendvec processing - stakeCollector := &stakeIndexCollector{ - entries: make([]accountsdb.StakeIndexEntry, 0, 1000000), // Pre-allocate for ~1M stake accounts + // Collect stake and vote pubkeys while parsing appendvecs. + accountCollector := &snapshotAccountCollector{ + stakeEntries: make([]accountsdb.StakeIndexEntry, 0, 1000000), // Pre-allocate for ~1M stake accounts } - pools, err := initWorkerPools(wg, sl, manifest, incrementalManifest, accountsDbDir, &largestFileId, stakeCollector) + pools, err := initWorkerPools(wg, sl, manifest, incrementalManifest, accountsDbDir, &largestFileId, accountCollector) if err != nil { return nil, nil, fmt.Errorf("initializing worker pools: %w", err) } @@ -417,7 +418,7 @@ func BuildAccountsDbPaths( // Write stake pubkey index file (with appendvec location hints) stakeIndexPath := filepath.Join(accountsDbDir, "stake_pubkeys.idx") - if err := accountsdb.WriteStakePubkeyIndex(stakeIndexPath, stakeCollector.entries); err != nil { + if err := accountsdb.WriteStakePubkeyIndex(stakeIndexPath, accountCollector.stakeEntries); err != nil { return nil, nil, fmt.Errorf("writing stake pubkey index: %w", err) } @@ -432,6 +433,10 @@ func BuildAccountsDbPaths( if err != nil { return nil, nil, err } + if err := accountsDb.SeedVoteAccountPubkeys(accountCollector.votePubkeys); err != nil { + accountsDb.CloseDb() + return nil, nil, fmt.Errorf("seeding vote pubkey index: %w", err) + } if incrementalManifest != nil { return accountsDb, incrementalManifest, nil @@ -647,29 +652,22 @@ func invokeSnapshotTask(wg *sync.WaitGroup, pool *ants.PoolWithFunc, task any) ( return err } -// stakeIndexCollector aggregates stake account pubkeys from multiple worker goroutines -// during appendvec processing. Used to build the stake pubkey index file. -// -// WHY: The manifest's delegation list can be stale/incomplete (Firedancer notes: -// "the cache in the manifest is partially incomplete"). Instead of trusting manifest -// data, we: -// 1. Collect stake pubkeys during appendvec parsing (by checking owner == StakeProgramAddr) -// 2. Write them to stake_pubkeys.idx after snapshot processing -// 3. At startup, load pubkeys from index and read ALL delegation fields from AccountsDB -// -// This ensures stake cache contains fresh data from AccountsDB, not potentially stale -// manifest data. -type stakeIndexCollector struct { - mu sync.Mutex - entries []accountsdb.StakeIndexEntry +// snapshotAccountCollector records stake and vote pubkeys while workers parse +// appendvecs. The manifest delegation list can be incomplete, and vote accounts +// without stake are absent from it. +type snapshotAccountCollector struct { + mu sync.Mutex + stakeEntries []accountsdb.StakeIndexEntry + votePubkeys []solana.PublicKey } -func (c *stakeIndexCollector) Add(entries []accountsdb.StakeIndexEntry) { - if len(entries) == 0 { +func (c *snapshotAccountCollector) Add(stakes []accountsdb.StakeIndexEntry, votes []solana.PublicKey) { + if len(stakes) == 0 && len(votes) == 0 { return } c.mu.Lock() - c.entries = append(c.entries, entries...) + c.stakeEntries = append(c.stakeEntries, stakes...) + c.votePubkeys = append(c.votePubkeys, votes...) c.mu.Unlock() } @@ -680,7 +678,7 @@ func initWorkerPools( incrementalManifest *SnapshotManifest, accountsDbDir string, largestFileId *atomic.Uint64, - stakeCollector *stakeIndexCollector, + accountCollector *snapshotAccountCollector, ) (*snapshotWorkerPools, error) { indexEntryCommitterWorkers := snapshotIndexEntryCommitterWorkers() indexEntryBuilderWorkers := snapshotIndexEntryBuilderWorkers() @@ -723,14 +721,14 @@ func initWorkerPools( return } task := i.(indexEntryBuilderTask) - pubkeys, entries, stakeEntries, err := accountsdb.BuildIndexEntriesFromAppendVecs(task.Data, task.FileSize, task.Slot, task.FileId) + pubkeys, entries, stakeEntries, votePubkeys, err := accountsdb.BuildIndexEntriesFromAppendVecs(task.Data, task.FileSize, task.Slot, task.FileId) if err != nil { workerErrors.Record(fmt.Errorf("building index entries: %w", err)) return } - // Collect stake entries with appendvec location hints for building stake index - stakeCollector.Add(stakeEntries) + // Collect stake locations and vote pubkeys for their indexes. + accountCollector.Add(stakeEntries, votePubkeys) commitTask := indexEntryCommitterTask{IndexEntries: entries, Pubkeys: pubkeys} statsd.Timing(statsd.TasksIndexEntryBuilderLatency, uint64(time.Since(start)), nil) diff --git a/pkg/snapshot/build_db_with_incr.go b/pkg/snapshot/build_db_with_incr.go index 7afb8b9b..9b737b7c 100644 --- a/pkg/snapshot/build_db_with_incr.go +++ b/pkg/snapshot/build_db_with_incr.go @@ -81,12 +81,12 @@ func BuildAccountsDbAuto( defer cleanupIndexWorkDir() sl := NewShardLogger(numShards, logsDir) - // Create stake pubkey collector for building stake index during appendvec processing - stakeCollector := &stakeIndexCollector{ - entries: make([]accountsdb.StakeIndexEntry, 0, 1000000), // Pre-allocate for ~1M stake accounts + // Collect stake and vote pubkeys while parsing appendvecs. + accountCollector := &snapshotAccountCollector{ + stakeEntries: make([]accountsdb.StakeIndexEntry, 0, 1000000), // Pre-allocate for ~1M stake accounts } - pools, err := initWorkerPools(wg, sl, manifest, incrementalManifest, accountsDbDir, &largestFileId, stakeCollector) + pools, err := initWorkerPools(wg, sl, manifest, incrementalManifest, accountsDbDir, &largestFileId, accountCollector) if err != nil { return nil, nil, fmt.Errorf("initializing worker pools: %w", err) } @@ -287,7 +287,7 @@ func BuildAccountsDbAuto( // Write stake pubkey index file (with appendvec location hints) stakeIndexPath := filepath.Join(accountsDbDir, "stake_pubkeys.idx") - if err := accountsdb.WriteStakePubkeyIndex(stakeIndexPath, stakeCollector.entries); err != nil { + if err := accountsdb.WriteStakePubkeyIndex(stakeIndexPath, accountCollector.stakeEntries); err != nil { return nil, nil, fmt.Errorf("writing stake pubkey index: %w", err) } @@ -302,6 +302,10 @@ func BuildAccountsDbAuto( if err != nil { return nil, nil, err } + if err := accountsDb.SeedVoteAccountPubkeys(accountCollector.votePubkeys); err != nil { + accountsDb.CloseDb() + return nil, nil, fmt.Errorf("seeding vote pubkey index: %w", err) + } rpcClient := rpcclient.NewRpcClient(rpcEndpoints[0]) latestSlot, err := rpcClient.GetSlot() diff --git a/pkg/snapshot/build_db_worker_test.go b/pkg/snapshot/build_db_worker_test.go index e387caf7..df1a15a9 100644 --- a/pkg/snapshot/build_db_worker_test.go +++ b/pkg/snapshot/build_db_worker_test.go @@ -60,7 +60,7 @@ func TestSnapshotWorkerParseFailurePropagatesAfterDrain(t *testing.T) { nil, accountsDir, &atomic.Uint64{}, - &stakeIndexCollector{}, + &snapshotAccountCollector{}, ) require.NoError(t, err) t.Cleanup(pools.Release) @@ -139,7 +139,7 @@ func TestSnapshotWorkersSupportArchiveSizedAppendVecs(t *testing.T) { t.Cleanup(closeShardLogger) wg := &sync.WaitGroup{} largestFileID := &atomic.Uint64{} - stakes := &stakeIndexCollector{} + stakes := &snapshotAccountCollector{} pools, err := initWorkerPools(wg, shardLogger, tt.full, tt.incremental, accountsDir, largestFileID, stakes) require.NoError(t, err) t.Cleanup(pools.Release) @@ -155,8 +155,8 @@ func TestSnapshotWorkersSupportArchiveSizedAppendVecs(t *testing.T) { closeShardLogger() require.Equal(t, int64(tt.wantEntries*(32+vlen)), shardLogger.TotalBytes()) require.Equal(t, fileID, largestFileID.Load()) - require.Len(t, stakes.entries, tt.wantEntries) - for i, entry := range stakes.entries { + require.Len(t, stakes.stakeEntries, tt.wantEntries) + for i, entry := range stakes.stakeEntries { require.Equal(t, keys[i], entry.Pubkey) require.Equal(t, fileID, entry.FileId) require.Equal(t, uint64(i)*oneAccountSize, entry.Offset)