From 2344cb40a30fd25e025bcb1d5dd61008411e088c Mon Sep 17 00:00:00 2001 From: Neeraj Godiyal Date: Wed, 23 Sep 2026 23:30:08 +0530 Subject: [PATCH 01/14] rpc: expose rooted validator metrics for Solana Exporter --- cmd/mithril/node/node.go | 27 ++- pkg/accountsdb/accountsdb.go | 15 ++ pkg/accountsdb/fold.go | 4 +- pkg/replay/block.go | 18 +- pkg/rpcserver/get_account_info.go | 42 +++-- pkg/rpcserver/get_balance.go | 90 ++++++++++ pkg/rpcserver/get_block_production.go | 183 +++++++++++++++++++ pkg/rpcserver/get_epoch_info.go | 60 +++++-- pkg/rpcserver/get_leader_schedule.go | 113 ++++++++++++ pkg/rpcserver/get_vote_accounts.go | 249 ++++++++++++++++++++++++++ pkg/rpcserver/optional_params.go | 10 ++ pkg/rpcserver/parse_vote_account.go | 127 +++++++++++++ pkg/rpcserver/rpcserver.go | 91 ++++++++++ 13 files changed, 1001 insertions(+), 28 deletions(-) create mode 100644 pkg/rpcserver/get_balance.go create mode 100644 pkg/rpcserver/get_block_production.go create mode 100644 pkg/rpcserver/get_leader_schedule.go create mode 100644 pkg/rpcserver/get_vote_accounts.go create mode 100644 pkg/rpcserver/optional_params.go create mode 100644 pkg/rpcserver/parse_vote_account.go diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index c452d494e..337a31bd5 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -2517,6 +2517,7 @@ postBootstrap: klog.Fatalf("invalid port: %d", rpcPort) } else if rpcPort != 0 { rpcServer = rpcserver.NewRpcServer(accountsDb, uint16(rpcPort), epochScheduleFromState(mithrilState), solana.MustHashFromBase58(networkGenesisHash)) + publishRPCBankState(rpcServer, mithrilState) rpcServer.Start() mlog.Log.Infof("Started RPC server on port %d", rpcPort) } @@ -2877,7 +2878,7 @@ postBootstrap: } } - var slotCtxSetter replay.SlotCtxSetter + var slotCtxSetter replay.RPCStateSetter if rpcServer != nil { slotCtxSetter = rpcServer } @@ -4348,7 +4349,7 @@ func runReplayWithRecovery( useTurbine bool, dbgOpts *replay.DebugOptions, metricsWriter io.Writer, - rpcServer replay.SlotCtxSetter, + rpcServer replay.RPCStateSetter, mithrilState *state.MithrilState, blockFetchOpts *replay.BlockFetchOpts, consensusOpts *replay.ConsensusOpts, @@ -4550,6 +4551,7 @@ func runReplayWithRecovery( mlog.Log.Errorf("fork switch: no rooted checkpoint context to re-replay from; halting") break } + publishRPCBankState(rpcServer, mithrilState) if mithrilState.LastRootedSlot > prevRooted { // progress since last attempt -> fresh budget attempt = 0 prevRooted = mithrilState.LastRootedSlot @@ -4589,3 +4591,24 @@ func runReplayWithRecovery( } return result } + +type rpcBankStateSetter interface { + SetRootedBankState(slot, blockHeight, transactionCount uint64) +} + +func publishRPCBankState(rpcServer rpcBankStateSetter, mithrilState *state.MithrilState) { + if rpcServer == nil || mithrilState == nil { + return + } + rootedSlot := mithrilState.ManifestParentSlot + rootedBlockHeight := mithrilState.ManifestBlockHeight + rootedTransactionCount := mithrilState.ManifestTransactionCount + if rootedCtx := mithrilState.LastRootedContext; rootedCtx != nil { + rootedSlot = rootedCtx.Slot + rootedBlockHeight = rootedCtx.BlockHeight + if rootedCtx.TransactionCount != nil { + rootedTransactionCount = *rootedCtx.TransactionCount + } + } + rpcServer.SetRootedBankState(rootedSlot, rootedBlockHeight, rootedTransactionCount) +} diff --git a/pkg/accountsdb/accountsdb.go b/pkg/accountsdb/accountsdb.go index 49714f731..4c1fd2239 100644 --- a/pkg/accountsdb/accountsdb.go +++ b/pkg/accountsdb/accountsdb.go @@ -97,6 +97,21 @@ func (accountsDb *AccountsDb) StoreQueueLen() int { return accountsDb.inProgressStoreRequests.Len() } +// DurableThrough returns the highest slot whose fold is fully committed. +func (accountsDb *AccountsDb) DurableThrough() uint64 { + return accountsDb.durableThrough.Load() +} + +// VoteAccountPubkeys returns the vote accounts in the current cache view. +func (accountsDb *AccountsDb) VoteAccountPubkeys() []solana.PublicKey { + pubkeys := make([]solana.PublicKey, 0, accountsDb.VoteAcctCache.Size()) + accountsDb.VoteAcctCache.Range(func(pubkey solana.PublicKey, _ *accounts.Account) bool { + pubkeys = append(pubkeys, pubkey) + return true + }) + return pubkeys +} + // silentLogger implements pebble.Logger but discards all messages. // This suppresses verbose WAL recovery messages on startup. type silentLogger struct{} diff --git a/pkg/accountsdb/fold.go b/pkg/accountsdb/fold.go index 096a59a16..7e01dfe8d 100644 --- a/pkg/accountsdb/fold.go +++ b/pkg/accountsdb/fold.go @@ -304,6 +304,9 @@ func (db *AccountsDb) CommitBatch( // ordinary concurrent-safe cache operations finish. db.refreshReadCacheEntries(live) + // Advance the watermark before removing pendingFold so rooted readers keep + // waiting until the matching bank state is published. + db.durableThrough.Store(throughSlot) db.readCacheEpochMu.Lock() db.pendingFold = nil db.readCacheEpochMu.Unlock() @@ -311,7 +314,6 @@ func (db *AccountsDb) CommitBatch( // (8) Publish. db.lastBatchSeq = batchSeq - db.durableThrough.Store(throughSlot) return BatchCommitResult{ BatchSeq: batchSeq, diff --git a/pkg/replay/block.go b/pkg/replay/block.go index 495ed53a8..8af6b03fd 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -49,9 +49,11 @@ import ( "github.com/panjf2000/ants/v2" ) -// SlotCtxSetter is implemented by types that accept a SlotCtx update (e.g. RpcServer). -type SlotCtxSetter interface { +// RPCStateSetter is implemented by the RPC server so replay can publish both +// its live execution bank and the latest bank durably folded into AccountsDB. +type RPCStateSetter interface { SetSlotCtx(slotCtx *sealevel.SlotCtx) + SetRootedBankState(slot, blockHeight, transactionCount uint64) } // BlockFetchOpts contains options for parallel block fetching @@ -1670,7 +1672,7 @@ func ReplayBlocks( useTurbine bool, dbgOpts *DebugOptions, metricsWriter io.Writer, - rpcServer SlotCtxSetter, + rpcServer RPCStateSetter, blockFetchOpts *BlockFetchOpts, consensusOpts *ConsensusOpts, // nil = use defaults (max_depth=64, policy="halt") onCancelWriteState OnCancelWriteState, // callback to write state immediately on cancellation (can be nil) @@ -2063,6 +2065,13 @@ func ReplayBlocks( mithrilState.LastRootedSlot = promotedThrough mithrilState.LastRootedBankhash = rootedCtx.Bankhash mithrilState.LastRootedContext = rootedCtx + if rpcServer != nil { + var transactionCount uint64 + if rootedCtx.TransactionCount != nil { + transactionCount = *rootedCtx.TransactionCount + } + rpcServer.SetRootedBankState(promotedThrough, rootedCtx.BlockHeight, transactionCount) + } if transactionStatuses.Root(promotedThrough) { mlog.Log.Infof("transaction status cache reconstructed complete %d-root coverage through durable slot %d", maxTransactionStatusRoots, promotedThrough) @@ -3085,6 +3094,9 @@ func ReplayBlocks( if rpcServer != nil { rpcServer.SetSlotCtx(lastSlotCtx) + if unrootedTailState == nil { + rpcServer.SetRootedBankState(block.Slot, global.BlockHeight(), global.TransactionCount()) + } } replayCtx.Capitalization -= lastSlotCtx.LamportsBurnt diff --git a/pkg/rpcserver/get_account_info.go b/pkg/rpcserver/get_account_info.go index 11d01e4c2..3909505e0 100644 --- a/pkg/rpcserver/get_account_info.go +++ b/pkg/rpcserver/get_account_info.go @@ -3,11 +3,13 @@ package rpcserver import ( "context" "encoding/base64" + "errors" "fmt" "reflect" "github.com/DataDog/zstd" - "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/accountsdb" + "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/Overclock-Validator/mithril/pkg/mlog" "github.com/Overclock-Validator/mithril/pkg/safemath" "github.com/filecoin-project/go-jsonrpc" @@ -73,18 +75,31 @@ func (rpcServer *RpcServer) GetAccountInfo(ctx context.Context, p jsonrpc.RawPar return GetAccountInfoResp{}, fmt.Errorf("invalid base58 encoding") } - acct, err := rpcServer.acctsDb.GetAccount(0, pk) - if err != nil { - return GetAccountInfoResp{}, nil - } - conf, err := parseGetAccountInfoConfMap(params) if err != nil { return GetAccountInfoResp{}, err } + rooted, acct, err := rpcServer.readRootedAccount(ctx, pk) + if err != nil && !errors.Is(err, accountsdb.ErrNoAccount) { + return GetAccountInfoResp{}, fmt.Errorf("read account at rooted slot %d: %w", rooted.Slot, err) + } + if conf.MinContextSlot > rooted.Slot { + return GetAccountInfoResp{}, &MinContextSlotNotReachedError{ContextSlot: rooted.Slot} + } + if errors.Is(err, accountsdb.ErrNoAccount) { + return GetAccountInfoResp{Context: GetAccountInfoRespContext{ApiVersion: "mithril 0.1", Slot: rooted.Slot}}, nil + } var acctData interface{} - acctData, err = encodeAcctDataWithConfig(acct.Data, conf) + if conf.EncodingType != nil && *conf.EncodingType == GetAccountEncodingJson && conf.DataSlice != nil { + return GetAccountInfoResp{}, fmt.Errorf("cannot use jsonParsed with dataSlice") + } + if conf.EncodingType != nil && *conf.EncodingType == GetAccountEncodingJson && acct.Owner == addresses.VoteProgramAddr { + acctData, err = parsedVoteAccountData(acct.Data, pk) + } + if acctData == nil || err != nil { + acctData, err = encodeAcctDataWithConfig(acct.Data, conf) + } if err != nil { return GetAccountInfoResp{}, err } @@ -97,7 +112,7 @@ func (rpcServer *RpcServer) GetAccountInfo(ctx context.Context, p jsonrpc.RawPar RentEpoch: acct.RentEpoch, Space: uint64(len(acct.Data))} - return GetAccountInfoResp{Context: GetAccountInfoRespContext{ApiVersion: "mithril 0.1", Slot: global.Slot()}, Value: val}, nil + return GetAccountInfoResp{Context: GetAccountInfoRespContext{ApiVersion: "mithril 0.1", Slot: rooted.Slot}, Value: val}, nil } func parseGetAccountInfoConfMap(params []interface{}) (*GetAccountInfoConfig, error) { @@ -117,12 +132,19 @@ func parseGetAccountInfoConfMap(params []interface{}) (*GetAccountInfoConfig, er commitmentObj, ok := confMap["commitment"] if ok { commitmentStr, ok := commitmentObj.(string) - if !ok { + if !ok || (commitmentStr != "processed" && commitmentStr != "confirmed" && commitmentStr != "finalized") { return nil, fmt.Errorf("invalid commitment") } conf.Commitment = commitmentStr } + if minContextSlot, ok := confMap["minContextSlot"]; ok { + conf.MinContextSlot, err = rpcUint64(minContextSlot, "minContextSlot") + if err != nil { + return nil, err + } + } + // encoding type is optional. defaults to base58. encodingObj, ok := confMap["encoding"] if ok { @@ -199,7 +221,7 @@ func parseGetAcctDataEncodingType(encodingStr string) (int, error) { return GetAccountEncodingBase64Zstd, nil case "jsonParsed": - return GetAccountEncodingBase64, nil + return GetAccountEncodingJson, nil default: return 0, fmt.Errorf("invalid data encoding %s", encodingStr) diff --git a/pkg/rpcserver/get_balance.go b/pkg/rpcserver/get_balance.go new file mode 100644 index 000000000..dcc9036b0 --- /dev/null +++ b/pkg/rpcserver/get_balance.go @@ -0,0 +1,90 @@ +package rpcserver + +import ( + "context" + "errors" + "fmt" + + "github.com/Overclock-Validator/mithril/pkg/accountsdb" + "github.com/filecoin-project/go-jsonrpc" + "github.com/gagliardetto/solana-go" +) + +type GetBalanceResp struct { + Context GetBalanceRespContext `json:"context"` + Value uint64 `json:"value"` +} + +type GetBalanceRespContext struct { + APIVersion string `json:"apiVersion"` + Slot uint64 `json:"slot"` +} + +func (rpcServer *RpcServer) GetBalance(ctx context.Context, p jsonrpc.RawParams) (GetBalanceResp, error) { + if err := ctx.Err(); err != nil { + return GetBalanceResp{}, err + } + + params, err := jsonrpc.DecodeParams[[]interface{}](p) + if err != nil { + return GetBalanceResp{}, &InvalidParamsError{Message: fmt.Sprintf("decoding params: %v", err)} + } + if len(params) < 1 || len(params) > 2 { + return GetBalanceResp{}, &InvalidParamsError{Message: "getBalance requires an address and optional config"} + } + address, ok := params[0].(string) + if !ok { + return GetBalanceResp{}, &InvalidParamsError{Message: "getBalance requires an address as first parameter"} + } + pubkey, err := solana.PublicKeyFromBase58(address) + if err != nil { + return GetBalanceResp{}, &InvalidParamsError{Message: "Invalid param: Invalid"} + } + + minContextSlot, err := parseBalanceConfig(params) + if err != nil { + return GetBalanceResp{}, err + } + rooted, account, err := rpcServer.readRootedAccount(ctx, pubkey) + if err != nil && !errors.Is(err, accountsdb.ErrNoAccount) { + return GetBalanceResp{}, fmt.Errorf("read balance at rooted slot %d: %w", rooted.Slot, err) + } + if minContextSlot != nil && rooted.Slot < *minContextSlot { + return GetBalanceResp{}, &MinContextSlotNotReachedError{ContextSlot: rooted.Slot} + } + + var lamports uint64 + if account != nil { + lamports = account.Lamports + } + + return GetBalanceResp{ + Context: GetBalanceRespContext{APIVersion: "mithril 0.1", Slot: rooted.Slot}, + Value: lamports, + }, nil +} + +func parseBalanceConfig(params []interface{}) (*uint64, error) { + if len(params) == 1 || params[1] == nil { + return nil, nil + } + config, ok := params[1].(map[string]interface{}) + if !ok { + return nil, &InvalidParamsError{Message: "invalid getBalance config"} + } + if commitment, exists := config["commitment"]; exists { + value, ok := commitment.(string) + if !ok || (value != "processed" && value != "confirmed" && value != "finalized") { + return nil, &InvalidParamsError{Message: "invalid commitment"} + } + } + value, exists := config["minContextSlot"] + if !exists { + return nil, nil + } + minContextSlot, err := rpcUint64(value, "minContextSlot") + if err != nil { + return nil, err + } + return &minContextSlot, nil +} diff --git a/pkg/rpcserver/get_block_production.go b/pkg/rpcserver/get_block_production.go new file mode 100644 index 000000000..54385ccf1 --- /dev/null +++ b/pkg/rpcserver/get_block_production.go @@ -0,0 +1,183 @@ +package rpcserver + +import ( + "context" + "fmt" + "math" + + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/filecoin-project/go-jsonrpc" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" +) + +type GetBlockProductionResp struct { + Context GetBalanceRespContext `json:"context"` + Value BlockProductionResults `json:"value"` +} + +type BlockProductionResults struct { + ByIdentity map[string][2]uint64 `json:"byIdentity"` + Range BlockProductionRange `json:"range"` +} + +type BlockProductionRange struct { + FirstSlot uint64 `json:"firstSlot"` + LastSlot uint64 `json:"lastSlot"` +} + +type getBlockProductionConfig struct { + identity *solana.PublicKey + firstSlot *uint64 + lastSlot *uint64 +} + +func (rpcServer *RpcServer) GetBlockProduction(ctx context.Context, p jsonrpc.RawParams) (GetBlockProductionResp, error) { + if err := ctx.Err(); err != nil { + return GetBlockProductionResp{}, err + } + params, err := decodeOptionalParams(p) + if err != nil { + return GetBlockProductionResp{}, &InvalidParamsError{Message: fmt.Sprintf("decoding params: %v", err)} + } + config, err := parseBlockProductionConfig(params) + if err != nil { + return GetBlockProductionResp{}, err + } + if rpcServer.epochSchedule == nil { + return GetBlockProductionResp{}, fmt.Errorf("node has no epoch schedule available") + } + rooted, historyAccount, err := rpcServer.readRootedAccount(ctx, sealevel.SysvarSlotHistoryAddr) + if err != nil { + return GetBlockProductionResp{}, fmt.Errorf("read slot history at rooted slot %d: %w", rooted.Slot, err) + } + + firstSlot := rpcServer.epochSchedule.FirstSlotInEpoch(rpcServer.epochSchedule.GetEpoch(rooted.Slot)) + if config.firstSlot != nil { + firstSlot = *config.firstSlot + } + lastSlot := rooted.Slot + if config.lastSlot != nil { + lastSlot = *config.lastSlot + } + if lastSlot < firstSlot { + return GetBlockProductionResp{}, &InvalidParamsError{Message: fmt.Sprintf("lastSlot, %d, cannot be less than firstSlot, %d", lastSlot, firstSlot)} + } + + var history sealevel.SysvarSlotHistory + if err := history.UnmarshalWithDecoder(bin.NewBinDecoder(historyAccount.Data)); err != nil { + return GetBlockProductionResp{}, fmt.Errorf("decode slot history at rooted slot %d: %w", rooted.Slot, err) + } + oldest, newest, err := slotHistoryBounds(&history) + if err != nil { + return GetBlockProductionResp{}, err + } + if firstSlot < oldest { + return GetBlockProductionResp{}, &InvalidParamsError{Message: fmt.Sprintf("firstSlot, %d, is too small; min %d", firstSlot, oldest)} + } + if lastSlot > newest { + return GetBlockProductionResp{}, &InvalidParamsError{Message: fmt.Sprintf("lastSlot, %d, is too large; max %d", lastSlot, newest)} + } + + byIdentity := make(map[string][2]uint64) + for slot := firstSlot; ; slot++ { + leader, ok := global.LeaderForSlot(slot) + if !ok { + return GetBlockProductionResp{}, fmt.Errorf("leader schedule unavailable for slot %d", slot) + } + if config.identity == nil || leader == *config.identity { + production := byIdentity[leader.String()] + production[0]++ + if slotHistoryContains(&history, slot) { + production[1]++ + } + byIdentity[leader.String()] = production + } + if slot == lastSlot { + break + } + } + + return GetBlockProductionResp{ + Context: GetBalanceRespContext{APIVersion: "mithril 0.1", Slot: rooted.Slot}, + Value: BlockProductionResults{ + ByIdentity: byIdentity, + Range: BlockProductionRange{FirstSlot: firstSlot, LastSlot: lastSlot}, + }, + }, nil +} + +func parseBlockProductionConfig(params []interface{}) (getBlockProductionConfig, error) { + var config getBlockProductionConfig + if len(params) > 1 { + return config, &InvalidParamsError{Message: "getBlockProduction accepts at most one config object"} + } + if len(params) == 0 || params[0] == nil { + return config, nil + } + values, ok := params[0].(map[string]interface{}) + if !ok { + return config, &InvalidParamsError{Message: "invalid getBlockProduction config"} + } + if commitment, exists := values["commitment"]; exists { + value, ok := commitment.(string) + if !ok || (value != "processed" && value != "confirmed" && value != "finalized") { + return config, &InvalidParamsError{Message: "invalid commitment"} + } + } + if raw, exists := values["identity"]; exists { + value, ok := raw.(string) + if !ok { + return config, &InvalidParamsError{Message: "invalid identity"} + } + identity, err := solana.PublicKeyFromBase58(value) + if err != nil { + return config, &InvalidParamsError{Message: "invalid identity"} + } + config.identity = &identity + } + if raw, exists := values["range"]; exists { + rangeValues, ok := raw.(map[string]interface{}) + if !ok { + return config, &InvalidParamsError{Message: "invalid range"} + } + first, exists := rangeValues["firstSlot"] + if !exists { + return config, &InvalidParamsError{Message: "range requires firstSlot"} + } + firstSlot, err := rpcUint64(first, "firstSlot") + if err != nil { + return config, err + } + config.firstSlot = &firstSlot + if last, exists := rangeValues["lastSlot"]; exists && last != nil { + lastSlot, err := rpcUint64(last, "lastSlot") + if err != nil { + return config, err + } + config.lastSlot = &lastSlot + } + } + return config, nil +} + +func rpcUint64(raw interface{}, name string) (uint64, error) { + value, ok := raw.(float64) + if !ok || value < 0 || value >= math.Exp2(64) || math.Trunc(value) != value { + return 0, &InvalidParamsError{Message: fmt.Sprintf("invalid %s", name)} + } + return uint64(value), nil +} + +func slotHistoryBounds(history *sealevel.SysvarSlotHistory) (uint64, uint64, error) { + if history == nil || history.Bits.Len == 0 || history.Bits.Bits.BlocksLen == 0 || len(history.Bits.Bits.Blocks) < int(history.Bits.Bits.BlocksLen) || history.NextSlot == 0 { + return 0, 0, fmt.Errorf("node has no usable slot history") + } + return history.NextSlot - min(history.NextSlot, history.Bits.Len), history.NextSlot - 1, nil +} + +func slotHistoryContains(history *sealevel.SysvarSlotHistory, slot uint64) bool { + block := (slot / 64) % history.Bits.Bits.BlocksLen + return history.Bits.Bits.Blocks[block]&(uint64(1)<<(slot%64)) != 0 +} diff --git a/pkg/rpcserver/get_epoch_info.go b/pkg/rpcserver/get_epoch_info.go index 2ee4a9cdc..5f4ee494e 100644 --- a/pkg/rpcserver/get_epoch_info.go +++ b/pkg/rpcserver/get_epoch_info.go @@ -18,25 +18,61 @@ type GetEpochInfoResp struct { } func (rpcServer *RpcServer) GetEpochInfo(ctx context.Context, p jsonrpc.RawParams) (GetEpochInfoResp, error) { - params, err := jsonrpc.DecodeParams[[]interface{}](p) + params, err := decodeOptionalParams(p) if err != nil { - return GetEpochInfoResp{}, fmt.Errorf("decoding params: %w", err) + return GetEpochInfoResp{}, &InvalidParamsError{Message: fmt.Sprintf("decoding params: %v", err)} + } + if len(params) > 1 { + return GetEpochInfoResp{}, &InvalidParamsError{Message: "getEpochInfo accepts at most one config object"} + } + commitment := "finalized" + var minContextSlot *uint64 + if len(params) == 1 && params[0] != nil { + config, ok := params[0].(map[string]interface{}) + if !ok { + return GetEpochInfoResp{}, &InvalidParamsError{Message: "invalid getEpochInfo config"} + } + if raw, exists := config["commitment"]; exists { + value, ok := raw.(string) + if !ok || (value != "processed" && value != "confirmed" && value != "finalized") { + return GetEpochInfoResp{}, &InvalidParamsError{Message: "invalid commitment"} + } + commitment = value + } + if raw, exists := config["minContextSlot"]; exists { + value, err := rpcUint64(raw, "minContextSlot") + if err != nil { + return GetEpochInfoResp{}, err + } + minContextSlot = &value + } + } + if rpcServer.epochSchedule == nil { + return GetEpochInfoResp{}, fmt.Errorf("node has no epoch schedule available") } - - _ = params - - epoch := global.Epoch() slot := global.Slot() - firstSlotInEpoch := rpcServer.epochSchedule.FirstSlotInEpoch(epoch) - slotIndex := slot - firstSlotInEpoch + blockHeight := global.BlockHeight() + transactionCount := global.TransactionCount() + if commitment != "processed" { + rooted, ok := rpcServer.getRootedBankState() + if !ok { + return GetEpochInfoResp{}, fmt.Errorf("node has no rooted bank available") + } + slot, blockHeight, transactionCount = rooted.Slot, rooted.BlockHeight, rooted.TransactionCount + } + if minContextSlot != nil && slot < *minContextSlot { + return GetEpochInfoResp{}, &MinContextSlotNotReachedError{ContextSlot: slot} + } + epoch := rpcServer.epochSchedule.GetEpoch(slot) + slotIndex := slot - rpcServer.epochSchedule.FirstSlotInEpoch(epoch) resp := GetEpochInfoResp{ - AbsoluteSlot: global.Slot(), - BlockHeight: global.BlockHeight(), - Epoch: global.Epoch(), + AbsoluteSlot: slot, + BlockHeight: blockHeight, + Epoch: epoch, SlotIndex: slotIndex, SlotsInEpoch: 432000, - TransactionCount: global.TransactionCount(), + TransactionCount: transactionCount, } return resp, nil diff --git a/pkg/rpcserver/get_leader_schedule.go b/pkg/rpcserver/get_leader_schedule.go new file mode 100644 index 000000000..0310d3f2e --- /dev/null +++ b/pkg/rpcserver/get_leader_schedule.go @@ -0,0 +1,113 @@ +package rpcserver + +import ( + "context" + "fmt" + + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/filecoin-project/go-jsonrpc" + "github.com/gagliardetto/solana-go" +) + +type getLeaderScheduleConfig struct { + identity *solana.PublicKey +} + +func (rpcServer *RpcServer) GetLeaderSchedule(ctx context.Context, p jsonrpc.RawParams) (map[string][]uint64, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + params, err := decodeOptionalParams(p) + if err != nil { + return nil, &InvalidParamsError{Message: fmt.Sprintf("decoding params: %v", err)} + } + if len(params) > 2 { + return nil, &InvalidParamsError{Message: "getLeaderSchedule accepts an optional slot and config"} + } + rooted, ok := rpcServer.getRootedBankState() + if !ok { + return nil, fmt.Errorf("node has no rooted bank available") + } + if rpcServer.epochSchedule == nil { + return nil, fmt.Errorf("node has no epoch schedule available") + } + + slot := rooted.Slot + var rawConfig interface{} + if len(params) > 0 && params[0] != nil { + if _, ok := params[0].(map[string]interface{}); ok { + if len(params) != 1 { + return nil, &InvalidParamsError{Message: "leader schedule config cannot be provided twice"} + } + rawConfig = params[0] + } else { + value, err := rpcUint64(params[0], "slot") + if err != nil { + return nil, err + } + slot = value + } + } + if len(params) == 2 { + rawConfig = params[1] + } + config, err := parseLeaderScheduleConfig(rawConfig) + if err != nil { + return nil, err + } + + epoch := rpcServer.epochSchedule.GetEpoch(slot) + firstSlot := rpcServer.epochSchedule.FirstSlotInEpoch(epoch) + slotsInEpoch := rpcServer.epochSchedule.SlotsInEpoch(epoch) + schedule := make(map[string][]uint64) + for index := uint64(0); index < slotsInEpoch; index++ { + leader, ok := global.LeaderForSlot(firstSlot + index) + if !ok { + return nil, nil + } + if config.identity != nil && leader != *config.identity { + continue + } + key := leader.String() + schedule[key] = append(schedule[key], index) + } + return schedule, nil +} + +func parseLeaderScheduleConfig(raw interface{}) (getLeaderScheduleConfig, error) { + var config getLeaderScheduleConfig + if raw == nil { + return config, nil + } + values, ok := raw.(map[string]interface{}) + if !ok { + return config, &InvalidParamsError{Message: "invalid getLeaderSchedule config"} + } + if commitment, exists := values["commitment"]; exists { + value, ok := commitment.(string) + if !ok || (value != "processed" && value != "confirmed" && value != "finalized") { + return config, &InvalidParamsError{Message: "invalid commitment"} + } + } + if raw, exists := values["identity"]; exists { + value, ok := raw.(string) + if !ok { + return config, &InvalidParamsError{Message: "invalid identity"} + } + identity, err := solana.PublicKeyFromBase58(value) + if err != nil { + return config, &InvalidParamsError{Message: "invalid identity"} + } + config.identity = &identity + } + if raw, exists := values["keyByVoteAccount"]; exists { + value, ok := raw.(bool) + if !ok { + return config, &InvalidParamsError{Message: "invalid keyByVoteAccount"} + } + if value { + return config, &InvalidParamsError{Message: "keyByVoteAccount is not supported"} + } + } + return config, nil +} diff --git a/pkg/rpcserver/get_vote_accounts.go b/pkg/rpcserver/get_vote_accounts.go new file mode 100644 index 000000000..b6eddddfe --- /dev/null +++ b/pkg/rpcserver/get_vote_accounts.go @@ -0,0 +1,249 @@ +package rpcserver + +import ( + "context" + "fmt" + "slices" + "sync" + + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/filecoin-project/go-jsonrpc" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" +) + +const ( + delinquentValidatorSlotDistance = 128 + maxRPCEpochCreditsHistory = 5 +) + +type GetVoteAccountsResp struct { + Current []VoteAccountInfo `json:"current"` + Delinquent []VoteAccountInfo `json:"delinquent"` +} + +type VoteAccountInfo struct { + VotePubkey string `json:"votePubkey"` + NodePubkey string `json:"nodePubkey"` + ActivatedStake uint64 `json:"activatedStake"` + Commission uint8 `json:"commission"` + InflationRewardsCommissionBPS uint16 `json:"inflationRewardsCommissionBps"` + EpochCredits [][3]uint64 `json:"epochCredits"` + EpochVoteAccount bool `json:"epochVoteAccount"` + LastVote uint64 `json:"lastVote"` + RootSlot uint64 `json:"rootSlot"` +} + +type getVoteAccountsConfig struct { + votePubkey *solana.PublicKey + delinquentSlotDistance uint64 +} + +func (rpcServer *RpcServer) GetVoteAccounts(ctx context.Context, p jsonrpc.RawParams) (GetVoteAccountsResp, error) { + if err := ctx.Err(); err != nil { + return GetVoteAccountsResp{}, err + } + params, err := decodeOptionalParams(p) + if err != nil { + return GetVoteAccountsResp{}, &InvalidParamsError{Message: fmt.Sprintf("decoding params: %v", err)} + } + config, err := parseGetVoteAccountsConfig(params) + if err != nil { + return GetVoteAccountsResp{}, err + } + if rpcServer.epochSchedule == nil { + return GetVoteAccountsResp{}, fmt.Errorf("node has no epoch schedule available") + } + if rpcServer.acctsDb == nil { + return GetVoteAccountsResp{}, fmt.Errorf("node has no accounts database available") + } + + for { + rooted, ok := rpcServer.getRootedBankState() + if !ok { + return GetVoteAccountsResp{}, fmt.Errorf("node has no rooted bank available") + } + epoch := rpcServer.epochSchedule.GetEpoch(rooted.Slot) + stakes, ok := global.EpochStakesSnapshot(epoch) + if !ok { + return GetVoteAccountsResp{}, fmt.Errorf("epoch stakes unavailable for rooted epoch %d", epoch) + } + activeStakes, err := rpcServer.rootedActivatedStakes(ctx, rooted.Slot, epoch) + if err != nil { + return GetVoteAccountsResp{}, err + } + + votePubkeys := rpcServer.acctsDb.VoteAccountPubkeys() + // A fresh snapshot can have an empty vote cache before any vote account is read. + for votePubkey := range stakes.Stakes { + votePubkeys = append(votePubkeys, votePubkey) + } + slices.SortFunc(votePubkeys, func(left, right solana.PublicKey) int { + return slices.Compare(left[:], right[:]) + }) + votePubkeys = slices.Compact(votePubkeys) + filtered := votePubkeys[:0] + for _, votePubkey := range votePubkeys { + if config.votePubkey == nil || votePubkey == *config.votePubkey { + filtered = append(filtered, votePubkey) + } + } + votePubkeys = filtered + published, accounts, err := rpcServer.readRootedAccounts(ctx, votePubkeys) + if err != nil { + return GetVoteAccountsResp{}, fmt.Errorf("read vote accounts at rooted slot %d: %w", rooted.Slot, err) + } + if published.Slot != rooted.Slot { + continue + } + + response := GetVoteAccountsResp{ + Current: make([]VoteAccountInfo, 0, len(votePubkeys)), + Delinquent: make([]VoteAccountInfo, 0), + } + for index, votePubkey := range votePubkeys { + account := accounts[index] + if account == nil || account.Owner != addresses.VoteProgramAddr { + return GetVoteAccountsResp{}, fmt.Errorf("vote account %s is unavailable at rooted slot %d", votePubkey, rooted.Slot) + } + versioned, err := sealevel.UnmarshalVersionedVoteState(account.Data) + if err != nil { + return GetVoteAccountsResp{}, fmt.Errorf("decode vote account %s: %w", votePubkey, err) + } + voteState := versioned.ConvertToCurrent() + lastVote, _ := voteState.LastVotedSlot() + var rootSlot uint64 + if voteState.RootSlot != nil { + rootSlot = *voteState.RootSlot + } + creditsStart := max(0, len(voteState.EpochCredits)-maxRPCEpochCreditsHistory) + creditsHistory := voteState.EpochCredits[creditsStart:] + epochCredits := make([][3]uint64, len(creditsHistory)) + for i, credits := range creditsHistory { + epochCredits[i] = [3]uint64{credits.Epoch, credits.Credits, credits.PrevCredits} + } + commission, commissionBPS := voteCommission(versioned, voteState) + _, epochVoteAccount := stakes.Stakes[votePubkey] + info := VoteAccountInfo{ + VotePubkey: votePubkey.String(), + NodePubkey: voteState.NodePubkey.String(), + ActivatedStake: activeStakes[votePubkey], + Commission: commission, + InflationRewardsCommissionBPS: commissionBPS, + EpochCredits: epochCredits, + EpochVoteAccount: epochVoteAccount, + LastVote: lastVote, + RootSlot: rootSlot, + } + current := lastVote > 0 + if rooted.Slot >= config.delinquentSlotDistance { + current = lastVote > rooted.Slot-config.delinquentSlotDistance + } + if current { + response.Current = append(response.Current, info) + } else if info.ActivatedStake > 0 { + response.Delinquent = append(response.Delinquent, info) + } + } + return response, nil + } +} + +func (rpcServer *RpcServer) rootedActivatedStakes(ctx context.Context, slot, epoch uint64) (map[solana.PublicKey]uint64, error) { + rooted, accounts, err := rpcServer.readRootedAccounts(ctx, []solana.PublicKey{sealevel.SysvarStakeHistoryAddr}) + if err != nil { + return nil, fmt.Errorf("read rooted stake history: %w", err) + } + if rooted.Slot != slot { + return nil, nil + } + if accounts[0] == nil { + return nil, fmt.Errorf("stake history unavailable at rooted slot %d", slot) + } + var history sealevel.SysvarStakeHistory + if err := history.UnmarshalWithDecoder(bin.NewBinDecoder(accounts[0].Data)); err != nil { + return nil, fmt.Errorf("decode stake history at rooted slot %d: %w", slot, err) + } + slotCtx := rpcServer.getSlotCtx() + if slotCtx == nil || slotCtx.Features == nil { + return nil, fmt.Errorf("node features unavailable for stake activation") + } + activationEpoch := sealevel.NewWarmupCooldownRateEpochWithSlotCtx(slotCtx, rpcServer.epochSchedule) + stakes := make(map[solana.PublicKey]uint64) + var mu sync.Mutex + _, err = global.StreamStakeAccounts(rpcServer.acctsDb, slot, func(_ solana.PublicKey, delegation *sealevel.Delegation, _ uint64) { + active := delegation.Stake(epoch, &history, activationEpoch) + if active == 0 { + return + } + mu.Lock() + stakes[delegation.VoterPubkey] += active + mu.Unlock() + }) + if err != nil { + return nil, fmt.Errorf("scan rooted stake accounts at slot %d: %w", slot, err) + } + if err := ctx.Err(); err != nil { + return nil, err + } + return stakes, nil +} + +func parseGetVoteAccountsConfig(params []interface{}) (getVoteAccountsConfig, error) { + config := getVoteAccountsConfig{delinquentSlotDistance: delinquentValidatorSlotDistance} + if len(params) > 1 { + return config, &InvalidParamsError{Message: "getVoteAccounts accepts at most one config object"} + } + if len(params) == 0 || params[0] == nil { + return config, nil + } + values, ok := params[0].(map[string]interface{}) + if !ok { + return config, &InvalidParamsError{Message: "invalid getVoteAccounts config"} + } + if commitment, exists := values["commitment"]; exists { + value, ok := commitment.(string) + if !ok || (value != "processed" && value != "confirmed" && value != "finalized") { + return config, &InvalidParamsError{Message: "invalid commitment"} + } + } + if raw, exists := values["votePubkey"]; exists { + value, ok := raw.(string) + if !ok { + return config, &InvalidParamsError{Message: "invalid votePubkey"} + } + pubkey, err := solana.PublicKeyFromBase58(value) + if err != nil { + return config, &InvalidParamsError{Message: "invalid votePubkey"} + } + config.votePubkey = &pubkey + } + if raw, exists := values["keepUnstakedDelinquents"]; exists { + value, ok := raw.(bool) + if !ok { + return config, &InvalidParamsError{Message: "invalid keepUnstakedDelinquents"} + } + if value { + return config, &InvalidParamsError{Message: "keepUnstakedDelinquents is not supported"} + } + } + if raw, exists := values["delinquentSlotDistance"]; exists { + value, err := rpcUint64(raw, "delinquentSlotDistance") + if err != nil { + return config, err + } + config.delinquentSlotDistance = value + } + return config, nil +} + +func voteCommission(versioned *sealevel.VoteStateVersions, current *sealevel.VoteState) (uint8, uint16) { + if versioned.Type == sealevel.VoteStateVersionV4 { + basisPoints := versioned.V4.InflationRewardsCommissionBps + percent := (uint32(basisPoints) + 99) / 100 + return uint8(min(uint32(255), percent)), basisPoints + } + return current.Commission, uint16(current.Commission) * 100 +} diff --git a/pkg/rpcserver/optional_params.go b/pkg/rpcserver/optional_params.go new file mode 100644 index 000000000..74c000e25 --- /dev/null +++ b/pkg/rpcserver/optional_params.go @@ -0,0 +1,10 @@ +package rpcserver + +import "github.com/filecoin-project/go-jsonrpc" + +func decodeOptionalParams(p jsonrpc.RawParams) ([]interface{}, error) { + if len(p) == 0 { + return nil, nil + } + return jsonrpc.DecodeParams[[]interface{}](p) +} diff --git a/pkg/rpcserver/parse_vote_account.go b/pkg/rpcserver/parse_vote_account.go new file mode 100644 index 000000000..6dd415536 --- /dev/null +++ b/pkg/rpcserver/parse_vote_account.go @@ -0,0 +1,127 @@ +package rpcserver + +import ( + "strconv" + + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/mr-tron/base58" +) + +type parsedVoteAccount struct { + Program string `json:"program"` + Parsed parsedVoteAccountBody `json:"parsed"` + Space uint64 `json:"space"` +} + +type parsedVoteAccountBody struct { + Type string `json:"type"` + Info parsedVoteAccountInfo `json:"info"` +} + +type parsedVoteAccountInfo struct { + NodePubkey string `json:"nodePubkey"` + AuthorizedWithdrawer string `json:"authorizedWithdrawer"` + Commission uint8 `json:"commission"` + Votes []parsedVote `json:"votes"` + RootSlot *uint64 `json:"rootSlot"` + AuthorizedVoters []parsedAuthorizedVoter `json:"authorizedVoters"` + PriorVoters []parsedPriorVoter `json:"priorVoters"` + EpochCredits []parsedEpochCredits `json:"epochCredits"` + LastTimestamp parsedBlockTimestamp `json:"lastTimestamp"` + InflationRewardsCommissionBPS uint16 `json:"inflationRewardsCommissionBps"` + InflationRewardsCollector string `json:"inflationRewardsCollector"` + BlockRevenueCollector string `json:"blockRevenueCollector"` + BlockRevenueCommissionBPS uint16 `json:"blockRevenueCommissionBps"` + PendingDelegatorRewards string `json:"pendingDelegatorRewards"` + BLSPubkeyCompressed *string `json:"blsPubkeyCompressed"` +} + +type parsedVote struct { + Latency uint8 `json:"latency"` + Slot uint64 `json:"slot"` + ConfirmationCount uint32 `json:"confirmationCount"` +} + +type parsedAuthorizedVoter struct { + Epoch uint64 `json:"epoch"` + AuthorizedVoter string `json:"authorizedVoter"` +} + +type parsedPriorVoter struct { + AuthorizedPubkey string `json:"authorizedPubkey"` + EpochOfLastAuthorizedSwitch uint64 `json:"epochOfLastAuthorizedSwitch"` + TargetEpoch uint64 `json:"targetEpoch"` +} + +type parsedEpochCredits struct { + Epoch uint64 `json:"epoch"` + Credits string `json:"credits"` + PreviousCredits string `json:"previousCredits"` +} + +type parsedBlockTimestamp struct { + Slot uint64 `json:"slot"` + Timestamp int64 `json:"timestamp"` +} + +func parsedVoteAccountData(data []byte, votePubkey solana.PublicKey) (parsedVoteAccount, error) { + versioned, err := sealevel.UnmarshalVersionedVoteState(data) + if err != nil { + return parsedVoteAccount{}, err + } + state := versioned.ConvertToCurrent() + info := parsedVoteAccountInfo{ + NodePubkey: state.NodePubkey.String(), + AuthorizedWithdrawer: state.AuthorizedWithdrawer.String(), + Commission: state.Commission, + Votes: make([]parsedVote, 0, state.Votes.Len()), + RootSlot: state.RootSlot, + AuthorizedVoters: make([]parsedAuthorizedVoter, 0, state.AuthorizedVoters.AuthorizedVoters.Len()), + PriorVoters: make([]parsedPriorVoter, 0), + EpochCredits: make([]parsedEpochCredits, 0, len(state.EpochCredits)), + LastTimestamp: parsedBlockTimestamp{ + Slot: state.LastTimestamp.Slot, Timestamp: state.LastTimestamp.Timestamp, + }, + InflationRewardsCommissionBPS: uint16(state.Commission) * 100, + InflationRewardsCollector: votePubkey.String(), + BlockRevenueCollector: state.NodePubkey.String(), + BlockRevenueCommissionBPS: 10000, + PendingDelegatorRewards: "0", + } + for i := 0; i < state.Votes.Len(); i++ { + vote := state.Votes.At(i) + info.Votes = append(info.Votes, parsedVote{ + Latency: vote.Latency, + Slot: vote.Lockout.Slot, + ConfirmationCount: vote.Lockout.ConfirmationCount, + }) + } + epochs, voters := state.AuthorizedVoters.AuthorizedVoters.KeyValues() + for i := range epochs { + info.AuthorizedVoters = append(info.AuthorizedVoters, parsedAuthorizedVoter{ + Epoch: epochs[i], AuthorizedVoter: voters[i].String(), + }) + } + for _, credits := range state.EpochCredits { + info.EpochCredits = append(info.EpochCredits, parsedEpochCredits{ + Epoch: credits.Epoch, Credits: strconv.FormatUint(credits.Credits, 10), PreviousCredits: strconv.FormatUint(credits.PrevCredits, 10), + }) + } + if versioned.Type == sealevel.VoteStateVersionV4 { + info.Commission, info.InflationRewardsCommissionBPS = voteCommission(versioned, state) + info.InflationRewardsCollector = versioned.V4.InflationRewardsCollector.String() + info.BlockRevenueCollector = versioned.V4.BlockRevenueCollector.String() + info.BlockRevenueCommissionBPS = versioned.V4.BlockRevenueCommissionBps + info.PendingDelegatorRewards = strconv.FormatUint(versioned.V4.PendingDelegatorRewards, 10) + if versioned.V4.BlsPubkeyCompressed != nil { + encoded := base58.Encode(versioned.V4.BlsPubkeyCompressed[:]) + info.BLSPubkeyCompressed = &encoded + } + } + return parsedVoteAccount{ + Program: "vote", + Parsed: parsedVoteAccountBody{Type: "vote", Info: info}, + Space: uint64(len(data)), + }, nil +} diff --git a/pkg/rpcserver/rpcserver.go b/pkg/rpcserver/rpcserver.go index 8a3dd7afe..2a50624a0 100644 --- a/pkg/rpcserver/rpcserver.go +++ b/pkg/rpcserver/rpcserver.go @@ -11,8 +11,10 @@ import ( "net/http/httptest" "strings" "sync" + "sync/atomic" "time" + "github.com/Overclock-Validator/mithril/pkg/accounts" "github.com/Overclock-Validator/mithril/pkg/accountsdb" "github.com/Overclock-Validator/mithril/pkg/mlog" "github.com/Overclock-Validator/mithril/pkg/sealevel" @@ -31,6 +33,7 @@ type RpcServer struct { slotCtx *sealevel.SlotCtx slotCtxMu sync.RWMutex genesisHash string + rootedBank atomic.Pointer[rootedBankState] leaderTPUCacheMu sync.RWMutex leaderTPUByIdentity map[solana.PublicKey]tpuEndpoint @@ -45,19 +48,107 @@ type RpcServer struct { sendTransactionLeaderForwardCount uint64 } +// rootedBankState identifies the durable AccountsDB view served by state RPCs. +// Publishing it only after a successful fold keeps response context and account +// values on the same finalized bank. +type rootedBankState struct { + Slot uint64 + BlockHeight uint64 + TransactionCount uint64 +} + const maxQuietMethodProbeBody = 64 << 10 var supportedRPCMethods = map[string]struct{}{ "getAccountInfo": {}, + "getBalance": {}, "getBankHash": {}, + "getBlockProduction": {}, "getBlockHeight": {}, "getEpochInfo": {}, "getGenesisHash": {}, "getLatestBlockhash": {}, + "getLeaderSchedule": {}, + "getVoteAccounts": {}, "sendTransaction": {}, "simulateTransaction": {}, } +func (rpcServer *RpcServer) SetRootedBankState(slot, blockHeight, transactionCount uint64) { + rpcServer.rootedBank.Store(&rootedBankState{ + Slot: slot, + BlockHeight: blockHeight, + TransactionCount: transactionCount, + }) +} + +func (rpcServer *RpcServer) getRootedBankState() (rootedBankState, bool) { + rooted := rpcServer.rootedBank.Load() + if rooted == nil { + return rootedBankState{}, false + } + return *rooted, true +} + +func (rpcServer *RpcServer) readRootedAccount(ctx context.Context, pubkey solana.PublicKey) (rootedBankState, *accounts.Account, error) { + rooted, accountSet, err := rpcServer.readRootedAccounts(ctx, []solana.PublicKey{pubkey}) + if err != nil { + return rooted, nil, err + } + if len(accountSet) != 1 || accountSet[0] == nil { + return rooted, nil, accountsdb.ErrNoAccount + } + return rooted, accountSet[0], nil +} + +func (rpcServer *RpcServer) readRootedAccounts(ctx context.Context, pubkeys []solana.PublicKey) (rootedBankState, []*accounts.Account, error) { + for { + rooted, ok := rpcServer.getRootedBankState() + if !ok { + return rootedBankState{}, nil, fmt.Errorf("node has no rooted bank available") + } + if rpcServer.acctsDb == nil { + return rootedBankState{}, nil, fmt.Errorf("node has no accounts database available") + } + if rpcServer.rootedPublicationPending(rooted.Slot) { + if err := waitForRootedPublication(ctx); err != nil { + return rooted, nil, err + } + continue + } + accounts, stats, err := rpcServer.acctsDb.GetAccountsBatchSharedWithStats(ctx, rooted.Slot, pubkeys) + latest, stillPublished := rpcServer.getRootedBankState() + if stats.PendingFoldHits > 0 || !stillPublished || latest.Slot != rooted.Slot || + rpcServer.rootedPublicationPending(rooted.Slot) { + if err := waitForRootedPublication(ctx); err != nil { + return rooted, nil, err + } + continue + } + return rooted, accounts, err + } +} + +func (rpcServer *RpcServer) rootedPublicationPending(rootedSlot uint64) bool { + if !rpcServer.acctsDb.RootedDurable { + return false + } + durableThrough := rpcServer.acctsDb.DurableThrough() + // Zero means the snapshot baseline is active and no fold has committed yet. + return durableThrough != 0 && durableThrough != rootedSlot +} + +func waitForRootedPublication(ctx context.Context) error { + timer := time.NewTimer(time.Millisecond) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case <-timer.C: + return nil + } +} + func NewRpcServer(acctsDb *accountsdb.AccountsDb, port uint16, epochSchedule *sealevel.SysvarEpochSchedule, genesisHash solana.Hash) *RpcServer { var err error rpcServer := &RpcServer{genesisHash: genesisHash.String()} From 3cef30748f415849392835d0b17c7f8b9777e284 Mon Sep 17 00:00:00 2001 From: Neeraj Godiyal Date: Wed, 23 Sep 2026 23:30:30 +0530 Subject: [PATCH 02/14] test(rpc): cover vote accounts, balances, schedules, and production --- cmd/mithril/node/checkpoint_recovery_test.go | 28 +++ pkg/accountsdb/fold_test.go | 13 ++ pkg/rpcserver/get_account_info_parsed_test.go | 128 ++++++++++++ pkg/rpcserver/get_balance_test.go | 112 +++++++++++ pkg/rpcserver/get_block_production_test.go | 86 ++++++++ pkg/rpcserver/get_epoch_info_rooted_test.go | 33 ++++ pkg/rpcserver/get_leader_schedule_test.go | 83 ++++++++ pkg/rpcserver/get_vote_accounts_test.go | 185 ++++++++++++++++++ pkg/rpcserver/optional_params_test.go | 45 +++++ 9 files changed, 713 insertions(+) create mode 100644 pkg/rpcserver/get_account_info_parsed_test.go create mode 100644 pkg/rpcserver/get_balance_test.go create mode 100644 pkg/rpcserver/get_block_production_test.go create mode 100644 pkg/rpcserver/get_epoch_info_rooted_test.go create mode 100644 pkg/rpcserver/get_leader_schedule_test.go create mode 100644 pkg/rpcserver/get_vote_accounts_test.go create mode 100644 pkg/rpcserver/optional_params_test.go diff --git a/cmd/mithril/node/checkpoint_recovery_test.go b/cmd/mithril/node/checkpoint_recovery_test.go index d6ccc1f2b..ff5a1588c 100644 --- a/cmd/mithril/node/checkpoint_recovery_test.go +++ b/cmd/mithril/node/checkpoint_recovery_test.go @@ -134,6 +134,34 @@ func TestRecoveredManifestContextAdvancesStaleStateBeforeIntegrity(t *testing.T) assert.Equal(t, manifestCtx, s.LastRootedContext) } +type recordingRPCBankState struct { + slot uint64 + blockHeight uint64 + transactionCount uint64 +} + +func (state *recordingRPCBankState) SetRootedBankState(slot, blockHeight, transactionCount uint64) { + state.slot = slot + state.blockHeight = blockHeight + state.transactionCount = transactionCount +} + +func TestPublishRPCBankStateUsesRecoveredRoot(t *testing.T) { + transactionCount := uint64(789) + server := new(recordingRPCBankState) + publishRPCBankState(server, &state.MithrilState{ + ManifestParentSlot: 10, + ManifestBlockHeight: 20, + ManifestTransactionCount: 30, + LastRootedContext: &state.ResumeContext{ + Slot: 123, + BlockHeight: 456, + TransactionCount: &transactionCount, + }, + }) + assert.Equal(t, recordingRPCBankState{slot: 123, blockHeight: 456, transactionCount: 789}, *server) +} + func writeCheckpointManifest(t *testing.T, accountsDbPath, manifestsDir string, seq, root, fileID uint64, suffix string) *state.TransactionStatusCheckpointRef { t.Helper() payload := []byte(fmt.Sprintf("checkpoint-payload-%d", root)) diff --git a/pkg/accountsdb/fold_test.go b/pkg/accountsdb/fold_test.go index fbdff0ade..64f01d0f4 100644 --- a/pkg/accountsdb/fold_test.go +++ b/pkg/accountsdb/fold_test.go @@ -116,6 +116,19 @@ func TestCommitBatchReadsBackAndDedupes(t *testing.T) { assert.Equal(t, []byte("ctx-108"), manifest.ResumeCtx) } +func TestCommitBatchPublishesDurableThroughWithIndex(t *testing.T) { + db, _ := newFoldTestDb(t) + defer db.CloseDb() + + db.foldHooks.afterIndexCommit = func() { + require.Equal(t, uint64(42), db.DurableThrough()) + } + _, err := db.CommitBatch(foldDeltas( + accounts.SlotDelta{Slot: 42, Delta: []*accounts.Account{foldAcct(1, 1, nil)}}, + ), 42, nil, nil) + require.NoError(t, err) +} + // Manifest encode/decode round-trip, and CRC detection of a torn manifest. func TestSegmentManifestRoundTripAndTornDetection(t *testing.T) { dir := t.TempDir() diff --git a/pkg/rpcserver/get_account_info_parsed_test.go b/pkg/rpcserver/get_account_info_parsed_test.go new file mode 100644 index 000000000..9f9e7f08d --- /dev/null +++ b/pkg/rpcserver/get_account_info_parsed_test.go @@ -0,0 +1,128 @@ +package rpcserver + +import ( + "encoding/json" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetAccountInfoParsesVoteAccountAtRoot(t *testing.T) { + const rootedSlot = 500 + votePubkey := solana.PublicKey{1} + nodePubkey := solana.PublicKey{2} + withdrawer := solana.PublicKey{3} + collector := solana.PublicKey{4} + blockCollector := solana.PublicKey{5} + authorized := solana.PublicKey{6} + rootSlot := uint64(490) + bls := [48]byte{7} + state := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4} + state.V4.NodePubkey = nodePubkey + state.V4.AuthorizedWithdrawer = withdrawer + state.V4.InflationRewardsCollector = collector + state.V4.BlockRevenueCollector = blockCollector + state.V4.InflationRewardsCommissionBps = 755 + state.V4.BlockRevenueCommissionBps = 321 + state.V4.PendingDelegatorRewards = 42 + state.V4.BlsPubkeyCompressed = &bls + state.V4.RootSlot = &rootSlot + state.V4.AuthorizedVoters.AuthorizedVoters.Set(3, authorized) + state.V4.Votes.PushBack(sealevel.LandedVote{Latency: 2, Lockout: sealevel.VoteLockout{Slot: 499, ConfirmationCount: 4}}) + state.V4.EpochCredits = []sealevel.EpochCredits{{Epoch: 3, Credits: 9, PrevCredits: 5}} + state.V4.LastTimestamp = sealevel.BlockTimestamp{Slot: 499, Timestamp: 1_700_000_000} + + db := newRPCAccountsDB(t) + account := voteAccountForRPC(t, votePubkey, &state) + account.Lamports = 27_074_400 + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: rootedSlot, Delta: []*accounts.Account{account}}}, rootedSlot, nil, nil) + require.NoError(t, err) + + server := &RpcServer{acctsDb: db} + server.SetRootedBankState(rootedSlot, 495, 100) + got, err := server.GetAccountInfo(t.Context(), mustRawParams(t, []interface{}{ + votePubkey.String(), + map[string]interface{}{"commitment": "finalized", "encoding": "jsonParsed", "minContextSlot": float64(rootedSlot)}, + })) + require.NoError(t, err) + require.Equal(t, uint64(rootedSlot), got.Context.Slot) + require.NotNil(t, got.Value) + require.Equal(t, uint64(27_074_400), got.Value.Lamports) + + raw, err := json.Marshal(got.Value.Data) + require.NoError(t, err) + require.Contains(t, string(raw), `"lastTimestamp":{"slot":499,"timestamp":1700000000}`) + var decoded struct { + Program string `json:"program"` + Parsed struct { + Type string `json:"type"` + Info parsedVoteAccountInfo `json:"info"` + } `json:"parsed"` + Space uint64 `json:"space"` + } + require.NoError(t, json.Unmarshal(raw, &decoded)) + require.Equal(t, "vote", decoded.Program) + require.Equal(t, "vote", decoded.Parsed.Type) + require.Equal(t, nodePubkey.String(), decoded.Parsed.Info.NodePubkey) + require.Equal(t, withdrawer.String(), decoded.Parsed.Info.AuthorizedWithdrawer) + require.Equal(t, uint8(8), decoded.Parsed.Info.Commission) + require.Equal(t, uint16(755), decoded.Parsed.Info.InflationRewardsCommissionBPS) + require.Equal(t, "42", decoded.Parsed.Info.PendingDelegatorRewards) + require.Equal(t, parsedBlockTimestamp{Slot: 499, Timestamp: 1_700_000_000}, decoded.Parsed.Info.LastTimestamp) + require.Equal(t, []parsedAuthorizedVoter{{Epoch: 3, AuthorizedVoter: authorized.String()}}, decoded.Parsed.Info.AuthorizedVoters) + require.Equal(t, []parsedVote{{Latency: 2, Slot: 499, ConfirmationCount: 4}}, decoded.Parsed.Info.Votes) + require.Equal(t, []parsedEpochCredits{{Epoch: 3, Credits: "9", PreviousCredits: "5"}}, decoded.Parsed.Info.EpochCredits) +} + +func TestGetAccountInfoJsonParsedFallsBackToBase64(t *testing.T) { + db := newRPCAccountsDB(t) + address := solana.PublicKey{9} + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 10, Delta: []*accounts.Account{{Key: address, Lamports: 1, Data: []byte{1, 2, 3}}}}}, 10, nil, nil) + require.NoError(t, err) + server := &RpcServer{acctsDb: db} + server.SetRootedBankState(10, 9, 0) + + got, err := server.GetAccountInfo(t.Context(), mustRawParams(t, []interface{}{ + address.String(), map[string]interface{}{"encoding": "jsonParsed"}, + })) + require.NoError(t, err) + require.Equal(t, []string{"AQID", "base64"}, got.Value.Data) +} + +func TestGetAccountInfoParsesLegacyVoteAccountWithV4Defaults(t *testing.T) { + votePubkey := solana.PublicKey{1} + nodePubkey := solana.PublicKey{2} + state := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionCurrent} + state.Current.NodePubkey = nodePubkey + state.Current.Commission = 7 + + db := newRPCAccountsDB(t) + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 10, Delta: []*accounts.Account{ + voteAccountForRPC(t, votePubkey, &state), + }}}, 10, nil, nil) + require.NoError(t, err) + server := &RpcServer{acctsDb: db} + server.SetRootedBankState(10, 9, 0) + + got, err := server.GetAccountInfo(t.Context(), mustRawParams(t, []interface{}{ + votePubkey.String(), map[string]interface{}{"encoding": "jsonParsed"}, + })) + require.NoError(t, err) + raw, err := json.Marshal(got.Value.Data) + require.NoError(t, err) + var decoded struct { + Parsed struct { + Info parsedVoteAccountInfo `json:"info"` + } `json:"parsed"` + } + require.NoError(t, json.Unmarshal(raw, &decoded)) + require.Equal(t, uint8(7), decoded.Parsed.Info.Commission) + require.Equal(t, uint16(700), decoded.Parsed.Info.InflationRewardsCommissionBPS) + require.Equal(t, votePubkey.String(), decoded.Parsed.Info.InflationRewardsCollector) + require.Equal(t, nodePubkey.String(), decoded.Parsed.Info.BlockRevenueCollector) + require.Equal(t, uint16(10000), decoded.Parsed.Info.BlockRevenueCommissionBPS) + require.Equal(t, "0", decoded.Parsed.Info.PendingDelegatorRewards) +} diff --git a/pkg/rpcserver/get_balance_test.go b/pkg/rpcserver/get_balance_test.go new file mode 100644 index 000000000..04ae95c97 --- /dev/null +++ b/pkg/rpcserver/get_balance_test.go @@ -0,0 +1,112 @@ +package rpcserver + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/accountsdb" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetBalanceUsesPublishedRootedBank(t *testing.T) { + db := newRPCAccountsDB(t) + address := solana.PublicKey{7} + _, err := db.CommitBatch([]accounts.SlotDelta{{ + Slot: 42, + Delta: []*accounts.Account{{ + Key: address, + Lamports: 123_456_789, + }}, + }}, 42, nil, nil) + require.NoError(t, err) + + server := &RpcServer{acctsDb: db} + server.SetRootedBankState(42, 40, 10) + got, err := server.GetBalance(t.Context(), mustRawParams(t, []interface{}{ + address.String(), + map[string]interface{}{"commitment": "confirmed", "minContextSlot": float64(42)}, + })) + require.NoError(t, err) + require.Equal(t, uint64(42), got.Context.Slot) + require.Equal(t, uint64(123_456_789), got.Value) + + _, err = server.GetBalance(t.Context(), mustRawParams(t, []interface{}{ + address.String(), + map[string]interface{}{"minContextSlot": float64(43)}, + })) + var minSlotErr *MinContextSlotNotReachedError + require.True(t, errors.As(err, &minSlotErr)) + require.Equal(t, uint64(42), minSlotErr.ContextSlot) +} + +func TestGetBalanceMissingAccountIsZero(t *testing.T) { + server := &RpcServer{acctsDb: newRPCAccountsDB(t)} + server.SetRootedBankState(9, 8, 0) + + got, err := server.GetBalance(t.Context(), mustRawParams(t, []interface{}{solana.PublicKey{8}.String()})) + require.NoError(t, err) + require.Equal(t, uint64(9), got.Context.Slot) + require.Zero(t, got.Value) +} + +func TestGetBalanceDoesNotMixNewAccountsWithOldRoot(t *testing.T) { + db := newRPCAccountsDB(t) + address := solana.PublicKey{9} + _, err := db.CommitBatch([]accounts.SlotDelta{{ + Slot: 43, + Delta: []*accounts.Account{{ + Key: address, + Lamports: 43, + }}, + }}, 43, nil, nil) + require.NoError(t, err) + + server := &RpcServer{acctsDb: db} + server.SetRootedBankState(42, 40, 10) + deadline, cancel := context.WithTimeout(t.Context(), 5*time.Millisecond) + defer cancel() + _, err = server.GetBalance(deadline, mustRawParams(t, []interface{}{address.String()})) + require.ErrorIs(t, err, context.DeadlineExceeded) + + server.SetRootedBankState(43, 41, 11) + got, err := server.GetBalance(t.Context(), mustRawParams(t, []interface{}{address.String()})) + require.NoError(t, err) + require.Equal(t, uint64(43), got.Context.Slot) + require.Equal(t, uint64(43), got.Value) +} + +func TestGetBalanceRejectsInvalidParams(t *testing.T) { + server := &RpcServer{} + for _, params := range [][]interface{}{ + {}, + {true}, + {"not-a-pubkey"}, + {solana.PublicKey{1}.String(), true}, + {solana.PublicKey{1}.String(), map[string]interface{}{"commitment": "unknown"}}, + {solana.PublicKey{1}.String(), map[string]interface{}{"minContextSlot": 1.5}}, + } { + _, err := server.GetBalance(t.Context(), mustRawParams(t, params)) + var invalid *InvalidParamsError + require.True(t, errors.As(err, &invalid), "params: %#v; error: %v", params, err) + } +} + +func newRPCAccountsDB(t *testing.T) *accountsdb.AccountsDb { + t.Helper() + dir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(dir, "accounts"), 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "largest_file_id"), make([]byte, 8), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(dir, "bootstrap_high_file_id"), make([]byte, 8), 0o644)) + db, err := accountsdb.OpenDb(dir) + require.NoError(t, err) + db.RootedDurable = true + db.InitCaches() + t.Cleanup(db.CloseDb) + return db +} diff --git a/pkg/rpcserver/get_block_production_test.go b/pkg/rpcserver/get_block_production_test.go new file mode 100644 index 000000000..19e7e396c --- /dev/null +++ b/pkg/rpcserver/get_block_production_test.go @@ -0,0 +1,86 @@ +package rpcserver + +import ( + "errors" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/leaderschedule" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetBlockProductionUsesRootedSlotHistory(t *testing.T) { + const rootedSlot = uint64(13) + firstLeader := solana.PublicKey{1} + secondLeader := solana.PublicKey{2} + global.SetLeaderScheduleForEpoch(0, leaderschedule.NewLeaderScheduleFromKeyedSlots( + map[solana.PublicKey][]uint64{ + firstLeader: {10, 11}, + secondLeader: {12, 13}, + }, + 0, + )) + t.Cleanup(func() { global.SetLeaderScheduleForEpoch(0, nil) }) + + history := sealevel.SysvarSlotHistory{ + Bits: sealevel.SlotHistoryBitvec{ + Bits: sealevel.SlotHistoryInner{BlocksLen: 1, Blocks: []uint64{(1 << 10) | (1 << 12) | (1 << 13)}}, + Len: 64, + }, + NextSlot: rootedSlot + 1, + } + db := newRPCAccountsDB(t) + _, err := db.CommitBatch([]accounts.SlotDelta{{ + Slot: rootedSlot, + Delta: []*accounts.Account{{ + Key: sealevel.SysvarSlotHistoryAddr, + Lamports: 1, + Data: history.MustMarshal(), + }}, + }}, rootedSlot, nil, nil) + require.NoError(t, err) + + server := &RpcServer{ + acctsDb: db, + epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 32}, + } + server.SetRootedBankState(rootedSlot, 12, 0) + got, err := server.GetBlockProduction(t.Context(), mustRawParams(t, []interface{}{ + map[string]interface{}{ + "commitment": "finalized", + "range": map[string]interface{}{ + "firstSlot": float64(10), + "lastSlot": float64(rootedSlot), + }, + }, + })) + require.NoError(t, err) + require.Equal(t, rootedSlot, got.Context.Slot) + require.Equal(t, BlockProductionRange{FirstSlot: 10, LastSlot: 13}, got.Value.Range) + require.Equal(t, [2]uint64{2, 1}, got.Value.ByIdentity[firstLeader.String()]) + require.Equal(t, [2]uint64{2, 2}, got.Value.ByIdentity[secondLeader.String()]) +} + +func TestGetBlockProductionRejectsInvalidRange(t *testing.T) { + server := &RpcServer{ + acctsDb: newRPCAccountsDB(t), + epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 32}, + } + server.SetRootedBankState(13, 12, 0) + for _, params := range [][]interface{}{ + {true}, + {map[string]interface{}{"commitment": "unknown"}}, + {map[string]interface{}{"identity": "invalid"}}, + {map[string]interface{}{"range": true}}, + {map[string]interface{}{"range": map[string]interface{}{}}}, + {map[string]interface{}{"range": map[string]interface{}{"firstSlot": 1.5}}}, + {map[string]interface{}{}, map[string]interface{}{}}, + } { + _, err := server.GetBlockProduction(t.Context(), mustRawParams(t, params)) + var invalid *InvalidParamsError + require.True(t, errors.As(err, &invalid), "params: %#v; error: %v", params, err) + } +} diff --git a/pkg/rpcserver/get_epoch_info_rooted_test.go b/pkg/rpcserver/get_epoch_info_rooted_test.go new file mode 100644 index 000000000..0d1336411 --- /dev/null +++ b/pkg/rpcserver/get_epoch_info_rooted_test.go @@ -0,0 +1,33 @@ +package rpcserver + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/filecoin-project/go-jsonrpc" + "github.com/stretchr/testify/require" +) + +func TestGetEpochInfoUsesRootedBankForFinalized(t *testing.T) { + oldSlot, oldEpoch := global.Slot(), global.Epoch() + oldHeight, oldTransactions := global.BlockHeight(), global.TransactionCount() + t.Cleanup(func() { + global.SetSlot(oldSlot) + global.SetEpoch(oldEpoch) + global.SetBlockHeight(oldHeight) + global.SetTransactionCount(oldTransactions) + }) + global.SetSlot(432250) + global.SetEpoch(1) + global.SetBlockHeight(432000) + global.SetTransactionCount(2000) + + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 432000, LeaderScheduleSlotOffset: 432000}} + server.SetRootedBankState(432123, 431900, 1900) + got, err := server.GetEpochInfo(t.Context(), jsonrpc.RawParams(`[{"commitment":"finalized"}]`)) + require.NoError(t, err) + require.Equal(t, uint64(432123), got.AbsoluteSlot) + require.Equal(t, uint64(431900), got.BlockHeight) + require.Equal(t, uint64(1900), got.TransactionCount) +} diff --git a/pkg/rpcserver/get_leader_schedule_test.go b/pkg/rpcserver/get_leader_schedule_test.go new file mode 100644 index 000000000..836924bb0 --- /dev/null +++ b/pkg/rpcserver/get_leader_schedule_test.go @@ -0,0 +1,83 @@ +package rpcserver + +import ( + "errors" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/leaderschedule" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetLeaderScheduleReturnsRelativeEpochSlots(t *testing.T) { + epochSchedule := &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 4} + firstSlot := epochSchedule.FirstSlotInEpoch(3) + firstLeader := solana.PublicKey{1} + secondLeader := solana.PublicKey{2} + global.SetLeaderScheduleForEpoch(3, leaderschedule.NewLeaderScheduleFromKeyedSlots( + map[solana.PublicKey][]uint64{ + firstLeader: {0, 2}, + secondLeader: {1, 3}, + }, + firstSlot, + )) + t.Cleanup(func() { global.SetLeaderScheduleForEpoch(3, nil) }) + + server := &RpcServer{epochSchedule: epochSchedule} + server.SetRootedBankState(firstSlot+2, firstSlot+2, 0) + got, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{ + float64(firstSlot + 1), + map[string]interface{}{"commitment": "confirmed"}, + })) + require.NoError(t, err) + require.Equal(t, []uint64{0, 2}, got[firstLeader.String()]) + require.Equal(t, []uint64{1, 3}, got[secondLeader.String()]) + + filtered, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{ + float64(firstSlot), + map[string]interface{}{"identity": secondLeader.String()}, + })) + require.NoError(t, err) + require.Equal(t, map[string][]uint64{secondLeader.String(): {1, 3}}, filtered) + + filtered, err = server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{ + map[string]interface{}{"commitment": "finalized", "identity": secondLeader.String()}, + })) + require.NoError(t, err) + require.Equal(t, map[string][]uint64{secondLeader.String(): {1, 3}}, filtered) + + missing, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{ + float64(firstSlot), map[string]interface{}{"identity": solana.PublicKey{9}.String()}, + })) + require.NoError(t, err) + require.Empty(t, missing) +} + +func TestGetLeaderScheduleReturnsNullWhenEpochScheduleIsUnavailable(t *testing.T) { + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 4}} + server.SetRootedBankState(40_002, 1, 0) + + got, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{float64(40_002)})) + require.NoError(t, err) + require.Nil(t, got) +} + +func TestGetLeaderScheduleRejectsInvalidParams(t *testing.T) { + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 4}} + server.SetRootedBankState(1, 1, 0) + for _, params := range [][]interface{}{ + {-1.0}, + {1.5}, + {float64(1), true}, + {float64(1), map[string]interface{}{"commitment": "unknown"}}, + {float64(1), map[string]interface{}{"identity": "invalid"}}, + {float64(1), map[string]interface{}{"keyByVoteAccount": true}}, + {float64(1), nil, nil}, + } { + _, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, params)) + var invalid *InvalidParamsError + require.True(t, errors.As(err, &invalid), "params: %#v; error: %v", params, err) + } +} diff --git a/pkg/rpcserver/get_vote_accounts_test.go b/pkg/rpcserver/get_vote_accounts_test.go new file mode 100644 index 000000000..d6f9d35c3 --- /dev/null +++ b/pkg/rpcserver/get_vote_accounts_test.go @@ -0,0 +1,185 @@ +package rpcserver + +import ( + "bytes" + "errors" + "math" + "path/filepath" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetVoteAccountsUsesRootedVoteState(t *testing.T) { + const ( + rootedSlot = 500 + epoch = 0 + ) + currentVote := solana.PublicKey{1} + delinquentVote := solana.PublicKey{2} + newVote := solana.PublicKey{3} + currentNode := solana.PublicKey{11} + delinquentNode := solana.PublicKey{12} + newNode := solana.PublicKey{13} + currentRoot := uint64(498) + delinquentRoot := uint64(299) + + currentState := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4} + currentState.V4.NodePubkey = currentNode + currentState.V4.InflationRewardsCommissionBps = 755 + currentState.V4.RootSlot = ¤tRoot + currentState.V4.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: 499}}) + currentState.V4.EpochCredits = []sealevel.EpochCredits{ + {Epoch: 0, Credits: 1}, + {Epoch: 1, Credits: 2, PrevCredits: 1}, + {Epoch: 2, Credits: 3, PrevCredits: 2}, + {Epoch: 3, Credits: 4, PrevCredits: 3}, + {Epoch: 4, Credits: 5, PrevCredits: 4}, + {Epoch: 5, Credits: 9, PrevCredits: 5}, + } + + delinquentState := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionCurrent} + delinquentState.Current.NodePubkey = delinquentNode + delinquentState.Current.Commission = 5 + delinquentState.Current.RootSlot = &delinquentRoot + delinquentState.Current.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: 300}}) + + newState := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionCurrent} + newState.Current.NodePubkey = newNode + newState.Current.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: 498}}) + + db := newRPCAccountsDB(t) + global.ClearPendingStakePubkeys() + t.Cleanup(func() { + global.EnqueuePendingStakePubkey(rootedSlot, solana.PublicKey{21}) + _, err := global.FlushPendingStakePubkeys(filepath.Join(db.AcctsDir, "..")) + require.NoError(t, err) + global.ClearPendingStakePubkeys() + }) + stakeHistory := sealevel.SysvarStakeHistory{} + var historyData bytes.Buffer + require.NoError(t, stakeHistory.MarshalWithEncoder(bin.NewBinEncoder(&historyData))) + stakeAccount := func(key, vote solana.PublicKey, lamports uint64) *accounts.Account { + state := &sealevel.StakeStateV2{Status: sealevel.StakeStateV2StatusStake} + state.Stake.Stake.Delegation = sealevel.Delegation{ + VoterPubkey: vote, StakeLamports: lamports, + ActivationEpoch: math.MaxUint64, DeactivationEpoch: math.MaxUint64, + } + data, err := sealevel.MarshalStakeStake(state) + require.NoError(t, err) + global.EnqueuePendingStakePubkey(rootedSlot, key) + return &accounts.Account{Key: key, Owner: addresses.StakeProgramAddr, Lamports: lamports, Data: data} + } + _, err := db.CommitBatch([]accounts.SlotDelta{{ + Slot: rootedSlot, + Delta: []*accounts.Account{ + voteAccountForRPC(t, currentVote, ¤tState), + voteAccountForRPC(t, delinquentVote, &delinquentState), + voteAccountForRPC(t, newVote, &newState), + stakeAccount(solana.PublicKey{21}, currentVote, 125), + stakeAccount(solana.PublicKey{22}, delinquentVote, 200), + stakeAccount(solana.PublicKey{23}, solana.PublicKey{4}, 50), + {Key: sealevel.SysvarStakeHistoryAddr, Owner: addresses.SysvarOwnerAddr, Lamports: 1, Data: historyData.Bytes()}, + }, + }}, rootedSlot, nil, nil) + require.NoError(t, err) + _, err = global.FlushPendingStakePubkeys(filepath.Join(db.AcctsDir, "..")) + require.NoError(t, err) + + previous, hadPrevious := global.EpochStakesSnapshot(epoch) + global.PutEpochStakes(epoch, + map[solana.PublicKey]uint64{currentVote: 100, delinquentVote: 200}, + map[solana.PublicKey]*epochstakes.VoteAccount{ + currentVote: {NodePubkey: currentNode}, + delinquentVote: {NodePubkey: delinquentNode}, + }, + 300, + ) + t.Cleanup(func() { + if hadPrevious { + global.PutEpochStakes(epoch, previous.Stakes, previous.VoteAccounts, previous.TotalStake) + } else { + global.ClearEpochStakes(epoch) + } + }) + + server := &RpcServer{ + acctsDb: db, + epochSchedule: &sealevel.SysvarEpochSchedule{ + SlotsPerEpoch: 1_000, + }, + } + server.SetSlotCtx(&sealevel.SlotCtx{Features: features.NewFeaturesDefault()}) + server.SetRootedBankState(rootedSlot, 490, 1_000) + got, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{ + map[string]interface{}{"commitment": "confirmed"}, + })) + require.NoError(t, err) + require.Len(t, got.Current, 2) + require.Len(t, got.Delinquent, 1) + require.Equal(t, currentVote.String(), got.Current[0].VotePubkey) + require.Equal(t, currentNode.String(), got.Current[0].NodePubkey) + require.Equal(t, uint64(125), got.Current[0].ActivatedStake, "live stake differs from frozen epoch stake") + require.Equal(t, uint8(8), got.Current[0].Commission) + require.Equal(t, uint16(755), got.Current[0].InflationRewardsCommissionBPS) + require.Equal(t, uint64(499), got.Current[0].LastVote) + require.Equal(t, [][3]uint64{{1, 2, 1}, {2, 3, 2}, {3, 4, 3}, {4, 5, 4}, {5, 9, 5}}, got.Current[0].EpochCredits) + require.True(t, got.Current[0].EpochVoteAccount) + require.Equal(t, newVote.String(), got.Current[1].VotePubkey) + require.Equal(t, uint64(0), got.Current[1].ActivatedStake) + require.False(t, got.Current[1].EpochVoteAccount) + require.Equal(t, delinquentVote.String(), got.Delinquent[0].VotePubkey) + require.Equal(t, uint8(5), got.Delinquent[0].Commission) + + filtered, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{ + map[string]interface{}{"votePubkey": currentVote.String()}, + })) + require.NoError(t, err) + require.Len(t, filtered.Current, 1) + require.Empty(t, filtered.Delinquent) + + db.VoteAcctCache.Clear() + cold, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{})) + require.NoError(t, err) + require.Len(t, cold.Current, 1) + require.Equal(t, currentVote.String(), cold.Current[0].VotePubkey) + require.Len(t, cold.Delinquent, 1) + require.Equal(t, delinquentVote.String(), cold.Delinquent[0].VotePubkey) +} + +func TestGetVoteAccountsRejectsInvalidConfig(t *testing.T) { + server := &RpcServer{} + for _, params := range [][]interface{}{ + {true}, + {map[string]interface{}{"commitment": "unknown"}}, + {map[string]interface{}{"votePubkey": "invalid"}}, + {map[string]interface{}{"keepUnstakedDelinquents": "yes"}}, + {map[string]interface{}{"keepUnstakedDelinquents": true}}, + {map[string]interface{}{"delinquentSlotDistance": 1.5}}, + {map[string]interface{}{}, map[string]interface{}{}}, + } { + _, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, params)) + var invalid *InvalidParamsError + require.True(t, errors.As(err, &invalid), "params: %#v; error: %v", params, err) + } +} + +func voteAccountForRPC(t *testing.T, key solana.PublicKey, state *sealevel.VoteStateVersions) *accounts.Account { + t.Helper() + data, err := sealevel.MarshalVersionedVoteState(state) + require.NoError(t, err) + return &accounts.Account{ + Key: key, + Lamports: 1, + Owner: addresses.VoteProgramAddr, + Data: data, + } +} diff --git a/pkg/rpcserver/optional_params_test.go b/pkg/rpcserver/optional_params_test.go new file mode 100644 index 000000000..b0cfa31a1 --- /dev/null +++ b/pkg/rpcserver/optional_params_test.go @@ -0,0 +1,45 @@ +package rpcserver + +import ( + "context" + "testing" + + "github.com/filecoin-project/go-jsonrpc" + "github.com/stretchr/testify/require" +) + +func TestOptionalRPCParamsCanBeOmitted(t *testing.T) { + server := &RpcServer{} + methods := map[string]func(context.Context, jsonrpc.RawParams) error{ + "getEpochInfo": func(ctx context.Context, p jsonrpc.RawParams) error { + _, err := server.GetEpochInfo(ctx, p) + return err + }, + "getVoteAccounts": func(ctx context.Context, p jsonrpc.RawParams) error { + _, err := server.GetVoteAccounts(ctx, p) + return err + }, + "getLeaderSchedule": func(ctx context.Context, p jsonrpc.RawParams) error { + _, err := server.GetLeaderSchedule(ctx, p) + return err + }, + "getBlockProduction": func(ctx context.Context, p jsonrpc.RawParams) error { + _, err := server.GetBlockProduction(ctx, p) + return err + }, + } + for name, call := range methods { + t.Run(name, func(t *testing.T) { + // Missing node state should fail identically for each empty parameter form. + want := call(t.Context(), jsonrpc.RawParams(`[]`)) + require.Error(t, want) + for _, params := range []jsonrpc.RawParams{nil, jsonrpc.RawParams(`null`)} { + require.EqualError(t, call(t.Context(), params), want.Error()) + } + for _, params := range []jsonrpc.RawParams{jsonrpc.RawParams(`{}`), jsonrpc.RawParams(`true`)} { + var invalid *InvalidParamsError + require.ErrorAs(t, call(t.Context(), params), &invalid) + } + }) + } +} From bd11a01a3b9c9fa3252dfd25d2c23c702e8a3581 Mon Sep 17 00:00:00 2001 From: Neeraj Godiyal Date: Thu, 24 Sep 2026 13:10:48 +0530 Subject: [PATCH 03/14] rpc: persist vote-account candidates and use rooted feature state --- pkg/accountsdb/accountsdb.go | 11 +-- pkg/accountsdb/compact.go | 2 +- pkg/accountsdb/fold.go | 11 ++- pkg/accountsdb/index.go | 11 ++- pkg/accountsdb/segment.go | 16 ++- pkg/accountsdb/vote_index.go | 153 +++++++++++++++++++++++++++++ pkg/rpcserver/get_vote_accounts.go | 38 ++++--- pkg/snapshot/build_db.go | 52 +++++----- pkg/snapshot/build_db_with_incr.go | 14 ++- 9 files changed, 246 insertions(+), 62 deletions(-) create mode 100644 pkg/accountsdb/vote_index.go diff --git a/pkg/accountsdb/accountsdb.go b/pkg/accountsdb/accountsdb.go index 4c1fd2239..f738d5c23 100644 --- a/pkg/accountsdb/accountsdb.go +++ b/pkg/accountsdb/accountsdb.go @@ -32,6 +32,7 @@ type AccountsDb struct { AcctsDir string LargestFileId atomic.Uint64 VoteAcctCache otter.Cache[solana.PublicKey, *accounts.Account] + voteIndexMu sync.Mutex CommonAcctsCache otter.Cache[solana.PublicKey, *accounts.Account] ProgramCache otter.Cache[solana.PublicKey, *ProgramCacheEntry] // Otter permits concurrent ordinary operations but not Clear. Rewind takes @@ -102,16 +103,6 @@ func (accountsDb *AccountsDb) DurableThrough() uint64 { return accountsDb.durableThrough.Load() } -// VoteAccountPubkeys returns the vote accounts in the current cache view. -func (accountsDb *AccountsDb) VoteAccountPubkeys() []solana.PublicKey { - pubkeys := make([]solana.PublicKey, 0, accountsDb.VoteAcctCache.Size()) - accountsDb.VoteAcctCache.Range(func(pubkey solana.PublicKey, _ *accounts.Account) bool { - pubkeys = append(pubkeys, pubkey) - return true - }) - return pubkeys -} - // silentLogger implements pebble.Logger but discards all messages. // This suppresses verbose WAL recovery messages on startup. type silentLogger struct{} diff --git a/pkg/accountsdb/compact.go b/pkg/accountsdb/compact.go index 6b167f7ed..9e2f61481 100644 --- a/pkg/accountsdb/compact.go +++ b/pkg/accountsdb/compact.go @@ -268,7 +268,7 @@ func (db *AccountsDb) compactFile(c compactCandidate, minDeadFraction float64) ( return false, 0, err } - pubkeys, idxEntries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), c.slot, c.fileId) + pubkeys, idxEntries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), c.slot, c.fileId) if err != nil { return false, 0, err } diff --git a/pkg/accountsdb/fold.go b/pkg/accountsdb/fold.go index 7e01dfe8d..49391695b 100644 --- a/pkg/accountsdb/fold.go +++ b/pkg/accountsdb/fold.go @@ -13,6 +13,7 @@ import ( "sort" "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/cockroachdb/pebble" "golang.org/x/sync/errgroup" ) @@ -181,7 +182,10 @@ func (db *AccountsDb) CommitBatch( segErr := func() error { for _, k := range keys { v := union[k] - records = append(records, ManifestRecord{Pubkey: k, Offset: dataLen, OwnerSlot: v.ownerSlot}) + records = append(records, ManifestRecord{ + Pubkey: k, Offset: dataLen, OwnerSlot: v.ownerSlot, + Vote: v.acct.Lamports > 0 && v.acct.Owner == addresses.VoteProgramAddr, + }) ava := AppendVecAccount{ DataLen: uint64(len(v.acct.Data)), Pubkey: v.acct.Key, @@ -339,6 +343,11 @@ func (db *AccountsDb) applyManifestToIndex(m *SegmentManifest) error { if err := batch.Set(r.Pubkey[:], idxBuf[:], nil); err != nil { return err } + if r.Vote { + if err := batch.Set(voteIndexKey(r.Pubkey), nil, nil); err != nil { + return err + } + } } if err := batch.Set(metaKeyLastBatch, encodeFoldMeta(foldMeta{ BatchSeq: m.BatchSeq, diff --git a/pkg/accountsdb/index.go b/pkg/accountsdb/index.go index 7601d73f4..0a064517a 100644 --- a/pkg/accountsdb/index.go +++ b/pkg/accountsdb/index.go @@ -100,10 +100,12 @@ func WriteStakePubkeyIndex(path string, entries []StakeIndexEntry) error { // - pubkeys: all account pubkeys // - acctIdxEntries: index entries for each account // - stakeEntries: stake account pubkeys with their appendvec location hints -func BuildIndexEntriesFromAppendVecs(data []byte, fileSize uint64, slot uint64, fileId uint64) ([]solana.PublicKey, []AccountIndexEntry, []StakeIndexEntry, error) { +// - votePubkeys: vote-program account candidates, including accounts without active stake +func BuildIndexEntriesFromAppendVecs(data []byte, fileSize uint64, slot uint64, fileId uint64) ([]solana.PublicKey, []AccountIndexEntry, []StakeIndexEntry, []solana.PublicKey, error) { pubkeys := make([]solana.PublicKey, 0, 20000) acctIdxEntries := make([]AccountIndexEntry, 0, 20000) stakeEntries := make([]StakeIndexEntry, 0, 1000) + var votePubkeys []solana.PublicKey parser := &appendVecParser{Buf: data, FileSize: fileSize, FileId: fileId, Slot: slot} var owner solana.PublicKey @@ -117,7 +119,7 @@ func BuildIndexEntriesFromAppendVecs(data []byte, fileSize uint64, slot uint64, if errors.Is(err, io.EOF) { break } - return nil, nil, nil, fmt.Errorf("parse appendvec slot=%d file_id=%d: %w", slot, fileId, err) + return nil, nil, nil, nil, fmt.Errorf("parse appendvec slot=%d file_id=%d: %w", slot, fileId, err) } // Collect stake account entries with appendvec location hints if bytes.Equal(owner[:], addresses.StakeProgramAddr[:]) { @@ -128,7 +130,10 @@ func BuildIndexEntriesFromAppendVecs(data []byte, fileSize uint64, slot uint64, Offset: acctIdxEntries[idx].Offset, }) } + if owner == addresses.VoteProgramAddr { + votePubkeys = append(votePubkeys, pubkeys[len(pubkeys)-1]) + } } - return pubkeys, acctIdxEntries, stakeEntries, nil + return pubkeys, acctIdxEntries, stakeEntries, votePubkeys, nil } diff --git a/pkg/accountsdb/segment.go b/pkg/accountsdb/segment.go index 9359f7841..5cc189f8a 100644 --- a/pkg/accountsdb/segment.go +++ b/pkg/accountsdb/segment.go @@ -62,6 +62,7 @@ type ManifestRecord struct { Offset uint64 // record offset within this segment's data file OwnerSlot uint64 // slot that produced this version (observability) PrevValid bool // false => the key was absent from the index before this batch + Vote bool // this version is a nonzero-lamport vote-program account Prev AccountIndexEntry // index entry before this batch overwrote it } @@ -138,11 +139,14 @@ func (m *SegmentManifest) encode() []byte { body.Write(r.Pubkey[:]) put64(r.Offset) put64(r.OwnerSlot) + var flags byte if r.PrevValid { - body.WriteByte(1) - } else { - body.WriteByte(0) + flags |= 1 } + if r.Vote { + flags |= 2 + } + body.WriteByte(flags) r.Prev.Marshal(&prevBuf) body.Write(prevBuf[:]) } @@ -324,7 +328,11 @@ func ReadSegmentManifest(path string) (*SegmentManifest, error) { if err != nil { return nil, err } - r.PrevValid = pv == 1 + if pv&^3 != 0 { + return nil, ErrTornManifest + } + r.PrevValid = pv&1 != 0 + r.Vote = pv&2 != 0 prevRaw, err := d.bytes(24) if err != nil { return nil, err diff --git a/pkg/accountsdb/vote_index.go b/pkg/accountsdb/vote_index.go new file mode 100644 index 000000000..982fd8f62 --- /dev/null +++ b/pkg/accountsdb/vote_index.go @@ -0,0 +1,153 @@ +package accountsdb + +import ( + "context" + "fmt" + + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/cockroachdb/pebble" + "github.com/gagliardetto/solana-go" +) + +// Candidate keys are separate from the 32-byte account-index keys. They are +// append-only; callers check the current account state before returning it. +const voteIndexPrefix = "\x00mithril.vote." + +var voteIndexReadyKey = []byte("\x00mithril.meta.vote_index_ready") + +func voteIndexKey(pubkey solana.PublicKey) []byte { + key := make([]byte, len(voteIndexPrefix)+len(pubkey)) + copy(key, voteIndexPrefix) + copy(key[len(voteIndexPrefix):], pubkey[:]) + return key +} + +// SeedVoteAccountPubkeys seeds a freshly built snapshot index with vote-program +// account candidates observed while parsing its appendvecs. +func (db *AccountsDb) SeedVoteAccountPubkeys(pubkeys []solana.PublicKey) error { + batch := db.Index.NewBatch() + defer batch.Close() + for _, pubkey := range pubkeys { + if err := batch.Set(voteIndexKey(pubkey), nil, nil); err != nil { + return err + } + } + if err := batch.Set(voteIndexReadyKey, nil, nil); err != nil { + return err + } + if err := batch.Commit(pebble.Sync); err != nil { + return err + } + if db.IndexWALDisabled { + return db.Index.Flush() + } + return nil +} + +// VoteAccountPubkeys returns every known vote-account candidate. Existing +// stores built before this index was added are migrated once, on first use. +func (db *AccountsDb) VoteAccountPubkeys(ctx context.Context) ([]solana.PublicKey, error) { + db.voteIndexMu.Lock() + defer db.voteIndexMu.Unlock() + if _, closer, err := db.Index.Get(voteIndexReadyKey); err == nil { + if err := closer.Close(); err != nil { + return nil, err + } + } else if err == pebble.ErrNotFound { + if err := db.migrateVoteIndex(ctx); err != nil { + return nil, err + } + } else { + return nil, err + } + + iter, err := db.Index.NewIter(&pebble.IterOptions{ + LowerBound: []byte(voteIndexPrefix), + UpperBound: []byte("\x00mithril.vote/"), + }) + if err != nil { + return nil, err + } + var pubkeys []solana.PublicKey + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + iter.Close() + return nil, err + } + key := iter.Key() + if len(key) != len(voteIndexPrefix)+32 { + iter.Close() + return nil, fmt.Errorf("invalid vote-index key length %d", len(key)) + } + pubkeys = append(pubkeys, solana.PublicKeyFromBytes(key[len(voteIndexPrefix):])) + } + iterErr := iter.Error() + closeErr := iter.Close() + if iterErr != nil { + return nil, iterErr + } + return pubkeys, closeErr +} + +// migrateVoteIndex scans an older store once. Finalized folds can run during +// the scan: they add their new vote candidates to the same append-only index. +func (db *AccountsDb) migrateVoteIndex(ctx context.Context) error { + iter, err := db.Index.NewIter(nil) + if err != nil { + return err + } + defer iter.Close() + batch := db.Index.NewBatch() + defer batch.Close() + keys := make([]solana.PublicKey, 0, 512) + flush := func() error { + if len(keys) == 0 { + return nil + } + accounts, err := db.GetAccountsBatch(ctx, db.DurableThrough(), keys) + if err != nil { + return err + } + for i, account := range accounts { + if account != nil && account.Lamports > 0 && account.Owner == addresses.VoteProgramAddr { + if err := batch.Set(voteIndexKey(keys[i]), nil, nil); err != nil { + return err + } + } + } + keys = keys[:0] + return nil + } + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + if key := iter.Key(); len(key) == 32 { + keys = append(keys, solana.PublicKeyFromBytes(key)) + if len(keys) == cap(keys) { + if err := flush(); err != nil { + return err + } + } + } + } + if err := iter.Error(); err != nil { + return err + } + if err := flush(); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + if err := batch.Set(voteIndexReadyKey, nil, nil); err != nil { + return err + } + if err := batch.Commit(pebble.Sync); err != nil { + return err + } + if db.IndexWALDisabled { + return db.Index.Flush() + } + return nil +} diff --git a/pkg/rpcserver/get_vote_accounts.go b/pkg/rpcserver/get_vote_accounts.go index b6eddddfe..1025fef08 100644 --- a/pkg/rpcserver/get_vote_accounts.go +++ b/pkg/rpcserver/get_vote_accounts.go @@ -7,6 +7,7 @@ import ( "sync" "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/features" "github.com/Overclock-Validator/mithril/pkg/global" "github.com/Overclock-Validator/mithril/pkg/sealevel" "github.com/filecoin-project/go-jsonrpc" @@ -75,8 +76,11 @@ func (rpcServer *RpcServer) GetVoteAccounts(ctx context.Context, p jsonrpc.RawPa return GetVoteAccountsResp{}, err } - votePubkeys := rpcServer.acctsDb.VoteAccountPubkeys() - // A fresh snapshot can have an empty vote cache before any vote account is read. + votePubkeys, err := rpcServer.acctsDb.VoteAccountPubkeys(ctx) + if err != nil { + return GetVoteAccountsResp{}, fmt.Errorf("list vote accounts: %w", err) + } + // Epoch stakes also cover stores created before the durable index was built. for votePubkey := range stakes.Stakes { votePubkeys = append(votePubkeys, votePubkey) } @@ -105,12 +109,12 @@ func (rpcServer *RpcServer) GetVoteAccounts(ctx context.Context, p jsonrpc.RawPa } for index, votePubkey := range votePubkeys { account := accounts[index] - if account == nil || account.Owner != addresses.VoteProgramAddr { - return GetVoteAccountsResp{}, fmt.Errorf("vote account %s is unavailable at rooted slot %d", votePubkey, rooted.Slot) + if account == nil || account.Lamports == 0 || account.Owner != addresses.VoteProgramAddr { + continue // An old candidate may have been deleted or changed owner. } versioned, err := sealevel.UnmarshalVersionedVoteState(account.Data) - if err != nil { - return GetVoteAccountsResp{}, fmt.Errorf("decode vote account %s: %w", votePubkey, err) + if err != nil || !versioned.IsInitialized() { + continue // Exclude invalid and uninitialized vote states. } voteState := versioned.ConvertToCurrent() lastVote, _ := voteState.LastVotedSlot() @@ -152,7 +156,12 @@ func (rpcServer *RpcServer) GetVoteAccounts(ctx context.Context, p jsonrpc.RawPa } func (rpcServer *RpcServer) rootedActivatedStakes(ctx context.Context, slot, epoch uint64) (map[solana.PublicKey]uint64, error) { - rooted, accounts, err := rpcServer.readRootedAccounts(ctx, []solana.PublicKey{sealevel.SysvarStakeHistoryAddr}) + // Replay may not have published a slot context after snapshot boot; both + // inputs to stake activation must come from the same rooted bank instead. + rooted, accounts, err := rpcServer.readRootedAccounts(ctx, []solana.PublicKey{ + sealevel.SysvarStakeHistoryAddr, + features.ReduceStakeWarmupCooldown.Address, + }) if err != nil { return nil, fmt.Errorf("read rooted stake history: %w", err) } @@ -166,11 +175,18 @@ func (rpcServer *RpcServer) rootedActivatedStakes(ctx context.Context, slot, epo if err := history.UnmarshalWithDecoder(bin.NewBinDecoder(accounts[0].Data)); err != nil { return nil, fmt.Errorf("decode stake history at rooted slot %d: %w", slot, err) } - slotCtx := rpcServer.getSlotCtx() - if slotCtx == nil || slotCtx.Features == nil { - return nil, fmt.Errorf("node features unavailable for stake activation") + var activationEpoch *uint64 + featureAccount := accounts[1] + if featureAccount != nil && featureAccount.Lamports > 0 && featureAccount.Owner == addresses.FeatureAddr { + var feature features.FeatureAcct + if err := feature.UnmarshalWithDecoder(bin.NewBinDecoder(featureAccount.Data)); err != nil { + return nil, fmt.Errorf("decode rooted stake warmup feature: %w", err) + } + if feature.ActivatedAt != nil && *feature.ActivatedAt <= rooted.Slot { + epoch := rpcServer.epochSchedule.GetEpoch(*feature.ActivatedAt) + activationEpoch = &epoch + } } - activationEpoch := sealevel.NewWarmupCooldownRateEpochWithSlotCtx(slotCtx, rpcServer.epochSchedule) stakes := make(map[solana.PublicKey]uint64) var mu sync.Mutex _, err = global.StreamStakeAccounts(rpcServer.acctsDb, slot, func(_ solana.PublicKey, delegation *sealevel.Delegation, _ uint64) { diff --git a/pkg/snapshot/build_db.go b/pkg/snapshot/build_db.go index a7e2ed934..4853b2d78 100644 --- a/pkg/snapshot/build_db.go +++ b/pkg/snapshot/build_db.go @@ -19,6 +19,7 @@ import ( "github.com/Overclock-Validator/mithril/pkg/statsd" "github.com/Overclock-Validator/mithril/pkg/txstatus" "github.com/cockroachdb/pebble" + "github.com/gagliardetto/solana-go" "github.com/panjf2000/ants/v2" ) @@ -309,12 +310,12 @@ func BuildAccountsDbPaths( numShards := snapshotIndexShards() sl := NewShardLogger(numShards, logsDir) - // Create stake pubkey collector for building stake index during appendvec processing - stakeCollector := &stakeIndexCollector{ - entries: make([]accountsdb.StakeIndexEntry, 0, 1000000), // Pre-allocate for ~1M stake accounts + // Collect stake and vote pubkeys while parsing appendvecs. + accountCollector := &snapshotAccountCollector{ + stakeEntries: make([]accountsdb.StakeIndexEntry, 0, 1000000), // Pre-allocate for ~1M stake accounts } - pools, err := initWorkerPools(wg, sl, manifest, incrementalManifest, accountsDbDir, &largestFileId, stakeCollector) + pools, err := initWorkerPools(wg, sl, manifest, incrementalManifest, accountsDbDir, &largestFileId, accountCollector) if err != nil { return nil, nil, fmt.Errorf("initializing worker pools: %w", err) } @@ -416,7 +417,7 @@ func BuildAccountsDbPaths( // Write stake pubkey index file (with appendvec location hints) stakeIndexPath := filepath.Join(accountsDbDir, "stake_pubkeys.idx") - if err := accountsdb.WriteStakePubkeyIndex(stakeIndexPath, stakeCollector.entries); err != nil { + if err := accountsdb.WriteStakePubkeyIndex(stakeIndexPath, accountCollector.stakeEntries); err != nil { return nil, nil, fmt.Errorf("writing stake pubkey index: %w", err) } @@ -431,6 +432,10 @@ func BuildAccountsDbPaths( if err != nil { return nil, nil, err } + if err := accountsDb.SeedVoteAccountPubkeys(accountCollector.votePubkeys); err != nil { + accountsDb.CloseDb() + return nil, nil, fmt.Errorf("seeding vote pubkey index: %w", err) + } if incrementalManifest != nil { return accountsDb, incrementalManifest, nil @@ -646,29 +651,22 @@ func invokeSnapshotTask(wg *sync.WaitGroup, pool *ants.PoolWithFunc, task any) ( return err } -// stakeIndexCollector aggregates stake account pubkeys from multiple worker goroutines -// during appendvec processing. Used to build the stake pubkey index file. -// -// WHY: The manifest's delegation list can be stale/incomplete (Firedancer notes: -// "the cache in the manifest is partially incomplete"). Instead of trusting manifest -// data, we: -// 1. Collect stake pubkeys during appendvec parsing (by checking owner == StakeProgramAddr) -// 2. Write them to stake_pubkeys.idx after snapshot processing -// 3. At startup, load pubkeys from index and read ALL delegation fields from AccountsDB -// -// This ensures stake cache contains fresh data from AccountsDB, not potentially stale -// manifest data. -type stakeIndexCollector struct { - mu sync.Mutex - entries []accountsdb.StakeIndexEntry +// snapshotAccountCollector records stake and vote pubkeys while workers parse +// appendvecs. The manifest delegation list can be incomplete, and vote accounts +// without stake are absent from it. +type snapshotAccountCollector struct { + mu sync.Mutex + stakeEntries []accountsdb.StakeIndexEntry + votePubkeys []solana.PublicKey } -func (c *stakeIndexCollector) Add(entries []accountsdb.StakeIndexEntry) { - if len(entries) == 0 { +func (c *snapshotAccountCollector) Add(stakes []accountsdb.StakeIndexEntry, votes []solana.PublicKey) { + if len(stakes) == 0 && len(votes) == 0 { return } c.mu.Lock() - c.entries = append(c.entries, entries...) + c.stakeEntries = append(c.stakeEntries, stakes...) + c.votePubkeys = append(c.votePubkeys, votes...) c.mu.Unlock() } @@ -679,7 +677,7 @@ func initWorkerPools( incrementalManifest *SnapshotManifest, accountsDbDir string, largestFileId *atomic.Uint64, - stakeCollector *stakeIndexCollector, + accountCollector *snapshotAccountCollector, ) (*snapshotWorkerPools, error) { indexEntryCommitterWorkers := snapshotIndexEntryCommitterWorkers() indexEntryBuilderWorkers := snapshotIndexEntryBuilderWorkers() @@ -722,14 +720,14 @@ func initWorkerPools( return } task := i.(indexEntryBuilderTask) - pubkeys, entries, stakeEntries, err := accountsdb.BuildIndexEntriesFromAppendVecs(task.Data, task.FileSize, task.Slot, task.FileId) + pubkeys, entries, stakeEntries, votePubkeys, err := accountsdb.BuildIndexEntriesFromAppendVecs(task.Data, task.FileSize, task.Slot, task.FileId) if err != nil { workerErrors.Record(fmt.Errorf("building index entries: %w", err)) return } - // Collect stake entries with appendvec location hints for building stake index - stakeCollector.Add(stakeEntries) + // Collect stake locations and vote pubkeys for their indexes. + accountCollector.Add(stakeEntries, votePubkeys) commitTask := indexEntryCommitterTask{IndexEntries: entries, Pubkeys: pubkeys} statsd.Timing(statsd.TasksIndexEntryBuilderLatency, uint64(time.Since(start)), nil) diff --git a/pkg/snapshot/build_db_with_incr.go b/pkg/snapshot/build_db_with_incr.go index 7afb8b9bf..9b737b7cf 100644 --- a/pkg/snapshot/build_db_with_incr.go +++ b/pkg/snapshot/build_db_with_incr.go @@ -81,12 +81,12 @@ func BuildAccountsDbAuto( defer cleanupIndexWorkDir() sl := NewShardLogger(numShards, logsDir) - // Create stake pubkey collector for building stake index during appendvec processing - stakeCollector := &stakeIndexCollector{ - entries: make([]accountsdb.StakeIndexEntry, 0, 1000000), // Pre-allocate for ~1M stake accounts + // Collect stake and vote pubkeys while parsing appendvecs. + accountCollector := &snapshotAccountCollector{ + stakeEntries: make([]accountsdb.StakeIndexEntry, 0, 1000000), // Pre-allocate for ~1M stake accounts } - pools, err := initWorkerPools(wg, sl, manifest, incrementalManifest, accountsDbDir, &largestFileId, stakeCollector) + pools, err := initWorkerPools(wg, sl, manifest, incrementalManifest, accountsDbDir, &largestFileId, accountCollector) if err != nil { return nil, nil, fmt.Errorf("initializing worker pools: %w", err) } @@ -287,7 +287,7 @@ func BuildAccountsDbAuto( // Write stake pubkey index file (with appendvec location hints) stakeIndexPath := filepath.Join(accountsDbDir, "stake_pubkeys.idx") - if err := accountsdb.WriteStakePubkeyIndex(stakeIndexPath, stakeCollector.entries); err != nil { + if err := accountsdb.WriteStakePubkeyIndex(stakeIndexPath, accountCollector.stakeEntries); err != nil { return nil, nil, fmt.Errorf("writing stake pubkey index: %w", err) } @@ -302,6 +302,10 @@ func BuildAccountsDbAuto( if err != nil { return nil, nil, err } + if err := accountsDb.SeedVoteAccountPubkeys(accountCollector.votePubkeys); err != nil { + accountsDb.CloseDb() + return nil, nil, fmt.Errorf("seeding vote pubkey index: %w", err) + } rpcClient := rpcclient.NewRpcClient(rpcEndpoints[0]) latestSlot, err := rpcClient.GetSlot() From f09f3574e84927228074a1a0329fcd307d8f1add Mon Sep 17 00:00:00 2001 From: Neeraj Godiyal Date: Thu, 24 Sep 2026 13:11:10 +0530 Subject: [PATCH 04/14] test(rpc): cover vote index recovery and snapshot-start metrics --- pkg/accountsdb/appendvec_test.go | 24 ++++++--- pkg/accountsdb/fold_test.go | 2 +- pkg/accountsdb/vote_index_test.go | 70 +++++++++++++++++++++++++ pkg/rpcserver/get_vote_accounts_test.go | 46 ++++++++++++++-- pkg/snapshot/build_db_worker_test.go | 2 +- 5 files changed, 133 insertions(+), 11 deletions(-) create mode 100644 pkg/accountsdb/vote_index_test.go diff --git a/pkg/accountsdb/appendvec_test.go b/pkg/accountsdb/appendvec_test.go index 9314740a1..09c6c4a39 100644 --- a/pkg/accountsdb/appendvec_test.go +++ b/pkg/accountsdb/appendvec_test.go @@ -5,6 +5,7 @@ import ( "encoding/binary" "testing" + "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/gagliardetto/solana-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -21,7 +22,7 @@ func TestBuildIndexEntriesDoesNotReadPastSliceLen(t *testing.T) { copy(backing[len(real):], phantom) data := backing[:len(real)] // len excludes phantom; cap and manifest include it. - pubkeys, entries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(cap(data)), 7, 9) + pubkeys, entries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(cap(data)), 7, 9) require.NoError(t, err) require.Equal(t, []solana.PublicKey{realKey}, pubkeys) require.Len(t, entries, 1) @@ -37,7 +38,7 @@ func TestBuildIndexEntriesBoundsOversizedManifestByDataLen(t *testing.T) { var pubkeys []solana.PublicKey require.NotPanics(t, func() { var err error - pubkeys, _, _, err = BuildIndexEntriesFromAppendVecs(data, uint64(len(data)+hdrLen), 8, 10) + pubkeys, _, _, _, err = BuildIndexEntriesFromAppendVecs(data, uint64(len(data)+hdrLen), 8, 10) require.NoError(t, err) }) require.Equal(t, []solana.PublicKey{key}, pubkeys) @@ -54,7 +55,7 @@ func TestBuildIndexEntriesStopsAtDefaultZeroLamportTerminator(t *testing.T) { binary.LittleEndian.PutUint64(terminator[dataLenOffset:dataLenOffset+8], ^uint64(0)) data := append(append(append([]byte{}, first...), terminator...), afterTerminator...) - pubkeys, entries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 12, 13) + pubkeys, entries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 12, 13) require.NoError(t, err) require.Equal(t, []solana.PublicKey{firstKey}, pubkeys) require.Len(t, entries, 1) @@ -67,7 +68,7 @@ func TestBuildIndexEntriesTerminatorRequiresDefaultKeyAndZeroLamports(t *testing zeroLamportAccount := marshalAppendVecTestAccount(t, zeroLamportKey, 0, []byte("tombstone")) data := append(defaultKeyAccount, zeroLamportAccount...) - pubkeys, entries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 18, 19) + pubkeys, entries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 18, 19) require.NoError(t, err) require.Equal(t, []solana.PublicKey{defaultKey, zeroLamportKey}, pubkeys) require.Len(t, entries, 2) @@ -80,7 +81,7 @@ func TestBuildIndexEntriesRejectsTruncatedAccountData(t *testing.T) { copy(data[pubkeyOffset:pubkeyOffset+32], key[:]) binary.LittleEndian.PutUint64(data[lamportsOffset:lamportsOffset+8], 1) - pubkeys, entries, stakeEntries, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 14, 15) + pubkeys, entries, stakeEntries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 14, 15) require.ErrorContains(t, err, "truncated appendvec account data") assert.Nil(t, pubkeys) assert.Nil(t, entries) @@ -92,12 +93,23 @@ func TestBuildIndexEntriesAcceptsFinalAccountWithoutAlignmentPadding(t *testing. encoded := marshalAppendVecTestAccount(t, key, 77, []byte{1, 2, 3}) data := encoded[: hdrLen+3 : hdrLen+3] - pubkeys, entries, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(encoded)), 16, 17) + pubkeys, entries, _, _, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(encoded)), 16, 17) require.NoError(t, err) require.Equal(t, []solana.PublicKey{key}, pubkeys) require.Len(t, entries, 1) } +func TestBuildIndexEntriesCollectsVoteAccounts(t *testing.T) { + vote := appendVecTestPubkey(10) + ordinary := appendVecTestPubkey(11) + voteData := marshalAppendVecTestAccount(t, vote, 1, nil) + copy(voteData[ownerOffset:ownerOffset+32], addresses.VoteProgramAddr[:]) + data := append(voteData, marshalAppendVecTestAccount(t, ordinary, 1, nil)...) + _, _, _, votes, err := BuildIndexEntriesFromAppendVecs(data, uint64(len(data)), 20, 21) + require.NoError(t, err) + require.Equal(t, []solana.PublicKey{vote}, votes) +} + func appendVecTestPubkey(firstByte byte) solana.PublicKey { var key solana.PublicKey key[0] = firstByte diff --git a/pkg/accountsdb/fold_test.go b/pkg/accountsdb/fold_test.go index 64f01d0f4..3f9c24033 100644 --- a/pkg/accountsdb/fold_test.go +++ b/pkg/accountsdb/fold_test.go @@ -144,7 +144,7 @@ func TestSegmentManifestRoundTripAndTornDetection(t *testing.T) { Bankhashes: []SlotBankhash{{Slot: 100, Bankhash: bh(100)}, {Slot: 130, Bankhash: bh(130)}}, Records: []ManifestRecord{ {Pubkey: [32]byte{1}, Offset: 0, OwnerSlot: 100, PrevValid: true, Prev: AccountIndexEntry{Slot: 90, FileId: 3, Offset: 77}}, - {Pubkey: [32]byte{2}, Offset: 136, OwnerSlot: 130}, + {Pubkey: [32]byte{2}, Offset: 136, OwnerSlot: 130, Vote: true}, }, ResumeCtx: []byte(`{"slot":130}`), } diff --git a/pkg/accountsdb/vote_index_test.go b/pkg/accountsdb/vote_index_test.go new file mode 100644 index 000000000..bdbbc3006 --- /dev/null +++ b/pkg/accountsdb/vote_index_test.go @@ -0,0 +1,70 @@ +package accountsdb + +import ( + "context" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/cockroachdb/pebble" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestVoteIndexMigratesExistingStoreAndPersists(t *testing.T) { + db, dir := newFoldTestDb(t) + vote := &accounts.Account{Key: solana.PublicKey{1}, Owner: addresses.VoteProgramAddr, Lamports: 1} + ordinary := &accounts.Account{Key: solana.PublicKey{2}, Lamports: 1} + _, err := db.CommitBatch(foldDeltas(accounts.SlotDelta{Slot: 10, Delta: []*accounts.Account{vote, ordinary}}), 10, nil, nil) + require.NoError(t, err) + // Mimic a database written before the vote index existed. + require.NoError(t, db.Index.Delete(voteIndexKey(vote.Key), pebble.Sync)) + db.VoteAcctCache.Clear() + got, err := db.VoteAccountPubkeys(context.Background()) + require.NoError(t, err) + require.Equal(t, []solana.PublicKey{vote.Key}, got) + + db = reopenFoldTestDb(t, db, dir) + defer db.CloseDb() + _, err = db.RecoverFoldState() + require.NoError(t, err) + got, err = db.VoteAccountPubkeys(context.Background()) + require.NoError(t, err) + require.Equal(t, []solana.PublicKey{vote.Key}, got) +} + +func TestVoteIndexSnapshotSeedAndFoldRecovery(t *testing.T) { + db, dir := newFoldTestDb(t) + seedVote := solana.PublicKey{1} + foldVote := &accounts.Account{Key: solana.PublicKey{2}, Owner: addresses.VoteProgramAddr, Lamports: 1} + require.NoError(t, db.SeedVoteAccountPubkeys([]solana.PublicKey{seedVote})) + _, err := db.CommitBatch(foldDeltas(accounts.SlotDelta{Slot: 10, Delta: []*accounts.Account{foldVote}}), 10, nil, nil) + require.NoError(t, err) + got, err := db.VoteAccountPubkeys(context.Background()) + require.NoError(t, err) + require.Equal(t, []solana.PublicKey{seedVote, foldVote.Key}, got) + + // Simulate a lost index tail: recovery must restore the candidate from the + // durable fold manifest along with the account's primary index entry. + require.NoError(t, db.Index.Delete(voteIndexKey(foldVote.Key), pebble.Sync)) + require.NoError(t, db.Index.Delete(metaKeyLastBatch, pebble.Sync)) + db = reopenFoldTestDb(t, db, dir) + defer db.CloseDb() + _, err = db.RecoverFoldState() + require.NoError(t, err) + got, err = db.VoteAccountPubkeys(context.Background()) + require.NoError(t, err) + require.Equal(t, []solana.PublicKey{seedVote, foldVote.Key}, got) +} + +func TestVoteIndexMigrationCanRetryAfterCancellation(t *testing.T) { + db, _ := newFoldTestDb(t) + defer db.CloseDb() + ctx, cancel := context.WithCancel(context.Background()) + cancel() + _, err := db.VoteAccountPubkeys(ctx) + require.ErrorIs(t, err, context.Canceled) + got, err := db.VoteAccountPubkeys(context.Background()) + require.NoError(t, err) + require.Empty(t, got) +} diff --git a/pkg/rpcserver/get_vote_accounts_test.go b/pkg/rpcserver/get_vote_accounts_test.go index d6f9d35c3..b8671f449 100644 --- a/pkg/rpcserver/get_vote_accounts_test.go +++ b/pkg/rpcserver/get_vote_accounts_test.go @@ -26,6 +26,7 @@ func TestGetVoteAccountsUsesRootedVoteState(t *testing.T) { currentVote := solana.PublicKey{1} delinquentVote := solana.PublicKey{2} newVote := solana.PublicKey{3} + invalidVote := solana.PublicKey{5} currentNode := solana.PublicKey{11} delinquentNode := solana.PublicKey{12} newNode := solana.PublicKey{13} @@ -34,6 +35,7 @@ func TestGetVoteAccountsUsesRootedVoteState(t *testing.T) { currentState := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4} currentState.V4.NodePubkey = currentNode + currentState.V4.AuthorizedVoters.AuthorizedVoters.Set(0, currentNode) currentState.V4.InflationRewardsCommissionBps = 755 currentState.V4.RootSlot = ¤tRoot currentState.V4.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: 499}}) @@ -48,13 +50,17 @@ func TestGetVoteAccountsUsesRootedVoteState(t *testing.T) { delinquentState := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionCurrent} delinquentState.Current.NodePubkey = delinquentNode + delinquentState.Current.AuthorizedVoters.AuthorizedVoters.Set(0, delinquentNode) delinquentState.Current.Commission = 5 delinquentState.Current.RootSlot = &delinquentRoot delinquentState.Current.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: 300}}) newState := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionCurrent} newState.Current.NodePubkey = newNode + newState.Current.AuthorizedVoters.AuthorizedVoters.Set(0, newNode) newState.Current.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: 498}}) + uninitializedVote := solana.PublicKey{4} + uninitializedState := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionCurrent} db := newRPCAccountsDB(t) global.ClearPendingStakePubkeys() @@ -84,9 +90,11 @@ func TestGetVoteAccountsUsesRootedVoteState(t *testing.T) { voteAccountForRPC(t, currentVote, ¤tState), voteAccountForRPC(t, delinquentVote, &delinquentState), voteAccountForRPC(t, newVote, &newState), + voteAccountForRPC(t, uninitializedVote, &uninitializedState), + {Key: invalidVote, Owner: addresses.VoteProgramAddr, Lamports: 1}, stakeAccount(solana.PublicKey{21}, currentVote, 125), stakeAccount(solana.PublicKey{22}, delinquentVote, 200), - stakeAccount(solana.PublicKey{23}, solana.PublicKey{4}, 50), + stakeAccount(solana.PublicKey{23}, uninitializedVote, 50), {Key: sealevel.SysvarStakeHistoryAddr, Owner: addresses.SysvarOwnerAddr, Lamports: 1, Data: historyData.Bytes()}, }, }}, rootedSlot, nil, nil) @@ -117,7 +125,6 @@ func TestGetVoteAccountsUsesRootedVoteState(t *testing.T) { SlotsPerEpoch: 1_000, }, } - server.SetSlotCtx(&sealevel.SlotCtx{Features: features.NewFeaturesDefault()}) server.SetRootedBankState(rootedSlot, 490, 1_000) got, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{ map[string]interface{}{"commitment": "confirmed"}, @@ -149,10 +156,43 @@ func TestGetVoteAccountsUsesRootedVoteState(t *testing.T) { db.VoteAcctCache.Clear() cold, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{})) require.NoError(t, err) - require.Len(t, cold.Current, 1) + require.Len(t, cold.Current, 2) require.Equal(t, currentVote.String(), cold.Current[0].VotePubkey) + require.Equal(t, newVote.String(), cold.Current[1].VotePubkey) require.Len(t, cold.Delinquent, 1) require.Equal(t, delinquentVote.String(), cold.Delinquent[0].VotePubkey) + + // Candidate keys are append-only, but an account that changes owner must + // no longer be returned as a vote account. + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 501, Delta: []*accounts.Account{{ + Key: newVote, Lamports: 1, Owner: addresses.SystemProgramAddr, + }}}}, 501, nil, nil) + require.NoError(t, err) + server.SetRootedBankState(501, 490, 1_000) + changed, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{})) + require.NoError(t, err) + require.Len(t, changed.Current, 1) + require.Equal(t, currentVote.String(), changed.Current[0].VotePubkey) + + activationSlot := uint64(0) + featureData, err := features.MarshalFeatureAcct(&features.FeatureAcct{ActivatedAt: &activationSlot}) + require.NoError(t, err) + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 502, Delta: []*accounts.Account{{ + Key: features.ReduceStakeWarmupCooldown.Address, Owner: addresses.FeatureAddr, Lamports: 1, Data: featureData, + }}}}, 502, nil, nil) + require.NoError(t, err) + server.SetRootedBankState(502, 490, 1_000) + withFeature, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{})) + require.NoError(t, err) + require.Equal(t, uint64(125), withFeature.Current[0].ActivatedStake) + + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 503, Delta: []*accounts.Account{{ + Key: features.ReduceStakeWarmupCooldown.Address, Owner: addresses.FeatureAddr, Lamports: 1, Data: []byte{1}, + }}}}, 503, nil, nil) + require.NoError(t, err) + server.SetRootedBankState(503, 490, 1_000) + _, err = server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{})) + require.ErrorContains(t, err, "decode rooted stake warmup feature") } func TestGetVoteAccountsRejectsInvalidConfig(t *testing.T) { diff --git a/pkg/snapshot/build_db_worker_test.go b/pkg/snapshot/build_db_worker_test.go index a8ab894d6..9d36fc3c9 100644 --- a/pkg/snapshot/build_db_worker_test.go +++ b/pkg/snapshot/build_db_worker_test.go @@ -59,7 +59,7 @@ func TestSnapshotWorkerParseFailurePropagatesAfterDrain(t *testing.T) { nil, accountsDir, &atomic.Uint64{}, - &stakeIndexCollector{}, + &snapshotAccountCollector{}, ) require.NoError(t, err) t.Cleanup(pools.Release) From 5651a73782cda1c7f4a5781bc21718bae8b9aef0 Mon Sep 17 00:00:00 2001 From: Neeraj Godiyal Date: Sat, 26 Sep 2026 15:15:24 +0530 Subject: [PATCH 05/14] Preserve replay state setter name for MCP integration --- cmd/mithril/node/node.go | 4 ++-- pkg/replay/block.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index f9871e85f..cf992c9cf 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -2932,7 +2932,7 @@ postBootstrap: } } - var slotCtxSetter replay.RPCStateSetter + var slotCtxSetter replay.SlotCtxSetter if rpcServer != nil { slotCtxSetter = rpcServer } @@ -4405,7 +4405,7 @@ func runReplayWithRecovery( useTurbine bool, dbgOpts *replay.DebugOptions, metricsWriter io.Writer, - rpcServer replay.RPCStateSetter, + rpcServer replay.SlotCtxSetter, mithrilState *state.MithrilState, blockFetchOpts *replay.BlockFetchOpts, consensusOpts *replay.ConsensusOpts, diff --git a/pkg/replay/block.go b/pkg/replay/block.go index 5c962a264..a3b7ae2ad 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -47,9 +47,9 @@ import ( "github.com/panjf2000/ants/v2" ) -// RPCStateSetter is implemented by the RPC server so replay can publish both +// SlotCtxSetter is implemented by the RPC server so replay can publish both // its live execution bank and the latest bank durably folded into AccountsDB. -type RPCStateSetter interface { +type SlotCtxSetter interface { SetSlotCtx(slotCtx *sealevel.SlotCtx) SetRootedBankState(slot, blockHeight, transactionCount uint64) } @@ -1684,7 +1684,7 @@ func ReplayBlocks( useTurbine bool, dbgOpts *DebugOptions, metricsWriter io.Writer, - rpcServer RPCStateSetter, + rpcServer SlotCtxSetter, blockFetchOpts *BlockFetchOpts, consensusOpts *ConsensusOpts, // nil = use defaults (max_depth=64, policy="halt") onCancelWriteState OnCancelWriteState, // callback to write state immediately on cancellation (can be nil) From ab30fded26c7dc0645776d7263f717faa9b00d9f Mon Sep 17 00:00:00 2001 From: Neeraj Godiyal Date: Sun, 27 Sep 2026 19:24:05 +0530 Subject: [PATCH 06/14] rpc: add commitment-aware getSlot with minimum context validation Serve the live replay slot for processed requests and the published rooted slot for confirmed and finalized requests. Decode minContextSlot as an exact uint64 and return the existing structured RPC errors. Add HTTP coverage for commitment selection, minimum-context boundaries, invalid inputs, numeric precision, missing roots and cancellation. Document the supported options. --- README.md | 6 ++ pkg/rpcserver/get_slot.go | 56 +++++++++++++++ pkg/rpcserver/get_slot_test.go | 121 ++++++++++++++++++++++++++++++++ pkg/rpcserver/rpcserver.go | 1 + pkg/rpcserver/rpcserver_test.go | 4 +- 5 files changed, 186 insertions(+), 2 deletions(-) create mode 100644 pkg/rpcserver/get_slot.go create mode 100644 pkg/rpcserver/get_slot_test.go diff --git a/README.md b/README.md index dcbb27af2..dc143441a 100644 --- a/README.md +++ b/README.md @@ -223,6 +223,12 @@ curl http://YOUR_MITHRIL_IP:8899 -X POST -H "Content-Type: application/json" -d - `getBlockHeight` - Get current block height - `getEpochInfo` - Get current epoch info - `getLatestBlockhash` - Get recent blockhash +- `getSlot` - Get the slot at the requested commitment (defaults to finalized) + +`getSlot` accepts an optional config object with `commitment` and `minContextSlot`. +On Alpenglow, `processed` uses the live replay slot; `confirmed` and `finalized` +use the published rooted slot. If that slot is below `minContextSlot`, the request +returns error `-32016` with the available slot in `data.contextSlot`. We're actively expanding RPC method coverage. Upcoming methods include transaction simulation, send transaction, and get leader schedule. diff --git a/pkg/rpcserver/get_slot.go b/pkg/rpcserver/get_slot.go new file mode 100644 index 000000000..7dbb52fba --- /dev/null +++ b/pkg/rpcserver/get_slot.go @@ -0,0 +1,56 @@ +package rpcserver + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/filecoin-project/go-jsonrpc" +) + +// GetSlot returns the slot at the requested commitment, defaulting to finalized. +func (rpcServer *RpcServer) GetSlot(ctx context.Context, p jsonrpc.RawParams) (uint64, error) { + if err := ctx.Err(); err != nil { + return 0, err + } + var params []json.RawMessage + if len(p) != 0 { + if err := json.Unmarshal(p, ¶ms); err != nil { + return 0, &InvalidParamsError{Message: fmt.Sprintf("decoding params: %v", err)} + } + } + if len(params) > 1 { + return 0, &InvalidParamsError{Message: "getSlot accepts at most one config object"} + } + var config struct { + Commitment *string `json:"commitment"` + MinContextSlot *uint64 `json:"minContextSlot"` + } + if len(params) == 1 { + if err := json.Unmarshal(params[0], &config); err != nil { + return 0, &InvalidParamsError{Message: fmt.Sprintf("invalid getSlot config: %v", err)} + } + } + commitment := "finalized" + if config.Commitment != nil { + commitment = *config.Commitment + } + if commitment != "processed" && commitment != "confirmed" && commitment != "finalized" { + return 0, &InvalidParamsError{Message: "invalid commitment"} + } + + slot := global.Slot() + if commitment != "processed" { + // Match getEpochInfo's published rooted view for confirmed and finalized. + rooted, ok := rpcServer.getRootedBankState() + if !ok { + return 0, fmt.Errorf("node has no rooted bank available") + } + slot = rooted.Slot + } + if config.MinContextSlot != nil && slot < *config.MinContextSlot { + return 0, &MinContextSlotNotReachedError{ContextSlot: slot} + } + return slot, nil +} diff --git a/pkg/rpcserver/get_slot_test.go b/pkg/rpcserver/get_slot_test.go new file mode 100644 index 000000000..a61e7780f --- /dev/null +++ b/pkg/rpcserver/get_slot_test.go @@ -0,0 +1,121 @@ +package rpcserver + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetSlotHTTP(t *testing.T) { + oldSlot := global.Slot() + t.Cleanup(func() { global.SetSlot(oldSlot) }) + global.SetSlot(150) + server := NewRpcServer(nil, 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 512}, solana.Hash{}) + t.Cleanup(func() { require.NoError(t, server.listener.Close()) }) + server.SetRootedBankState(100, 90, 0) + httpServer := httptest.NewServer(server) + t.Cleanup(httpServer.Close) + + check := func(t *testing.T, params string, want uint64, code int) { + t.Helper() + body := `{"jsonrpc":"2.0","id":7,"method":"getSlot"` + if params != "" { + body += `,"params":` + params + } + resp, err := httpServer.Client().Post(httpServer.URL, "application/json", strings.NewReader(body+`}`)) + require.NoError(t, err) + defer resp.Body.Close() + var got struct { + JSONRPC string `json:"jsonrpc"` + ID int `json:"id"` + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Data struct { + ContextSlot uint64 `json:"contextSlot"` + } `json:"data"` + } `json:"error"` + } + require.NoError(t, json.NewDecoder(resp.Body).Decode(&got)) + require.Equal(t, "2.0", got.JSONRPC) + require.Equal(t, 7, got.ID) + if code != 0 { + require.NotNil(t, got.Error) + require.Equal(t, code, got.Error.Code) + require.Empty(t, got.Result) + if code == -32016 { + require.Equal(t, want, got.Error.Data.ContextSlot) + } + return + } + require.Equal(t, http.StatusOK, resp.StatusCode) + require.Nil(t, got.Error) + require.Equal(t, fmt.Sprint(want), string(got.Result)) + } + + for _, tc := range []struct { + name string + params string + want uint64 + code int + }{ + {"omitted", "", 100, 0}, + {"empty", `[]`, 100, 0}, + {"null params", `null`, 100, 0}, + {"null config", `[null]`, 100, 0}, + {"empty config", `[{}]`, 100, 0}, + {"null fields", `[{"commitment":null,"minContextSlot":null}]`, 100, 0}, + {"finalized", `[{"commitment":"finalized"}]`, 100, 0}, + {"confirmed", `[{"commitment":"confirmed"}]`, 100, 0}, + {"processed", `[{"commitment":"processed"}]`, 150, 0}, + {"root boundary", `[{"minContextSlot":100}]`, 100, 0}, + {"root too old", `[{"minContextSlot":101}]`, 100, -32016}, + {"confirmed too old", `[{"commitment":"confirmed","minContextSlot":101}]`, 100, -32016}, + {"processed boundary", `[{"commitment":"processed","minContextSlot":150}]`, 150, 0}, + {"processed too old", `[{"commitment":"processed","minContextSlot":151}]`, 150, -32016}, + {"unknown field", `[{"futureOption":true}]`, 100, 0}, + {"extra argument", `[{},{}]`, 0, -32602}, + {"scalar config", `[true]`, 0, -32602}, + {"array config", `[[]]`, 0, -32602}, + {"unknown commitment", `[{"commitment":"invalid"}]`, 0, -32602}, + {"empty commitment", `[{"commitment":""}]`, 0, -32602}, + {"typed commitment", `[{"commitment":1}]`, 0, -32602}, + {"negative slot", `[{"minContextSlot":-1}]`, 0, -32602}, + {"fractional slot", `[{"minContextSlot":1.5}]`, 0, -32602}, + {"string slot", `[{"minContextSlot":"100"}]`, 0, -32602}, + {"overflow slot", `[{"minContextSlot":18446744073709551616}]`, 0, -32602}, + } { + t.Run(tc.name, func(t *testing.T) { check(t, tc.params, tc.want, tc.code) }) + } + + t.Run("exact uint64", func(t *testing.T) { + server.SetRootedBankState(1<<53, 0, 0) + check(t, `[{"minContextSlot":9007199254740993}]`, 1<<53, -32016) + server.SetRootedBankState(^uint64(0), 0, 0) + check(t, `[{"minContextSlot":18446744073709551615}]`, ^uint64(0), 0) + }) + t.Run("no rooted bank", func(t *testing.T) { + server.rootedBank.Store(nil) + check(t, `[]`, 0, 1) + check(t, `[{"commitment":"processed"}]`, 150, 0) + }) + t.Run("zero slot", func(t *testing.T) { + server.SetRootedBankState(0, 0, 0) + check(t, `[]`, 0, 0) + }) + t.Run("canceled request", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err := server.GetSlot(ctx, nil) + require.ErrorIs(t, err, context.Canceled) + }) +} diff --git a/pkg/rpcserver/rpcserver.go b/pkg/rpcserver/rpcserver.go index 2a50624a0..3f49e8886 100644 --- a/pkg/rpcserver/rpcserver.go +++ b/pkg/rpcserver/rpcserver.go @@ -69,6 +69,7 @@ var supportedRPCMethods = map[string]struct{}{ "getGenesisHash": {}, "getLatestBlockhash": {}, "getLeaderSchedule": {}, + "getSlot": {}, "getVoteAccounts": {}, "sendTransaction": {}, "simulateTransaction": {}, diff --git a/pkg/rpcserver/rpcserver_test.go b/pkg/rpcserver/rpcserver_test.go index 8ec2c4deb..7720037f7 100644 --- a/pkg/rpcserver/rpcserver_test.go +++ b/pkg/rpcserver/rpcserver_test.go @@ -10,7 +10,7 @@ import ( func TestServeHTTPQuietlyHandlesUnsupportedMethod(t *testing.T) { rpcServer := &RpcServer{} - req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"jsonrpc":"2.0","method":"getSlot","id":7}`)) + req := httptest.NewRequest(http.MethodPost, "/", strings.NewReader(`{"jsonrpc":"2.0","method":"unsupportedMethod","id":7}`)) rec := httptest.NewRecorder() rpcServer.ServeHTTP(rec, req) @@ -33,7 +33,7 @@ func TestServeHTTPQuietlyHandlesUnsupportedMethod(t *testing.T) { if resp.JSONRPC != "2.0" || resp.ID != 7 { t.Fatalf("unexpected response identity: %+v", resp) } - if resp.Error.Code != -32601 || resp.Error.Message != "method 'getSlot' not found" { + if resp.Error.Code != -32601 || resp.Error.Message != "method 'unsupportedMethod' not found" { t.Fatalf("unexpected error response: %+v", resp.Error) } } From 07281a5ded62f6a35f259da30024b54c011fd5f0 Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Tue, 29 Sep 2026 04:32:36 -0500 Subject: [PATCH 07/14] rpc: maintain committed stake totals for vote-account queries --- docs/rooted-stake-rpc-cache.md | 58 +++ pkg/accountsdb/account_observer.go | 67 +++ pkg/accountsdb/account_observer_test.go | 74 +++ pkg/accountsdb/accountsdb.go | 12 + pkg/accountsdb/fold.go | 6 + pkg/accountsdb/recovery.go | 14 + pkg/accountsdb/rewind.go | 10 + pkg/rpcserver/get_balance_test.go | 2 +- pkg/rpcserver/get_vote_accounts.go | 109 ++--- pkg/rpcserver/rooted_stake_cache.go | 445 ++++++++++++++++++ .../rooted_stake_cache_bench_test.go | 104 ++++ pkg/rpcserver/rooted_stake_cache_test.go | 332 +++++++++++++ pkg/rpcserver/rpcserver.go | 6 +- 13 files changed, 1164 insertions(+), 75 deletions(-) create mode 100644 docs/rooted-stake-rpc-cache.md create mode 100644 pkg/accountsdb/account_observer.go create mode 100644 pkg/accountsdb/account_observer_test.go create mode 100644 pkg/rpcserver/rooted_stake_cache.go create mode 100644 pkg/rpcserver/rooted_stake_cache_bench_test.go create mode 100644 pkg/rpcserver/rooted_stake_cache_test.go diff --git a/docs/rooted-stake-rpc-cache.md b/docs/rooted-stake-rpc-cache.md new file mode 100644 index 000000000..306ef2de0 --- /dev/null +++ b/docs/rooted-stake-rpc-cache.md @@ -0,0 +1,58 @@ +# Rooted stake totals for getVoteAccounts + +`getVoteAccounts` maintains activated stake by vote account in memory. The +initial request loads the existing stake-candidate index and reads accounts in +batches of 512. Subsequent account commits update only changed delegations. +Filtered requests read one vote account and one total; unfiltered requests still +enumerate vote accounts, but neither path rescans stake accounts on each scrape. + +The retained data is one compact delegation/contribution per live delegated +stake account, plus totals per vote account. Full account buffers are not retained. +Memory is proportional to delegated stake accounts. Bootstrap temporarily tracks +changed keys while loading candidates; recalculation temporarily holds a key list. + +## Publication and recovery contract + +- AccountsDB brackets writes with a process-local version. An odd version, + pending legacy write, or failed write cannot serve a committed cache view. +- Observers receive borrowed changed accounts after successful writes, before the + version becomes readable. They perform no I/O and retain only decoded values. +- A response must match both the account version and the RPC's published bank + identity from start to finish. A durable fold or completed legacy write alone + does not publish its account values under the previous RPC bank context. +- Initialization merges commits while scanning. Placeholders cannot resurrect + closed accounts, and accounts created after the candidate snapshot are supplied + by the observer. No unchanged-root interval spanning the full scan is required. +- Epoch, stake-history, or warmup-feature changes trigger a RAM-only recalculation + in bounded chunks. Per-entry calculation tags count concurrent updates exactly + once; new activation inputs cancel an unfinished calculation. +- Rewind, recovery, and failed writes invalidate the derived view. Recovery also + clears old account-read caches before a rebuild. Reusing the same slot on another + fork cannot reuse its old stake totals. Failed writes require successful repair + or a subsequent successful write before the account version is readable again. + +This cache adds no durable index, disk writes, or manifest format changes. It is +reconstructed after process restart. It does not address the separate manifest +format compatibility issue in the PR on which it is based. + +## Work and validation + +The first request and recovery still require account reads. Epoch/input changes +require work proportional to delegated stake accounts, but no disk scan. Normal +commits inspect changed accounts and decode only stake candidates; this cost is +paid by the account writer. RPC callers waiting for initialization can cancel. +The cache checks cancellation between read/calculation batches. + +```sh +GOMAXPROCS=2 go test -race ./pkg/accountsdb ./pkg/rpcserver +GOMAXPROCS=2 go vet ./pkg/accountsdb ./pkg/rpcserver +GOMAXPROCS=2 go test ./pkg/rpcserver -run '^$' \ + -bench '^BenchmarkRootedStakeTotals$' -benchtime=500ms -count=3 +``` + +The benchmark uses 20,000 stake accounts and 1,000 vote accounts. It compares the +old per-query scan against warm aggregate reads, and separately measures applying +64 stake updates among 20,000 changed accounts. It reports time and allocations. +These are component measurements: they exclude JSON encoding, network latency, +initial cache loading, and an epoch transition. They do not establish mainnet +capacity or live validator performance. diff --git a/pkg/accountsdb/account_observer.go b/pkg/accountsdb/account_observer.go new file mode 100644 index 000000000..83987ce8c --- /dev/null +++ b/pkg/accountsdb/account_observer.go @@ -0,0 +1,67 @@ +package accountsdb + +import "github.com/Overclock-Validator/mithril/pkg/accounts" + +// AccountStateObserver maintains disposable derived state, not durable state. +// Calls are serialized after a successful write, before its version becomes +// readable. Reset is true on rewind, recovery, or any failed/partial write. +// Accounts are borrowed for this call only. Observers must not retain mutable +// account data, call back into AccountsDb, or perform I/O. +type AccountStateObserver interface { + AccountsChanged(version uint64, changed []*accounts.Account, reset bool) +} + +// ObserveAccountState installs an observer for subsequent write completions. It is intended for +// node-lifetime derived indexes; the returned function unregisters it. +func (db *AccountsDb) ObserveAccountState(observer AccountStateObserver) func() { + db.accountObserversMu.Lock() + if db.accountObservers == nil { + db.accountObservers = make(map[AccountStateObserver]struct{}) + } + db.accountObservers[observer] = struct{}{} + db.accountObserversMu.Unlock() + return func() { + db.accountObserversMu.Lock() + delete(db.accountObservers, observer) + db.accountObserversMu.Unlock() + } +} + +// CommittedAccountVersion is a process-local coherence fence, not a slot or an +// on-disk format version. Readers must check it before AND after reading derived +// state and accounts, and independently match the RPC's published bank. An odd +// version or queued legacy write cannot serve a committed view. +func (db *AccountsDb) CommittedAccountVersion() (uint64, bool) { + version := db.accountStateVersion.Load() + if version&1 != 0 || db.accountStateUncertain.Load() || db.StoreQueueLen() != 0 { + return version, false + } + return version, db.accountStateVersion.Load() == version +} + +// CommittedAccountSlot identifies the last completed write. Zero is the +// snapshot baseline, whose slot is supplied by the RPC's initial bank state. +// Use only inside a stable CommittedAccountVersion fence. +func (db *AccountsDb) CommittedAccountSlot() uint64 { return db.accountCommittedSlot.Load() } + +// Existing fold/legacy locks are acquired first. This additional serialization +// also covers tests/tools that mix the two write APIs. No RPC read holds this +// mutex while scanning or loading accounts. +func (db *AccountsDb) beginAccountChange() func([]*accounts.Account, uint64, bool, bool) { + db.accountStateMu.Lock() + db.accountStateVersion.Add(1) + return func(changed []*accounts.Account, slot uint64, success, reset bool) { + version := db.accountStateVersion.Load() + 1 + db.accountObserversMu.Lock() + for observer := range db.accountObservers { + observer.AccountsChanged(version, changed, reset || !success) + } + db.accountObserversMu.Unlock() + if success { + db.accountCommittedSlot.Store(slot) + } + db.accountStateUncertain.Store(!success) + db.accountStateVersion.Store(version) + db.accountStateMu.Unlock() + } +} diff --git a/pkg/accountsdb/account_observer_test.go b/pkg/accountsdb/account_observer_test.go new file mode 100644 index 000000000..30605bf8f --- /dev/null +++ b/pkg/accountsdb/account_observer_test.go @@ -0,0 +1,74 @@ +package accountsdb + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/stretchr/testify/require" +) + +type accountObserverFunc func(uint64, []*accounts.Account, bool) + +func (f *accountObserverFunc) AccountsChanged(version uint64, changed []*accounts.Account, reset bool) { + (*f)(version, changed, reset) +} + +func TestAccountObserverCommitFailureRecoveryAndUnregister(t *testing.T) { + db, _ := newFoldTestDb(t) + defer db.CloseDb() + notifications := 0 + resetCount := 0 + var lastVersion uint64 + observer := accountObserverFunc(func(version uint64, changed []*accounts.Account, reset bool) { + _, stable := db.CommittedAccountVersion() + require.False(t, stable, "derived state must be updated before the version is readable") + require.Greater(t, version, lastVersion) + lastVersion = version + notifications++ + if reset { + resetCount++ + } else { + require.Len(t, changed, 1) + } + }) + unregister := db.ObserveAccountState(&observer) + db.foldHooks.afterPublicationStart = func() { _, stable := db.CommittedAccountVersion(); require.False(t, stable) } + _, err := db.CommitBatch(foldDeltas(accounts.SlotDelta{Slot: 10, Delta: []*accounts.Account{foldAcct(1, 100, nil)}}), 10, nil, nil) + require.NoError(t, err) + version, stable := db.CommittedAccountVersion() + require.True(t, stable) + require.Equal(t, lastVersion, version) + require.Equal(t, 1, notifications) + require.Zero(t, resetCount) + cached, err := db.GetAccount(10, foldAcct(1, 0, nil).Key) + require.NoError(t, err) + db.CommonAcctsCache.Set(cached.Key, cached) + // Rejecting invalid arguments does not poison a previously valid view. + _, err = db.CommitBatch(foldDeltas(accounts.SlotDelta{Slot: 12}, accounts.SlotDelta{Slot: 11}), 12, nil, nil) + require.Error(t, err) + got, stable := db.CommittedAccountVersion() + require.True(t, stable) + require.Equal(t, version, got) + + db.foldHooks.afterPublicationStart = func() { panic("crash after publication began") } + require.Panics(t, func() { + _, _ = db.CommitBatch(foldDeltas(accounts.SlotDelta{Slot: 11, Delta: []*accounts.Account{foldAcct(1, 200, nil)}}), 11, nil, nil) + }) + _, stable = db.CommittedAccountVersion() + require.False(t, stable, "partial writes fail closed until repaired") + require.Equal(t, 1, resetCount) + db.foldHooks.afterPublicationStart = nil + _, err = db.RecoverFoldState() + require.NoError(t, err) + _, stable = db.CommittedAccountVersion() + require.True(t, stable) + require.Equal(t, 2, resetCount) + account, err := db.GetAccount(11, foldAcct(1, 0, nil).Key) + require.NoError(t, err) + require.Equal(t, uint64(200), account.Lamports) + unregister() + before := notifications + _, err = db.CommitBatch(foldDeltas(accounts.SlotDelta{Slot: 12}), 12, nil, nil) + require.NoError(t, err) + require.Equal(t, before, notifications) +} diff --git a/pkg/accountsdb/accountsdb.go b/pkg/accountsdb/accountsdb.go index b33e379b4..81ad3cc10 100644 --- a/pkg/accountsdb/accountsdb.go +++ b/pkg/accountsdb/accountsdb.go @@ -41,6 +41,14 @@ type AccountsDb struct { // the write side; hot-path program cache operations take the read side. programCacheMu sync.RWMutex + // Coherent, process-local derived views of committed account state. + accountStateMu sync.Mutex + accountStateVersion atomic.Uint64 + accountCommittedSlot atomic.Uint64 + accountStateUncertain atomic.Bool + accountObserversMu sync.Mutex + accountObservers map[AccountStateObserver]struct{} + // readCacheEpochMu protects the cache epoch and the pending fold view. // CommitBatch publishes its immutable newest-wins union here before the // Pebble index commit. Readers use that view for changed keys while the @@ -700,11 +708,15 @@ func (accountsDb *AccountsDb) storeAccountsSync(accts []*accounts.Account, slot accountsDb.refreshReadCaches(accts) accountsDb.appendVecReadMu.Lock() defer accountsDb.appendVecReadMu.Unlock() + finish := accountsDb.beginAccountChange() + success := false + defer func() { finish(accts, slot, success, false) }() if StoreAccountsWorkers == 1 { accountsDb.storeAccountsInternal(accts, slot) } else { accountsDb.parallelStoreAccounts(StoreAccountsWorkers, accts, slot) } + success = true } // refreshReadCaches keeps already-hot common entries coherent after a store, diff --git a/pkg/accountsdb/fold.go b/pkg/accountsdb/fold.go index 49391695b..54f1be0e4 100644 --- a/pkg/accountsdb/fold.go +++ b/pkg/accountsdb/fold.go @@ -159,6 +159,11 @@ func (db *AccountsDb) CommitBatch( } sort.Slice(keys, func(i, j int) bool { return bytes.Compare(keys[i][:], keys[j][:]) < 0 }) + finish := db.beginAccountChange() + var committed []*accounts.Account + success := false + defer func() { finish(committed, throughSlot, success, false) }() + // (2) fileId allocation; persist high-water BEFORE the first data byte (I7). fileId := db.LargestFileId.Add(1) if err := db.persistLargestFileId(); err != nil { @@ -318,6 +323,7 @@ func (db *AccountsDb) CommitBatch( // (8) Publish. db.lastBatchSeq = batchSeq + committed, success = live, true return BatchCommitResult{ BatchSeq: batchSeq, diff --git a/pkg/accountsdb/recovery.go b/pkg/accountsdb/recovery.go index cac4a38e5..f854ad118 100644 --- a/pkg/accountsdb/recovery.go +++ b/pkg/accountsdb/recovery.go @@ -53,6 +53,9 @@ type RecoveryResult struct { func (db *AccountsDb) RecoverFoldState() (RecoveryResult, error) { db.foldMu.Lock() defer db.foldMu.Unlock() + finish := db.beginAccountChange() + success := false + defer func() { finish(nil, db.durableThrough.Load(), success, true) }() // Derived indexes must rebuild, including on partial failures. res := RecoveryResult{} @@ -165,8 +168,19 @@ func (db *AccountsDb) RecoverFoldState() (RecoveryResult, error) { // reconciliation instead of condemning the store as data loss. res.RewindInProgress = hasParkedRewindManifests(db.AcctsDir) + // Recovery can also run in-process after a partial fold. Discard cached + // account bytes before rebuilding any derived view against the repaired index. + db.readCacheEpochMu.Lock() + db.readCacheEpoch++ + db.pendingFold = nil + if db.commonAdmission != nil { + db.resetReadCachesLocked() + } + db.readCacheEpochMu.Unlock() + db.lastBatchSeq = res.BatchSeq db.durableThrough.Store(res.DurableThrough) + success = true return res, nil } diff --git a/pkg/accountsdb/rewind.go b/pkg/accountsdb/rewind.go index 7dd5cce10..de8864702 100644 --- a/pkg/accountsdb/rewind.go +++ b/pkg/accountsdb/rewind.go @@ -153,6 +153,10 @@ func (db *AccountsDb) RewindToBatchBoundary(throughSlot uint64) (RewindResult, e return res, fmt.Errorf("accountsdb: rewind: store has no fold meta (nothing folded)") } if meta.ThroughSlot == throughSlot { + finish := db.beginAccountChange() + success := false + defer func() { finish(nil, db.durableThrough.Load(), success, true) }() // Derived indexes must rebuild, including on partial failures. + // This also repairs the in-memory publication state after a prior // WAL-less commit succeeded but its explicit Flush reported an error. db.readCacheEpochMu.Lock() @@ -176,6 +180,7 @@ func (db *AccountsDb) RewindToBatchBoundary(throughSlot uint64) (RewindResult, e // crashed before moving the undone files aside — finalize any leftover // parked manifests now so recovery stops reporting a rewind in progress. db.finalizeParkedRewindLeftoversLocked(throughSlot) + success = true return res, nil // already at the target boundary } @@ -229,6 +234,10 @@ func (db *AccountsDb) RewindToBatchBoundary(throughSlot uint64) (RewindResult, e } } + finish := db.beginAccountChange() + success := false + defer func() { finish(nil, db.durableThrough.Load(), success, true) }() // Derived indexes must rebuild, including on partial failures. + // Step 1: park suffix manifests (ascending) so recovery treats the // batches as undone even if we crash mid-rewind. for seq := seqT + 1; seq <= meta.BatchSeq; seq++ { @@ -318,5 +327,6 @@ func (db *AccountsDb) RewindToBatchBoundary(throughSlot uint64) (RewindResult, e res.NewThrough = target.manifest.ThroughSlot res.ResumeCtx = target.manifest.ResumeCtx mlog.Log.Warnf("accountsdb: REWOUND %d fold batch(es) (%d keys) — durable state restored to slot %d", res.UndoneBatches, res.UndoneKeys, res.NewThrough) + success = true return res, nil } diff --git a/pkg/rpcserver/get_balance_test.go b/pkg/rpcserver/get_balance_test.go index 04ae95c97..8ce0831f3 100644 --- a/pkg/rpcserver/get_balance_test.go +++ b/pkg/rpcserver/get_balance_test.go @@ -97,7 +97,7 @@ func TestGetBalanceRejectsInvalidParams(t *testing.T) { } } -func newRPCAccountsDB(t *testing.T) *accountsdb.AccountsDb { +func newRPCAccountsDB(t testing.TB) *accountsdb.AccountsDb { t.Helper() dir := t.TempDir() require.NoError(t, os.MkdirAll(filepath.Join(dir, "accounts"), 0o755)) diff --git a/pkg/rpcserver/get_vote_accounts.go b/pkg/rpcserver/get_vote_accounts.go index 1025fef08..8c561fda5 100644 --- a/pkg/rpcserver/get_vote_accounts.go +++ b/pkg/rpcserver/get_vote_accounts.go @@ -2,16 +2,14 @@ package rpcserver import ( "context" + "errors" "fmt" "slices" - "sync" "github.com/Overclock-Validator/mithril/pkg/addresses" - "github.com/Overclock-Validator/mithril/pkg/features" "github.com/Overclock-Validator/mithril/pkg/global" "github.com/Overclock-Validator/mithril/pkg/sealevel" "github.com/filecoin-project/go-jsonrpc" - bin "github.com/gagliardetto/binary" "github.com/gagliardetto/solana-go" ) @@ -62,6 +60,17 @@ func (rpcServer *RpcServer) GetVoteAccounts(ctx context.Context, p jsonrpc.RawPa } for { + if err := ctx.Err(); err != nil { + return GetVoteAccountsResp{}, err + } + bank := rpcServer.rootedBank.Load() + version, stable := rpcServer.acctsDb.CommittedAccountVersion() + if !stable { + if err := waitForRootedPublication(ctx); err != nil { + return GetVoteAccountsResp{}, err + } + continue + } rooted, ok := rpcServer.getRootedBankState() if !ok { return GetVoteAccountsResp{}, fmt.Errorf("node has no rooted bank available") @@ -71,30 +80,32 @@ func (rpcServer *RpcServer) GetVoteAccounts(ctx context.Context, p jsonrpc.RawPa if !ok { return GetVoteAccountsResp{}, fmt.Errorf("epoch stakes unavailable for rooted epoch %d", epoch) } - activeStakes, err := rpcServer.rootedActivatedStakes(ctx, rooted.Slot, epoch) + activeStakes, err := rpcServer.rootedActivatedStakes(ctx, rooted.Slot, epoch, config.votePubkey) + if errors.Is(err, errStakeViewChanged) { + if err := waitForRootedPublication(ctx); err != nil { + return GetVoteAccountsResp{}, err + } + continue + } if err != nil { return GetVoteAccountsResp{}, err } - votePubkeys, err := rpcServer.acctsDb.VoteAccountPubkeys(ctx) - if err != nil { - return GetVoteAccountsResp{}, fmt.Errorf("list vote accounts: %w", err) - } - // Epoch stakes also cover stores created before the durable index was built. - for votePubkey := range stakes.Stakes { - votePubkeys = append(votePubkeys, votePubkey) - } - slices.SortFunc(votePubkeys, func(left, right solana.PublicKey) int { - return slices.Compare(left[:], right[:]) - }) - votePubkeys = slices.Compact(votePubkeys) - filtered := votePubkeys[:0] - for _, votePubkey := range votePubkeys { - if config.votePubkey == nil || votePubkey == *config.votePubkey { - filtered = append(filtered, votePubkey) + var votePubkeys []solana.PublicKey + if config.votePubkey != nil { + votePubkeys = []solana.PublicKey{*config.votePubkey} + } else { + votePubkeys, err = rpcServer.acctsDb.VoteAccountPubkeys(ctx) + if err != nil { + return GetVoteAccountsResp{}, fmt.Errorf("list vote accounts: %w", err) + } + // Epoch stakes also cover stores created before the durable index was built. + for key := range stakes.Stakes { + votePubkeys = append(votePubkeys, key) } + slices.SortFunc(votePubkeys, func(a, b solana.PublicKey) int { return slices.Compare(a[:], b[:]) }) + votePubkeys = slices.Compact(votePubkeys) } - votePubkeys = filtered published, accounts, err := rpcServer.readRootedAccounts(ctx, votePubkeys) if err != nil { return GetVoteAccountsResp{}, fmt.Errorf("read vote accounts at rooted slot %d: %w", rooted.Slot, err) @@ -151,60 +162,12 @@ func (rpcServer *RpcServer) GetVoteAccounts(ctx context.Context, p jsonrpc.RawPa response.Delinquent = append(response.Delinquent, info) } } - return response, nil - } -} - -func (rpcServer *RpcServer) rootedActivatedStakes(ctx context.Context, slot, epoch uint64) (map[solana.PublicKey]uint64, error) { - // Replay may not have published a slot context after snapshot boot; both - // inputs to stake activation must come from the same rooted bank instead. - rooted, accounts, err := rpcServer.readRootedAccounts(ctx, []solana.PublicKey{ - sealevel.SysvarStakeHistoryAddr, - features.ReduceStakeWarmupCooldown.Address, - }) - if err != nil { - return nil, fmt.Errorf("read rooted stake history: %w", err) - } - if rooted.Slot != slot { - return nil, nil - } - if accounts[0] == nil { - return nil, fmt.Errorf("stake history unavailable at rooted slot %d", slot) - } - var history sealevel.SysvarStakeHistory - if err := history.UnmarshalWithDecoder(bin.NewBinDecoder(accounts[0].Data)); err != nil { - return nil, fmt.Errorf("decode stake history at rooted slot %d: %w", slot, err) - } - var activationEpoch *uint64 - featureAccount := accounts[1] - if featureAccount != nil && featureAccount.Lamports > 0 && featureAccount.Owner == addresses.FeatureAddr { - var feature features.FeatureAcct - if err := feature.UnmarshalWithDecoder(bin.NewBinDecoder(featureAccount.Data)); err != nil { - return nil, fmt.Errorf("decode rooted stake warmup feature: %w", err) - } - if feature.ActivatedAt != nil && *feature.ActivatedAt <= rooted.Slot { - epoch := rpcServer.epochSchedule.GetEpoch(*feature.ActivatedAt) - activationEpoch = &epoch - } - } - stakes := make(map[solana.PublicKey]uint64) - var mu sync.Mutex - _, err = global.StreamStakeAccounts(rpcServer.acctsDb, slot, func(_ solana.PublicKey, delegation *sealevel.Delegation, _ uint64) { - active := delegation.Stake(epoch, &history, activationEpoch) - if active == 0 { - return + latest, stable := rpcServer.acctsDb.CommittedAccountVersion() + if !stable || latest != version || rpcServer.rootedBank.Load() != bank || rpcServer.rootedPublicationPending(rooted.Slot) { + continue } - mu.Lock() - stakes[delegation.VoterPubkey] += active - mu.Unlock() - }) - if err != nil { - return nil, fmt.Errorf("scan rooted stake accounts at slot %d: %w", slot, err) - } - if err := ctx.Err(); err != nil { - return nil, err + return response, nil } - return stakes, nil } func parseGetVoteAccountsConfig(params []interface{}) (getVoteAccountsConfig, error) { diff --git a/pkg/rpcserver/rooted_stake_cache.go b/pkg/rpcserver/rooted_stake_cache.go new file mode 100644 index 000000000..e10b75413 --- /dev/null +++ b/pkg/rpcserver/rooted_stake_cache.go @@ -0,0 +1,445 @@ +package rpcserver + +import ( + "context" + "errors" + "fmt" + "maps" + "path/filepath" + "sync" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/accountsdb" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" +) + +var errStakeViewChanged = errors.New("rooted stake view changed") + +const stakeCacheBatchSize = 512 + +type cachedDelegation struct { + delegation sealevel.Delegation + active uint64 + calculation uint64 + loaded bool +} + +// rootedStakeCache is disposable process-local derived state. It owns compact +// delegations and totals, never account buffers. Committed changes replace each +// delegation's contribution; speculative replay does not. Epoch/history/feature +// changes require RAM-only recalculation. Rewind, recovery and failed writes +// discard the view. No on-disk index or manifest format is changed here. +// +// RPC publication is separate: both bank identity and account version must +// match over the entire response. A commit alone cannot expose an unannounced +// bank. Initialization/recalculation merge concurrent commits in bounded chunks; +// they do not need replay to stop or one root to last for an entire stake scan. +type rootedStakeCache struct { + mu sync.Mutex + building chan struct{} + version uint64 + generation uint64 + calculation uint64 + dataReady bool + totalsReady bool + byStake map[solana.PublicKey]cachedDelegation + totals map[solana.PublicKey]uint64 + // While candidate keys are being loaded/seeded, remember touched keys so an + // old candidate cannot resurrect a concurrently closed account. Released as + // soon as placeholders are seeded, before the account scan starts. + bootstrapChanged map[solana.PublicKey]struct{} + epoch uint64 + history sealevel.SysvarStakeHistory + activationEpoch *uint64 + // Work counters used by tests/benchmarks; protected by mu. + scans uint64 + recalculations uint64 + updates uint64 + // Deterministic interleaving hooks; nil outside tests. + afterBootstrapSeed func() + beforeRecalculationChunk func(int) +} + +func stakeDelegation(account *accounts.Account) (sealevel.Delegation, bool) { + if account == nil || account.Lamports == 0 || account.Owner != addresses.StakeProgramAddr { + return sealevel.Delegation{}, false + } + state, err := sealevel.UnmarshalStakeState(account.Data) + if err != nil || state.Status != sealevel.StakeStateV2StatusStake { + return sealevel.Delegation{}, false + } + return state.Stake.Stake.Delegation, true +} + +// AccountsChanged runs on the account writer: no I/O or whole-cache scan. +// Invalidate activation inputs before applying this batch's delegations, so +// correctness does not depend on the order of accounts in the batch. +func (c *rootedStakeCache) AccountsChanged(version uint64, changed []*accounts.Account, reset bool) { + c.mu.Lock() + defer c.mu.Unlock() + if reset || c.byStake == nil || c.version+2 != version { + c.byStake, c.totals, c.bootstrapChanged = nil, nil, nil + c.dataReady, c.totalsReady = false, false + c.generation++ + c.version = version + return + } + for _, account := range changed { + if account != nil && (account.Key == sealevel.SysvarStakeHistoryAddr || account.Key == features.ReduceStakeWarmupCooldown.Address) { + c.totals, c.totalsReady = nil, false + c.calculation++ + break + } + } + for _, account := range changed { + if account == nil { + continue + } + if c.bootstrapChanged != nil { + c.bootstrapChanged[account.Key] = struct{}{} + } + old, known := c.byStake[account.Key] + if !known && account.Owner != addresses.StakeProgramAddr { + continue + } + if known && old.loaded && old.calculation == c.calculation && c.totals != nil { + c.subtract(old) + } + delete(c.byStake, account.Key) + if delegation, ok := stakeDelegation(account); ok { + entry := cachedDelegation{delegation: delegation, loaded: true} + if c.totals != nil { + entry.active = delegation.Stake(c.epoch, &c.history, c.activationEpoch) + entry.calculation = c.calculation + if entry.active != 0 { + c.totals[delegation.VoterPubkey] += entry.active + } + } + c.byStake[account.Key] = entry + } + c.updates++ + } + c.version = version +} + +func (c *rootedStakeCache) subtract(entry cachedDelegation) { + voter := entry.delegation.VoterPubkey + total := c.totals[voter] - entry.active + if total == 0 { + delete(c.totals, voter) + } else { + c.totals[voter] = total + } +} + +// selectTotals copies only voter totals, or one total for a filtered request. +// Callers never receive mutable cache maps. +func (c *rootedStakeCache) selectTotals(vote *solana.PublicKey) map[solana.PublicKey]uint64 { + if vote != nil { + return map[solana.PublicKey]uint64{*vote: c.totals[*vote]} + } + return maps.Clone(c.totals) +} + +func (s *RpcServer) rootedActivatedStakes(ctx context.Context, slot, epoch uint64, vote *solana.PublicKey) (map[solana.PublicKey]uint64, error) { + s.stakeCacheOnce.Do(func() { s.acctsDb.ObserveAccountState(&s.stakeCache) }) + c := &s.stakeCache + for { + if err := ctx.Err(); err != nil { + return nil, err + } + bank := s.rootedBank.Load() + version, stable := s.acctsDb.CommittedAccountVersion() + if bank == nil || bank.Slot != slot || !stable || s.rootedPublicationPending(slot) { + return nil, errStakeViewChanged + } + c.mu.Lock() + // Do not clear a newer cache if a write completed between the initial + // version read and acquiring the cache lock. + current, valid := s.acctsDb.CommittedAccountVersion() + if !valid || current != version || s.rootedBank.Load() != bank { + c.mu.Unlock() + return nil, errStakeViewChanged + } + if c.dataReady && c.totalsReady && c.version == version && c.epoch == epoch { + totals := c.selectTotals(vote) + c.mu.Unlock() + return totals, nil + } + if done := c.building; done != nil { + c.mu.Unlock() + select { + case <-done: + continue + case <-ctx.Done(): + return nil, ctx.Err() + } + } + c.building = make(chan struct{}) + cold := !c.dataReady || c.version != version + if cold { + c.generation++ + c.byStake = make(map[solana.PublicKey]cachedDelegation) + c.bootstrapChanged = make(map[solana.PublicKey]struct{}) + c.dataReady, c.totalsReady = false, false + c.totals = nil + c.version = version + } + generation := c.generation + c.mu.Unlock() + + var err error + if cold { + err = s.bootstrapStakeCache(ctx, generation) + } + if err == nil { + err = s.recalculateStakeTotals(ctx, slot, epoch, generation) + } + c.mu.Lock() + latest, valid := s.acctsDb.CommittedAccountVersion() + if err == nil && (!valid || c.version != latest || s.rootedBank.Load() != bank || s.rootedPublicationPending(slot)) { + err = errStakeViewChanged + } + var totals map[solana.PublicKey]uint64 + if err == nil { + totals = c.selectTotals(vote) + } + close(c.building) + c.building = nil + c.mu.Unlock() + return totals, err + } +} +func (s *RpcServer) rootedStakeInputs(ctx context.Context, slot uint64) (sealevel.SysvarStakeHistory, *uint64, error) { + var history sealevel.SysvarStakeHistory + rooted, values, err := s.readRootedAccounts(ctx, []solana.PublicKey{sealevel.SysvarStakeHistoryAddr, features.ReduceStakeWarmupCooldown.Address}) + if err != nil { + return history, nil, fmt.Errorf("read rooted stake history: %w", err) + } + if rooted.Slot != slot { + return history, nil, errStakeViewChanged + } + if values[0] == nil || values[0].Lamports == 0 { + return history, nil, fmt.Errorf("stake history unavailable at rooted slot %d", slot) + } + if err := history.UnmarshalWithDecoder(bin.NewBinDecoder(values[0].Data)); err != nil { + return history, nil, fmt.Errorf("decode stake history at rooted slot %d: %w", slot, err) + } + var activation *uint64 + account := values[1] + if account != nil && account.Lamports > 0 && account.Owner == addresses.FeatureAddr { + var feature features.FeatureAcct + if err := feature.UnmarshalWithDecoder(bin.NewBinDecoder(account.Data)); err != nil { + return history, nil, fmt.Errorf("decode rooted stake warmup feature: %w", err) + } + if feature.ActivatedAt != nil && *feature.ActivatedAt <= slot { + epoch := s.epochSchedule.GetEpoch(*feature.ActivatedAt) + activation = &epoch + } + } + return history, activation, nil +} + +// bootstrapStakeCache seeds placeholders from the existing candidate index. +// Incoming commits win over placeholders, including deletion/owner changes. +// Each disk batch is coherent, but batches may come from different roots: the +// observer reconciles touched accounts to the latest committed state. Only a +// reset (rewind/recovery/failure), not ordinary root advancement, cancels this. +func (s *RpcServer) bootstrapStakeCache(ctx context.Context, generation uint64) (err error) { + c := &s.stakeCache + defer func() { + if err != nil { + c.mu.Lock() + if c.generation == generation { + c.byStake, c.bootstrapChanged = nil, nil + c.dataReady = false + } + c.mu.Unlock() + } + }() + // Read pending FIRST: if a flush moves entries to disk between the two + // reads, either the pending snapshot or the subsequent file load contains them. + pending := global.PendingStakeEntriesSnapshot() + entries, err := global.LoadStakePubkeyIndex(filepath.Join(s.acctsDb.AcctsDir, "..")) + if err != nil { + return fmt.Errorf("load stake account index: %w", err) + } + for _, list := range [][]accountsdb.StakeIndexEntry{entries, pending} { + for start := 0; start < len(list); start += stakeCacheBatchSize { + if err = ctx.Err(); err != nil { + return err + } + c.mu.Lock() + if c.generation != generation { + c.mu.Unlock() + return errStakeViewChanged + } + for _, entry := range list[start:min(start+stakeCacheBatchSize, len(list))] { + if _, changed := c.bootstrapChanged[entry.Pubkey]; !changed { + if _, exists := c.byStake[entry.Pubkey]; !exists { + c.byStake[entry.Pubkey] = cachedDelegation{} + } + } + } + c.mu.Unlock() + } + } + c.mu.Lock() + if c.generation != generation { + c.mu.Unlock() + return errStakeViewChanged + } + c.bootstrapChanged = nil + c.mu.Unlock() + if c.afterBootstrapSeed != nil { + c.afterBootstrapSeed() + } + for _, list := range [][]accountsdb.StakeIndexEntry{entries, pending} { + for start := 0; start < len(list); start += stakeCacheBatchSize { + keys := make([]solana.PublicKey, 0, stakeCacheBatchSize) + c.mu.Lock() + if c.generation != generation { + c.mu.Unlock() + return errStakeViewChanged + } + for _, candidate := range list[start:min(start+stakeCacheBatchSize, len(list))] { + if entry, exists := c.byStake[candidate.Pubkey]; exists && !entry.loaded { + keys = append(keys, candidate.Pubkey) + } + } + c.mu.Unlock() + if len(keys) == 0 { + continue + } + for { + if err = ctx.Err(); err != nil { + return err + } + version, stable := s.acctsDb.CommittedAccountVersion() + if !stable { + if err = waitForRootedPublication(ctx); err != nil { + return err + } + continue + } + _, values, readErr := s.readRootedAccounts(ctx, keys) + if readErr != nil { + return readErr + } + c.mu.Lock() + if c.generation != generation { + c.mu.Unlock() + return errStakeViewChanged + } + latest, stable := s.acctsDb.CommittedAccountVersion() + if !stable || latest != version { + c.mu.Unlock() + continue + } + for i, account := range values { + key := keys[i] + // A commit already installed the latest value (or deleted it). + if entry, exists := c.byStake[key]; !exists || entry.loaded { + continue + } + delete(c.byStake, key) + if delegation, ok := stakeDelegation(account); ok { + c.byStake[key] = cachedDelegation{delegation: delegation, loaded: true} + } + } + c.mu.Unlock() + break + } + } + } + c.mu.Lock() + defer c.mu.Unlock() + if c.generation != generation { + return errStakeViewChanged + } + c.dataReady = true + c.scans++ + return nil +} + +// Recalculate in bounded chunks. A per-entry calculation tag makes concurrent +// committed updates part of the same sum exactly once. New accounts omitted +// from the starting key list are already included by the observer. An activation +// input change cancels this sum; ordinary account changes do not restart it. +func (s *RpcServer) recalculateStakeTotals(ctx context.Context, slot, epoch, generation uint64) (err error) { + c := &s.stakeCache + bank := s.rootedBank.Load() + version, stable := s.acctsDb.CommittedAccountVersion() + if bank == nil || bank.Slot != slot || !stable { + return errStakeViewChanged + } + history, activation, err := s.rootedStakeInputs(ctx, slot) + if err != nil { + return err + } + c.mu.Lock() + latest, stable := s.acctsDb.CommittedAccountVersion() + if !stable || latest != version || c.version != version || c.generation != generation || s.rootedBank.Load() != bank { + c.mu.Unlock() + return errStakeViewChanged + } + c.calculation++ + calculation := c.calculation + c.epoch, c.history, c.activationEpoch = epoch, history, activation + c.totals = make(map[solana.PublicKey]uint64) + c.totalsReady = false + keys := make([]solana.PublicKey, 0, len(c.byStake)) + for key := range c.byStake { + keys = append(keys, key) + } + c.mu.Unlock() + defer func() { + if err != nil { + c.mu.Lock() + if c.generation == generation && c.calculation == calculation { + c.totals = nil + c.totalsReady = false + } + c.mu.Unlock() + } + }() + for start := 0; start < len(keys); start += stakeCacheBatchSize { + if c.beforeRecalculationChunk != nil { + c.beforeRecalculationChunk(start) + } + if err = ctx.Err(); err != nil { + return err + } + c.mu.Lock() + if c.generation != generation || c.calculation != calculation { + c.mu.Unlock() + return errStakeViewChanged + } + for _, key := range keys[start:min(start+stakeCacheBatchSize, len(keys))] { + entry, exists := c.byStake[key] + if !exists || entry.calculation == calculation { + continue + } + entry.active = entry.delegation.Stake(epoch, &history, activation) + entry.calculation = calculation + c.byStake[key] = entry + if entry.active != 0 { + c.totals[entry.delegation.VoterPubkey] += entry.active + } + } + c.mu.Unlock() + } + c.mu.Lock() + defer c.mu.Unlock() + if c.generation != generation || c.calculation != calculation { + return errStakeViewChanged + } + c.totalsReady = true + c.recalculations++ + return nil +} diff --git a/pkg/rpcserver/rooted_stake_cache_bench_test.go b/pkg/rpcserver/rooted_stake_cache_bench_test.go new file mode 100644 index 000000000..7fc6d7ff7 --- /dev/null +++ b/pkg/rpcserver/rooted_stake_cache_bench_test.go @@ -0,0 +1,104 @@ +package rpcserver + +import ( + "encoding/binary" + "math" + "path/filepath" + "sync" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// BenchmarkRootedStakeTotals compares the old query-time scan with the new +// warm cache on the same database. It measures stake aggregation, not the full +// JSON-RPC response. Cold initialization and epoch recalculation are excluded +// from warm timings. No live validator or mainnet-scale performance is implied. +func BenchmarkRootedStakeTotals(b *testing.B) { + const stakeCount = 20_000 + const voterCount = 1_000 + db := newRPCAccountsDB(b) + global.ClearPendingStakePubkeys() + b.Cleanup(global.ClearPendingStakePubkeys) + payloads := make([][]byte, voterCount) + for i := range payloads { + voter := solana.PublicKey{0xb0} + binary.LittleEndian.PutUint64(voter[1:], uint64(i)) + state := &sealevel.StakeStateV2{Status: sealevel.StakeStateV2StatusStake} + state.Stake.Stake.Delegation = sealevel.Delegation{VoterPubkey: voter, StakeLamports: 1_000_000, ActivationEpoch: math.MaxUint64, DeactivationEpoch: math.MaxUint64} + data, err := sealevel.MarshalStakeStake(state) + require.NoError(b, err) + payloads[i] = data + } + delta := make([]*accounts.Account, 0, stakeCount+1) + for i := range stakeCount { + key := solana.PublicKey{0xa0} + binary.LittleEndian.PutUint64(key[1:], uint64(i)) + delta = append(delta, &accounts.Account{Key: key, Owner: addresses.StakeProgramAddr, Lamports: 1_000_001, Data: payloads[i%voterCount]}) + global.EnqueuePendingStakePubkey(10, key) + } + delta = append(delta, cacheHistoryAccount(b, nil)) + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 10, Delta: delta}}, 10, nil, nil) + require.NoError(b, err) + _, err = global.FlushPendingStakePubkeys(filepath.Dir(db.AcctsDir)) + require.NoError(b, err) + s := &RpcServer{acctsDb: db, epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 100}} + s.SetRootedBankState(10, 10, 0) + expected, err := s.rootedActivatedStakes(b.Context(), 10, 0, nil) + require.NoError(b, err) + require.Len(b, expected, voterCount) + voter := solana.PublicKey{0xb0} + b.Run("LegacyScanAll", func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + totals := make(map[solana.PublicKey]uint64) + history := sealevel.SysvarStakeHistory{} + var mu sync.Mutex + _, err := global.StreamStakeAccounts(db, 10, func(_ solana.PublicKey, d *sealevel.Delegation, _ uint64) { + active := d.Stake(0, &history, nil) + mu.Lock() + totals[d.VoterPubkey] += active + mu.Unlock() + }) + if err != nil || totals[voter] != 20_000_000 { + b.Fatalf("scan: %v, total: %d", err, totals[voter]) + } + } + }) + for _, tc := range []struct { + name string + filter *solana.PublicKey + }{{"CachedAll", nil}, {"CachedOne", &voter}} { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for b.Loop() { + totals, err := s.rootedActivatedStakes(b.Context(), 10, 0, tc.filter) + if err != nil || totals[voter] != 20_000_000 { + b.Fatalf("cache: %v, total: %d", err, totals[voter]) + } + } + }) + } + // Include writer-side maintenance: 20k changed accounts, only 64 stakes. + mixed := make([]*accounts.Account, stakeCount) + copy(mixed, delta[:64]) + for i := 64; i < len(mixed); i++ { + key := solana.PublicKey{0xc0} + binary.LittleEndian.PutUint64(key[1:], uint64(i)) + mixed[i] = &accounts.Account{Key: key, Owner: addresses.SystemProgramAddr, Lamports: 1} + } + b.Run("Apply64StakesAmong20000Changes", func(b *testing.B) { + b.ReportAllocs() + version := s.stakeCache.version + for b.Loop() { + version += 2 + s.stakeCache.AccountsChanged(version, mixed, false) + } + }) + +} diff --git a/pkg/rpcserver/rooted_stake_cache_test.go b/pkg/rpcserver/rooted_stake_cache_test.go new file mode 100644 index 000000000..1d795b18e --- /dev/null +++ b/pkg/rpcserver/rooted_stake_cache_test.go @@ -0,0 +1,332 @@ +package rpcserver + +import ( + "bytes" + "context" + "math" + "math/rand" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func cacheStakeAccount(t testing.TB, key, voter byte, amount uint64, activation, deactivation uint64) *accounts.Account { + t.Helper() + state := &sealevel.StakeStateV2{Status: sealevel.StakeStateV2StatusStake} + state.Stake.Stake.Delegation = sealevel.Delegation{VoterPubkey: solana.PublicKey{voter}, StakeLamports: amount, ActivationEpoch: activation, DeactivationEpoch: deactivation, WarmupCooldownRate: 0.25} + data, err := sealevel.MarshalStakeStake(state) + require.NoError(t, err) + return &accounts.Account{Key: solana.PublicKey{key}, Owner: addresses.StakeProgramAddr, Lamports: amount + 1, Data: data} +} + +func cacheHistoryAccount(t testing.TB, history sealevel.SysvarStakeHistory) *accounts.Account { + t.Helper() + var buf bytes.Buffer + require.NoError(t, history.MarshalWithEncoder(bin.NewBinEncoder(&buf))) + return &accounts.Account{Key: sealevel.SysvarStakeHistoryAddr, Owner: addresses.SysvarOwnerAddr, Lamports: 1, Data: buf.Bytes()} +} + +func seedStakeCacheServer(t *testing.T, initial ...*accounts.Account) *RpcServer { + t.Helper() + db := newRPCAccountsDB(t) + global.ClearPendingStakePubkeys() + for _, account := range initial { + global.EnqueuePendingStakePubkey(10, account.Key) + } + // Use existing candidate-index publication, including its cache invalidation. + _, err := global.FlushPendingStakePubkeys(filepath.Dir(db.AcctsDir)) + require.NoError(t, err) + initial = append(initial, cacheHistoryAccount(t, nil)) + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 10, Delta: initial}}, 10, nil, nil) + require.NoError(t, err) + s := &RpcServer{acctsDb: db, epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 100}} + s.SetRootedBankState(10, 10, 0) + t.Cleanup(global.ClearPendingStakePubkeys) + return s +} + +func commitCacheAccounts(t *testing.T, s *RpcServer, slot uint64, changed ...*accounts.Account) { + t.Helper() + for _, account := range changed { + if account.Owner == addresses.StakeProgramAddr { + global.EnqueuePendingStakePubkey(slot, account.Key) + } + } + _, err := s.acctsDb.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: changed}}, slot, nil, nil) + require.NoError(t, err) + s.SetRootedBankState(slot, slot, 0) +} + +func cacheTotals(t *testing.T, s *RpcServer) map[solana.PublicKey]uint64 { + t.Helper() + bank, _ := s.getRootedBankState() + totals, err := s.rootedActivatedStakes(t.Context(), bank.Slot, s.epochSchedule.GetEpoch(bank.Slot), nil) + require.NoError(t, err) + return totals +} + +func TestRootedStakeCacheIncrementalChangesAndRewind(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64), cacheStakeAccount(t, 22, 2, 200, math.MaxUint64, math.MaxUint64)) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 100, {2}: 200}, cacheTotals(t, s)) + require.Equal(t, uint64(1), s.stakeCache.scans) + // RPC callers cannot mutate the retained totals. + got := cacheTotals(t, s) + got[solana.PublicKey{1}] = 999 + voter := solana.PublicKey{1} + filtered, err := s.rootedActivatedStakes(t.Context(), 10, 0, &voter) + require.NoError(t, err) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 100}, filtered) + + commitCacheAccounts(t, s, 11, cacheStakeAccount(t, 21, 2, 150, math.MaxUint64, math.MaxUint64), cacheStakeAccount(t, 23, 1, 300, math.MaxUint64, math.MaxUint64)) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 300, {2}: 350}, cacheTotals(t, s)) + commitCacheAccounts(t, s, 12, &accounts.Account{Key: solana.PublicKey{22}, Owner: addresses.SystemProgramAddr, Lamports: 5}, &accounts.Account{Key: solana.PublicKey{23}}) + require.Equal(t, map[solana.PublicKey]uint64{{2}: 150}, cacheTotals(t, s)) + commitCacheAccounts(t, s, 13, &accounts.Account{Key: solana.PublicKey{21}, Owner: addresses.StakeProgramAddr, Lamports: 1, Data: []byte{0}}) + require.Empty(t, cacheTotals(t, s)) + require.Equal(t, uint64(1), s.stakeCache.scans, "all updates must be incremental") + require.Equal(t, uint64(1), s.stakeCache.recalculations) + + _, err = s.acctsDb.RewindToBatchBoundary(10) + require.NoError(t, err) + s.SetRootedBankState(10, 10, 0) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 100, {2}: 200}, cacheTotals(t, s)) + require.Equal(t, uint64(2), s.stakeCache.scans) + // A different fork can reuse exactly the same slots: version fencing must not. + commitCacheAccounts(t, s, 11, cacheStakeAccount(t, 21, 1, 500, math.MaxUint64, math.MaxUint64)) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 500, {2}: 200}, cacheTotals(t, s)) + _, err = s.acctsDb.RecoverFoldState() + require.NoError(t, err) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 500, {2}: 200}, cacheTotals(t, s)) + require.Equal(t, uint64(3), s.stakeCache.scans) +} + +func TestRootedStakeCacheEpochHistoryAndFeatureChanges(t *testing.T) { + stake := cacheStakeAccount(t, 21, 1, 100, 0, 2) + s := seedStakeCacheServer(t, stake) + require.Empty(t, cacheTotals(t, s), "activation epoch has no effective stake") + history := sealevel.SysvarStakeHistory{{Epoch: 0, Entry: sealevel.StakeHistoryEntry{Effective: 1000, Activating: 1000}}} + commitCacheAccounts(t, s, 100, cacheHistoryAccount(t, history)) + delegation, _ := stakeDelegation(stake) + require.Equal(t, delegation.Stake(1, &history, nil), cacheTotals(t, s)[solana.PublicKey{1}]) + require.Equal(t, uint64(25), cacheTotals(t, s)[solana.PublicKey{1}]) + activation := uint64(0) + data, err := features.MarshalFeatureAcct(&features.FeatureAcct{ActivatedAt: &activation}) + require.NoError(t, err) + // Include a stake update in the same batch BEFORE the feature account: input + // invalidation cannot depend on the serialized account order. + changed := cacheStakeAccount(t, 21, 1, 200, 0, 2) + commitCacheAccounts(t, s, 101, changed, &accounts.Account{Key: features.ReduceStakeWarmupCooldown.Address, Owner: addresses.FeatureAddr, Lamports: 1, Data: data}) + delegation, _ = stakeDelegation(changed) + require.Equal(t, delegation.Stake(1, &history, &activation), cacheTotals(t, s)[solana.PublicKey{1}]) + require.Equal(t, uint64(18), cacheTotals(t, s)[solana.PublicKey{1}]) + commitCacheAccounts(t, s, 300) // Epoch changes even without a history account write. + require.Zero(t, cacheTotals(t, s)[solana.PublicKey{1}]) + require.Equal(t, uint64(1), s.stakeCache.scans, "activation changes must not reread all accounts") + require.Equal(t, uint64(4), s.stakeCache.recalculations) +} + +func TestRootedStakeCacheWaitersCancelAndDoNotPublishEarly(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64)) + cacheTotals(t, s) + _, err := s.acctsDb.CommitBatch([]accounts.SlotDelta{{Slot: 11, Delta: []*accounts.Account{cacheStakeAccount(t, 21, 1, 200, math.MaxUint64, math.MaxUint64)}}}, 11, nil, nil) + require.NoError(t, err) + _, err = s.rootedActivatedStakes(t.Context(), 10, 0, nil) + require.ErrorIs(t, err, errStakeViewChanged, "DB commit alone must not publish a new RPC bank") + s.SetRootedBankState(11, 11, 0) + require.Equal(t, uint64(200), cacheTotals(t, s)[solana.PublicKey{1}]) + c := &s.stakeCache + c.mu.Lock() + c.byStake = nil + c.dataReady = false + c.building = make(chan struct{}) + c.mu.Unlock() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + _, err = s.rootedActivatedStakes(ctx, 11, 0, nil) + require.ErrorIs(t, err, context.Canceled) + ctx, cancel = context.WithTimeout(t.Context(), 20*time.Millisecond) + defer cancel() + _, err = s.rootedActivatedStakes(ctx, 11, 0, nil) + require.ErrorIs(t, err, context.DeadlineExceeded) + c.mu.Lock() + close(c.building) + c.building = nil + c.mu.Unlock() + require.Equal(t, uint64(200), cacheTotals(t, s)[solana.PublicKey{1}]) +} + +func TestRootedStakeCacheConcurrentBootstrap(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64)) + var wg sync.WaitGroup + for range 12 { + wg.Go(func() { + _, err := s.rootedActivatedStakes(t.Context(), 10, 0, nil) + if err != nil { + t.Error(err) + } + }) + } + wg.Wait() + require.Equal(t, uint64(1), s.stakeCache.scans) +} + +func TestRootedStakeCacheLegacyStore(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64)) + cacheTotals(t, s) + s.acctsDb.RootedDurable = false + done := make(chan struct{}) + require.NoError(t, s.acctsDb.StoreAccounts([]*accounts.Account{cacheStakeAccount(t, 21, 2, 500, math.MaxUint64, math.MaxUint64)}, 11, func() { close(done) })) + <-done + require.Eventually(t, func() bool { _, ok := s.acctsDb.CommittedAccountVersion(); return ok }, time.Second, time.Millisecond) + _, err := s.rootedActivatedStakes(t.Context(), 10, 0, nil) + require.ErrorIs(t, err, errStakeViewChanged, "a completed legacy write still needs RPC bank publication") + s.SetRootedBankState(11, 11, 0) + require.Equal(t, map[solana.PublicKey]uint64{{2}: 500}, cacheTotals(t, s)) + require.Equal(t, uint64(1), s.stakeCache.scans) +} + +// Compare many independent delegation mutations with complete recomputation, +// including redelegation, closure, invalid state, and ownership changes. +func TestRootedStakeCacheDifferentialUpdates(t *testing.T) { + c := &rootedStakeCache{version: 0, epoch: 3, byStake: make(map[solana.PublicKey]cachedDelegation), totals: make(map[solana.PublicKey]uint64)} + live := make(map[solana.PublicKey]*accounts.Account) + rng := rand.New(rand.NewSource(904)) + for step := uint64(1); step <= 500; step++ { + key := byte(1 + rng.Intn(80)) + voter := byte(100 + rng.Intn(10)) + account := cacheStakeAccount(t, key, voter, uint64(1+rng.Intn(10000)), math.MaxUint64, math.MaxUint64) + switch rng.Intn(5) { + case 0: + account.Lamports = 0 + case 1: + account.Owner = addresses.SystemProgramAddr + case 2: + account.Data = []byte{255} + } + live[account.Key] = account + c.AccountsChanged(step*2, []*accounts.Account{account}, false) + expected := make(map[solana.PublicKey]uint64) + for _, account := range live { + if delegation, ok := stakeDelegation(account); ok { + expected[delegation.VoterPubkey] += delegation.Stake(3, &c.history, nil) + } + } + require.Equal(t, expected, c.totals, "step %d", step) + } +} + +func TestGetVoteAccountsConcurrentCommitConsistency(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 1000, math.MaxUint64, math.MaxUint64)) + previous, hadPrevious := global.EpochStakesSnapshot(0) + global.PutEpochStakes(0, map[solana.PublicKey]uint64{{1}: 1000}, map[solana.PublicKey]*epochstakes.VoteAccount{{1}: {NodePubkey: solana.PublicKey{2}}}, 1000) + t.Cleanup(func() { + if hadPrevious { + global.PutEpochStakes(0, previous.Stakes, previous.VoteAccounts, previous.TotalStake) + } else { + global.ClearEpochStakes(0) + } + }) + voteAccount := func(slot uint64) *accounts.Account { + state := sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionCurrent} + state.Current.NodePubkey = solana.PublicKey{2} + state.Current.AuthorizedVoters.AuthorizedVoters.Set(0, solana.PublicKey{2}) + state.Current.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: slot}}) + return voteAccountForRPC(t, solana.PublicKey{1}, &state) + } + commitCacheAccounts(t, s, 11, voteAccount(11), cacheStakeAccount(t, 21, 1, 1100, math.MaxUint64, math.MaxUint64)) + cacheTotals(t, s) + var wg sync.WaitGroup + wg.Go(func() { + for slot := uint64(12); slot < 50; slot++ { + changed := []*accounts.Account{voteAccount(slot), cacheStakeAccount(t, 21, 1, slot*100, math.MaxUint64, math.MaxUint64)} + _, err := s.acctsDb.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: changed}}, slot, nil, nil) + if err != nil { + t.Error(err) + return + } + s.SetRootedBankState(slot, slot, 0) + } + }) + for worker := range 4 { + wg.Go(func() { + params := mustRawParams(t, []interface{}{}) + if worker%2 == 0 { + params = mustRawParams(t, []interface{}{map[string]interface{}{"votePubkey": solana.PublicKey{1}.String()}}) + } + for range 30 { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + got, err := s.GetVoteAccounts(ctx, params) + cancel() + if err != nil { + t.Error(err) + return + } + if len(got.Current) != 1 { + t.Errorf("unexpected response: %+v", got) + return + } + if got.Current[0].ActivatedStake != got.Current[0].LastVote*100 { + t.Errorf("mixed committed banks: %+v", got.Current[0]) + return + } + } + }) + } + wg.Wait() + require.Equal(t, uint64(1), s.stakeCache.scans) +} + +func TestRootedStakeCacheBootstrapMergesConcurrentChanges(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64), cacheStakeAccount(t, 22, 2, 200, math.MaxUint64, math.MaxUint64)) + s.stakeCache.afterBootstrapSeed = func() { + commitCacheAccounts(t, s, 11, &accounts.Account{Key: solana.PublicKey{21}, Owner: addresses.SystemProgramAddr, Lamports: 1}, cacheStakeAccount(t, 22, 1, 250, math.MaxUint64, math.MaxUint64), cacheStakeAccount(t, 23, 3, 300, math.MaxUint64, math.MaxUint64)) + } + _, err := s.rootedActivatedStakes(t.Context(), 10, 0, nil) + require.ErrorIs(t, err, errStakeViewChanged, "request must retry its bank, not its completed bootstrap") + require.True(t, s.stakeCache.dataReady) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 250, {3}: 300}, cacheTotals(t, s)) + require.Equal(t, uint64(1), s.stakeCache.scans) +} + +func TestRootedStakeCacheRecalculationMergesConcurrentChanges(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64), cacheStakeAccount(t, 22, 2, 200, math.MaxUint64, math.MaxUint64)) + s.stakeCache.beforeRecalculationChunk = func(start int) { + if start == 0 { + commitCacheAccounts(t, s, 11, cacheStakeAccount(t, 21, 1, 150, math.MaxUint64, math.MaxUint64), &accounts.Account{Key: solana.PublicKey{22}}, cacheStakeAccount(t, 23, 3, 300, math.MaxUint64, math.MaxUint64)) + } + } + _, err := s.rootedActivatedStakes(t.Context(), 10, 0, nil) + require.ErrorIs(t, err, errStakeViewChanged) + require.True(t, s.stakeCache.totalsReady) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 150, {3}: 300}, cacheTotals(t, s)) + require.Equal(t, uint64(1), s.stakeCache.scans) + require.Equal(t, uint64(1), s.stakeCache.recalculations, "root advancement must not restart the full RAM calculation") +} + +func TestRootedStakeCacheRewindCancelsBootstrap(t *testing.T) { + s := seedStakeCacheServer(t, cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64)) + commitCacheAccounts(t, s, 11, cacheStakeAccount(t, 21, 1, 200, math.MaxUint64, math.MaxUint64)) + var once sync.Once + s.stakeCache.afterBootstrapSeed = func() { + once.Do(func() { + _, err := s.acctsDb.RewindToBatchBoundary(10) + require.NoError(t, err) + s.SetRootedBankState(10, 10, 0) + }) + } + _, err := s.rootedActivatedStakes(t.Context(), 11, 0, nil) + require.ErrorIs(t, err, errStakeViewChanged) + require.False(t, s.stakeCache.dataReady) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 100}, cacheTotals(t, s)) +} diff --git a/pkg/rpcserver/rpcserver.go b/pkg/rpcserver/rpcserver.go index 3f49e8886..9eade2f2b 100644 --- a/pkg/rpcserver/rpcserver.go +++ b/pkg/rpcserver/rpcserver.go @@ -35,6 +35,9 @@ type RpcServer struct { genesisHash string rootedBank atomic.Pointer[rootedBankState] + stakeCacheOnce sync.Once + stakeCache rootedStakeCache + leaderTPUCacheMu sync.RWMutex leaderTPUByIdentity map[solana.PublicKey]tpuEndpoint leaderTPUCacheUpdatedAt time.Time @@ -132,7 +135,8 @@ func (rpcServer *RpcServer) readRootedAccounts(ctx context.Context, pubkeys []so func (rpcServer *RpcServer) rootedPublicationPending(rootedSlot uint64) bool { if !rpcServer.acctsDb.RootedDurable { - return false + committed := rpcServer.acctsDb.CommittedAccountSlot() + return committed != 0 && committed != rootedSlot } durableThrough := rpcServer.acctsDb.DurableThrough() // Zero means the snapshot baseline is active and no fold has committed yet. From 566bd769966d9d26bc057edcdb73f5b983ab9d1f Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Tue, 29 Sep 2026 05:12:36 -0500 Subject: [PATCH 08/14] accountsdb: preserve v1 manifest undo encoding for vote accounts --- docs/rooted-stake-rpc-cache.md | 19 ++- pkg/accountsdb/fold_test.go | 2 +- pkg/accountsdb/manifest_compat_test.go | 170 +++++++++++++++++++++++++ pkg/accountsdb/recovery.go | 43 +++++++ pkg/accountsdb/segment.go | 16 ++- 5 files changed, 240 insertions(+), 10 deletions(-) create mode 100644 pkg/accountsdb/manifest_compat_test.go diff --git a/docs/rooted-stake-rpc-cache.md b/docs/rooted-stake-rpc-cache.md index 306ef2de0..8b91a13ba 100644 --- a/docs/rooted-stake-rpc-cache.md +++ b/docs/rooted-stake-rpc-cache.md @@ -32,8 +32,23 @@ changed keys while loading candidates; recalculation temporarily holds a key lis or a subsequent successful write before the account version is readable again. This cache adds no durable index, disk writes, or manifest format changes. It is -reconstructed after process restart. It does not address the separate manifest -format compatibility issue in the PR on which it is based. +reconstructed after process restart. + +The durable vote-candidate index also keeps the original version-1 manifest +format: `PrevValid` is exactly 0 or 1, as required by older rewind readers. +Vote membership is an in-memory hint during normal commits. Recovery derives it +from account headers in the CRC-checked segment before atomically installing the +primary index, vote candidates, and fold watermark. This adds header reads only +for batches whose index application must be replayed. Metadata/read failures +leave that batch's files intact and do not advance its watermark. + +The reader still accepts the 2/3 flag bytes emitted by earlier revisions of the +RPC branch, preserving their undo pointers. Existing files are not automatically +rewritten: stores that already contain those flags must use the corrected reader +for recovery/rewind, or be rebuilt before downgrading to a pre-RPC binary. Newly +written manifests are compatible with the original reader. A format-version bump +alone would not make downgrade safe, because older recovery treats unknown +manifest headers as orphans. ## Work and validation diff --git a/pkg/accountsdb/fold_test.go b/pkg/accountsdb/fold_test.go index 3f9c24033..64f01d0f4 100644 --- a/pkg/accountsdb/fold_test.go +++ b/pkg/accountsdb/fold_test.go @@ -144,7 +144,7 @@ func TestSegmentManifestRoundTripAndTornDetection(t *testing.T) { Bankhashes: []SlotBankhash{{Slot: 100, Bankhash: bh(100)}, {Slot: 130, Bankhash: bh(130)}}, Records: []ManifestRecord{ {Pubkey: [32]byte{1}, Offset: 0, OwnerSlot: 100, PrevValid: true, Prev: AccountIndexEntry{Slot: 90, FileId: 3, Offset: 77}}, - {Pubkey: [32]byte{2}, Offset: 136, OwnerSlot: 130, Vote: true}, + {Pubkey: [32]byte{2}, Offset: 136, OwnerSlot: 130}, }, ResumeCtx: []byte(`{"slot":130}`), } diff --git a/pkg/accountsdb/manifest_compat_test.go b/pkg/accountsdb/manifest_compat_test.go new file mode 100644 index 000000000..d99fa3453 --- /dev/null +++ b/pkg/accountsdb/manifest_compat_test.go @@ -0,0 +1,170 @@ +package accountsdb + +import ( + "context" + "encoding/binary" + "hash/crc32" + "os" + "path/filepath" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/cockroachdb/pebble" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Pin the original v1 wire layout independently of the current decoder. In +// particular the old reader used equality with 1, not a bit mask, for undo. +func TestManifestV1BooleanCompatibility(t *testing.T) { + for _, prevValid := range []bool{false, true} { + for _, vote := range []bool{false, true} { + m := &SegmentManifest{Kind: ManifestKindFold, ThroughSlot: 20, FileId: 2, + Records: []ManifestRecord{{Pubkey: [32]byte{1}, PrevValid: prevValid, Vote: vote, + Prev: AccountIndexEntry{Slot: 10, FileId: 1, Offset: 136}}}} + raw := m.encode() + require.Equal(t, uint32(1), binary.LittleEndian.Uint32(raw[4:8])) + // Fixed prefix + empty bankhash count + empty context length + record count. + const recordStart = 4 + 4 + 1 + 5*8 + 4 + 4 + 4 + 8 + const booleanOffset = recordStart + 32 + 8 + 8 + require.LessOrEqual(t, raw[booleanOffset], byte(1)) + require.Equal(t, prevValid, raw[booleanOffset] == 1, "legacy undo decision") + var previous AccountIndexEntry + previous.Unmarshal((*[24]byte)(raw[booleanOffset+1 : booleanOffset+25])) + require.Equal(t, m.Records[0].Prev, previous) + require.Equal(t, crc32.ChecksumIEEE(raw[:len(raw)-4]), binary.LittleEndian.Uint32(raw[len(raw)-4:])) + + // Earlier versions of this RPC branch emitted bit 1 as a vote hint. The + // fixed reader accepts those files; an explicit rewrite emits a boolean. + raw[booleanOffset] |= 2 + binary.LittleEndian.PutUint32(raw[len(raw)-4:], crc32.ChecksumIEEE(raw[:len(raw)-4])) + dir := t.TempDir() + path := segmentManifestPath(dir, 20, 2) + require.NoError(t, os.WriteFile(path, raw, 0600)) + got, err := ReadSegmentManifest(path) + require.NoError(t, err) + require.Equal(t, prevValid, got.Records[0].PrevValid) + require.True(t, got.Records[0].Vote) + require.NoError(t, WriteSegmentManifest(dir, got)) + canonical, err := os.ReadFile(path) + require.NoError(t, err) + require.LessOrEqual(t, canonical[booleanOffset], byte(1)) + } + } +} + +func TestManifestRecoveryDerivesVoteCandidates(t *testing.T) { + for name, legacyFlags := range map[string]bool{"original-v1": false, "legacy-rpc-flags": true} { + t.Run(name, func(t *testing.T) { + db, dir := newFoldTestDb(t) + require.NoError(t, db.SeedVoteAccountPubkeys(nil)) // No migration scan may mask recovery bugs. + vote := foldAcct(1, 10, []byte("vote")) + vote.Owner = addresses.VoteProgramAddr + closed := foldAcct(2, 0, nil) + closed.Owner = addresses.VoteProgramAddr + ordinary := foldAcct(3, 10, []byte("ordinary")) + res := commitTestBatch(t, db, 10, vote, closed, ordinary) + manifest, err := ReadSegmentManifest(segmentManifestPath(db.AcctsDir, 10, res.FileId)) + require.NoError(t, err) + for _, r := range manifest.Records { + require.False(t, r.Vote, "vote hints must not be serialized") + } + if legacyFlags { + // Even a stale legacy vote hint must not override the segment's owner + // or zero balance. Mark all three records as votes in this fixture. + path := segmentManifestPath(db.AcctsDir, 10, res.FileId) + raw, err := os.ReadFile(path) + require.NoError(t, err) + recordStart := len(raw) - 4 - len(manifest.Records)*manifestRecordSize + for i := range manifest.Records { + raw[recordStart+i*manifestRecordSize+32+8+8] |= 2 + } + binary.LittleEndian.PutUint32(raw[len(raw)-4:], crc32.ChecksumIEEE(raw[:len(raw)-4])) + require.NoError(t, os.WriteFile(path, raw, 0600)) + } + require.NoError(t, db.Index.Delete(voteIndexKey(vote.Key), pebble.Sync)) + require.NoError(t, db.Index.Delete(metaKeyLastBatch, pebble.Sync)) + for _, a := range []*accounts.Account{vote, closed, ordinary} { + require.NoError(t, db.Index.Delete(a.Key[:], pebble.Sync)) + } + db = reopenFoldTestDb(t, db, dir) + defer db.CloseDb() + recovered, err := db.RecoverFoldState() + require.NoError(t, err) + require.Equal(t, []uint64{1}, recovered.ReplayedBatches) + got, err := db.VoteAccountPubkeys(context.Background()) + require.NoError(t, err) + require.Equal(t, []solana.PublicKey{vote.Key}, got) + require.Equal(t, uint64(10), mustColdRead(t, db, 10, vote.Key).Lamports) + again, err := db.RecoverFoldState() + require.NoError(t, err) + require.Empty(t, again.ReplayedBatches) + }) + } +} + +func TestVoteManifestRewindRestoresExistingAccount(t *testing.T) { + db, _ := newFoldTestDb(t) + defer db.CloseDb() + old := foldAcct(1, 10, []byte("old")) + old.Owner = addresses.VoteProgramAddr + commitTestBatch(t, db, 10, old) + newer := foldAcct(1, 20, []byte("new")) + newer.Owner = addresses.VoteProgramAddr + created := foldAcct(2, 30, nil) + created.Owner = addresses.VoteProgramAddr + commitTestBatch(t, db, 20, newer, created) + _, err := db.RewindToBatchBoundary(10) + require.NoError(t, err) + restored := mustColdRead(t, db, 10, old.Key) + require.Equal(t, old.Lamports, restored.Lamports) + require.Equal(t, old.Data, restored.Data) + _, closer, err := db.Index.Get(created.Key[:]) + if closer != nil { + closer.Close() + } + require.ErrorIs(t, err, pebble.ErrNotFound) +} + +// A structurally valid, CRC-valid manifest with invalid record metadata must +// fail before changing the index, and must not be deleted as an orphan. +func TestManifestVoteRecoveryMetadataErrorPreservesFiles(t *testing.T) { + for _, kind := range []string{"offset", "pubkey", "payload"} { + t.Run(kind, func(t *testing.T) { + db, _ := newFoldTestDb(t) + defer db.CloseDb() + vote := foldAcct(1, 10, nil) + vote.Owner = addresses.VoteProgramAddr + res := commitTestBatch(t, db, 10, vote) + path := segmentManifestPath(db.AcctsDir, 10, res.FileId) + m, err := ReadSegmentManifest(path) + require.NoError(t, err) + dataPath := filepath.Join(db.AcctsDir, SegmentDataName(10, res.FileId)) + switch kind { + case "offset": + m.Records[0].Offset = ^uint64(0) + case "pubkey": + m.Records[0].Pubkey[0]++ + case "payload": + data, err := os.ReadFile(dataPath) + require.NoError(t, err) + binary.LittleEndian.PutUint64(data[dataLenOffset:], ^uint64(0)) + require.NoError(t, os.WriteFile(dataPath, data, 0600)) + m.DataCRC = crc32.ChecksumIEEE(data) + } + require.NoError(t, WriteSegmentManifest(db.AcctsDir, m)) + require.NoError(t, db.Index.Delete(metaKeyLastBatch, pebble.Sync)) + require.NoError(t, db.Index.Delete(voteIndexKey(vote.Key), pebble.Sync)) + before := snapshotIndex(t, db) + _, err = db.RecoverFoldState() + require.ErrorContains(t, err, "restore vote candidates") + require.Equal(t, before, snapshotIndex(t, db)) + require.FileExists(t, path) + require.FileExists(t, dataPath) + _, haveMeta, err := db.readFoldMeta() + require.NoError(t, err) + require.False(t, haveMeta) + }) + } +} diff --git a/pkg/accountsdb/recovery.go b/pkg/accountsdb/recovery.go index f854ad118..0586fa843 100644 --- a/pkg/accountsdb/recovery.go +++ b/pkg/accountsdb/recovery.go @@ -7,6 +7,7 @@ import ( "path/filepath" "strings" + "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/Overclock-Validator/mithril/pkg/mlog" ) @@ -98,6 +99,12 @@ func (db *AccountsDb) RecoverFoldState() (RecoveryResult, error) { mlog.Log.Warnf("accountsdb: fold manifest seq %d failed verification (%v); treating batch and everything above as undecided", seq, verr) break } + // Vote membership is derived from the CRC-checked segment, keeping the + // durable manifest compatible with pre-vote-index binaries. A read or + // metadata error must leave this decided batch intact for diagnosis/retry. + if err := db.restoreManifestVoteFlags(manifest); err != nil { + return res, fmt.Errorf("accountsdb: restore vote candidates for fold seq %d: %w", seq, err) + } if err := db.applyManifestToIndex(manifest); err != nil { return res, fmt.Errorf("accountsdb: replay fold manifest seq %d: %w", seq, err) } @@ -314,3 +321,39 @@ func (db *AccountsDb) removeStaleTmpManifests() ([]string, error) { } return removed, nil } + +// restoreManifestVoteFlags reconstructs the transient vote-index hints before +// replaying a durable manifest. Normal commits already have these hints from +// the accounts being written and do not reread the segment. Read fixed headers +// only, without loading account payloads or consulting the possibly stale index. +// The caller must first verify the segment length and CRC. +func (db *AccountsDb) restoreManifestVoteFlags(m *SegmentManifest) error { + if len(m.Records) == 0 { + return nil + } + f, err := os.Open(filepath.Join(db.AcctsDir, SegmentDataName(m.ThroughSlot, m.FileId))) + if err != nil { + return err + } + defer f.Close() + var header [hdrLen]byte + for i := range m.Records { + r := &m.Records[i] + if r.Offset > m.DataLen || m.DataLen-r.Offset < hdrLen || r.Offset > uint64(1<<63-1)-hdrLen { + return fmt.Errorf("account %x: header offset %d outside segment", r.Pubkey, r.Offset) + } + if _, err := f.ReadAt(header[:], int64(r.Offset)); err != nil { + return err + } + if [32]byte(header[pubkeyOffset:pubkeyOffset+32]) != r.Pubkey { + return fmt.Errorf("account %x: segment pubkey mismatch", r.Pubkey) + } + dataLen := binary.LittleEndian.Uint64(header[dataLenOffset : dataLenOffset+8]) + if dataLen > m.DataLen-r.Offset-hdrLen { + return fmt.Errorf("account %x: payload outside segment", r.Pubkey) + } + r.Vote = binary.LittleEndian.Uint64(header[lamportsOffset:lamportsOffset+8]) > 0 && + [32]byte(header[ownerOffset:ownerOffset+32]) == addresses.VoteProgramAddr + } + return nil +} diff --git a/pkg/accountsdb/segment.go b/pkg/accountsdb/segment.go index 5cc189f8a..da767b667 100644 --- a/pkg/accountsdb/segment.go +++ b/pkg/accountsdb/segment.go @@ -62,7 +62,7 @@ type ManifestRecord struct { Offset uint64 // record offset within this segment's data file OwnerSlot uint64 // slot that produced this version (observability) PrevValid bool // false => the key was absent from the index before this batch - Vote bool // this version is a nonzero-lamport vote-program account + Vote bool // derived in memory; not encoded in the v1 manifest Prev AccountIndexEntry // index entry before this batch overwrote it } @@ -139,14 +139,13 @@ func (m *SegmentManifest) encode() []byte { body.Write(r.Pubkey[:]) put64(r.Offset) put64(r.OwnerSlot) - var flags byte + // Version 1 encodes a boolean, not a bit field. Older rewind readers + // use pv == 1; adding a vote flag here would silently lose undo pointers. + var prevValid byte if r.PrevValid { - flags |= 1 + prevValid = 1 } - if r.Vote { - flags |= 2 - } - body.WriteByte(flags) + body.WriteByte(prevValid) r.Prev.Marshal(&prevBuf) body.Write(prevBuf[:]) } @@ -331,6 +330,9 @@ func ReadSegmentManifest(path string) (*SegmentManifest, error) { if pv&^3 != 0 { return nil, ErrTornManifest } + // Accept the flags emitted by the earlier RPC branch so those stores + // remain readable. New writes use only 0/1; recovery derives Vote from + // the segment even for these older flagged manifests. r.PrevValid = pv&1 != 0 r.Vote = pv&2 != 0 prevRaw, err := d.bytes(24) From b9a867f700db710b6f8c59cf79c3f9d22c0363e9 Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Tue, 29 Sep 2026 05:20:10 -0500 Subject: [PATCH 09/14] rpc: return null for missing and deleted accounts --- pkg/rpcserver/get_account_info_test.go | 74 ++++++++++++++++++++++++++ pkg/rpcserver/rpcserver.go | 4 +- 2 files changed, 77 insertions(+), 1 deletion(-) create mode 100644 pkg/rpcserver/get_account_info_test.go diff --git a/pkg/rpcserver/get_account_info_test.go b/pkg/rpcserver/get_account_info_test.go new file mode 100644 index 000000000..c70f2fe98 --- /dev/null +++ b/pkg/rpcserver/get_account_info_test.go @@ -0,0 +1,74 @@ +package rpcserver + +import ( + "encoding/json" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetAccountInfoAccountExistence(t *testing.T) { + for _, state := range []string{"never-created", "funded-empty-data", "deleted"} { + t.Run(state, func(t *testing.T) { + db := newRPCAccountsDB(t) + server := &RpcServer{acctsDb: db} + address := solana.PublicKey{87} + slot := uint64(10) + lamports := uint64(0) + server.SetRootedBankState(slot, slot, 0) + if state != "never-created" { + lamports = 42 + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: []*accounts.Account{{Key: address, Lamports: lamports}}}}, slot, nil, nil) + require.NoError(t, err) + // Prime the account-read cache before deleting, and confirm that a live + // account with empty data is not mistaken for a missing placeholder. + funded, err := server.GetAccountInfo(t.Context(), mustRawParams(t, []interface{}{address.String()})) + require.NoError(t, err) + require.NotNil(t, funded.Value) + require.Equal(t, lamports, funded.Value.Lamports) + } + if state == "deleted" { + slot++ + lamports = 0 + // A zero-lamport tombstone is absent even if it retains payload bytes. + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: []*accounts.Account{{Key: address, Data: []byte{1, 2, 3}}}}}, slot, nil, nil) + require.NoError(t, err) + server.SetRootedBankState(slot, slot, 0) + } + for _, encoding := range []string{"", "base58", "base64", "base64+zstd", "jsonParsed"} { + params := []interface{}{address.String()} + if encoding != "" { + params = append(params, map[string]interface{}{"encoding": encoding}) + } + got, err := server.GetAccountInfo(t.Context(), mustRawParams(t, params)) + require.NoError(t, err) + require.Equal(t, slot, got.Context.Slot) + if lamports == 0 { + require.Nil(t, got.Value, "state=%s encoding=%s", state, encoding) + wire, err := json.Marshal(got) + require.NoError(t, err) + var decoded map[string]json.RawMessage + require.NoError(t, json.Unmarshal(wire, &decoded)) + require.Equal(t, "null", string(decoded["value"]), "value must be present as JSON null") + } else { + require.NotNil(t, got.Value) + require.Equal(t, lamports, got.Value.Lamports) + } + } + balance, err := server.GetBalance(t.Context(), mustRawParams(t, []interface{}{address.String()})) + require.NoError(t, err) + require.Equal(t, slot, balance.Context.Slot) + require.Equal(t, lamports, balance.Value) + + params := mustRawParams(t, []interface{}{address.String(), map[string]interface{}{"minContextSlot": slot + 1}}) + _, err = server.GetAccountInfo(t.Context(), params) + var minSlotErr *MinContextSlotNotReachedError + require.ErrorAs(t, err, &minSlotErr, "missing accounts must still enforce minContextSlot") + require.Equal(t, slot, minSlotErr.ContextSlot) + _, err = server.GetBalance(t.Context(), params) + require.ErrorAs(t, err, &minSlotErr) + }) + } +} diff --git a/pkg/rpcserver/rpcserver.go b/pkg/rpcserver/rpcserver.go index 9eade2f2b..9dc8f4145 100644 --- a/pkg/rpcserver/rpcserver.go +++ b/pkg/rpcserver/rpcserver.go @@ -99,7 +99,9 @@ func (rpcServer *RpcServer) readRootedAccount(ctx context.Context, pubkey solana if err != nil { return rooted, nil, err } - if len(accountSet) != 1 || accountSet[0] == nil { + // Batch reads use zero-lamport placeholders for absent/deleted accounts. + // Translate that loader representation into absence at the RPC boundary. + if len(accountSet) != 1 || accountSet[0] == nil || accountSet[0].Lamports == 0 { return rooted, nil, accountsdb.ErrNoAccount } return rooted, accountSet[0], nil From 3164ac3831ab8e845704b1d9cfaf2571c2d39e36 Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Tue, 29 Sep 2026 05:37:52 -0500 Subject: [PATCH 10/14] rpc: add opt-in stake-cache sizing and concurrent epoch tests --- docs/rooted-stake-rpc-cache.md | 28 ++ .../rooted_stake_cache_scale_test.go | 304 ++++++++++++++++++ 2 files changed, 332 insertions(+) create mode 100644 pkg/rpcserver/rooted_stake_cache_scale_test.go diff --git a/docs/rooted-stake-rpc-cache.md b/docs/rooted-stake-rpc-cache.md index 8b91a13ba..f028ac267 100644 --- a/docs/rooted-stake-rpc-cache.md +++ b/docs/rooted-stake-rpc-cache.md @@ -71,3 +71,31 @@ old per-query scan against warm aggregate reads, and separately measures applyin These are component measurements: they exclude JSON encoding, network latency, initial cache loading, and an epoch transition. They do not establish mainnet capacity or live validator performance. + +## Larger on-disk sizing experiment + +The opt-in `TestRootedStakeCacheScale` exercises 1,000 initialized vote accounts +and a configurable count of delegated stake accounts (up to 2 million). Its +synthetic mix is 80% active, 10% activating, and 10% deactivating, with stake-history +changes at epoch boundaries. Run each size in a fresh process: + +```sh +MITHRIL_STAKE_SCALE=1000000 GOMAXPROCS=2 GOMEMLIMIT=4GiB \ + go test ./pkg/rpcserver -run '^TestRootedStakeCacheScale$' \ + -count=1 -v -timeout=10m +``` + +JSON log records report initial loading, warm all-voter/single-voter RPC handler +latencies and allocations, epoch recalculation, and concurrent bootstrap and +epoch changes. Four callers pause 50 ms between requests while 100 durable folds +update 64 stake accounts each, with a 20 ms pause between folds. A separate +no-reader stage measures fold latency. Every stage checks totals against a full +account scan; concurrent root advancement must not restart the bootstrap scan. + +Handler timings include JSON response encoding but exclude HTTP. Startup clears +application account caches; fixture creation still warms the OS page cache, so +this is not a cold-storage benchmark. The retained-cache estimate is the change +in live Go heap after dropping only the derived cache and collecting garbage. +Whole-process peak memory includes fixture construction and full-scan oracles; +it must not be described as the cache size or a complete RPC-node memory budget. +Reported tail percentiles describe these short samples, not production SLOs. diff --git a/pkg/rpcserver/rooted_stake_cache_scale_test.go b/pkg/rpcserver/rooted_stake_cache_scale_test.go new file mode 100644 index 000000000..8d97ba4b4 --- /dev/null +++ b/pkg/rpcserver/rooted_stake_cache_scale_test.go @@ -0,0 +1,304 @@ +package rpcserver + +import ( + "context" + "encoding/binary" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + "runtime" + "sort" + "strconv" + "sync" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Opt-in resource experiment; run each size in a fresh process. This measures +// local fixture scaling, not mainnet capacity. Fixture writes warm the OS page +// cache. RPC timings include response construction/JSON, but exclude HTTP. +// +// MITHRIL_STAKE_SCALE=1000000 GOMAXPROCS=2 go test ./pkg/rpcserver \ +// -run '^TestRootedStakeCacheScale$' -count=1 -v -timeout=10m +func TestRootedStakeCacheScale(t *testing.T) { + size := os.Getenv("MITHRIL_STAKE_SCALE") + if size == "" { + t.Skip("set MITHRIL_STAKE_SCALE to run the on-disk sizing experiment") + } + count, err := strconv.Atoi(size) + require.NoError(t, err) + require.GreaterOrEqual(t, count, 1000) + require.LessOrEqual(t, count, 2_000_000) + const voters = 1000 + const epochSlots = uint64(1_000_000) + key := func(prefix byte, i int) solana.PublicKey { + k := solana.PublicKey{prefix} + binary.LittleEndian.PutUint64(k[1:], uint64(i)) + return k + } + emit := func(stage string, fields map[string]any) { + fields["stage"], fields["stakes"], fields["voters"] = stage, count, voters + raw, err := json.Marshal(fields) + require.NoError(t, err) + t.Log(string(raw)) + } + heap := func() runtime.MemStats { runtime.GC(); var m runtime.MemStats; runtime.ReadMemStats(&m); return m } + quantiles := func(samples []time.Duration) map[string]any { + sort.Slice(samples, func(i, j int) bool { return samples[i] < samples[j] }) + out := map[string]any{"samples": len(samples)} + if len(samples) > 0 { + for name, q := range map[string]float64{"p50_ms": .5, "p95_ms": .95, "p99_ms": .99, "max_ms": 1} { + out[name] = float64(samples[max(0, int(math.Ceil(float64(len(samples))*q))-1)]) / float64(time.Millisecond) + } + } + return out + } + db := newRPCAccountsDB(t) + global.ClearPendingStakePubkeys() + t.Cleanup(global.ClearPendingStakePubkeys) + // 80% bootstrap-active, 10% activating, 10% deactivating at epoch 100. + makeStake := func(i int, amount uint64) *accounts.Account { + activation, deactivation := uint64(math.MaxUint64), uint64(math.MaxUint64) + switch (i / voters) % 10 { + case 0: + activation = 98 + case 1: + deactivation = 100 + } + state := &sealevel.StakeStateV2{Status: sealevel.StakeStateV2StatusStake} + state.Stake.Stake.Delegation = sealevel.Delegation{VoterPubkey: key(0xb0, i%voters), StakeLamports: amount, ActivationEpoch: activation, DeactivationEpoch: deactivation, WarmupCooldownRate: .25} + data, err := sealevel.MarshalStakeStake(state) + require.NoError(t, err) + return &accounts.Account{Key: key(0xa0, i), Owner: addresses.StakeProgramAddr, Lamports: amount + 1, Data: data} + } + fixtureStart := time.Now() + for first := 0; first < count; first += 10_000 { + delta := make([]*accounts.Account, 0, min(10_000, count-first)) + slot := uint64(first/10_000 + 1) + for i := first; i < min(first+10_000, count); i++ { + a := makeStake(i, 1_000_000) + delta = append(delta, a) + global.EnqueuePendingStakePubkey(slot, a.Key) + } + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: delta}}, slot, nil, nil) + require.NoError(t, err) + } + _, err = global.FlushPendingStakePubkeys(filepath.Dir(db.AcctsDir)) + require.NoError(t, err) + var history sealevel.SysvarStakeHistory + entry := sealevel.StakeHistoryEntry{Effective: 10_000_000_000, Activating: 2_000_000_000, Deactivating: 1_000_000_000} + for e := uint64(100); e > 68; e-- { + history = append(history, sealevel.StakeHistoryPair{Epoch: e - 1, Entry: entry}) + } + delta := []*accounts.Account{cacheHistoryAccount(t, history)} + stakes := make(map[solana.PublicKey]uint64, voters) + for i := 0; i < voters; i++ { + voter := key(0xb0, i) + stakes[voter] = 1 + state := &sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4} + state.V4.NodePubkey = key(0xc0, i) + state.V4.AuthorizedVoters.AuthorizedVoters.Set(0, state.V4.NodePubkey) + state.V4.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: 100 * epochSlots}}) + delta = append(delta, voteAccountForRPC(t, voter, state)) + } + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 100 * epochSlots, Delta: delta}}, 100*epochSlots, nil, nil) + require.NoError(t, err) + require.NoError(t, db.SeedVoteAccountPubkeys(nil)) + for epoch := uint64(100); epoch <= 102; epoch++ { + previous, hadPrevious := global.EpochStakesSnapshot(epoch) + global.PutEpochStakes(epoch, stakes, map[solana.PublicKey]*epochstakes.VoteAccount{}, voters) + t.Cleanup(func() { + if hadPrevious { + global.PutEpochStakes(epoch, previous.Stakes, previous.VoteAccounts, previous.TotalStake) + } else { + global.ClearEpochStakes(epoch) + } + }) + } + s := &RpcServer{acctsDb: db, epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: epochSlots}} + s.SetRootedBankState(100*epochSlots, 100*epochSlots, 0) + db.CommonAcctsCache.Clear() + db.VoteAcctCache.Clear() + delta = nil + before := heap() + emit("fixture", map[string]any{"seconds": time.Since(fixtureStart).Seconds(), "heap_bytes": before.HeapAlloc}) + allParams := mustRawParams(t, []interface{}{}) + oneParams := mustRawParams(t, []interface{}{map[string]interface{}{"votePubkey": key(0xb0, 0).String()}}) + query := func(ctx context.Context, filtered bool) error { + params := allParams + if filtered { + params = oneParams + } + response, err := s.GetVoteAccounts(ctx, params) + if err != nil { + return err + } + want := voters + if filtered { + want = 1 + } + if len(response.Current)+len(response.Delinquent) != want { + return fmt.Errorf("got %d votes, want %d", len(response.Current)+len(response.Delinquent), want) + } + _, err = json.Marshal(response) + return err + } + start := time.Now() + require.NoError(t, query(t.Context(), false)) + coldDuration := time.Since(start) + after := heap() + emit("cold", map[string]any{"seconds": coldDuration.Seconds(), "heap_bytes": after.HeapAlloc, "heap_growth_bytes": int64(after.HeapAlloc) - int64(before.HeapAlloc), "allocated_bytes": after.TotalAlloc - before.TotalAlloc, "allocations": after.Mallocs - before.Mallocs, "gc_cycles": after.NumGC - before.NumGC}) + for _, filtered := range []bool{false, true} { + samples := make([]time.Duration, 0, 100) + var a, b runtime.MemStats + runtime.ReadMemStats(&a) + for i := 0; i < 100; i++ { + start := time.Now() + require.NoError(t, query(t.Context(), filtered)) + samples = append(samples, time.Since(start)) + } + runtime.ReadMemStats(&b) + stats := quantiles(samples) + stats["bytes_per_query"] = (b.TotalAlloc - a.TotalAlloc) / 100 + stats["allocs_per_query"] = (b.Mallocs - a.Mallocs) / 100 + name := "warm_all" + if filtered { + name = "warm_one" + } + emit(name, stats) + } + // Keep a direct, full-scan aggregation as the independent update oracle. + verify := func(stage string) { + bank := s.rootedBank.Load() + epoch := s.epochSchedule.GetEpoch(bank.Slot) + got, err := s.rootedActivatedStakes(t.Context(), bank.Slot, epoch, nil) + require.NoError(t, err) + expected := make(map[solana.PublicKey]uint64) + var mu sync.Mutex + start := time.Now() + _, err = global.StreamStakeAccounts(db, bank.Slot, func(_ solana.PublicKey, d *sealevel.Delegation, _ uint64) { + active := d.Stake(epoch, &history, nil) + if active > 0 { + mu.Lock() + expected[d.VoterPubkey] += active + mu.Unlock() + } + }) + require.NoError(t, err) + require.Equal(t, expected, got) + emit(stage+"_oracle", map[string]any{"seconds": time.Since(start).Seconds()}) + } + verify("cold") + // Measure a quiet epoch recalculation before adding concurrent traffic. + history = append(sealevel.SysvarStakeHistory{{Epoch: 100, Entry: entry}}, history...) + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 101 * epochSlots, Delta: []*accounts.Account{cacheHistoryAccount(t, history)}}}, 101*epochSlots, nil, nil) + require.NoError(t, err) + s.SetRootedBankState(101*epochSlots, 101*epochSlots, 0) + start = time.Now() + require.NoError(t, query(t.Context(), false)) + emit("epoch_quiet", map[string]any{"seconds": time.Since(start).Seconds()}) + verify("epoch_quiet") + // Four paced callers mix whole-cluster and single-voter requests while 100 + // durable folds update 64 stakes apiece. Include wall time waiting for roots. + concurrent := func(stage string, readers int) { + ctx, cancel := context.WithTimeout(t.Context(), 90*time.Second) + var wg sync.WaitGroup + defer func() { cancel(); wg.Wait() }() + var mu sync.Mutex + var all, one []time.Duration + failures := make(chan error, 4) + for worker := 0; worker < readers; worker++ { + wg.Add(1) + go func() { + defer wg.Done() + for { + if ctx.Err() != nil { + return + } + start := time.Now() + err := query(ctx, worker%2 == 1) + elapsed := time.Since(start) + if err != nil { + if ctx.Err() == nil { + failures <- err + } + return + } + mu.Lock() + if worker%2 == 1 { + one = append(one, elapsed) + } else { + all = append(all, elapsed) + } + mu.Unlock() + select { + case <-ctx.Done(): + return + case <-time.After(50 * time.Millisecond): + } + } + }() + } + var commits []time.Duration + startSlot := s.rootedBank.Load().Slot + for n := 0; n < 100; n++ { + changed := make([]*accounts.Account, 64) + for i := range changed { + changed[i] = makeStake((n*64+i)%count, uint64(1_000_000+n+1)) + } + slot := startSlot + uint64(n+1) + start := time.Now() + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: changed}}, slot, nil, nil) + if err != nil { + cancel() + wg.Wait() + t.Fatal(err) + } + s.SetRootedBankState(slot, slot, 0) + commits = append(commits, time.Since(start)) + time.Sleep(20 * time.Millisecond) + } + // Let in-flight initial requests finish before stopping the readers. + require.NoError(t, query(ctx, false)) + time.Sleep(200 * time.Millisecond) + cancel() + wg.Wait() + close(failures) + for err := range failures { + require.NoError(t, err) + } + emit(stage+"_all", quantiles(all)) + emit(stage+"_one", quantiles(one)) + emit(stage+"_commit", quantiles(commits)) + verify(stage) + } + concurrent("quiet_commits", 0) + concurrent("warm_concurrent", 4) + history = append(sealevel.SysvarStakeHistory{{Epoch: 101, Entry: entry}}, history...) + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 102 * epochSlots, Delta: []*accounts.Account{cacheHistoryAccount(t, history)}}}, 102*epochSlots, nil, nil) + require.NoError(t, err) + s.SetRootedBankState(102*epochSlots, 102*epochSlots, 0) + concurrent("epoch_concurrent", 4) + version, _ := db.CommittedAccountVersion() + s.stakeCache.AccountsChanged(version, nil, true) + db.CommonAcctsCache.Clear() + db.VoteAcctCache.Clear() + concurrent("bootstrap_concurrent", 4) + require.Equal(t, uint64(2), s.stakeCache.scans, "concurrent root changes must not restart the account scan") + retained := heap() + version, _ = db.CommittedAccountVersion() + s.stakeCache.AccountsChanged(version, nil, true) + released := heap() + emit("memory", map[string]any{"heap_before_drop_bytes": retained.HeapAlloc, "heap_after_drop_bytes": released.HeapAlloc, "derived_cache_retained_estimate_bytes": int64(retained.HeapAlloc) - int64(released.HeapAlloc), "scans": s.stakeCache.scans, "recalculations": s.stakeCache.recalculations}) + runtime.KeepAlive(s) +} From a9b0010d3dd0b4eac81c826d5630adf6a63ce26c Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Tue, 29 Sep 2026 06:06:21 -0500 Subject: [PATCH 11/14] rpc: cache parsed rooted vote-account records --- docs/rooted-stake-rpc-cache.md | 27 ++- pkg/rpcserver/get_vote_accounts.go | 51 ++--- pkg/rpcserver/get_vote_accounts_test.go | 2 +- pkg/rpcserver/rpcserver.go | 6 +- pkg/rpcserver/vote_record_cache.go | 145 ++++++++++++++ pkg/rpcserver/vote_record_cache_bench_test.go | 96 ++++++++++ pkg/rpcserver/vote_record_cache_test.go | 180 ++++++++++++++++++ 7 files changed, 466 insertions(+), 41 deletions(-) create mode 100644 pkg/rpcserver/vote_record_cache.go create mode 100644 pkg/rpcserver/vote_record_cache_bench_test.go create mode 100644 pkg/rpcserver/vote_record_cache_test.go diff --git a/docs/rooted-stake-rpc-cache.md b/docs/rooted-stake-rpc-cache.md index f028ac267..e47860614 100644 --- a/docs/rooted-stake-rpc-cache.md +++ b/docs/rooted-stake-rpc-cache.md @@ -3,8 +3,19 @@ `getVoteAccounts` maintains activated stake by vote account in memory. The initial request loads the existing stake-candidate index and reads accounts in batches of 512. Subsequent account commits update only changed delegations. -Filtered requests read one vote account and one total; unfiltered requests still +Filtered requests select one vote record and one total; unfiltered requests still enumerate vote accounts, but neither path rescans stake accounts on each scrape. +Initialized vote-account records are decoded lazily and reused until a committed +write touches that account. The cache owns the RPC fields and up to five credit +entries, not account buffers. Stake totals, epoch membership and delinquency are +applied per request. Returned credit slices are copies owned by the caller. + +The writer only invalidates touched vote records; it performs no additional vote +decoding or I/O. Rewind, recovery and failed writes clear all parsed records. Loads +check the account version under the invalidation lock before entering the cache, +so an old in-flight decode cannot resurrect a record after a concurrent commit. +The final response still checks bank identity and account version even on a fully +cached read. Unknown or invalid filtered keys do not create negative cache entries. The retained data is one compact delegation/contribution per live delegated stake account, plus totals per vote account. Full account buffers are not retained. @@ -72,6 +83,20 @@ These are component measurements: they exclude JSON encoding, network latency, initial cache loading, and an epoch transition. They do not establish mainnet capacity or live validator performance. +`BenchmarkGetVoteAccountsRecords` measures the warm handler plus JSON encoding +for 1,000 V4 vote accounts (31 lockouts and 64 credit-history entries each), or one +filtered account. It isolates vote records with initialized empty stake totals. +The changed-record cases invalidate 10% or 100% of decoded records before every +request, including invalidation cost, while account buffers remain warm. They +simulate cache churn without disk commits; they do not measure writer contention. +Reuse benefits depend on how many vote accounts change between requests. A full +refresh still pays decoding plus cache maintenance and response ownership costs. + +```sh +GOMAXPROCS=2 go test ./pkg/rpcserver -run '^$' \ + -bench '^BenchmarkGetVoteAccountsRecords$' -benchtime=700ms -count=3 +``` + ## Larger on-disk sizing experiment The opt-in `TestRootedStakeCacheScale` exercises 1,000 initialized vote accounts diff --git a/pkg/rpcserver/get_vote_accounts.go b/pkg/rpcserver/get_vote_accounts.go index 8c561fda5..ab45b9aff 100644 --- a/pkg/rpcserver/get_vote_accounts.go +++ b/pkg/rpcserver/get_vote_accounts.go @@ -6,7 +6,6 @@ import ( "fmt" "slices" - "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/Overclock-Validator/mithril/pkg/global" "github.com/Overclock-Validator/mithril/pkg/sealevel" "github.com/filecoin-project/go-jsonrpc" @@ -106,52 +105,30 @@ func (rpcServer *RpcServer) GetVoteAccounts(ctx context.Context, p jsonrpc.RawPa slices.SortFunc(votePubkeys, func(a, b solana.PublicKey) int { return slices.Compare(a[:], b[:]) }) votePubkeys = slices.Compact(votePubkeys) } - published, accounts, err := rpcServer.readRootedAccounts(ctx, votePubkeys) + records, err := rpcServer.rootedVoteRecords(ctx, rooted.Slot, version, votePubkeys) + if errors.Is(err, errStakeViewChanged) { + continue + } if err != nil { return GetVoteAccountsResp{}, fmt.Errorf("read vote accounts at rooted slot %d: %w", rooted.Slot, err) } - if published.Slot != rooted.Slot { - continue - } response := GetVoteAccountsResp{ Current: make([]VoteAccountInfo, 0, len(votePubkeys)), Delinquent: make([]VoteAccountInfo, 0), } for index, votePubkey := range votePubkeys { - account := accounts[index] - if account == nil || account.Lamports == 0 || account.Owner != addresses.VoteProgramAddr { - continue // An old candidate may have been deleted or changed owner. - } - versioned, err := sealevel.UnmarshalVersionedVoteState(account.Data) - if err != nil || !versioned.IsInitialized() { - continue // Exclude invalid and uninitialized vote states. - } - voteState := versioned.ConvertToCurrent() - lastVote, _ := voteState.LastVotedSlot() - var rootSlot uint64 - if voteState.RootSlot != nil { - rootSlot = *voteState.RootSlot - } - creditsStart := max(0, len(voteState.EpochCredits)-maxRPCEpochCreditsHistory) - creditsHistory := voteState.EpochCredits[creditsStart:] - epochCredits := make([][3]uint64, len(creditsHistory)) - for i, credits := range creditsHistory { - epochCredits[i] = [3]uint64{credits.Epoch, credits.Credits, credits.PrevCredits} - } - commission, commissionBPS := voteCommission(versioned, voteState) - _, epochVoteAccount := stakes.Stakes[votePubkey] - info := VoteAccountInfo{ - VotePubkey: votePubkey.String(), - NodePubkey: voteState.NodePubkey.String(), - ActivatedStake: activeStakes[votePubkey], - Commission: commission, - InflationRewardsCommissionBPS: commissionBPS, - EpochCredits: epochCredits, - EpochVoteAccount: epochVoteAccount, - LastVote: lastVote, - RootSlot: rootSlot, + record := records[index] + if record == nil { + continue } + info := *record + // Response slices are owned by the caller, never by the retained cache. + info.EpochCredits = make([][3]uint64, len(record.EpochCredits)) + copy(info.EpochCredits, record.EpochCredits) + info.ActivatedStake = activeStakes[votePubkey] + _, info.EpochVoteAccount = stakes.Stakes[votePubkey] + lastVote := info.LastVote current := lastVote > 0 if rooted.Slot >= config.delinquentSlotDistance { current = lastVote > rooted.Slot-config.delinquentSlotDistance diff --git a/pkg/rpcserver/get_vote_accounts_test.go b/pkg/rpcserver/get_vote_accounts_test.go index b8671f449..d71d709c2 100644 --- a/pkg/rpcserver/get_vote_accounts_test.go +++ b/pkg/rpcserver/get_vote_accounts_test.go @@ -212,7 +212,7 @@ func TestGetVoteAccountsRejectsInvalidConfig(t *testing.T) { } } -func voteAccountForRPC(t *testing.T, key solana.PublicKey, state *sealevel.VoteStateVersions) *accounts.Account { +func voteAccountForRPC(t testing.TB, key solana.PublicKey, state *sealevel.VoteStateVersions) *accounts.Account { t.Helper() data, err := sealevel.MarshalVersionedVoteState(state) require.NoError(t, err) diff --git a/pkg/rpcserver/rpcserver.go b/pkg/rpcserver/rpcserver.go index 9dc8f4145..7e90df08d 100644 --- a/pkg/rpcserver/rpcserver.go +++ b/pkg/rpcserver/rpcserver.go @@ -35,8 +35,10 @@ type RpcServer struct { genesisHash string rootedBank atomic.Pointer[rootedBankState] - stakeCacheOnce sync.Once - stakeCache rootedStakeCache + stakeCacheOnce sync.Once + stakeCache rootedStakeCache + voteRecordsOnce sync.Once + voteRecords rootedVoteRecordCache leaderTPUCacheMu sync.RWMutex leaderTPUByIdentity map[solana.PublicKey]tpuEndpoint diff --git a/pkg/rpcserver/vote_record_cache.go b/pkg/rpcserver/vote_record_cache.go new file mode 100644 index 000000000..1fc8a1146 --- /dev/null +++ b/pkg/rpcserver/vote_record_cache.go @@ -0,0 +1,145 @@ +package rpcserver + +import ( + "context" + "sync" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" +) + +// rootedVoteRecordCache retains only immutable, account-derived RPC fields, not +// borrowed account buffers, stake totals, epoch membership or delinquency. Those +// bank/query-dependent values are still calculated for each response. +// +// Writes invalidate touched records before the account version becomes readable. +// Decoding is lazy on the RPC caller, never on the account writer. Recovery, +// rewind and failed writes discard all records, including same-slot fork reuse. +// The enclosing RPC must still validate its account version and bank identity. +type rootedVoteRecordCache struct { + mu sync.Mutex + records map[solana.PublicKey]*VoteAccountInfo +} + +func (c *rootedVoteRecordCache) AccountsChanged(_ uint64, changed []*accounts.Account, reset bool) { + c.mu.Lock() + defer c.mu.Unlock() + if reset { + c.records = nil + return + } + for _, account := range changed { + if account != nil { + delete(c.records, account.Key) + } + } +} + +func decodeVoteRecord(key solana.PublicKey, account *accounts.Account) *VoteAccountInfo { + if account == nil || account.Lamports == 0 || account.Owner != addresses.VoteProgramAddr { + return nil + } + versioned, err := sealevel.UnmarshalVersionedVoteState(account.Data) + if err != nil || !versioned.IsInitialized() { + return nil + } + var voteState *sealevel.VoteState + var v4View sealevel.VoteState + if versioned.Type == sealevel.VoteStateVersionV4 { + // Only these read-only fields are needed by RPC. Avoid allocating the + // full legacy write-back conversion for every refreshed V4 account. + state := &versioned.V4 + v4View.NodePubkey, v4View.RootSlot = state.NodePubkey, state.RootSlot + v4View.Votes, v4View.EpochCredits = state.Votes, state.EpochCredits + voteState = &v4View + } else { + voteState = versioned.ConvertToCurrent() + } + lastVote, _ := voteState.LastVotedSlot() + var rootSlot uint64 + if voteState.RootSlot != nil { + rootSlot = *voteState.RootSlot + } + creditsStart := max(0, len(voteState.EpochCredits)-maxRPCEpochCreditsHistory) + creditsHistory := voteState.EpochCredits[creditsStart:] + // Keep the bounded credit history in the same allocation as the record. + // The returned pointer retains this storage; responses still copy the slice. + record := new(struct { + info VoteAccountInfo + credits [maxRPCEpochCreditsHistory][3]uint64 + }) + epochCredits := record.credits[:len(creditsHistory):len(creditsHistory)] + for i, credits := range creditsHistory { + epochCredits[i] = [3]uint64{credits.Epoch, credits.Credits, credits.PrevCredits} + } + commission, commissionBPS := voteCommission(versioned, voteState) + record.info = VoteAccountInfo{ + VotePubkey: key.String(), NodePubkey: voteState.NodePubkey.String(), + Commission: commission, InflationRewardsCommissionBPS: commissionBPS, + EpochCredits: epochCredits, LastVote: lastVote, RootSlot: rootSlot, + } + return &record.info +} + +func (s *RpcServer) rootedVoteRecords(ctx context.Context, slot, version uint64, keys []solana.PublicKey) ([]*VoteAccountInfo, error) { + s.voteRecordsOnce.Do(func() { s.acctsDb.ObserveAccountState(&s.voteRecords) }) + c := &s.voteRecords + records := make([]*VoteAccountInfo, len(keys)) + var missing []solana.PublicKey + var positions []int + c.mu.Lock() + for i, key := range keys { + records[i] = c.records[key] + if records[i] == nil { + missing = append(missing, key) + positions = append(positions, i) + } + } + c.mu.Unlock() + if len(missing) == 0 { + return records, nil // The enclosing RPC checks the final version/bank fence. + } + rooted, accountSet, err := s.readRootedAccounts(ctx, missing) + if err != nil { + return nil, err + } + if rooted.Slot != slot { + return nil, errStakeViewChanged + } + decoded := make([]*VoteAccountInfo, len(missing)) + for i, account := range accountSet { + if err := ctx.Err(); err != nil { + return nil, err + } + decoded[i] = decodeVoteRecord(missing[i], account) + records[positions[i]] = decoded[i] + } + if !s.storeVoteRecords(version, missing, decoded) { + return nil, errStakeViewChanged + } + return records, nil +} + +func (s *RpcServer) storeVoteRecords(version uint64, keys []solana.PublicKey, records []*VoteAccountInfo) bool { + c := &s.voteRecords + c.mu.Lock() + defer c.mu.Unlock() + // Check under the invalidation lock: a concurrent commit must either make + // this load stale or subsequently remove its entries before readers see it. + latest, stable := s.acctsDb.CommittedAccountVersion() + if !stable || latest != version { + return false + } + if c.records == nil { + c.records = make(map[solana.PublicKey]*VoteAccountInfo) + } + for i, record := range records { + // Do not accumulate negative entries for arbitrary filtered RPC keys. + if record != nil { + c.records[keys[i]] = record + } + } + return true +} diff --git a/pkg/rpcserver/vote_record_cache_bench_test.go b/pkg/rpcserver/vote_record_cache_bench_test.go new file mode 100644 index 000000000..3f075a677 --- /dev/null +++ b/pkg/rpcserver/vote_record_cache_bench_test.go @@ -0,0 +1,96 @@ +package rpcserver + +import ( + "encoding/binary" + "encoding/json" + "path/filepath" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/filecoin-project/go-jsonrpc" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Measures the warm RPC handler plus JSON encoding, not HTTP transport, startup, +// stake aggregation or commits. Vote accounts include lockouts and credit history +// so decoding costs are represented rather than using empty vote-state fixtures. +// Changed cases invalidate decoded records before each query (including that cost), +// simulating account updates with warm account buffers; they do not perform disk +// commits and must not be presented as end-to-end writer/RPC contention results. +func BenchmarkGetVoteAccountsRecords(b *testing.B) { + db := newRPCAccountsDB(b) + global.ClearPendingStakePubkeys() + b.Cleanup(global.ClearPendingStakePubkeys) + global.EnqueuePendingStakePubkey(500, solana.PublicKey{0xee}) + _, err := global.FlushPendingStakePubkeys(filepath.Dir(db.AcctsDir)) + require.NoError(b, err) + delta := []*accounts.Account{cacheHistoryAccount(b, nil)} + stakes := make(map[solana.PublicKey]uint64) + for i := range 1000 { + key := solana.PublicKey{0xb0} + binary.LittleEndian.PutUint64(key[1:], uint64(i)) + v := &sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4} + v.V4.NodePubkey = key + v.V4.AuthorizedVoters.AuthorizedVoters.Set(0, key) + for j := range 31 { + v.V4.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: uint64(470 + j)}}) + } + for j := range 64 { + v.V4.EpochCredits = append(v.V4.EpochCredits, sealevel.EpochCredits{Epoch: uint64(j), Credits: uint64(j + 1), PrevCredits: uint64(j)}) + } + delta = append(delta, voteAccountForRPC(b, key, v)) + stakes[key] = 1 + } + _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 500, Delta: delta}}, 500, nil, nil) + require.NoError(b, err) + require.NoError(b, db.SeedVoteAccountPubkeys(nil)) + previous, present := global.EpochStakesSnapshot(0) + global.PutEpochStakes(0, stakes, map[solana.PublicKey]*epochstakes.VoteAccount{}, 1000) + b.Cleanup(func() { + if present { + global.PutEpochStakes(0, previous.Stakes, previous.VoteAccounts, previous.TotalStake) + } else { + global.ClearEpochStakes(0) + } + }) + s := &RpcServer{acctsDb: db, epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 1000}} + s.SetRootedBankState(500, 500, 0) + for _, tc := range []struct { + name string + params []interface{} + count int + changed int + }{ + {"All", []interface{}{}, 1000, 0}, + {"One", []interface{}{map[string]interface{}{"votePubkey": (solana.PublicKey{0xb0}).String()}}, 1, 0}, + {"All10PercentChanged", []interface{}{}, 1000, 100}, + {"AllChanged", []interface{}{}, 1000, 1000}, + } { + b.Run(tc.name, func(b *testing.B) { + raw, err := json.Marshal(tc.params) + require.NoError(b, err) + params := jsonrpc.RawParams(raw) + response, err := s.GetVoteAccounts(b.Context(), params) + require.NoError(b, err) + require.Len(b, response.Current, tc.count) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if tc.changed > 0 { + invalidateVoteRecordsForBenchmark(s, delta[1:1+tc.changed]) + } + response, err := s.GetVoteAccounts(b.Context(), params) + if err != nil { + b.Fatal(err) + } + if _, err := json.Marshal(response); err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/pkg/rpcserver/vote_record_cache_test.go b/pkg/rpcserver/vote_record_cache_test.go new file mode 100644 index 000000000..ab0646bc4 --- /dev/null +++ b/pkg/rpcserver/vote_record_cache_test.go @@ -0,0 +1,180 @@ +package rpcserver + +import ( + "context" + "fmt" + "math" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func recordTestAccount(t testing.TB, slot uint64) *accounts.Account { + v := &sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4} + v.V4.NodePubkey = solana.PublicKey{2} + v.V4.AuthorizedVoters.AuthorizedVoters.Set(0, v.V4.NodePubkey) + v.V4.Votes.PushBack(sealevel.LandedVote{Lockout: sealevel.VoteLockout{Slot: slot}}) + v.V4.EpochCredits = []sealevel.EpochCredits{{Epoch: 0, Credits: slot}} + v.V4.RootSlot = &slot + v.V4.InflationRewardsCommissionBps = 755 + return voteAccountForRPC(t, solana.PublicKey{1}, v) +} + +func readTestVoteRecord(t *testing.T, s *RpcServer) *VoteAccountInfo { + t.Helper() + version, stable := s.acctsDb.CommittedAccountVersion() + require.True(t, stable) + bank := s.rootedBank.Load() + records, err := s.rootedVoteRecords(t.Context(), bank.Slot, version, []solana.PublicKey{{1}}) + require.NoError(t, err) + return records[0] +} + +func TestVoteRecordCacheInvalidationAndRecovery(t *testing.T) { + s := seedStakeCacheServer(t, recordTestAccount(t, 10)) + first := readTestVoteRecord(t, s) + require.NotNil(t, first) + require.Same(t, first, readTestVoteRecord(t, s)) + commitCacheAccounts(t, s, 11, &accounts.Account{Key: solana.PublicKey{9}, Lamports: 1, Owner: addresses.SystemProgramAddr}) + require.Same(t, first, readTestVoteRecord(t, s), "unrelated writes must preserve parsed records") + oldVersion, _ := s.acctsDb.CommittedAccountVersion() + commitCacheAccounts(t, s, 12, recordTestAccount(t, 12)) + require.False(t, s.storeVoteRecords(oldVersion, []solana.PublicKey{{1}}, []*VoteAccountInfo{first}), "an in-flight old decode cannot resurrect an invalidated record") + updated := readTestVoteRecord(t, s) + require.Equal(t, uint64(12), updated.LastVote) + require.Equal(t, uint64(10), first.LastVote, "in-flight readers retain immutable old records") + for _, account := range []*accounts.Account{ + {Key: solana.PublicKey{1}, Lamports: 1, Owner: addresses.SystemProgramAddr}, + {Key: solana.PublicKey{1}, Lamports: 0, Owner: addresses.VoteProgramAddr}, + {Key: solana.PublicKey{1}, Lamports: 1, Owner: addresses.VoteProgramAddr, Data: []byte{0}}, + voteAccountForRPC(t, solana.PublicKey{1}, &sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4}), + } { + slot := s.rootedBank.Load().Slot + 1 + commitCacheAccounts(t, s, slot, account) + require.Nil(t, readTestVoteRecord(t, s)) + commitCacheAccounts(t, s, slot+1, recordTestAccount(t, slot+1)) + require.Equal(t, slot+1, readTestVoteRecord(t, s).LastVote) + } + _, err := s.acctsDb.RewindToBatchBoundary(10) + require.NoError(t, err) + s.SetRootedBankState(10, 10, 0) + require.Empty(t, s.voteRecords.records) + require.Equal(t, first, readTestVoteRecord(t, s)) + commitCacheAccounts(t, s, 11, recordTestAccount(t, 9)) // Same slot, different fork. + require.Equal(t, uint64(9), readTestVoteRecord(t, s).LastVote) + _, err = s.acctsDb.RecoverFoldState() + require.NoError(t, err) + require.Empty(t, s.voteRecords.records) + require.Equal(t, uint64(9), readTestVoteRecord(t, s).LastVote) + // Failed writes use the same observer reset contract as recovery. + s.voteRecords.AccountsChanged(0, nil, true) + require.Empty(t, s.voteRecords.records) +} + +func TestVoteRecordCacheResponseOwnershipAndPublication(t *testing.T) { + s := seedStakeCacheServer(t, recordTestAccount(t, 10), cacheStakeAccount(t, 21, 1, 100, math.MaxUint64, math.MaxUint64)) + previous, present := global.EpochStakesSnapshot(0) + global.PutEpochStakes(0, map[solana.PublicKey]uint64{{1}: 1}, map[solana.PublicKey]*epochstakes.VoteAccount{}, 1) + t.Cleanup(func() { + if present { + global.PutEpochStakes(0, previous.Stakes, previous.VoteAccounts, previous.TotalStake) + } else { + global.ClearEpochStakes(0) + } + }) + params := mustRawParams(t, []interface{}{}) + first, err := s.GetVoteAccounts(t.Context(), params) + require.NoError(t, err) + require.Len(t, first.Current, 1) + first.Current[0].EpochCredits[0][1] = 999 + first.Current[0].NodePubkey = "mutated" + next, err := s.GetVoteAccounts(t.Context(), params) + require.NoError(t, err) + require.Equal(t, uint64(10), next.Current[0].EpochCredits[0][1]) + require.Equal(t, (solana.PublicKey{2}).String(), next.Current[0].NodePubkey) + record := s.voteRecords.records[solana.PublicKey{1}] + // Root-relative classification is not cached with the account record. + _, err = s.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{map[string]interface{}{"delinquentSlotDistance": 0}})) + require.NoError(t, err) + _, err = s.acctsDb.CommitBatch([]accounts.SlotDelta{{Slot: 90, Delta: []*accounts.Account{cacheStakeAccount(t, 21, 1, 200, math.MaxUint64, math.MaxUint64)}}}, 90, nil, nil) + require.NoError(t, err) + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond) + defer cancel() + _, err = s.GetVoteAccounts(ctx, params) + require.ErrorIs(t, err, context.DeadlineExceeded, "cached records must not bypass rooted publication") + s.SetRootedBankState(90, 90, 0) + next, err = s.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{map[string]interface{}{"delinquentSlotDistance": 20}})) + require.NoError(t, err) + require.Empty(t, next.Current) + require.Len(t, next.Delinquent, 1) + require.Equal(t, uint64(200), next.Delinquent[0].ActivatedStake) + require.Same(t, record, s.voteRecords.records[solana.PublicKey{1}]) +} + +func TestVoteRecordCacheLegacyAndUnknownKeys(t *testing.T) { + s := seedStakeCacheServer(t, recordTestAccount(t, 10)) + readTestVoteRecord(t, s) + version, _ := s.acctsDb.CommittedAccountVersion() + keys := make([]solana.PublicKey, 100) + for i := range keys { + keys[i] = solana.PublicKey{0xee, byte(i)} + } + records, err := s.rootedVoteRecords(t.Context(), 10, version, keys) + require.NoError(t, err) + for _, record := range records { + require.Nil(t, record) + } + require.Len(t, s.voteRecords.records, 1, "arbitrary nonexistent filters must not grow retained memory") + s.acctsDb.RootedDurable = false + done := make(chan struct{}) + require.NoError(t, s.acctsDb.StoreAccounts([]*accounts.Account{recordTestAccount(t, 11)}, 11, func() { close(done) })) + <-done + require.Eventually(t, func() bool { _, stable := s.acctsDb.CommittedAccountVersion(); return stable }, time.Second, time.Millisecond) + s.SetRootedBankState(11, 11, 0) + require.Equal(t, uint64(11), readTestVoteRecord(t, s).LastVote) +} + +func invalidateVoteRecordsForBenchmark(s *RpcServer, changed []*accounts.Account) { + s.voteRecords.AccountsChanged(0, changed, false) +} + +func TestDecodeVoteRecordVersions(t *testing.T) { + for version := uint32(0); version <= sealevel.VoteStateVersionV4; version++ { + t.Run(fmt.Sprint(version), func(t *testing.T) { + key, node := solana.PublicKey{1}, solana.PublicKey{2} + root := uint64(9) + credits := []sealevel.EpochCredits{{Epoch: 1, Credits: 12, PrevCredits: 3}} + lockout := sealevel.VoteLockout{Slot: 10} + v := &sealevel.VoteStateVersions{Type: version} + switch version { + case sealevel.VoteStateVersionV0_23_5: + v.V0_23_5 = sealevel.VoteState0_23_5{NodePubkey: node, AuthorizedVoter: node, Commission: 8, RootSlot: &root, EpochCredits: credits} + v.V0_23_5.Votes.PushBack(lockout) + case sealevel.VoteStateVersionV1_14_11: + v.V1_14_11 = sealevel.VoteState1_14_11{NodePubkey: node, Commission: 8, RootSlot: &root, EpochCredits: credits} + v.V1_14_11.AuthorizedVoters.AuthorizedVoters.Set(0, node) + v.V1_14_11.Votes.PushBack(lockout) + case sealevel.VoteStateVersionCurrent: + v.Current = sealevel.VoteState{NodePubkey: node, Commission: 8, RootSlot: &root, EpochCredits: credits} + v.Current.AuthorizedVoters.AuthorizedVoters.Set(0, node) + v.Current.Votes.PushBack(sealevel.LandedVote{Lockout: lockout}) + case sealevel.VoteStateVersionV4: + v.V4.NodePubkey, v.V4.RootSlot, v.V4.EpochCredits = node, &root, credits + v.V4.InflationRewardsCommissionBps = 800 + v.V4.AuthorizedVoters.AuthorizedVoters.Set(0, node) + v.V4.Votes.PushBack(sealevel.LandedVote{Lockout: lockout}) + } + account := voteAccountForRPC(t, key, v) + got := decodeVoteRecord(key, account) + clear(account.Data) // The cache must own everything it retains. + require.Equal(t, &VoteAccountInfo{VotePubkey: key.String(), NodePubkey: node.String(), Commission: 8, InflationRewardsCommissionBPS: 800, RootSlot: 9, LastVote: 10, EpochCredits: [][3]uint64{{1, 12, 3}}}, got) + }) + } +} From f521efcc7674c685f514b52f025812f9d22f3983 Mon Sep 17 00:00:00 2001 From: 7layermagik <7layermagik@users.noreply.github.com> Date: Tue, 29 Sep 2026 06:25:49 -0500 Subject: [PATCH 12/14] rpc: bound stake recalculation lock holds without copying keys --- docs/rooted-stake-rpc-cache.md | 26 +++- pkg/rpcserver/rooted_stake_cache.go | 67 +++++----- .../rooted_stake_recalculation_bench_test.go | 57 ++++++++ .../rooted_stake_recalculation_test.go | 124 ++++++++++++++++++ 4 files changed, 240 insertions(+), 34 deletions(-) create mode 100644 pkg/rpcserver/rooted_stake_recalculation_bench_test.go create mode 100644 pkg/rpcserver/rooted_stake_recalculation_test.go diff --git a/docs/rooted-stake-rpc-cache.md b/docs/rooted-stake-rpc-cache.md index e47860614..74dac8852 100644 --- a/docs/rooted-stake-rpc-cache.md +++ b/docs/rooted-stake-rpc-cache.md @@ -20,7 +20,8 @@ cached read. Unknown or invalid filtered keys do not create negative cache entri The retained data is one compact delegation/contribution per live delegated stake account, plus totals per vote account. Full account buffers are not retained. Memory is proportional to delegated stake accounts. Bootstrap temporarily tracks -changed keys while loading candidates; recalculation temporarily holds a key list. +changed keys while loading candidates. Recalculation walks the existing map +without allocating or copying a whole-cache key list. ## Publication and recovery contract @@ -37,6 +38,15 @@ changed keys while loading candidates; recalculation temporarily holds a key lis - Epoch, stake-history, or warmup-feature changes trigger a RAM-only recalculation in bounded chunks. Per-entry calculation tags count concurrent updates exactly once; new activation inputs cancel an unfinished calculation. +- Recalculation holds the cache mutex for every map iterator step, read and write, + releasing it after each 512 visited entries, including already-counted entries. + Commits between chunks may delete or insert keys: the observer removes old + contributions and counts updated/new entries with the current calculation tag. + Those tags prevent double counting whether the iterator sees an insertion or + skips it, as permitted by [Go's map range semantics](https://go.dev/ref/spec#For_range). + After each lock reacquisition, generation and calculation checks reject resets + or changed activation inputs before the iterator advances again. This bounds + entries processed per lock hold, not a hard wall-clock latency guarantee. - Rewind, recovery, and failed writes invalidate the derived view. Recovery also clears old account-read caches before a rebuild. Reusing the same slot on another fork cannot reuse its old stake totals. Failed writes require successful repair @@ -83,6 +93,20 @@ These are component measurements: they exclude JSON encoding, network latency, initial cache loading, and an epoch transition. They do not establish mainnet capacity or live validator performance. +`BenchmarkRootedStakeRecalculation` isolates RAM traversal at 20,000 and 2 million +delegations with 1,000 voters. The delegation cache is seeded directly and all +stakes are bootstrap-active; it measures recalculation time and allocations, +not startup, disk, or writer latency. For concurrent commits and mixed activation +history, use the on-disk sizing experiment below. An optional Go mutex profile +can attribute writer/reader lock wait; profile times sum waiting goroutines and +must not be reported as a single request's wall-clock latency. The sizing test's +full-scan correctness oracles also appear in whole-process profiles. + +```sh +GOMAXPROCS=2 GOMEMLIMIT=4GiB go test ./pkg/rpcserver -run '^$' \ + -bench '^BenchmarkRootedStakeRecalculation$' -benchtime=700ms -count=3 +``` + `BenchmarkGetVoteAccountsRecords` measures the warm handler plus JSON encoding for 1,000 V4 vote accounts (31 lockouts and 64 credit-history entries each), or one filtered account. It isolates vote records with initialized empty stake totals. diff --git a/pkg/rpcserver/rooted_stake_cache.go b/pkg/rpcserver/rooted_stake_cache.go index e10b75413..5d9130ab3 100644 --- a/pkg/rpcserver/rooted_stake_cache.go +++ b/pkg/rpcserver/rooted_stake_cache.go @@ -367,10 +367,12 @@ func (s *RpcServer) bootstrapStakeCache(ctx context.Context, generation uint64) return nil } -// Recalculate in bounded chunks. A per-entry calculation tag makes concurrent -// committed updates part of the same sum exactly once. New accounts omitted -// from the starting key list are already included by the observer. An activation -// input change cancels this sum; ordinary account changes do not restart it. +// Recalculate in bounded chunks. Every map iterator step and entry access holds +// c.mu; commits may modify the map only between chunks. Go's map range semantics +// permit those modifications: deleted entries disappear, and new entries may or +// may not be visited. The observer already includes new/updated entries in this +// calculation, so their tags prevent double counting either way. A reset or new +// activation input cancels the iterator before it can advance again. func (s *RpcServer) recalculateStakeTotals(ctx context.Context, slot, epoch, generation uint64) (err error) { c := &s.stakeCache bank := s.rootedBank.Load() @@ -393,38 +395,37 @@ func (s *RpcServer) recalculateStakeTotals(ctx context.Context, slot, epoch, gen c.epoch, c.history, c.activationEpoch = epoch, history, activation c.totals = make(map[solana.PublicKey]uint64) c.totalsReady = false - keys := make([]solana.PublicKey, 0, len(c.byStake)) - for key := range c.byStake { - keys = append(keys, key) - } - c.mu.Unlock() + // From here every return owns c.mu, including errors from yield. Invalidate + // only this calculation; a reset may already have installed a different view. defer func() { - if err != nil { - c.mu.Lock() - if c.generation == generation && c.calculation == calculation { - c.totals = nil - c.totalsReady = false - } - c.mu.Unlock() + if err != nil && c.generation == generation && c.calculation == calculation { + c.totals = nil + c.totalsReady = false } + c.mu.Unlock() }() - for start := 0; start < len(keys); start += stakeCacheBatchSize { + // Enter and return locked. Never advance the map iterator while unlocked. + yield := func(visited int) error { + c.mu.Unlock() if c.beforeRecalculationChunk != nil { - c.beforeRecalculationChunk(start) - } - if err = ctx.Err(); err != nil { - return err + c.beforeRecalculationChunk(visited) } + canceled := ctx.Err() c.mu.Lock() + if canceled != nil { + return canceled + } if c.generation != generation || c.calculation != calculation { - c.mu.Unlock() return errStakeViewChanged } - for _, key := range keys[start:min(start+stakeCacheBatchSize, len(keys))] { - entry, exists := c.byStake[key] - if !exists || entry.calculation == calculation { - continue - } + return nil + } + if err = yield(0); err != nil { + return err + } + visited := 0 + for key, entry := range c.byStake { + if entry.calculation != calculation { entry.active = entry.delegation.Stake(epoch, &history, activation) entry.calculation = calculation c.byStake[key] = entry @@ -432,12 +433,12 @@ func (s *RpcServer) recalculateStakeTotals(ctx context.Context, slot, epoch, gen c.totals[entry.delegation.VoterPubkey] += entry.active } } - c.mu.Unlock() - } - c.mu.Lock() - defer c.mu.Unlock() - if c.generation != generation || c.calculation != calculation { - return errStakeViewChanged + visited++ + if visited%stakeCacheBatchSize == 0 { + if err = yield(visited); err != nil { + return err + } + } } c.totalsReady = true c.recalculations++ diff --git a/pkg/rpcserver/rooted_stake_recalculation_bench_test.go b/pkg/rpcserver/rooted_stake_recalculation_bench_test.go new file mode 100644 index 000000000..63b1fdf2b --- /dev/null +++ b/pkg/rpcserver/rooted_stake_recalculation_bench_test.go @@ -0,0 +1,57 @@ +package rpcserver + +import ( + "encoding/binary" + "fmt" + "math" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +// Isolates RAM-only recalculation and its allocations. The delegation cache is +// seeded directly; this is not a startup, disk, RPC or writer-latency benchmark. +// All stakes are bootstrap-active, so the comparison isolates traversal rather +// than the depth of activation-history calculations. Use the opt-in on-disk +// TestRootedStakeCacheScale for mixed activation states and concurrent commits. +func BenchmarkRootedStakeRecalculation(b *testing.B) { + for _, count := range []int{20_000, 2_000_000} { + b.Run(fmt.Sprint(count), func(b *testing.B) { + db := newRPCAccountsDB(b) + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 10, Delta: []*accounts.Account{cacheHistoryAccount(b, nil)}}}, 10, nil, nil) + require.NoError(b, err) + s := &RpcServer{acctsDb: db, epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 100}} + s.SetRootedBankState(10, 10, 0) + version, stable := db.CommittedAccountVersion() + require.True(b, stable) + c := &s.stakeCache + c.version, c.dataReady = version, true + c.byStake = make(map[solana.PublicKey]cachedDelegation, count) + for i := range count { + key, voter := solana.PublicKey{0xa0}, solana.PublicKey{0xb0} + binary.LittleEndian.PutUint64(key[1:], uint64(i)) + binary.LittleEndian.PutUint64(voter[1:], uint64(i%1000)) + c.byStake[key] = cachedDelegation{loaded: true, delegation: sealevel.Delegation{ + VoterPubkey: voter, StakeLamports: 1_000_000, + ActivationEpoch: math.MaxUint64, DeactivationEpoch: math.MaxUint64, + }} + } + require.NoError(b, s.recalculateStakeTotals(b.Context(), 10, 0, c.generation)) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + if err := s.recalculateStakeTotals(b.Context(), 10, 0, c.generation); err != nil { + b.Fatal(err) + } + } + b.StopTimer() + require.Len(b, c.totals, 1000) + for _, total := range c.totals { + require.Equal(b, uint64(count/1000)*1_000_000, total) + } + }) + } +} diff --git a/pkg/rpcserver/rooted_stake_recalculation_test.go b/pkg/rpcserver/rooted_stake_recalculation_test.go new file mode 100644 index 000000000..b7f62cfa2 --- /dev/null +++ b/pkg/rpcserver/rooted_stake_recalculation_test.go @@ -0,0 +1,124 @@ +package rpcserver + +import ( + "context" + "encoding/binary" + "math" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func recalculationStake(t testing.TB, index int, voter byte, amount uint64) *accounts.Account { + account := cacheStakeAccount(t, 0xa0, voter, amount, math.MaxUint64, math.MaxUint64) + binary.LittleEndian.PutUint64(account.Key[1:], uint64(index)) + return account +} + +func recalculationServer(t *testing.T) *RpcServer { + delta := make([]*accounts.Account, 2048) + for i := range delta { + delta[i] = recalculationStake(t, i, 1, 100) + } + s := seedStakeCacheServer(t, delta...) + require.Equal(t, map[solana.PublicKey]uint64{{1}: 204800}, cacheTotals(t, s)) + commitCacheAccounts(t, s, 100, cacheHistoryAccount(t, nil)) + return s +} + +func TestRootedStakeCacheRecalculationMapChurn(t *testing.T) { + s := recalculationServer(t) + expected := map[solana.PublicKey]uint64{{1}: 204800} + mutated := false + s.stakeCache.beforeRecalculationChunk = func(start int) { + if start != stakeCacheBatchSize || mutated { + return + } + mutated = true + // Select both already-counted and not-yet-counted entries without + // depending on randomized map iteration order. + var counted, pending []solana.PublicKey + c := &s.stakeCache + c.mu.Lock() + for key, entry := range c.byStake { + if entry.calculation == c.calculation && len(counted) < 2 { + counted = append(counted, key) + } else if entry.calculation != c.calculation && len(pending) < 2 { + pending = append(pending, key) + } + } + c.mu.Unlock() + require.Len(t, counted, 2) + require.Len(t, pending, 2) + var changed []*accounts.Account + for _, keys := range [][]solana.PublicKey{counted, pending} { + updated := recalculationStake(t, 0, 2, 250) + updated.Key = keys[0] + changed = append(changed, updated, &accounts.Account{Key: keys[1]}) + expected[solana.PublicKey{1}] -= 200 + expected[solana.PublicKey{2}] += 250 + } + // Force map growth while its iterator is suspended. New entries must + // contribute once whether the iterator later sees them or skips them. + for i := 2048; i < 8192; i++ { + changed = append(changed, recalculationStake(t, i, 3, 75)) + expected[solana.PublicKey{3}] += 75 + } + commitCacheAccounts(t, s, 101, changed...) + // Reinsert keys removed from both sides of the cursor, before resuming. + var recreated []*accounts.Account + for _, key := range []solana.PublicKey{counted[1], pending[1]} { + a := recalculationStake(t, 0, 4, 30) + a.Key = key + recreated = append(recreated, a) + expected[solana.PublicKey{4}] += 30 + } + commitCacheAccounts(t, s, 102, recreated...) + } + _, err := s.rootedActivatedStakes(t.Context(), 100, 1, nil) + require.ErrorIs(t, err, errStakeViewChanged, "the response bank changed, but the calculation may be reused") + require.True(t, mutated) + require.True(t, s.stakeCache.totalsReady) + require.Equal(t, expected, cacheTotals(t, s)) + require.Equal(t, uint64(2), s.stakeCache.recalculations) + require.Equal(t, uint64(1), s.stakeCache.scans) +} + +func TestRootedStakeCacheRecalculationInterruptedMidMap(t *testing.T) { + for _, action := range []string{"cancel", "history", "rewind"} { + t.Run(action, func(t *testing.T) { + s := recalculationServer(t) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + interrupted := false + s.stakeCache.beforeRecalculationChunk = func(start int) { + if start != stakeCacheBatchSize || interrupted { + return + } + interrupted = true + switch action { + case "cancel": + cancel() + case "history": + commitCacheAccounts(t, s, 101, cacheHistoryAccount(t, nil)) + case "rewind": + _, err := s.acctsDb.RewindToBatchBoundary(10) + require.NoError(t, err) + s.SetRootedBankState(10, 10, 0) + } + } + _, err := s.rootedActivatedStakes(ctx, 100, 1, nil) + if action == "cancel" { + require.ErrorIs(t, err, context.Canceled) + } else { + require.ErrorIs(t, err, errStakeViewChanged) + } + require.True(t, interrupted) + require.False(t, s.stakeCache.totalsReady) + s.stakeCache.beforeRecalculationChunk = nil + require.Equal(t, map[solana.PublicKey]uint64{{1}: 204800}, cacheTotals(t, s)) + }) + } +} From 808f892c7c2b07d9e3ac3d919bde7a423ff92fc3 Mon Sep 17 00:00:00 2001 From: Neeraj Godiyal Date: Wed, 30 Sep 2026 23:40:52 +0530 Subject: [PATCH 13/14] fix(rpc): keep account reads consistent with the published root --- README.md | 9 +- .../rooted_account_publication_test.go | 125 ++++++++++++++++++ pkg/rpcserver/rpc_registration_test.go | 90 +++++++++++++ pkg/rpcserver/rpcserver.go | 36 ++++- 4 files changed, 252 insertions(+), 8 deletions(-) create mode 100644 pkg/rpcserver/rooted_account_publication_test.go create mode 100644 pkg/rpcserver/rpc_registration_test.go diff --git a/README.md b/README.md index dc143441a..4ab027f9f 100644 --- a/README.md +++ b/README.md @@ -219,19 +219,24 @@ curl http://YOUR_MITHRIL_IP:8899 -X POST -H "Content-Type: application/json" -d **Currently supported RPC methods:** - `getAccountInfo` - Get account data and lamports +- `getBalance` - Get an account's lamport balance - `getBankHash` - Get bankhash for a slot (Mithril extension, not standard Solana RPC) - `getBlockHeight` - Get current block height +- `getBlockProduction` - Get produced and assigned block counts by validator - `getEpochInfo` - Get current epoch info +- `getGenesisHash` - Get the cluster genesis hash - `getLatestBlockhash` - Get recent blockhash +- `getLeaderSchedule` - Get the leader schedule for an epoch - `getSlot` - Get the slot at the requested commitment (defaults to finalized) +- `getVoteAccounts` - Get current and delinquent vote accounts +- `sendTransaction` - Submit a signed transaction +- `simulateTransaction` - Simulate a transaction `getSlot` accepts an optional config object with `commitment` and `minContextSlot`. On Alpenglow, `processed` uses the live replay slot; `confirmed` and `finalized` use the published rooted slot. If that slot is below `minContextSlot`, the request returns error `-32016` with the available slot in `data.contextSlot`. -We're actively expanding RPC method coverage. Upcoming methods include transaction simulation, send transaction, and get leader schedule. - ### Current Limitations - **RPC still required**: live near-tip blocks stream from turbine shreds, but RPC `getBlock` is still used for catchup and by the trailing execution verifier (Alpenglow certificates attest block *data*, not execution results, so an external oracle cross-checks execution until peer bankhash cross-checking lands). diff --git a/pkg/rpcserver/rooted_account_publication_test.go b/pkg/rpcserver/rooted_account_publication_test.go new file mode 100644 index 000000000..8baad9869 --- /dev/null +++ b/pkg/rpcserver/rooted_account_publication_test.go @@ -0,0 +1,125 @@ +package rpcserver + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestGetBalanceHTTPWaitsForFailedRewindRepair(t *testing.T) { + db := newRPCAccountsDB(t) + key := solana.PublicKey{204} + for _, slot := range []uint64{10, 20} { + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: []*accounts.Account{{Key: key, Lamports: slot}}}}, slot, nil, nil) + require.NoError(t, err) + } + server := NewRpcServer(db, 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 100}, solana.Hash{1}) + t.Cleanup(func() { _ = server.listener.Close() }) + server.SetRootedBankState(20, 20, 0) + endpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 100*time.Millisecond) + defer cancel() + server.ServeHTTP(w, r.WithContext(ctx)) + })) + t.Cleanup(endpoint.Close) + endpoint.Client().Timeout = time.Second + request, err := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": "getBalance", "params": []any{key.String()}}) + require.NoError(t, err) + call := func() (*GetBalanceResp, *rpcProbeError) { + t.Helper() + response, err := endpoint.Client().Post(endpoint.URL, "application/json", bytes.NewReader(request)) + require.NoError(t, err) + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + require.NoError(t, err) + t.Logf("HTTP status=%d body=%s", response.StatusCode, body) + var reply struct { + Result *GetBalanceResp `json:"result"` + Error *rpcProbeError `json:"error"` + } + require.NoError(t, json.Unmarshal(body, &reply)) + return reply.Result, reply.Error + } + rewoundPath := filepath.Join(db.AcctsDir, "rewound") + require.NoError(t, os.WriteFile(rewoundPath, nil, 0o644)) + _, err = db.RewindToBatchBoundary(10) + require.Error(t, err) + result, rpcError := call() + require.Nil(t, result, "partial rewind must not publish an account under the previous bank") + require.NotNil(t, rpcError) + require.Contains(t, rpcError.Message, context.DeadlineExceeded.Error()) + + require.NoError(t, os.Remove(rewoundPath)) + _, err = db.RewindToBatchBoundary(10) + require.NoError(t, err) + server.SetRootedBankState(10, 10, 0) + result, rpcError = call() + require.Nil(t, rpcError) + require.NotNil(t, result) + require.Equal(t, uint64(10), result.Context.Slot) + require.Equal(t, uint64(10), result.Value) +} + +func TestRootedAccountRPCsWaitForFailedRewindRepair(t *testing.T) { + key := solana.PublicKey{203} + for _, test := range []struct { + name string + read func(context.Context, *RpcServer) error + }{ + {"balance", func(ctx context.Context, server *RpcServer) error { + _, err := server.GetBalance(ctx, mustRawParams(t, []any{key.String()})) + return err + }}, + {"account info", func(ctx context.Context, server *RpcServer) error { + _, err := server.GetAccountInfo(ctx, mustRawParams(t, []any{key.String(), map[string]any{"encoding": "base64"}})) + return err + }}, + {"block production", func(ctx context.Context, server *RpcServer) error { + _, err := server.GetBlockProduction(ctx, nil) + return err + }}, + } { + t.Run(test.name, func(t *testing.T) { + db := newRPCAccountsDB(t) + for _, slot := range []uint64{10, 20} { + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: slot, Delta: []*accounts.Account{{Key: key, Lamports: slot}}}}, slot, nil, nil) + require.NoError(t, err) + } + server := &RpcServer{acctsDb: db, epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 100}} + server.SetRootedBankState(20, 20, 0) + // Fail forensic-directory creation after rewind's atomic index undo. + rewoundPath := filepath.Join(db.AcctsDir, "rewound") + require.NoError(t, os.WriteFile(rewoundPath, nil, 0o644)) + _, err := db.RewindToBatchBoundary(10) + require.Error(t, err) + _, stable := db.CommittedAccountVersion() + require.False(t, stable) + require.Equal(t, uint64(20), db.DurableThrough()) + ctx, cancel := context.WithTimeout(t.Context(), 100*time.Millisecond) + defer cancel() + require.ErrorIs(t, test.read(ctx, server), context.DeadlineExceeded, + "partial rewind must not publish account values under the previous bank") + + require.NoError(t, os.Remove(rewoundPath)) + _, err = db.RewindToBatchBoundary(10) + require.NoError(t, err) + server.SetRootedBankState(10, 10, 0) + rooted, account, err := server.readRootedAccount(t.Context(), key) + require.NoError(t, err) + require.Equal(t, uint64(10), rooted.Slot) + require.Equal(t, uint64(10), account.Lamports) + }) + } +} diff --git a/pkg/rpcserver/rpc_registration_test.go b/pkg/rpcserver/rpc_registration_test.go new file mode 100644 index 000000000..e06dd4e90 --- /dev/null +++ b/pkg/rpcserver/rpc_registration_test.go @@ -0,0 +1,90 @@ +package rpcserver + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" +) + +func TestRPCHandlerMethodSet(t *testing.T) { + typ := reflect.TypeOf(rpcHandler{}) + if typ.NumMethod() != len(supportedRPCMethods) { + t.Fatalf("registered %d methods, public method list has %d", typ.NumMethod(), len(supportedRPCMethods)) + } + for i := 0; i < typ.NumMethod(); i++ { + name := typ.Method(i).Name + name = strings.ToLower(name[:1]) + name[1:] + if _, ok := supportedRPCMethods[name]; !ok { + t.Fatalf("non-public method registered: %s", name) + } + } +} + +func TestInternalMethodsAreNotRPC(t *testing.T) { + for _, mode := range []string{"single", "mixed-batch", "oversized-single", "direct-dispatch"} { + t.Run(mode, func(t *testing.T) { + rpc := NewRpcServer(nil, 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 32}, solana.Hash{1}) + t.Cleanup(func() { _ = rpc.listener.Close() }) + rpc.SetRootedBankState(100, 90, 80) + call := map[string]any{"jsonrpc": "2.0", "id": 1, "method": "setRootedBankState", "params": []any{777, 666, 555}} + if mode == "oversized-single" { + call["padding"] = strings.Repeat("x", maxQuietMethodProbeBody+1) + } + var request any = call + if mode == "mixed-batch" { + request = []any{map[string]any{"jsonrpc": "2.0", "id": 2, "method": "getGenesisHash", "params": []any{}}, call} + } + body, err := json.Marshal(request) + if err != nil { + t.Fatal(err) + } + response := httptest.NewRecorder() + if mode == "direct-dispatch" { + rpc.rpcService.HandleRequest(context.Background(), bytes.NewReader(body), response) + } else { + rpc.ServeHTTP(response, httptest.NewRequest(http.MethodPost, "/", bytes.NewReader(body))) + } + var replies []struct { + ID int `json:"id"` + Result string `json:"result"` + Error *struct { + Code int `json:"code"` + } `json:"error"` + } + result := response.Body.Bytes() + if mode != "mixed-batch" { + result = append(append([]byte{'['}, result...), ']') + } + if err := json.Unmarshal(result, &replies); err != nil { + t.Fatalf("decode response %q: %v", result, err) + } + found := false + for _, reply := range replies { + if reply.ID == 1 { + found = true + if reply.Error == nil || reply.Error.Code != -32601 { + t.Fatalf("internal method was not rejected: %s", result) + } + } + if reply.ID == 2 && (reply.Error != nil || reply.Result != (solana.Hash{1}).String()) { + t.Fatalf("public batch method failed: %s", result) + } + } + if !found || (mode == "mixed-batch" && len(replies) != 2) { + t.Fatalf("missing response: %s", result) + } + root, _ := rpc.getRootedBankState() + if root.Slot != 100 { + t.Fatalf("internal request changed rooted bank: %d", root.Slot) + } + }) + } +} diff --git a/pkg/rpcserver/rpcserver.go b/pkg/rpcserver/rpcserver.go index 7e90df08d..de833021f 100644 --- a/pkg/rpcserver/rpcserver.go +++ b/pkg/rpcserver/rpcserver.go @@ -23,6 +23,28 @@ import ( "github.com/gagliardetto/solana-go" ) +// rpcHandler exposes only public RPC methods to go-jsonrpc's reflection-based +// registration. Internal setters and lifecycle hooks remain Go-only. +type rpcHandler struct { + rpcMethods +} + +type rpcMethods interface { + GetAccountInfo(context.Context, jsonrpc.RawParams) (GetAccountInfoResp, error) + GetBalance(context.Context, jsonrpc.RawParams) (GetBalanceResp, error) + GetBankHash(context.Context, jsonrpc.RawParams) (string, error) + GetBlockHeight(context.Context, jsonrpc.RawParams) (uint64, error) + GetBlockProduction(context.Context, jsonrpc.RawParams) (GetBlockProductionResp, error) + GetEpochInfo(context.Context, jsonrpc.RawParams) (GetEpochInfoResp, error) + GetGenesisHash(context.Context, jsonrpc.RawParams) (string, error) + GetLatestBlockhash(context.Context, jsonrpc.RawParams) (GetLatestBlockhashResp, error) + GetLeaderSchedule(context.Context, jsonrpc.RawParams) (map[string][]uint64, error) + GetSlot(context.Context, jsonrpc.RawParams) (uint64, error) + GetVoteAccounts(context.Context, jsonrpc.RawParams) (GetVoteAccountsResp, error) + SendTransaction(context.Context, jsonrpc.RawParams) (string, error) + SimulateTransaction(context.Context, jsonrpc.RawParams) (SimulateTransactionResp, error) +} + type RpcServer struct { isReady bool rpcService *jsonrpc.RPCServer @@ -111,22 +133,24 @@ func (rpcServer *RpcServer) readRootedAccount(ctx context.Context, pubkey solana func (rpcServer *RpcServer) readRootedAccounts(ctx context.Context, pubkeys []solana.PublicKey) (rootedBankState, []*accounts.Account, error) { for { - rooted, ok := rpcServer.getRootedBankState() - if !ok { + bank := rpcServer.rootedBank.Load() + if bank == nil { return rootedBankState{}, nil, fmt.Errorf("node has no rooted bank available") } + rooted := *bank if rpcServer.acctsDb == nil { return rootedBankState{}, nil, fmt.Errorf("node has no accounts database available") } - if rpcServer.rootedPublicationPending(rooted.Slot) { + version, stable := rpcServer.acctsDb.CommittedAccountVersion() + if !stable || rpcServer.rootedPublicationPending(rooted.Slot) { if err := waitForRootedPublication(ctx); err != nil { return rooted, nil, err } continue } accounts, stats, err := rpcServer.acctsDb.GetAccountsBatchSharedWithStats(ctx, rooted.Slot, pubkeys) - latest, stillPublished := rpcServer.getRootedBankState() - if stats.PendingFoldHits > 0 || !stillPublished || latest.Slot != rooted.Slot || + latest, stable := rpcServer.acctsDb.CommittedAccountVersion() + if !stable || latest != version || rpcServer.rootedBank.Load() != bank || stats.PendingFoldHits > 0 || rpcServer.rootedPublicationPending(rooted.Slot) { if err := waitForRootedPublication(ctx); err != nil { return rooted, nil, err @@ -177,7 +201,7 @@ func NewRpcServer(acctsDb *accountsdb.AccountsDb, port uint16, epochSchedule *se jsonrpc.WithServerErrors(rpcErrors), ) - rpcServer.rpcService.Register("MithrilRpc", rpcServer) + rpcServer.rpcService.Register("MithrilRpc", rpcHandler{rpcServer}) rpcServer.acctsDb = acctsDb if epochSchedule != nil { rpcServer.epochSchedule = epochSchedule From 2da5856aec0be2fa240a8f9760e382dc52e2b78f Mon Sep 17 00:00:00 2001 From: Neeraj Godiyal Date: Sun, 18 Oct 2026 23:11:32 +0530 Subject: [PATCH 14/14] fix(rpc): honor validator query options --- docs/rooted-stake-rpc-cache.md | 18 ++ pkg/rpcserver/get_account_info.go | 56 +++--- pkg/rpcserver/get_account_info_test.go | 55 ++++++ pkg/rpcserver/get_balance.go | 11 +- pkg/rpcserver/get_block_production.go | 24 +-- pkg/rpcserver/get_epoch_info.go | 4 +- pkg/rpcserver/get_leader_schedule.go | 42 +++-- pkg/rpcserver/get_leader_schedule_test.go | 49 +++++- pkg/rpcserver/get_vote_accounts.go | 24 ++- pkg/rpcserver/get_vote_accounts_test.go | 27 ++- pkg/rpcserver/optional_params.go | 39 ++++- pkg/rpcserver/rooted_commitment_test.go | 54 ++++++ pkg/rpcserver/rpc_parameter_contract_test.go | 175 +++++++++++++++++++ 13 files changed, 489 insertions(+), 89 deletions(-) create mode 100644 pkg/rpcserver/rooted_commitment_test.go create mode 100644 pkg/rpcserver/rpc_parameter_contract_test.go diff --git a/docs/rooted-stake-rpc-cache.md b/docs/rooted-stake-rpc-cache.md index 74dac8852..e725e6796 100644 --- a/docs/rooted-stake-rpc-cache.md +++ b/docs/rooted-stake-rpc-cache.md @@ -23,6 +23,24 @@ Memory is proportional to delegated stake accounts. Bootstrap temporarily tracks changed keys while loading candidates. Recalculation walks the existing map without allocating or copying a whole-cache key list. +## RPC options + +`getVoteAccounts`, `getBalance`, `getLeaderSchedule`, `getBlockProduction`, and +`getAccountInfo` read the published rooted state. They accept `confirmed` and +`finalized`; `processed` returns an unsupported-commitment error rather than a +rooted result mislabeled as processed. This does not provide a separate optimistic +confirmed account view for classic consensus. `getSlot` and `getEpochInfo` keep +their existing processed-counter support. + +`keepUnstakedDelinquents: true` includes unstaked accounts in the delinquent list. +The default is false. It does not change the current-account list. + +`getLeaderSchedule` supports `keyByVoteAccount: true` when the retained schedule +contains the selected vote account for each leader slot. The `identity` filter +still selects a node identity. A node-keyed compatibility schedule without vote +account provenance returns an error for this option; it does not infer a vote +account from the node identity. + ## Publication and recovery contract - AccountsDB brackets writes with a process-local version. An odd version, diff --git a/pkg/rpcserver/get_account_info.go b/pkg/rpcserver/get_account_info.go index 3909505e0..228861334 100644 --- a/pkg/rpcserver/get_account_info.go +++ b/pkg/rpcserver/get_account_info.go @@ -5,12 +5,10 @@ import ( "encoding/base64" "errors" "fmt" - "reflect" "github.com/DataDog/zstd" "github.com/Overclock-Validator/mithril/pkg/accountsdb" "github.com/Overclock-Validator/mithril/pkg/addresses" - "github.com/Overclock-Validator/mithril/pkg/mlog" "github.com/Overclock-Validator/mithril/pkg/safemath" "github.com/filecoin-project/go-jsonrpc" "github.com/gagliardetto/solana-go" @@ -56,7 +54,7 @@ const ( ) func (rpcServer *RpcServer) GetAccountInfo(ctx context.Context, p jsonrpc.RawParams) (GetAccountInfoResp, error) { - params, err := jsonrpc.DecodeParams[[]interface{}](p) + params, err := decodeOptionalParams(p) if err != nil { return GetAccountInfoResp{}, fmt.Errorf("decoding params: %w", err) } @@ -77,7 +75,7 @@ func (rpcServer *RpcServer) GetAccountInfo(ctx context.Context, p jsonrpc.RawPar conf, err := parseGetAccountInfoConfMap(params) if err != nil { - return GetAccountInfoResp{}, err + return GetAccountInfoResp{}, &InvalidParamsError{Message: err.Error()} } rooted, acct, err := rpcServer.readRootedAccount(ctx, pk) if err != nil && !errors.Is(err, accountsdb.ErrNoAccount) { @@ -116,7 +114,7 @@ func (rpcServer *RpcServer) GetAccountInfo(ctx context.Context, p jsonrpc.RawPar } func parseGetAccountInfoConfMap(params []interface{}) (*GetAccountInfoConfig, error) { - if len(params) < 2 { + if len(params) < 2 || params[1] == nil { return &GetAccountInfoConfig{}, nil } @@ -128,17 +126,14 @@ func parseGetAccountInfoConfMap(params []interface{}) (*GetAccountInfoConfig, er conf := &GetAccountInfoConfig{} - // commitment is optional - commitmentObj, ok := confMap["commitment"] - if ok { - commitmentStr, ok := commitmentObj.(string) - if !ok || (commitmentStr != "processed" && commitmentStr != "confirmed" && commitmentStr != "finalized") { - return nil, fmt.Errorf("invalid commitment") - } - conf.Commitment = commitmentStr + if err := validateRootedCommitment(confMap["commitment"]); err != nil { + return nil, err + } + if commitment, ok := confMap["commitment"].(string); ok { + conf.Commitment = commitment } - if minContextSlot, ok := confMap["minContextSlot"]; ok { + if minContextSlot, ok := confMap["minContextSlot"]; ok && minContextSlot != nil { conf.MinContextSlot, err = rpcUint64(minContextSlot, "minContextSlot") if err != nil { return nil, err @@ -147,7 +142,7 @@ func parseGetAccountInfoConfMap(params []interface{}) (*GetAccountInfoConfig, er // encoding type is optional. defaults to base58. encodingObj, ok := confMap["encoding"] - if ok { + if ok && encodingObj != nil { encodingStr, ok := encodingObj.(string) if ok { var encodingType int @@ -163,8 +158,7 @@ func parseGetAccountInfoConfMap(params []interface{}) (*GetAccountInfoConfig, er // dataSlice is optional. if present, it must contain both 'length' and 'offset' fields. dataSliceObj, ok := confMap["dataSlice"] - if ok { - mlog.Log.Infof("dataSliceObj: %+v, type: %s", dataSliceObj, reflect.TypeOf(dataSliceObj)) + if ok && dataSliceObj != nil { dsMap, ok := dataSliceObj.(map[string]interface{}) if ok { conf.DataSlice, err = parseDataSlice(dsMap) @@ -183,30 +177,22 @@ func parseDataSlice(dataSlice map[string]interface{}) (*GetAccountInfoDataSlice, lengthObj, hasLen := dataSlice["length"] offsetObj, hasOffset := dataSlice["offset"] - if hasLen && !hasOffset { + if !hasOffset { return nil, fmt.Errorf("missing dataSlice field offset") - } else if !hasLen && hasOffset { + } else if !hasLen { return nil, fmt.Errorf("missing dataSlice field length") } - ds := &GetAccountInfoDataSlice{} - if hasLen == hasOffset && hasLen { - length, ok := lengthObj.(float64) - if !ok { - return nil, fmt.Errorf("invalid dataSlice length, expected usize, got %s", reflect.TypeOf(lengthObj)) - } - offset, ok := offsetObj.(float64) - if !ok { - return nil, fmt.Errorf("invalid dataSlice offset, expected usize, got %s", reflect.TypeOf(offsetObj)) - } - - lengthUint := uint64(length) - offsetUint := uint64(offset) - ds.Len = &lengthUint - ds.Offset = &offsetUint + length, err := rpcUint64(lengthObj, "dataSlice length") + if err != nil { + return nil, err + } + offset, err := rpcUint64(offsetObj, "dataSlice offset") + if err != nil { + return nil, err } - return ds, nil + return &GetAccountInfoDataSlice{Len: &length, Offset: &offset}, nil } func parseGetAcctDataEncodingType(encodingStr string) (int, error) { diff --git a/pkg/rpcserver/get_account_info_test.go b/pkg/rpcserver/get_account_info_test.go index c70f2fe98..48ad909d8 100644 --- a/pkg/rpcserver/get_account_info_test.go +++ b/pkg/rpcserver/get_account_info_test.go @@ -2,13 +2,68 @@ package rpcserver import ( "encoding/json" + "fmt" + "net/http/httptest" + "strings" "testing" "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/sealevel" "github.com/gagliardetto/solana-go" "github.com/stretchr/testify/require" ) +func TestGetAccountInfoParameterHTTP(t *testing.T) { + db := newRPCAccountsDB(t) + key := solana.PublicKey{87} + _, err := db.CommitBatch([]accounts.SlotDelta{{Slot: 10, Delta: []*accounts.Account{{Key: key, Lamports: 42, Data: []byte("abcd")}}}}, 10, nil, nil) + require.NoError(t, err) + server := NewRpcServer(db, 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 432000}, solana.Hash{1}) + require.NoError(t, server.listener.Close()) + server.SetRootedBankState(10, 10, 0) + endpoint := httptest.NewServer(server) + t.Cleanup(endpoint.Close) + for _, test := range []struct { + config string + code int + data string + }{ + {`{"encoding":"base64","minContextSlot":10}`, 0, "YWJjZA=="}, + {`{"encoding":"base64","minContextSlot":10,"dataSlice":{"offset":1,"length":2}}`, 0, "YmM="}, + {`{"encoding":"base64","dataSlice":{"offset":18446744073709551615,"length":1}}`, 0, ""}, + {`{"encoding":"base64","commitment":null,"minContextSlot":null,"dataSlice":null}`, 0, "YWJjZA=="}, + {`{"minContextSlot":9007199254740993}`, -32016, ""}, + {`{"minContextSlot":1.0000000000000001}`, -32602, ""}, + {`{"dataSlice":{"offset":-1,"length":1}}`, -32602, ""}, + {`{"dataSlice":{"offset":0,"length":1.5}}`, -32602, ""}, + {`{"dataSlice":{}}`, -32602, ""}, + {`{"dataSlice":{"offset":0,"length":null}}`, -32602, ""}, + } { + t.Run(test.config, func(t *testing.T) { + body := fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":"getAccountInfo","params":[%q,%s]}`, key.String(), test.config) + response, err := endpoint.Client().Post(endpoint.URL, "application/json", strings.NewReader(body)) + require.NoError(t, err) + defer response.Body.Close() + var got struct { + Result GetAccountInfoResp `json:"result"` + Error *struct { + Code int `json:"code"` + } `json:"error"` + } + require.NoError(t, json.NewDecoder(response.Body).Decode(&got)) + if test.code != 0 { + require.NotNil(t, got.Error) + require.Equal(t, test.code, got.Error.Code) + return + } + require.Nil(t, got.Error) + require.Equal(t, uint64(10), got.Result.Context.Slot) + require.NotNil(t, got.Result.Value) + require.Equal(t, []interface{}{test.data, "base64"}, got.Result.Value.Data) + }) + } +} + func TestGetAccountInfoAccountExistence(t *testing.T) { for _, state := range []string{"never-created", "funded-empty-data", "deleted"} { t.Run(state, func(t *testing.T) { diff --git a/pkg/rpcserver/get_balance.go b/pkg/rpcserver/get_balance.go index dcc9036b0..23ec30501 100644 --- a/pkg/rpcserver/get_balance.go +++ b/pkg/rpcserver/get_balance.go @@ -25,7 +25,7 @@ func (rpcServer *RpcServer) GetBalance(ctx context.Context, p jsonrpc.RawParams) return GetBalanceResp{}, err } - params, err := jsonrpc.DecodeParams[[]interface{}](p) + params, err := decodeOptionalParams(p) if err != nil { return GetBalanceResp{}, &InvalidParamsError{Message: fmt.Sprintf("decoding params: %v", err)} } @@ -72,14 +72,11 @@ func parseBalanceConfig(params []interface{}) (*uint64, error) { if !ok { return nil, &InvalidParamsError{Message: "invalid getBalance config"} } - if commitment, exists := config["commitment"]; exists { - value, ok := commitment.(string) - if !ok || (value != "processed" && value != "confirmed" && value != "finalized") { - return nil, &InvalidParamsError{Message: "invalid commitment"} - } + if err := validateRootedCommitment(config["commitment"]); err != nil { + return nil, err } value, exists := config["minContextSlot"] - if !exists { + if !exists || value == nil { return nil, nil } minContextSlot, err := rpcUint64(value, "minContextSlot") diff --git a/pkg/rpcserver/get_block_production.go b/pkg/rpcserver/get_block_production.go index 54385ccf1..550963073 100644 --- a/pkg/rpcserver/get_block_production.go +++ b/pkg/rpcserver/get_block_production.go @@ -2,8 +2,9 @@ package rpcserver import ( "context" + "encoding/json" "fmt" - "math" + "strconv" "github.com/Overclock-Validator/mithril/pkg/global" "github.com/Overclock-Validator/mithril/pkg/sealevel" @@ -120,13 +121,10 @@ func parseBlockProductionConfig(params []interface{}) (getBlockProductionConfig, if !ok { return config, &InvalidParamsError{Message: "invalid getBlockProduction config"} } - if commitment, exists := values["commitment"]; exists { - value, ok := commitment.(string) - if !ok || (value != "processed" && value != "confirmed" && value != "finalized") { - return config, &InvalidParamsError{Message: "invalid commitment"} - } + if err := validateRootedCommitment(values["commitment"]); err != nil { + return config, err } - if raw, exists := values["identity"]; exists { + if raw, exists := values["identity"]; exists && raw != nil { value, ok := raw.(string) if !ok { return config, &InvalidParamsError{Message: "invalid identity"} @@ -137,7 +135,7 @@ func parseBlockProductionConfig(params []interface{}) (getBlockProductionConfig, } config.identity = &identity } - if raw, exists := values["range"]; exists { + if raw, exists := values["range"]; exists && raw != nil { rangeValues, ok := raw.(map[string]interface{}) if !ok { return config, &InvalidParamsError{Message: "invalid range"} @@ -163,11 +161,15 @@ func parseBlockProductionConfig(params []interface{}) (getBlockProductionConfig, } func rpcUint64(raw interface{}, name string) (uint64, error) { - value, ok := raw.(float64) - if !ok || value < 0 || value >= math.Exp2(64) || math.Trunc(value) != value { + value, ok := raw.(json.Number) + if !ok { + return 0, &InvalidParamsError{Message: fmt.Sprintf("invalid %s", name)} + } + parsed, err := strconv.ParseUint(string(value), 10, 64) + if err != nil { return 0, &InvalidParamsError{Message: fmt.Sprintf("invalid %s", name)} } - return uint64(value), nil + return parsed, nil } func slotHistoryBounds(history *sealevel.SysvarSlotHistory) (uint64, uint64, error) { diff --git a/pkg/rpcserver/get_epoch_info.go b/pkg/rpcserver/get_epoch_info.go index 5f4ee494e..f60c0a844 100644 --- a/pkg/rpcserver/get_epoch_info.go +++ b/pkg/rpcserver/get_epoch_info.go @@ -32,14 +32,14 @@ func (rpcServer *RpcServer) GetEpochInfo(ctx context.Context, p jsonrpc.RawParam if !ok { return GetEpochInfoResp{}, &InvalidParamsError{Message: "invalid getEpochInfo config"} } - if raw, exists := config["commitment"]; exists { + if raw, exists := config["commitment"]; exists && raw != nil { value, ok := raw.(string) if !ok || (value != "processed" && value != "confirmed" && value != "finalized") { return GetEpochInfoResp{}, &InvalidParamsError{Message: "invalid commitment"} } commitment = value } - if raw, exists := config["minContextSlot"]; exists { + if raw, exists := config["minContextSlot"]; exists && raw != nil { value, err := rpcUint64(raw, "minContextSlot") if err != nil { return GetEpochInfoResp{}, err diff --git a/pkg/rpcserver/get_leader_schedule.go b/pkg/rpcserver/get_leader_schedule.go index 0310d3f2e..aeaebc792 100644 --- a/pkg/rpcserver/get_leader_schedule.go +++ b/pkg/rpcserver/get_leader_schedule.go @@ -10,7 +10,8 @@ import ( ) type getLeaderScheduleConfig struct { - identity *solana.PublicKey + identity *solana.PublicKey + keyByVoteAccount bool } func (rpcServer *RpcServer) GetLeaderSchedule(ctx context.Context, p jsonrpc.RawParams) (map[string][]uint64, error) { @@ -61,15 +62,29 @@ func (rpcServer *RpcServer) GetLeaderSchedule(ctx context.Context, p jsonrpc.Raw slotsInEpoch := rpcServer.epochSchedule.SlotsInEpoch(epoch) schedule := make(map[string][]uint64) for index := uint64(0); index < slotsInEpoch; index++ { - leader, ok := global.LeaderForSlot(firstSlot + index) - if !ok { - return nil, nil + scheduledSlot := firstSlot + index + var leader, key solana.PublicKey + var ok bool + if config.keyByVoteAccount { + leader, key, ok = global.LeaderForSlotWithVoteAccount(scheduledSlot) + if !ok { + if _, exists := global.LeaderForSlot(scheduledSlot); exists { + return nil, fmt.Errorf("vote-account leader schedule unavailable for slot %d", scheduledSlot) + } + return nil, nil + } + } else { + leader, ok = global.LeaderForSlot(scheduledSlot) + if !ok { + return nil, nil + } + key = leader } if config.identity != nil && leader != *config.identity { continue } - key := leader.String() - schedule[key] = append(schedule[key], index) + keyString := key.String() + schedule[keyString] = append(schedule[keyString], index) } return schedule, nil } @@ -83,13 +98,10 @@ func parseLeaderScheduleConfig(raw interface{}) (getLeaderScheduleConfig, error) if !ok { return config, &InvalidParamsError{Message: "invalid getLeaderSchedule config"} } - if commitment, exists := values["commitment"]; exists { - value, ok := commitment.(string) - if !ok || (value != "processed" && value != "confirmed" && value != "finalized") { - return config, &InvalidParamsError{Message: "invalid commitment"} - } + if err := validateRootedCommitment(values["commitment"]); err != nil { + return config, err } - if raw, exists := values["identity"]; exists { + if raw, exists := values["identity"]; exists && raw != nil { value, ok := raw.(string) if !ok { return config, &InvalidParamsError{Message: "invalid identity"} @@ -100,14 +112,12 @@ func parseLeaderScheduleConfig(raw interface{}) (getLeaderScheduleConfig, error) } config.identity = &identity } - if raw, exists := values["keyByVoteAccount"]; exists { + if raw, exists := values["keyByVoteAccount"]; exists && raw != nil { value, ok := raw.(bool) if !ok { return config, &InvalidParamsError{Message: "invalid keyByVoteAccount"} } - if value { - return config, &InvalidParamsError{Message: "keyByVoteAccount is not supported"} - } + config.keyByVoteAccount = value } return config, nil } diff --git a/pkg/rpcserver/get_leader_schedule_test.go b/pkg/rpcserver/get_leader_schedule_test.go index 836924bb0..43ee1decd 100644 --- a/pkg/rpcserver/get_leader_schedule_test.go +++ b/pkg/rpcserver/get_leader_schedule_test.go @@ -4,6 +4,7 @@ import ( "errors" "testing" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" "github.com/Overclock-Validator/mithril/pkg/global" "github.com/Overclock-Validator/mithril/pkg/leaderschedule" "github.com/Overclock-Validator/mithril/pkg/sealevel" @@ -55,6 +56,52 @@ func TestGetLeaderScheduleReturnsRelativeEpochSlots(t *testing.T) { require.Empty(t, missing) } +func TestGetLeaderScheduleKeysBySelectedVoteAccount(t *testing.T) { + epochSchedule := &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 32} + const epoch = 13 + firstSlot := epochSchedule.FirstSlotInEpoch(epoch) + node := solana.PublicKey{31} + other := solana.PublicKey{32} + votes := map[solana.PublicKey]*epochstakes.VoteAccount{ + {41}: {NodePubkey: node}, + {42}: {NodePubkey: node}, + {43}: {NodePubkey: other}, + } + schedule := leaderschedule.New(votes, map[solana.PublicKey]uint64{{41}: 100, {42}: 100, {43}: 100}, epochSchedule, epoch, 32, 1) + global.SetLeaderScheduleForEpoch(epoch, schedule) + t.Cleanup(func() { global.SetLeaderScheduleForEpoch(epoch, nil) }) + server := &RpcServer{epochSchedule: epochSchedule} + server.SetRootedBankState(firstSlot, firstSlot, 0) + for _, identity := range []string{"", node.String(), other.String()} { + config := map[string]interface{}{"keyByVoteAccount": true} + if identity != "" { + config["identity"] = identity + } + want := make(map[string][]uint64) + for index := uint64(0); index < 32; index++ { + leader, vote, ok := schedule.LeaderForSlotWithVoteAccount(firstSlot + index) + require.True(t, ok) + if identity == "" || leader.String() == identity { + want[vote.String()] = append(want[vote.String()], index) + } + } + got, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{nil, config})) + require.NoError(t, err) + require.Equal(t, want, got) + if identity == node.String() { + require.Len(t, got, 2, "two vote accounts sharing an identity must stay distinct") + } + } + global.SetLeaderScheduleForEpoch(epoch, leaderschedule.NewLeaderScheduleFromKeyedSlots(map[solana.PublicKey][]uint64{node: {0}}, firstSlot)) + got, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{nil, map[string]interface{}{"keyByVoteAccount": true}})) + require.ErrorContains(t, err, "vote-account leader schedule unavailable") + require.Nil(t, got, "node-keyed compatibility schedules must not invent vote accounts") + global.SetLeaderScheduleForEpoch(epoch, nil) + got, err = server.GetLeaderSchedule(t.Context(), mustRawParams(t, []interface{}{nil, map[string]interface{}{"keyByVoteAccount": true}})) + require.NoError(t, err) + require.Nil(t, got, "missing epochs keep the standard null result") +} + func TestGetLeaderScheduleReturnsNullWhenEpochScheduleIsUnavailable(t *testing.T) { server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 4}} server.SetRootedBankState(40_002, 1, 0) @@ -73,7 +120,7 @@ func TestGetLeaderScheduleRejectsInvalidParams(t *testing.T) { {float64(1), true}, {float64(1), map[string]interface{}{"commitment": "unknown"}}, {float64(1), map[string]interface{}{"identity": "invalid"}}, - {float64(1), map[string]interface{}{"keyByVoteAccount": true}}, + {float64(1), map[string]interface{}{"keyByVoteAccount": "true"}}, {float64(1), nil, nil}, } { _, err := server.GetLeaderSchedule(t.Context(), mustRawParams(t, params)) diff --git a/pkg/rpcserver/get_vote_accounts.go b/pkg/rpcserver/get_vote_accounts.go index ab45b9aff..78ca737f5 100644 --- a/pkg/rpcserver/get_vote_accounts.go +++ b/pkg/rpcserver/get_vote_accounts.go @@ -35,8 +35,9 @@ type VoteAccountInfo struct { } type getVoteAccountsConfig struct { - votePubkey *solana.PublicKey - delinquentSlotDistance uint64 + votePubkey *solana.PublicKey + delinquentSlotDistance uint64 + keepUnstakedDelinquents bool } func (rpcServer *RpcServer) GetVoteAccounts(ctx context.Context, p jsonrpc.RawParams) (GetVoteAccountsResp, error) { @@ -135,7 +136,7 @@ func (rpcServer *RpcServer) GetVoteAccounts(ctx context.Context, p jsonrpc.RawPa } if current { response.Current = append(response.Current, info) - } else if info.ActivatedStake > 0 { + } else if info.ActivatedStake > 0 || config.keepUnstakedDelinquents { response.Delinquent = append(response.Delinquent, info) } } @@ -159,13 +160,10 @@ func parseGetVoteAccountsConfig(params []interface{}) (getVoteAccountsConfig, er if !ok { return config, &InvalidParamsError{Message: "invalid getVoteAccounts config"} } - if commitment, exists := values["commitment"]; exists { - value, ok := commitment.(string) - if !ok || (value != "processed" && value != "confirmed" && value != "finalized") { - return config, &InvalidParamsError{Message: "invalid commitment"} - } + if err := validateRootedCommitment(values["commitment"]); err != nil { + return config, err } - if raw, exists := values["votePubkey"]; exists { + if raw, exists := values["votePubkey"]; exists && raw != nil { value, ok := raw.(string) if !ok { return config, &InvalidParamsError{Message: "invalid votePubkey"} @@ -176,16 +174,14 @@ func parseGetVoteAccountsConfig(params []interface{}) (getVoteAccountsConfig, er } config.votePubkey = &pubkey } - if raw, exists := values["keepUnstakedDelinquents"]; exists { + if raw, exists := values["keepUnstakedDelinquents"]; exists && raw != nil { value, ok := raw.(bool) if !ok { return config, &InvalidParamsError{Message: "invalid keepUnstakedDelinquents"} } - if value { - return config, &InvalidParamsError{Message: "keepUnstakedDelinquents is not supported"} - } + config.keepUnstakedDelinquents = value } - if raw, exists := values["delinquentSlotDistance"]; exists { + if raw, exists := values["delinquentSlotDistance"]; exists && raw != nil { value, err := rpcUint64(raw, "delinquentSlotDistance") if err != nil { return config, err diff --git a/pkg/rpcserver/get_vote_accounts_test.go b/pkg/rpcserver/get_vote_accounts_test.go index d71d709c2..db5608c7c 100644 --- a/pkg/rpcserver/get_vote_accounts_test.go +++ b/pkg/rpcserver/get_vote_accounts_test.go @@ -162,6 +162,32 @@ func TestGetVoteAccountsUsesRootedVoteState(t *testing.T) { require.Len(t, cold.Delinquent, 1) require.Equal(t, delinquentVote.String(), cold.Delinquent[0].VotePubkey) + // A zero distance makes every fixture vote delinquent, including the + // initialized vote account with no stake or epoch membership. + for _, keep := range []interface{}{true, false, nil} { + config := map[string]interface{}{"delinquentSlotDistance": 0, "keepUnstakedDelinquents": keep} + all, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{config})) + require.NoError(t, err) + require.Empty(t, all.Current) + if keep == true { + require.Len(t, all.Delinquent, 3) + require.Equal(t, newVote.String(), all.Delinquent[2].VotePubkey) + require.Zero(t, all.Delinquent[2].ActivatedStake) + require.False(t, all.Delinquent[2].EpochVoteAccount) + } else { + require.Len(t, all.Delinquent, 2) + } + config["votePubkey"] = newVote.String() + filtered, err := server.GetVoteAccounts(t.Context(), mustRawParams(t, []interface{}{config})) + require.NoError(t, err) + require.Empty(t, filtered.Current) + if keep == true { + require.Len(t, filtered.Delinquent, 1) + } else { + require.Empty(t, filtered.Delinquent) + } + } + // Candidate keys are append-only, but an account that changes owner must // no longer be returned as a vote account. _, err = db.CommitBatch([]accounts.SlotDelta{{Slot: 501, Delta: []*accounts.Account{{ @@ -202,7 +228,6 @@ func TestGetVoteAccountsRejectsInvalidConfig(t *testing.T) { {map[string]interface{}{"commitment": "unknown"}}, {map[string]interface{}{"votePubkey": "invalid"}}, {map[string]interface{}{"keepUnstakedDelinquents": "yes"}}, - {map[string]interface{}{"keepUnstakedDelinquents": true}}, {map[string]interface{}{"delinquentSlotDistance": 1.5}}, {map[string]interface{}{}, map[string]interface{}{}}, } { diff --git a/pkg/rpcserver/optional_params.go b/pkg/rpcserver/optional_params.go index 74c000e25..f6a609b13 100644 --- a/pkg/rpcserver/optional_params.go +++ b/pkg/rpcserver/optional_params.go @@ -1,10 +1,45 @@ package rpcserver -import "github.com/filecoin-project/go-jsonrpc" +import ( + "bytes" + "encoding/json" + "fmt" + "io" + + "github.com/filecoin-project/go-jsonrpc" +) func decodeOptionalParams(p jsonrpc.RawParams) ([]interface{}, error) { if len(p) == 0 { return nil, nil } - return jsonrpc.DecodeParams[[]interface{}](p) + decoder := json.NewDecoder(bytes.NewReader(p)) + decoder.UseNumber() + var params []interface{} + if err := decoder.Decode(¶ms); err != nil { + return nil, err + } + if err := decoder.Decode(new(interface{})); err != io.EOF { + return nil, fmt.Errorf("parameters must contain one JSON value") + } + return params, nil +} + +// validateRootedCommitment rejects requests that need an unpublished account view. +func validateRootedCommitment(raw interface{}) error { + if raw == nil { + return nil + } + value, ok := raw.(string) + if !ok { + return &InvalidParamsError{Message: "invalid commitment"} + } + switch value { + case "confirmed", "finalized": + return nil + case "processed": + return &InvalidParamsError{Message: "processed commitment is not supported by this method; use confirmed or finalized"} + default: + return &InvalidParamsError{Message: "invalid commitment"} + } } diff --git a/pkg/rpcserver/rooted_commitment_test.go b/pkg/rpcserver/rooted_commitment_test.go new file mode 100644 index 000000000..4eef9063c --- /dev/null +++ b/pkg/rpcserver/rooted_commitment_test.go @@ -0,0 +1,54 @@ +package rpcserver + +import ( + "encoding/json" + "fmt" + "net/http/httptest" + "strings" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestRootedRPCRejectsProcessedHTTP(t *testing.T) { + server := NewRpcServer(newRPCAccountsDB(t), 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 1000}, solana.Hash{1}) + require.NoError(t, server.listener.Close()) + server.SetRootedBankState(500, 400, 0) + oldSlot := global.Slot() + global.SetSlot(600) + t.Cleanup(func() { global.SetSlot(oldSlot) }) + endpoint := httptest.NewServer(server) + t.Cleanup(endpoint.Close) + key := (solana.PublicKey{8}).String() + for _, test := range []struct{ method, params string }{ + {"getAccountInfo", `["` + key + `",{"commitment":"processed"}]`}, + {"getBalance", `["` + key + `",{"commitment":"processed"}]`}, + {"getVoteAccounts", `[{"commitment":"processed"}]`}, + {"getLeaderSchedule", `[{"commitment":"processed"}]`}, + {"getLeaderSchedule", `[null,{"commitment":"processed"}]`}, + {"getLeaderSchedule", `[500,{"commitment":"processed"}]`}, + {"getBlockProduction", `[{"commitment":"processed"}]`}, + } { + t.Run(test.method+test.params, func(t *testing.T) { + request := fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":%q,"params":%s}`, test.method, test.params) + response, err := endpoint.Client().Post(endpoint.URL, "application/json", strings.NewReader(request)) + require.NoError(t, err) + defer response.Body.Close() + var reply struct { + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error"` + } + require.NoError(t, json.NewDecoder(response.Body).Decode(&reply)) + require.NotNil(t, reply.Error, "must not substitute rooted state for processed") + require.Equal(t, -32602, reply.Error.Code) + require.Contains(t, reply.Error.Message, "processed commitment is not supported") + require.Empty(t, reply.Result) + }) + } +} diff --git a/pkg/rpcserver/rpc_parameter_contract_test.go b/pkg/rpcserver/rpc_parameter_contract_test.go new file mode 100644 index 000000000..353b85a8b --- /dev/null +++ b/pkg/rpcserver/rpc_parameter_contract_test.go @@ -0,0 +1,175 @@ +package rpcserver + +import ( + "encoding/json" + "errors" + "fmt" + "net/http/httptest" + "strings" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/filecoin-project/go-jsonrpc" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestSpecExactRPCParameters(t *testing.T) { + server := &RpcServer{epochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 432000}} + server.SetRootedBankState(9007199254740992, 4, 5) + t.Run("getSlot_exact_control", func(t *testing.T) { + _, err := server.GetSlot(t.Context(), jsonrpc.RawParams(`[{"minContextSlot":9007199254740993}]`)) + var behind *MinContextSlotNotReachedError + if !errors.As(err, &behind) { + t.Fatalf("expected minimum-context error, got %v", err) + } + }) + t.Run("epoch_minimum_must_not_round_down", func(t *testing.T) { + got, err := server.GetEpochInfo(t.Context(), jsonrpc.RawParams(`[{"minContextSlot":9007199254740993}]`)) + var behind *MinContextSlotNotReachedError + if !errors.As(err, &behind) { + t.Fatalf("requested minimum 9007199254740993; returned slot %d with error %v", got.AbsoluteSlot, err) + } + }) + t.Run("distance_accepts_full_u64", func(t *testing.T) { + params, err := decodeOptionalParams(jsonrpc.RawParams(`[{"delinquentSlotDistance":18446744073709551615}]`)) + if err != nil { + t.Fatal(err) + } + got, err := parseGetVoteAccountsConfig(params) + if err != nil || got.delinquentSlotDistance != ^uint64(0) { + t.Fatalf("valid maximum u64: value=%d error=%v", got.delinquentSlotDistance, err) + } + }) + t.Run("fraction_must_not_round_to_integer", func(t *testing.T) { + _, err := server.GetEpochInfo(t.Context(), jsonrpc.RawParams(`[{"minContextSlot":1.0000000000000001}]`)) + var invalid *InvalidParamsError + if !errors.As(err, &invalid) { + t.Fatalf("fractional u64 accepted: error=%v", err) + } + }) + t.Run("null_optional_minimum", func(t *testing.T) { + _, err := server.GetEpochInfo(t.Context(), jsonrpc.RawParams(`[{"minContextSlot":null}]`)) + if err != nil { + t.Fatalf("optional minimum null should use default: %v", err) + } + }) + t.Run("balance_minimum_must_not_round_down", func(t *testing.T) { + balanceServer := &RpcServer{acctsDb: newRPCAccountsDB(t)} + balanceServer.SetRootedBankState(9007199254740992, 4, 5) + params := jsonrpc.RawParams(`["` + (solana.PublicKey{8}).String() + `",{"minContextSlot":9007199254740993}]`) + got, err := balanceServer.GetBalance(t.Context(), params) + var behind *MinContextSlotNotReachedError + if !errors.As(err, &behind) { + t.Fatalf("requested minimum 9007199254740993; returned slot %d with error %v", got.Context.Slot, err) + } + }) +} + +func TestRPCParameterHTTPContract(t *testing.T) { + server := NewRpcServer(newRPCAccountsDB(t), 0, &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 432000}, solana.Hash{1}) + require.NoError(t, server.listener.Close()) + server.SetRootedBankState(9007199254740992, 4, 5) + endpoint := httptest.NewServer(server) + t.Cleanup(endpoint.Close) + key := (solana.PublicKey{8}).String() + for _, test := range []struct { + method, params string + code int + }{ + {"getEpochInfo", `[{"minContextSlot":9007199254740993}]`, -32016}, + {"getBalance", `["` + key + `",{"minContextSlot":9007199254740993}]`, -32016}, + {"getEpochInfo", `[{"commitment":null,"minContextSlot":null}]`, 0}, + {"getBalance", `["` + key + `",{"commitment":null,"minContextSlot":null}]`, 0}, + {"getEpochInfo", `[{"minContextSlot":1.0000000000000001}]`, -32602}, + {"getBalance", `["` + key + `",{"minContextSlot":1.0000000000000001}]`, -32602}, + {"getLeaderSchedule", `[1.0000000000000001]`, -32602}, + {"getVoteAccounts", `[{"delinquentSlotDistance":1.0000000000000001}]`, -32602}, + {"getBlockProduction", `[{"range":{"firstSlot":1.0000000000000001}}]`, -32602}, + {"getBlockProduction", `[{"range":{"firstSlot":0,"lastSlot":1.0000000000000001}}]`, -32602}, + {"getBlockProduction", `[{"range":{"firstSlot":null}}]`, -32602}, + {"getBlockProduction", `[{"range":{}}]`, -32602}, + {"getBalance", `[null]`, -32602}, + {"getEpochInfo", `[{"minContextSlot":18446744073709551616}]`, -32602}, + {"getEpochInfo", `[{"minContextSlot":-1}]`, -32602}, + {"getEpochInfo", `[{"minContextSlot":"42"}]`, -32602}, + {"getEpochInfo", `[{"minContextSlot":true}]`, -32602}, + {"getEpochInfo", `[{"minContextSlot":1e0}]`, -32602}, + {"getVoteAccounts", `[{"keepUnstakedDelinquents":"true"}]`, -32602}, + {"getLeaderSchedule", `[null,{"keyByVoteAccount":"true"}]`, -32602}, + {"getLeaderSchedule", `[null,{"keyByVoteAccount":true}]`, 0}, + {"getLeaderSchedule", `[null,{"keyByVoteAccount":null}]`, 0}, + } { + t.Run(test.method+"/"+test.params, func(t *testing.T) { + body := fmt.Sprintf(`{"jsonrpc":"2.0","id":1,"method":%q,"params":%s}`, test.method, test.params) + response, err := endpoint.Client().Post(endpoint.URL, "application/json", strings.NewReader(body)) + require.NoError(t, err) + defer response.Body.Close() + var got struct { + Result json.RawMessage `json:"result"` + Error *struct { + Code int `json:"code"` + Data struct { + ContextSlot uint64 `json:"contextSlot"` + } `json:"data"` + } `json:"error"` + } + require.NoError(t, json.NewDecoder(response.Body).Decode(&got)) + if test.code == 0 { + require.Nil(t, got.Error) + require.NotEmpty(t, got.Result) + if test.method == "getEpochInfo" { + var result GetEpochInfoResp + require.NoError(t, json.Unmarshal(got.Result, &result)) + require.Equal(t, uint64(9007199254740992), result.AbsoluteSlot) + } else if test.method == "getBalance" { + var result GetBalanceResp + require.NoError(t, json.Unmarshal(got.Result, &result)) + require.Equal(t, uint64(9007199254740992), result.Context.Slot) + require.Zero(t, result.Value) + } + } else { + require.NotNil(t, got.Error) + require.Equal(t, test.code, got.Error.Code) + if test.code == -32016 { + require.Equal(t, uint64(9007199254740992), got.Error.Data.ContextSlot) + } + } + }) + } +} + +func TestRPCOptionalConfigAndDecoderContract(t *testing.T) { + params, err := decodeOptionalParams(jsonrpc.RawParams(`[{"commitment":null,"identity":null,"votePubkey":null,"range":null,"keyByVoteAccount":null,"keepUnstakedDelinquents":null,"delinquentSlotDistance":null}]`)) + require.NoError(t, err) + vote, err := parseGetVoteAccountsConfig(params) + require.NoError(t, err) + require.Nil(t, vote.votePubkey) + require.Equal(t, uint64(128), vote.delinquentSlotDistance) + leaders, err := parseLeaderScheduleConfig(params[0]) + require.NoError(t, err) + require.Nil(t, leaders.identity) + production, err := parseBlockProductionConfig(params) + require.NoError(t, err) + require.Nil(t, production.identity) + require.Nil(t, production.firstSlot) + require.Nil(t, production.lastSlot) + + params, err = decodeOptionalParams(jsonrpc.RawParams(`[{"range":{"firstSlot":9007199254740993,"lastSlot":18446744073709551615}}]`)) + require.NoError(t, err) + production, err = parseBlockProductionConfig(params) + require.NoError(t, err) + require.Equal(t, uint64(9007199254740993), *production.firstSlot) + require.Equal(t, ^uint64(0), *production.lastSlot) + params, err = decodeOptionalParams(jsonrpc.RawParams(`[{"range":{"firstSlot":0,"lastSlot":null}}]`)) + require.NoError(t, err) + production, err = parseBlockProductionConfig(params) + require.NoError(t, err) + require.Nil(t, production.lastSlot) + for _, raw := range []string{`[] []`, `[] trailing`, `[`, `{}`, `true`} { + _, err := decodeOptionalParams(jsonrpc.RawParams(raw)) + require.Error(t, err, raw) + } + _, err = decodeOptionalParams(jsonrpc.RawParams("[] \n\t")) + require.NoError(t, err) +}