From d165d60b4b9ef730645f0f369dfaebb174cf1e82 Mon Sep 17 00:00:00 2001 From: smcio Date: Tue, 13 Oct 2026 18:44:18 +0000 Subject: [PATCH 1/2] fix epoch rounding bug & startup issue --- .../node/manifest_seed_refresh_test.go | 26 ++++ cmd/mithril/node/node.go | 13 +- pkg/epochstakes/commission_metadata.go | 51 ++++++ pkg/epochstakes/commission_metadata_test.go | 60 +++++++ pkg/epochstakes/epoch_stakes.go | 74 +++++---- pkg/global/global_ctx.go | 45 ------ pkg/replay/block.go | 38 ++++- pkg/replay/epoch.go | 34 ++-- pkg/replay/epoch_boundary_stakes_test.go | 38 +++++ pkg/replay/epoch_inflation_test.go | 16 ++ pkg/replay/epoch_stakes_seed.go | 35 +++++ pkg/replay/epoch_stakes_seed_test.go | 34 ++++ pkg/replay/initial_blockhash_test.go | 146 ++++++++++++++++++ pkg/replay/startup_epoch_boundary_test.go | 13 +- .../slot_452018181_snapshot_nonce.json | 99 ++++++++++++ pkg/rewards/delayed_commission_test.go | 72 +++++++++ pkg/rewards/inflation.go | 8 +- pkg/rewards/inflation_pow.go | 18 +++ pkg/rewards/inflation_pow_test.go | 19 +++ pkg/rewards/rewards.go | 67 ++++---- pkg/sealevel/vote_state.go | 17 ++ pkg/snapshot/manifest_decoder.go | 21 +-- pkg/snapshot/manifest_decoder_test.go | 37 +++++ pkg/snapshot/manifest_seed.go | 19 +-- 24 files changed, 840 insertions(+), 160 deletions(-) create mode 100644 pkg/epochstakes/commission_metadata.go create mode 100644 pkg/epochstakes/commission_metadata_test.go create mode 100644 pkg/replay/initial_blockhash_test.go create mode 100644 pkg/replay/testdata/slot_452018181_snapshot_nonce.json create mode 100644 pkg/rewards/delayed_commission_test.go create mode 100644 pkg/rewards/inflation_pow.go create mode 100644 pkg/rewards/inflation_pow_test.go diff --git a/cmd/mithril/node/manifest_seed_refresh_test.go b/cmd/mithril/node/manifest_seed_refresh_test.go index 48a1f41c8..b5895e24d 100644 --- a/cmd/mithril/node/manifest_seed_refresh_test.go +++ b/cmd/mithril/node/manifest_seed_refresh_test.go @@ -1,11 +1,14 @@ package node import ( + "encoding/json" "testing" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" "github.com/Overclock-Validator/mithril/pkg/sealevel" "github.com/Overclock-Validator/mithril/pkg/snapshot" "github.com/Overclock-Validator/mithril/pkg/state" + "github.com/gagliardetto/solana-go" "github.com/stretchr/testify/require" ) @@ -35,3 +38,26 @@ func TestRefreshManifestSeedRestoresClearedEpochStakes(t *testing.T) { require.Contains(t, s.ManifestEpochStakes, uint64(7)) } + +func TestRefreshManifestSeedRestoresDiscardedCommissions(t *testing.T) { + key := solana.PublicKey{32} + bps := uint16(10000) + manifest := &snapshot.SnapshotManifest{ + Bank: &snapshot.DeserializableVersionedBank{Slot: 451934700, Epoch: 1058, EpochSchedule: sealevel.SysvarEpochSchedule{SlotsPerEpoch: 432000, LeaderScheduleSlotOffset: 432000, Warmup: true, FirstNormalEpoch: 14, FirstNormalSlot: 524256}}, + VersionedEpochStakes: []snapshot.VersionedEpochStakesPair{{Epoch: 1058, Val: snapshot.VersionedEpochStakes{TotalStake: 42, Stakes: snapshot.Stake{VoteAccounts: []snapshot.VoteAccountsPair{{Key: key, Stake: 42, Value: snapshot.VoteAccount{InflationRewardsCommissionBps: &bps}}}}}}}, + } + s := state.NewReadyState(manifest.Bank.Slot, 1058, "", "", 0, 0) + snapshot.PopulateManifestSeed(s, manifest) + var legacy epochstakes.PersistedEpochStakes + require.NoError(t, json.Unmarshal([]byte(s.ManifestEpochStakes[1058]), &legacy)) + legacy.VoteAccts[key.String()].InflationRewardsCommissionBps = nil + raw, err := json.Marshal(legacy) + require.NoError(t, err) + s.ManifestEpochStakes[1058] = string(raw) + dir := t.TempDir() + refreshManifestSeedFromManifest(dir, s, manifest) + require.True(t, epochstakes.HasInflationCommissionMetadata([]byte(s.ManifestEpochStakes[1058]))) + restored, err := state.LoadState(dir) + require.NoError(t, err) + require.Equal(t, s.ManifestEpochStakes, restored.ManifestEpochStakes) +} diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index c9c964064..eab065b68 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -33,6 +33,7 @@ import ( "github.com/Overclock-Validator/mithril/pkg/blockstream" "github.com/Overclock-Validator/mithril/pkg/config" consensusengine "github.com/Overclock-Validator/mithril/pkg/consensus" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" "github.com/Overclock-Validator/mithril/pkg/features" "github.com/Overclock-Validator/mithril/pkg/global" "github.com/Overclock-Validator/mithril/pkg/gossip" @@ -490,8 +491,16 @@ func refreshManifestSeedFromManifest(accountsPath string, s *state.MithrilState, // ManifestEpochStakes is deliberately cleared after replay starts to release // the large JSON seed. A later graceful state save persists that cleared // value, so schedule metadata alone does not mean the manifest seed is - // complete on restart. Rehydrate when the stakes payload is absent. - if manifestEpochScheduleSeedMatches(s, manifest) && + // complete on restart. Rehydrate when the payload is absent or an older + // binary discarded its historical commissions. + commissionsComplete := true + for _, seed := range s.ManifestEpochStakes { + if !epochstakes.HasInflationCommissionMetadata([]byte(seed)) { + commissionsComplete = false + break + } + } + if commissionsComplete && manifestEpochScheduleSeedMatches(s, manifest) && s.SnapshotEpoch == snapshotEpoch && len(s.ManifestEpochStakes) > 0 { return } diff --git a/pkg/epochstakes/commission_metadata.go b/pkg/epochstakes/commission_metadata.go new file mode 100644 index 000000000..e11cb9ea1 --- /dev/null +++ b/pkg/epochstakes/commission_metadata.go @@ -0,0 +1,51 @@ +package epochstakes + +import ( + "encoding/json" + "fmt" +) + +// HasInflationCommissionMetadata distinguishes complete historical state from +// checkpoints written before commissions were retained. Explicit zero is known. +func HasInflationCommissionMetadata(data []byte) bool { + var persisted PersistedEpochStakes + if err := json.Unmarshal(data, &persisted); err != nil { + return false + } + for _, vote := range persisted.VoteAccts { + if vote == nil || vote.InflationRewardsCommissionBps == nil { + return false + } + } + return true +} + +// RestoreInflationCommissionMetadata fills only missing commission fields from +// the same historical generation in the retained snapshot. It never substitutes +// current vote state or changes already persisted values, stake, or membership. +func RestoreInflationCommissionMetadata(data, historical []byte) ([]byte, error) { + var persisted, source PersistedEpochStakes + if err := json.Unmarshal(data, &persisted); err != nil { + return nil, err + } + if err := json.Unmarshal(historical, &source); err != nil { + return nil, err + } + if persisted.Epoch != source.Epoch { + return nil, fmt.Errorf("cannot restore commission metadata for epoch %d from epoch %d", persisted.Epoch, source.Epoch) + } + changed := false + for pubkey, vote := range persisted.VoteAccts { + if vote == nil || vote.InflationRewardsCommissionBps != nil { + continue + } + if original := source.VoteAccts[pubkey]; original != nil && original.InflationRewardsCommissionBps != nil { + vote.InflationRewardsCommissionBps = original.InflationRewardsCommissionBps + changed = true + } + } + if !changed { + return data, nil + } + return json.Marshal(persisted) +} diff --git a/pkg/epochstakes/commission_metadata_test.go b/pkg/epochstakes/commission_metadata_test.go new file mode 100644 index 000000000..5495fde4d --- /dev/null +++ b/pkg/epochstakes/commission_metadata_test.go @@ -0,0 +1,60 @@ +package epochstakes + +import ( + "encoding/json" + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestCommissionMetadataSurvivesRestartAndOwnsItsMemory(t *testing.T) { + for _, bps := range []uint16{0, 1234, 10000, 65535} { + vote := solana.PublicKey{1} + commission := bps + cache := NewEpochStakesCache() + cache.PutEpoch(1058, map[solana.PublicKey]uint64{vote: 42}, map[solana.PublicKey]*VoteAccount{vote: {InflationRewardsCommissionBps: &commission}}, 42) + commission++ + *cache.EpochStakesAccts(1058)[vote].InflationRewardsCommissionBps = 7 + raw, err := cache.SerializeEpoch(1058) + require.NoError(t, err) + require.True(t, HasInflationCommissionMetadata(raw)) + restored := NewEpochStakesCache() + _, err = restored.DeserializeAndLoadEpoch(raw) + require.NoError(t, err) + got, _ := restored.Snapshot(1058) + require.Equal(t, bps, *got.VoteAccounts[vote].InflationRewardsCommissionBps) + var old PersistedEpochStakes + require.NoError(t, json.Unmarshal(raw, &old)) + old.VoteAccts[vote.String()].InflationRewardsCommissionBps = nil + legacy, err := json.Marshal(old) + require.NoError(t, err) + require.False(t, HasInflationCommissionMetadata(legacy)) + _, err = restored.DeserializeAndLoadEpoch(legacy) + require.NoError(t, err) + got, _ = restored.Snapshot(1058) + require.Nil(t, got.VoteAccounts[vote].InflationRewardsCommissionBps) + } +} + +func TestRestoreCommissionsOnlyFillsMissingFieldsInSameGeneration(t *testing.T) { + zero, full := uint16(0), uint16(10000) + old := PersistedEpochStakes{Epoch: 1058, TotalStake: 42, Stakes: map[string]uint64{"vote": 42}, VoteAccts: map[string]*VoteAccountJSON{ + "vote": {Lamports: 123}, "explicit-zero": {InflationRewardsCommissionBps: &zero}, "no-source": {}, + }} + source := PersistedEpochStakes{Epoch: 1058, TotalStake: 999, Stakes: map[string]uint64{"vote": 999}, VoteAccts: map[string]*VoteAccountJSON{ + "vote": {Lamports: 999, InflationRewardsCommissionBps: &full}, "explicit-zero": {InflationRewardsCommissionBps: &full}, "source-only": {InflationRewardsCommissionBps: &full}, + }} + encode := func(v any) []byte { b, e := json.Marshal(v); require.NoError(t, e); return b } + raw := encode(old) + repaired, err := RestoreInflationCommissionMetadata(raw, encode(source)) + require.NoError(t, err) + old.VoteAccts["vote"].InflationRewardsCommissionBps = &full + require.JSONEq(t, string(encode(old)), string(repaired)) + require.False(t, HasInflationCommissionMetadata(repaired), "missing history must stay unknown") + source.Epoch++ + _, err = RestoreInflationCommissionMetadata(raw, encode(source)) + require.ErrorContains(t, err, "from epoch 1059") + _, err = RestoreInflationCommissionMetadata([]byte("invalid"), encode(source)) + require.Error(t, err) +} diff --git a/pkg/epochstakes/epoch_stakes.go b/pkg/epochstakes/epoch_stakes.go index 44ea12bfe..921e88131 100644 --- a/pkg/epochstakes/epoch_stakes.go +++ b/pkg/epochstakes/epoch_stakes.go @@ -26,11 +26,14 @@ type VoteAccount struct { // Nil means an older persisted cache omitted this consensus-critical field. // Legacy vote states explicitly use the scheduled node identity. BlockRevenueCollector *solana.PublicKey - LastTimestampTs int64 - LastTimestampSlot uint64 - Owner solana.PublicKey - Executable byte - RentEpoch uint64 + // Nil means an older checkpoint omitted the historical rate; zero is a + // known zero commission and must not fall back to a newer epoch's rate. + InflationRewardsCommissionBps *uint16 + LastTimestampTs int64 + LastTimestampSlot uint64 + Owner solana.PublicKey + Executable byte + RentEpoch uint64 } // Snapshot is one immutable, atomically published view of an epoch's stake @@ -179,15 +182,16 @@ type PersistedEpochStakes struct { // VoteAccountJSON is the JSON-serializable format for vote account metadata. type VoteAccountJSON struct { - Lamports uint64 `json:"lamports"` - NodePubkey string `json:"node_pubkey"` - BlsPubkeyCompressed []byte `json:"bls_pubkey_compressed,omitempty"` - BlockRevenueCollector string `json:"block_revenue_collector,omitempty"` - LastTimestampTs int64 `json:"last_ts"` - LastTimestampSlot uint64 `json:"last_ts_slot"` - Owner string `json:"owner"` - Executable byte `json:"executable"` - RentEpoch uint64 `json:"rent_epoch"` + Lamports uint64 `json:"lamports"` + NodePubkey string `json:"node_pubkey"` + BlsPubkeyCompressed []byte `json:"bls_pubkey_compressed,omitempty"` + BlockRevenueCollector string `json:"block_revenue_collector,omitempty"` + InflationRewardsCommissionBps *uint16 `json:"inflation_rewards_commission_bps,omitempty"` + LastTimestampTs int64 `json:"last_ts"` + LastTimestampSlot uint64 `json:"last_ts_slot"` + Owner string `json:"owner"` + Executable byte `json:"executable"` + RentEpoch uint64 `json:"rent_epoch"` } // SerializeEpoch serializes the stakes for a single epoch to JSON. @@ -219,15 +223,16 @@ func (cache *EpochStakesCache) SerializeEpoch(epoch uint64) ([]byte, error) { collector = va.BlockRevenueCollector.String() } persisted.VoteAccts[pk.String()] = &VoteAccountJSON{ - BlockRevenueCollector: collector, - Lamports: va.Lamports, - NodePubkey: va.NodePubkey.String(), - BlsPubkeyCompressed: bls, - LastTimestampTs: va.LastTimestampTs, - LastTimestampSlot: va.LastTimestampSlot, - Owner: va.Owner.String(), - Executable: va.Executable, - RentEpoch: va.RentEpoch, + BlockRevenueCollector: collector, + InflationRewardsCommissionBps: va.InflationRewardsCommissionBps, + Lamports: va.Lamports, + NodePubkey: va.NodePubkey.String(), + BlsPubkeyCompressed: bls, + LastTimestampTs: va.LastTimestampTs, + LastTimestampSlot: va.LastTimestampSlot, + Owner: va.Owner.String(), + Executable: va.Executable, + RentEpoch: va.RentEpoch, } } } @@ -292,15 +297,16 @@ func (cache *EpochStakesCache) DeserializeAndLoadEpoch(data []byte) (uint64, err collector = &key } voteAccounts[pk] = &VoteAccount{ - BlockRevenueCollector: collector, - Lamports: vaJSON.Lamports, - NodePubkey: nodePubkey, - BlsPubkeyCompressed: bls, - LastTimestampTs: vaJSON.LastTimestampTs, - LastTimestampSlot: vaJSON.LastTimestampSlot, - Owner: owner, - Executable: vaJSON.Executable, - RentEpoch: vaJSON.RentEpoch, + BlockRevenueCollector: collector, + InflationRewardsCommissionBps: vaJSON.InflationRewardsCommissionBps, + Lamports: vaJSON.Lamports, + NodePubkey: nodePubkey, + BlsPubkeyCompressed: bls, + LastTimestampTs: vaJSON.LastTimestampTs, + LastTimestampSlot: vaJSON.LastTimestampSlot, + Owner: owner, + Executable: vaJSON.Executable, + RentEpoch: vaJSON.RentEpoch, } } @@ -378,6 +384,10 @@ func cloneVoteAccount(voteAccount *VoteAccount) *VoteAccount { return nil } cloned := *voteAccount + if voteAccount.InflationRewardsCommissionBps != nil { + commission := *voteAccount.InflationRewardsCommissionBps + cloned.InflationRewardsCommissionBps = &commission + } if voteAccount.BlockRevenueCollector != nil { collector := *voteAccount.BlockRevenueCollector cloned.BlockRevenueCollector = &collector diff --git a/pkg/global/global_ctx.go b/pkg/global/global_ctx.go index d99d9c5c6..6a6f6ff36 100644 --- a/pkg/global/global_ctx.go +++ b/pkg/global/global_ctx.go @@ -35,7 +35,6 @@ type GlobalCtx struct { cachedStakeEntries []accountsdb.StakeIndexEntry // Parsed+sorted index, populated on first load entriesFlushedSinceCompact int // Appended entries since last compaction voteCache map[solana.PublicKey]*sealevel.VoteStateVersions - epochVoteStateSnapshots map[uint64]map[solana.PublicKey]*sealevel.VoteStateVersions epochStakes *epochstakes.EpochStakesCache slotsConfirmed map[uint64]struct{} leaderSchedule *leaderschedule.LeaderSchedule // compatibility schedule installed by SetLeaderSchedule @@ -186,50 +185,6 @@ func VoteCacheSnapshot() map[solana.PublicKey]*sealevel.VoteStateVersions { return snapshot } -func PutEpochVoteStateSnapshot(epoch uint64, snapshot map[solana.PublicKey]*sealevel.VoteStateVersions) { - instance.voteCacheMutex.Lock() - defer instance.voteCacheMutex.Unlock() - - if instance.epochVoteStateSnapshots == nil { - instance.epochVoteStateSnapshots = make(map[uint64]map[solana.PublicKey]*sealevel.VoteStateVersions) - } - - snapshotCopy := make(map[solana.PublicKey]*sealevel.VoteStateVersions, len(snapshot)) - for pk, voteState := range snapshot { - snapshotCopy[pk] = voteState - } - instance.epochVoteStateSnapshots[epoch] = snapshotCopy - - for cachedEpoch := range instance.epochVoteStateSnapshots { - if cachedEpoch+2 < epoch { - delete(instance.epochVoteStateSnapshots, cachedEpoch) - } - } -} - -func EpochVoteStateSnapshot(epoch uint64) map[solana.PublicKey]*sealevel.VoteStateVersions { - instance.voteCacheMutex.RLock() - defer instance.voteCacheMutex.RUnlock() - - snapshot := instance.epochVoteStateSnapshots[epoch] - if snapshot == nil { - return nil - } - - snapshotCopy := make(map[solana.PublicKey]*sealevel.VoteStateVersions, len(snapshot)) - for pk, voteState := range snapshot { - snapshotCopy[pk] = voteState - } - return snapshotCopy -} - -func ClearEpochVoteStateSnapshots() { - instance.voteCacheMutex.Lock() - defer instance.voteCacheMutex.Unlock() - - instance.epochVoteStateSnapshots = nil -} - func PutEpochStakesEntry(epoch uint64, pubkey solana.PublicKey, stake uint64, voteAcct *epochstakes.VoteAccount) { instance.epochStakes.PutEntry(epoch, pubkey, stake, voteAcct) } diff --git a/pkg/replay/block.go b/pkg/replay/block.go index 5dc2e431f..9b7bf6510 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -1216,6 +1216,14 @@ func configureInitialBlock(acctsDb *accountsdb.AccountsDb, } block.ParentBankhash = parentBankhash block.ParentSlot = mithrilState.ManifestParentSlot + // The first bank inherits the snapshot's newest blockhash just like a + // child of an executed bank. Nonce instructions derive their next nonce + // from this field, independently of the global chain tip (Agave v4.4.0-beta.0 + // programs/system/src/system_instruction.rs:46-47). + block.LastBlockhash, err = manifestLastBlockhash(mithrilState) + if err != nil { + return err + } // LtHash: decode base64, restore with InitWithHash if mithrilState.ManifestAcctsLtHash != "" { @@ -1263,6 +1271,22 @@ func configureInitialBlock(acctsDb *accountsdb.AccountsDb, return nil } +// PopulateManifestSeed stores the blockhash queue newest first. Both the +// initial chain tip and the first execution context must use that same hash. +func manifestLastBlockhash(s *state.MithrilState) (solana.Hash, error) { + if s == nil || len(s.ManifestRecentBlockhashes) == 0 { + return solana.Hash{}, fmt.Errorf("snapshot state is missing manifest_recent_blockhashes") + } + hash, err := solana.HashFromBase58(s.ManifestRecentBlockhashes[0].Blockhash) + if err != nil { + return solana.Hash{}, fmt.Errorf("invalid newest manifest blockhash at snapshot slot %d: %w", s.ManifestParentSlot, err) + } + if hash == (solana.Hash{}) { + return solana.Hash{}, fmt.Errorf("zero newest manifest blockhash at snapshot slot %d", s.ManifestParentSlot) + } + return hash, nil +} + // reconstructFeeRateGovernor creates a FeeRateGovernor from state file manifest_* fields func reconstructFeeRateGovernor(s *state.MithrilState) *sealevel.FeeRateGovernor { if s.ManifestFeeRateGovernor == nil { @@ -1573,9 +1597,13 @@ func LoadInitialEpochStakesCache(mithrilState *state.MithrilState, resumeState * return fmt.Errorf("resume at epoch %d (snapshot epoch %d) but no persisted epoch stakes found - cannot use stale manifest stakes (need fresh snapshot)", startEpoch, snapshotEpoch) } if len(resumeState.ComputedEpochStakes) > 0 { + persisted, err := restorePersistedCommissionMetadata(mithrilState, resumeState.ComputedEpochStakes, startEpoch, snapshotEpoch) + if err != nil { + return err + } // Once replay has crossed a boundary, only its persisted effective // stakes are authoritative; never fall back to the snapshot manifest. - for epoch, data := range resumeState.ComputedEpochStakes { + for epoch, data := range persisted { loadedEpoch, err := global.DeserializeAndLoadEpochStakes(data) if err != nil { return fmt.Errorf("failed to load persisted epoch %d stakes: %w", epoch, err) @@ -1824,10 +1852,10 @@ func ReplayBlocks( initialLtHash.InitWithHash(raw) } initialNumSignatures = mithrilState.ManifestSignatureCount - if len(mithrilState.ManifestRecentBlockhashes) > 0 { - if raw, decodeErr := base58.DecodeFromString(mithrilState.ManifestRecentBlockhashes[0].Blockhash); decodeErr == nil { - initialLastBlockhash = solana.Hash(raw) - } + initialLastBlockhash, err = manifestLastBlockhash(mithrilState) + if err != nil { + result.Error = err + return result } } // Resume checkpoints persist the Alpenglow block ID and chained root, but diff --git a/pkg/replay/epoch.go b/pkg/replay/epoch.go index e138fc901..6260d47aa 100644 --- a/pkg/replay/epoch.go +++ b/pkg/replay/epoch.go @@ -283,7 +283,6 @@ func handleEpochTransition(acctsDb *accountsdb.AccountsDb, partitionedEpochRewar t1 := time.Now() updateEpochStakesAndRefreshVoteCache(leaderScheduleEpoch, block, acctsDb, prevSlotCtx.Slot, scanResult, f, epochSchedule) - global.PutEpochVoteStateSnapshot(newEpoch, global.VoteCacheSnapshot()) t2 := time.Now() if global.ManageLeaderSchedule() { @@ -398,8 +397,17 @@ func updateEpochStakesAndRefreshVoteCache(leaderScheduleEpoch uint64, b *block.B mlog.Log.Errorf("failed to rebuild vote cache at epoch boundary: %v", rebuildErr) } - // Skip epoch stakes storage if already cached (resume) - if hasEpochStakes { + // An older checkpoint can have cached this boundary's stakes without its + // commission fields. Recompute that new generation from the same durable + // parent; historical generations are restored only from their own snapshots. + commissionsComplete := true + for _, vote := range cachedEpochStakes.VoteAccounts { + if vote == nil || vote.InflationRewardsCommissionBps == nil { + commissionsComplete = false + break + } + } + if hasEpochStakes && commissionsComplete { mlog.Log.Infof("already had EpochStakes for epoch %d", leaderScheduleEpoch) // A restart can retain this generation from a previously attempted // boundary, while configuration still inherits the parent's stakes. @@ -442,16 +450,18 @@ func updateEpochStakesAndRefreshVoteCache(leaderScheduleEpoch uint64, b *block.B } epochStakes[votePk] = stake collector := voteAcct.BlockRevenueCollector() + commission := voteAcct.InflationRewardsCommissionBps() epochVoteAccounts[votePk] = &epochstakes.VoteAccount{ - BlockRevenueCollector: &collector, - Lamports: meta.Lamports, - NodePubkey: voteAcct.NodePubkey(), - BlsPubkeyCompressed: voteAcct.BlsPubkeyCompressed(), - LastTimestampTs: lastTimestampTs, - LastTimestampSlot: lastTimestampSlot, - Owner: meta.Owner, - Executable: executable, - RentEpoch: meta.RentEpoch, + BlockRevenueCollector: &collector, + InflationRewardsCommissionBps: &commission, + Lamports: meta.Lamports, + NodePubkey: voteAcct.NodePubkey(), + BlsPubkeyCompressed: voteAcct.BlsPubkeyCompressed(), + LastTimestampTs: lastTimestampTs, + LastTimestampSlot: lastTimestampSlot, + Owner: meta.Owner, + Executable: executable, + RentEpoch: meta.RentEpoch, } } } diff --git a/pkg/replay/epoch_boundary_stakes_test.go b/pkg/replay/epoch_boundary_stakes_test.go index fc4486222..f67414a29 100644 --- a/pkg/replay/epoch_boundary_stakes_test.go +++ b/pkg/replay/epoch_boundary_stakes_test.go @@ -11,6 +11,7 @@ import ( "github.com/Overclock-Validator/mithril/pkg/accountsdb" "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" "github.com/Overclock-Validator/mithril/pkg/features" "github.com/Overclock-Validator/mithril/pkg/global" "github.com/Overclock-Validator/mithril/pkg/sealevel" @@ -152,3 +153,40 @@ func epochBoundaryStakeDB(t *testing.T, slot uint64, delegations []sealevel.Dele }) return db } + +func TestEpochBoundaryRestoresCommissionInPreviouslyAttemptedGeneration(t *testing.T) { + const newEpoch = uint64(91040) + const slot = newEpoch * 54000 + key := solana.PublicKey{40} + db := epochBoundaryStakeDB(t, slot-1, []sealevel.Delegation{{VoterPubkey: key, StakeLamports: 1000, ActivationEpoch: math.MaxUint64, DeactivationEpoch: math.MaxUint64}}) + vote, err := db.GetAccount(slot-1, key) + require.NoError(t, err) + vote = vote.Clone() + vs, err := sealevel.UnmarshalVersionedVoteState(vote.Data) + require.NoError(t, err) + vs.V4.InflationRewardsCommissionBps = 1234 + vote.Data, err = sealevel.MarshalVersionedVoteState(vs) + require.NoError(t, err) + done := make(chan struct{}) + require.NoError(t, db.StoreAccounts([]*accounts.Account{vote}, slot-1, func() { close(done) })) + <-done + schedule := &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 54000, LeaderScheduleSlotOffset: 54000} + cacheEpoch := schedule.LeaderScheduleEpoch(slot) + t.Cleanup(func() { global.ClearEpochStakes(cacheEpoch) }) + // A failed attempt on the older binary persisted stake weights but omitted + // commissions. This new generation is reproducible from the durable parent. + global.PutEpochStakes(cacheEpoch, map[solana.PublicKey]uint64{key: 1000}, map[solana.PublicKey]*epochstakes.VoteAccount{key: {NodePubkey: key}}, 1000) + old, _ := global.EpochStakesSnapshot(cacheEpoch) + scan := &BoundaryStakeScanResult{VoteAcctStakes: map[solana.PublicKey]uint64{key: 1000}, EffectiveStakes: map[solana.PublicKey]uint64{key: 1000}, TotalEffectiveStake: 1000} + b := &block.Block{Slot: slot} + updateEpochStakesAndRefreshVoteCache(cacheEpoch, b, db, slot-1, scan, features.NewFeaturesDefault(), schedule) + got, _ := global.EpochStakesSnapshot(cacheEpoch) + require.Greater(t, got.Generation, old.Generation) + require.Equal(t, uint16(1234), *got.VoteAccounts[key].InflationRewardsCommissionBps) + require.Equal(t, old.Stakes, got.Stakes) + require.Equal(t, got.Stakes, b.EpochStakesPerVoteAcct) + // Once complete, this generation must stay immutable on another attempt. + updateEpochStakesAndRefreshVoteCache(cacheEpoch, b, db, slot-1, scan, features.NewFeaturesDefault(), schedule) + again, _ := global.EpochStakesSnapshot(cacheEpoch) + require.Equal(t, got.Generation, again.Generation) +} diff --git a/pkg/replay/epoch_inflation_test.go b/pkg/replay/epoch_inflation_test.go index fba7408e9..ff23b1164 100644 --- a/pkg/replay/epoch_inflation_test.go +++ b/pkg/replay/epoch_inflation_test.go @@ -282,3 +282,19 @@ func TestLoadEpochInflationAccountStateForReplayFallsBackToSpeculativeParent(t * require.NoError(t, err) require.Equal(t, parentState, loaded) } + +func TestEpochInflationBudgetTestnet451964256(t *testing.T) { + // Frozen inputs at the epoch 1058 -> 1059 boundary and Agave's on-chain + // vote_reward_account value. A one-ULP pow difference changes this by 1. + schedule := &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 432000, LeaderScheduleSlotOffset: 432000, Warmup: true, FirstNormalEpoch: 14, FirstNormalSlot: 524256} + f := features.NewFeaturesDefault() + f.EnableFeature(features.PicoInflation, 49772256) + f.EnableFeature(features.ReduceSlotTimeTo350ms, 426476256) + f.EnableFeature(features.ReduceSlotTimeTo300ms, 427340256) + f.EnableFeature(features.ReduceSlotTimeTo250ms, 428204256) + f.EnableFeature(features.ReduceSlotTimeTo200ms, 429068256) + inflation := rewards.Inflation{Initial: 0.15, Terminal: 0.015, Taper: 0.15} + got := newEpochInflationState(schedule, &inflation, 2139687087758323546, 387145715402013, 1059, 157784629.968, f) + require.Equal(t, uint64(392970281850805), got.MaxPossibleValidatorReward) + require.Equal(t, uint64(1059), got.Epoch) +} diff --git a/pkg/replay/epoch_stakes_seed.go b/pkg/replay/epoch_stakes_seed.go index 6b9b8f72c..1b8aea636 100644 --- a/pkg/replay/epoch_stakes_seed.go +++ b/pkg/replay/epoch_stakes_seed.go @@ -37,6 +37,41 @@ type manifestEpochStakeSeed struct { data []byte } +// An older checkpoint may have retained stake weights but discarded commission +// rates. Recover those fields from the exact same epoch-stakes generation in +// the snapshot, without replacing the checkpoint's stake or validator metadata. +func restorePersistedCommissionMetadata(mithrilState *state.MithrilState, persisted map[uint64][]byte, currentEpoch, snapshotEpoch uint64) (map[uint64][]byte, error) { + if mithrilState == nil || len(mithrilState.ManifestEpochStakes) == 0 { + return persisted, nil + } + needsRestore := false + for _, data := range persisted { + if !epochstakes.HasInflationCommissionMetadata(data) { + needsRestore = true + break + } + } + if !needsRestore { + return persisted, nil + } + seeds, _, err := prepareManifestEpochStakesForRuntime(mithrilState, currentEpoch, snapshotEpoch) + if err != nil { + return nil, err + } + restored := maps.Clone(persisted) + for _, seed := range seeds { + data, exists := restored[seed.runtimeEpoch] + if !exists || epochstakes.HasInflationCommissionMetadata(data) { + continue + } + restored[seed.runtimeEpoch], err = epochstakes.RestoreInflationCommissionMetadata(data, seed.data) + if err != nil { + return nil, fmt.Errorf("restore historical commissions for epoch %d: %w", seed.runtimeEpoch, err) + } + } + return restored, nil +} + func prepareManifestEpochStakesForRuntime(mithrilState *state.MithrilState, currentEpoch uint64, snapshotEpoch uint64) ([]manifestEpochStakeSeed, bool, error) { if mithrilState == nil || len(mithrilState.ManifestEpochStakes) == 0 { return nil, false, fmt.Errorf("state file missing manifest_epoch_stakes - delete AccountsDB and rebuild from snapshot") diff --git a/pkg/replay/epoch_stakes_seed_test.go b/pkg/replay/epoch_stakes_seed_test.go index 4a5b14d65..7851203bd 100644 --- a/pkg/replay/epoch_stakes_seed_test.go +++ b/pkg/replay/epoch_stakes_seed_test.go @@ -207,3 +207,37 @@ func persistedEpochStakeJSON(t *testing.T, epoch uint64) string { } return string(data) } + +func TestResumeRestoresOnlyMissingHistoricalCommissions(t *testing.T) { + const epoch = uint64(91030) + key := solana.PublicKey{30} + encode := func(e uint64, stake uint64, commission *uint16) []byte { + cache := epochstakes.NewEpochStakesCache() + cache.PutEpoch(e, map[solana.PublicKey]uint64{key: stake}, map[solana.PublicKey]*epochstakes.VoteAccount{key: {Lamports: stake, InflationRewardsCommissionBps: commission}}, stake) + raw, err := cache.SerializeEpoch(e) + require.NoError(t, err) + return raw + } + resume := &ResumeState{ComputedEpochStakes: map[uint64][]byte{ + epoch: encode(epoch, 42, nil), epoch + 1: encode(epoch+1, 43, new(uint16(0))), epoch + 2: encode(epoch+2, 44, nil), + }} + original := append([]byte(nil), resume.ComputedEpochStakes[epoch]...) + ms := &state.MithrilState{ManifestEpochStakes: map[uint64]string{ + epoch: string(encode(epoch, 999, new(uint16(10000)))), epoch + 1: string(encode(epoch+1, 999, new(uint16(5000)))), + }} + for e := epoch; e <= epoch+2; e++ { + global.ClearEpochStakes(e) + t.Cleanup(func() { global.ClearEpochStakes(e) }) + } + require.NoError(t, LoadInitialEpochStakesCache(ms, resume, epoch, epoch)) + old, ok := global.EpochStakesSnapshot(epoch) + require.True(t, ok) + require.Equal(t, uint16(10000), *old.VoteAccounts[key].InflationRewardsCommissionBps) + require.Equal(t, uint64(42), old.Stakes[key]) + require.Equal(t, uint64(42), old.VoteAccounts[key].Lamports) + next, _ := global.EpochStakesSnapshot(epoch + 1) + require.Zero(t, *next.VoteAccounts[key].InflationRewardsCommissionBps, "persisted zero is authoritative") + future, _ := global.EpochStakesSnapshot(epoch + 2) + require.Nil(t, future.VoteAccounts[key].InflationRewardsCommissionBps, "no cross-epoch backfill") + require.Equal(t, original, resume.ComputedEpochStakes[epoch], "upgrade must not mutate checkpoint input") +} diff --git a/pkg/replay/initial_blockhash_test.go b/pkg/replay/initial_blockhash_test.go new file mode 100644 index 000000000..28a9db0a3 --- /dev/null +++ b/pkg/replay/initial_blockhash_test.go @@ -0,0 +1,146 @@ +package replay + +import ( + "crypto/sha256" + "encoding/binary" + "encoding/json" + "math" + "os" + "sync" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/lthash" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/state" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/programs/system" + "github.com/stretchr/testify/require" +) + +func TestSnapshotStartupNonceSlot452018181(t *testing.T) { + var fixture struct { + Slot, Epoch uint64 + SnapshotSlot uint64 `json:"snapshot_slot"` + ParentBankhash solana.Hash `json:"parent_bankhash"` + ParentBlockhash solana.Hash `json:"parent_blockhash"` + Blockhash solana.Hash + Signatures uint64 `json:"num_signatures"` + BrokenHash solana.Hash `json:"broken_bankhash"` + ExpectedHash solana.Hash `json:"expected_bankhash"` + BrokenLtHash []byte `json:"broken_lthash"` + BrokenNonce *accounts.Account `json:"broken_nonce"` + ExpectedNonce *accounts.Account `json:"expected_nonce"` + } + raw, err := os.ReadFile("testdata/slot_452018181_snapshot_nonce.json") + require.NoError(t, err) + require.NoError(t, json.Unmarshal(raw, &fixture)) + schedule := &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 432000, LeaderScheduleSlotOffset: 432000, Warmup: true, FirstNormalEpoch: 14, FirstNormalSlot: 524256} + key := solana.PublicKey{0xC7} + db := epochBoundaryStakeDB(t, fixture.SnapshotSlot, []sealevel.Delegation{{VoterPubkey: key, StakeLamports: 1000, ActivationEpoch: math.MaxUint64, DeactivationEpoch: math.MaxUint64}}) + previousManage := global.ManageLeaderSchedule() + previousBlockhash := global.LatestBlockHash() + global.SetManageLeaderSchedule(false) + global.PutEpochStakes(fixture.Epoch+1, map[solana.PublicKey]uint64{key: 1000}, nil, 1000) + t.Cleanup(func() { + global.SetManageLeaderSchedule(previousManage) + global.SetLatestBlockHash(previousBlockhash) + global.ClearEpochStakes(fixture.Epoch + 1) + }) + ms := &state.MithrilState{ + ManifestParentSlot: fixture.SnapshotSlot, ManifestParentBankhash: fixture.ParentBankhash.String(), + ManifestEvictedBlockhash: solana.Hash{2}.String(), ManifestFeeRateGovernor: &state.ManifestFeeRateGovernorSeed{}, + ManifestRecentBlockhashes: []state.BlockhashEntry{{Blockhash: fixture.ParentBlockhash.String()}, {Blockhash: solana.Hash{3}.String()}}, + } + // Start with a genuinely unconfigured block. Older startup tests supplied + // LastBlockhash themselves, masking the snapshot initialization omission. + block := &b.Block{Slot: fixture.Slot, Epoch: fixture.Epoch, Blockhash: fixture.Blockhash} + require.NoError(t, configureInitialBlock(db, block, ms, schedule)) + require.Equal(t, [32]byte(fixture.ParentBlockhash), global.LatestBlockHash()) + require.NotEqual(t, fixture.ParentBankhash, fixture.ParentBlockhash) + require.NotEqual(t, fixture.Blockhash, fixture.ParentBlockhash) + + env, cleanup := newCommitTestSlotCtx() + defer cleanup() + block.Features, block.FeeRateGovernor = env.Features, env.FeeRateGovernor + ctx := newSlotCtx(block, env.Accounts, accounts.NewMemAccounts(), nil, nil, 0) + recent := sealevel.SysvarRecentBlockhashes{{Blockhash: fixture.ParentBlockhash, FeeCalculator: sealevel.FeeCalculator{LamportsPerSignature: 5000}}} + sealevel.SysvarCache.RecentBlockHashes.Sysvar = &recent + nonceKey, payer := solana.PublicKey{0xD7}, txfixture.PayerPubkey() + initialNonce := solana.Hash{0xAA} + nonce := sealevel.NonceStateVersions{Type: sealevel.NonceVersionCurrent, Current: sealevel.NonceData{ + IsInitialized: true, Authority: payer, DurableNonce: initialNonce, FeeCalculator: sealevel.FeeCalculator{LamportsPerSignature: 5000}, + }} + data, err := nonce.Marshal() + require.NoError(t, err) + require.NoError(t, ctx.SetAccount(nonceKey, &accounts.Account{Key: nonceKey, Lamports: 10_000_000, Data: data, Owner: addresses.SystemProgramAddr, RentEpoch: math.MaxUint64})) + tx, err := solana.NewTransaction([]solana.Instruction{system.NewAdvanceNonceAccountInstruction(nonceKey, solana.SysVarRecentBlockHashesPubkey, payer).Build()}, initialNonce, solana.TransactionPayer(payer)) + require.NoError(t, err) + privateKey := txfixture.PayerPrivateKey() + _, err = tx.Sign(func(pk solana.PublicKey) *solana.PrivateKey { + if pk == payer { + return &privateKey + } + return nil + }) + require.NoError(t, err) + var wg sync.WaitGroup + fee, _, err := ProcessTransaction(ctx, &wg, tx, nil, nil, nil, false) + wg.Wait() + require.NoError(t, err) + require.NotNil(t, fee) + advanced, err := ctx.GetAccount(nonceKey) + require.NoError(t, err) + actual, err := sealevel.UnmarshalNonceStateVersions(advanced.Data) + require.NoError(t, err) + expected, err := sealevel.UnmarshalNonceStateVersions(fixture.ExpectedNonce.Data) + require.NoError(t, err) + require.Equal(t, expected.State().DurableNonce, actual.State().DurableNonce) + + // The captured bank differed in exactly one account: transaction 92's + // successful nonce advance. Apply the nonce produced through fresh-start + // configuration to that captured account and recover the certified hash. + lt := new(lthash.LtHash).InitWithHash(fixture.BrokenLtHash) + bankHash := func() solana.Hash { + input := append([]byte(nil), fixture.ParentBankhash[:]...) + input = binary.LittleEndian.AppendUint64(input, fixture.Signatures) + input = append(input, fixture.Blockhash[:]...) + inner := sha256.Sum256(input) + return solana.Hash(sha256.Sum256(append(inner[:], lt.Hash()...))) + } + require.Equal(t, fixture.BrokenHash, bankHash()) + repaired := fixture.BrokenNonce.Clone() + captured, err := sealevel.UnmarshalNonceStateVersions(repaired.Data) + require.NoError(t, err) + captured.State().DurableNonce = actual.State().DurableNonce + repaired.Data, err = captured.Marshal() + require.NoError(t, err) + require.Equal(t, fixture.ExpectedNonce, repaired) + lt.Sub(new(lthash.LtHash).InitWithAcct(fixture.BrokenNonce)) + lt.Add(new(lthash.LtHash).InitWithAcct(repaired)) + require.Equal(t, fixture.ExpectedHash, bankHash()) +} + +func TestSnapshotStartupRejectsMissingOrMalformedParentBlockhash(t *testing.T) { + for _, tc := range []struct { + name string + entries []state.BlockhashEntry + }{ + {"absent", nil}, + {"empty", []state.BlockhashEntry{{}}}, + {"invalid base58", []state.BlockhashEntry{{Blockhash: "not-a-hash"}}}, + {"wrong length", []state.BlockhashEntry{{Blockhash: "123"}}}, + {"zero", []state.BlockhashEntry{{Blockhash: solana.Hash{}.String()}}}, + } { + t.Run(tc.name, func(t *testing.T) { + s := &state.MithrilState{ManifestParentSlot: 452018180, ManifestParentBankhash: solana.Hash{1}.String(), ManifestRecentBlockhashes: tc.entries} + // Even a prefilled candidate cannot replace missing snapshot metadata. + block := &b.Block{LastBlockhash: solana.Hash{99}} + require.Error(t, configureInitialBlock(nil, block, s, &sealevel.SysvarEpochSchedule{})) + }) + } +} diff --git a/pkg/replay/startup_epoch_boundary_test.go b/pkg/replay/startup_epoch_boundary_test.go index 88c728c24..07a69789a 100644 --- a/pkg/replay/startup_epoch_boundary_test.go +++ b/pkg/replay/startup_epoch_boundary_test.go @@ -87,17 +87,18 @@ func testStartupAcrossEpochBoundary(t *testing.T, mode string, offset uint64) { require.NoError(t, err) seeds[epoch] = string(data) } - parentHash, evictedHash := solana.Hash{1}, solana.Hash{2} + parentHash, evictedHash, parentBlockhash := solana.Hash{1}, solana.Hash{2}, solana.Hash{3} ms := &state.MithrilState{ ManifestParentSlot: parentSlot, ManifestParentBankhash: parentHash.String(), ManifestEvictedBlockhash: evictedHash.String(), ManifestEpochStakes: seeds, - ManifestFeeRateGovernor: &state.ManifestFeeRateGovernorSeed{}, + ManifestFeeRateGovernor: &state.ManifestFeeRateGovernorSeed{}, + ManifestRecentBlockhashes: []state.BlockhashEntry{{Blockhash: parentBlockhash.String()}}, } var resume *ResumeState if mode != "snapshot" { - recent := sealevel.SysvarRecentBlockhashes{{Blockhash: parentHash}} + recent := sealevel.SysvarRecentBlockhashes{{Blockhash: parentBlockhash}} resume = &ResumeState{ParentSlot: parentSlot, ParentBankhash: parentHash[:], - RecentBlockhashes: &recent, LastBlockhash: parentHash, EvictedBlockhash: evictedHash} + RecentBlockhashes: &recent, LastBlockhash: parentBlockhash, EvictedBlockhash: evictedHash} if mode == "persisted" || mode == "cached" { // A checkpoint whose original snapshot predates its rooted parent. ms.ManifestParentSlot -= 100 @@ -119,12 +120,14 @@ func testStartupAcrossEpochBoundary(t *testing.T, mode string, offset uint64) { parentBankEpoch := initialReplayEpoch(schedule, parentSlot+1, ms.ManifestParentSlot, resume) require.Equal(t, parentEpoch, parentBankEpoch) require.NoError(t, LoadInitialEpochStakesCache(ms, resume, parentBankEpoch, schedule.GetEpoch(ms.ManifestParentSlot))) - blk := &b.Block{Slot: parentSlot + 1 + offset, Epoch: enteringEpoch, LastBlockhash: parentHash} + blk := &b.Block{Slot: parentSlot + 1 + offset, Epoch: enteringEpoch} if resume == nil { require.NoError(t, configureInitialBlock(db, blk, ms, schedule)) } else { require.NoError(t, configureInitialBlockFromResume(db, blk, resume, ms, schedule)) } + require.Equal(t, [32]byte(parentBlockhash), blk.LastBlockhash) + require.Equal(t, [32]byte(parentBlockhash), global.LatestBlockHash()) require.Equal(t, voteKey, blk.Leader, "leader preparation must succeed before the epoch transition") require.Equal(t, parentStake, blk.TotalEpochStake, "initial configuration inherits the parent's execution stakes") require.Equal(t, parentStake, blk.EpochStakesPerVoteAcct[voteKey]) diff --git a/pkg/replay/testdata/slot_452018181_snapshot_nonce.json b/pkg/replay/testdata/slot_452018181_snapshot_nonce.json new file mode 100644 index 000000000..5aef0c781 --- /dev/null +++ b/pkg/replay/testdata/slot_452018181_snapshot_nonce.json @@ -0,0 +1,99 @@ +{ + "slot": 452018181, + "epoch": 1059, + "snapshot_slot": 452018180, + "parent_bankhash": "4WhuuVAGjGa1kzTXN57dwrcHW8PbPgJBWaeSMAT3osYk", + "parent_blockhash": "EknuovDKPJt63Y7DTS6ncaKas8qFH5aFopRNMkzf5TYQ", + "blockhash": "257x4HGQpdJTyZ1xoecEMaXwJRbCzrZZ4v4qAeYHdGgz", + "num_signatures": 127, + "broken_bankhash": "EuckeDw9YRedHJCMq6yVCWoqTUJgkCKAMxzbmyajiRkC", + "expected_bankhash": "9wqJASN8yEGyxEkFBH1iZzPARFC5VBYYsBuWH4yPpzzz", + "broken_lthash": "BOPxzAjNQI40pJEDSi8g1h9XpF5HQxNppbWLtsfd2UYbNkJOoHOqxEIGX/efdmIo4Qle4OlrMkc0vGKhBy3tuleYTwrXM2oVvEcTJXEuOQGOhm3XJCApXtdJRRh0C5kyQ48WOtPuhMgocESv0mNqYM2o/EQa9Ks90T7MLE4aO96ZLipw7MIb13w3dM8YA7mUdrh7Rz67afdkEYRacGFPu5eVwRAsn9+L01FuZSJP0EOh6DQ8GXwQNDYPoIS8sK8i7gOt7kZ8XEGWqnk70OXPDckvuvQeOD0EyhpcDv558MMOMp629SDhldHRdhAbe+5TyRYCxj1S3gRzP+2Q7ftP2FyRIttoggM5Ud3gy/pILMxjqEDNN98F+/NwPxndhOSstNhr5bcFyfYAtIOQBqpUG8cZKALCOkZDLdkigeWcEwam5C/IWqtJBY+Bhn7ZzWCTSJHV8/aP5R9/bZBhbY5WmoSSa/Y1OsyEJy/JbN91JHA7/42BiJF7wyVMLDj5FPpM7M3+WxfPQ3MD4DyO6UI5zIih5s2C3tAk/fQobuXcMWeMuLSV5jeRaa4xj6llLLDgN6iCFKnI+rs6jXwk6/6HAQ6hzJLbng4ZJ1PUai4ssuu3/JtviqVFdK1Av6gZL6eDIK14N5uky3NyhuxhkiIkvzJMVlHqIryraNg9JXdPfpq69MFffMby0nY7SIqAEzZBPhu99ZWKFVIkJlBPkdzxktvgFEbyMm0GbMgTHOgKq2E1VsGrb6tmpYrYyt8MWrY3pwTctYpHq6VGD6ULZuHL9CYH4r9336MS1d9L/T9SqkXcSw3XVqmikKQd/FSB1KjSvTIBpzl0rX3xCDWCVpA+NbLgKdYBtKJ5HSAPKm3BubADImMiSRsYXPQC1AzOguyVPjEAmjkf8ebrBx+ht1PYSEkwwUXQHn/DvJuyJ46hnW9fYjUBK0tqJ8P2TREOwHCBJu//x86jH7Qgks4Q/gUkjKVbMR3lmYtjuhZEgixmuEUsQhMbmfRm7nvJpvNi5ZQwUXXCRzXqwhHv9Ra5nzolsulytxlDv7ge1qj+cReFJMQE+FGrUzTKOa0M8gOes8FxyfS0CBHl0IsYt2A9mvvvQkL7eb/MXauAJ/8IMmyku2YhtKEM5HL7ciPCmiEQbohtjmalHc9SMqgtqQSBPkqq8kxmmoUESpqIpI71y6soI9P7WQP47ijSMxfb4E1CCQtL5Jma4F8TfV8knqA6fpSTLtnsDbuwpqyRJmhZTc9LO1nmsm/xXkv0oebk7acfP252GQG5XppVnIyDWk9S0Au6f0OFuz1xhYtnEog58egQxftc+bdxOzGCkBIIfFWALkUh/yKwLD4sX71j7dKVzrLseSE/YAy6BYTEyzhBt93nNUM3wEUFK5BYZ6oLtR+I2eeor4686rqhmq8EB7f/uiFopzmo3OO7xSvzX444KfgjOuLeEsD9B6m2566u/eaOK6hqmQ7U+hRJJfNdKJPx1raCGhi2tpbMzFfP9D907JGgLbangn0VzXdRz1OPi7tSs1a6GmR1JbN7ava9o5f4FitH1roCHmrfsb5mY62b+laW/VCik17R+JolZlmwiifVbdXsnZ23O27O1chHoi5+8mUdfyvmCVmTJZJdCP2ssW5tttux/FVxu/vZejiPKtBGC9I5F5YTbVyUngUTAN8gM4iaHbzSIHrNVIv3Y4oNtWWRY/RhwRl8wbyIS6tifakLFBhUAyDf1DvqGSI54PCLgVCM7+PvQrkJNFZjAu/5+X3usQRmzapHgb/XV1av0IkCxSD1i6WXVbZLXFK3/7C60bEp2KT+WpWKQON/RZu3dKKf1aAjSQCcFhXoUzSl+zNQlYZsvXqqK4Q5vCM9nTIVJw/6MZ7CN+Rf6rC22mggbi/M+Kmkz9DCSvZrijfCCE6V0JZwjhKmlatNTZHs+aFIliNXNpzpL1N4n40QxxOPtb2rD/M8ThGPEjp7ZGEKT7uyMkcC/MciRVxEOPv1OVQi99Q2EwWYJINjVYGBm1iWcIenBDU+JxaazBSht1b2eS3xzRoZGZWi66alLP8TMkW+5/hhTZqgYhuN3iWex2jyvuCHztRQ++2qDp9Z//tp6Jwt4/bnfmmytz2DvB7mPIPB/IFOabw5ZixG/7OAJYtxyKj5xnm8YnQi+Hos4dCVkVz2mrL+r9pb0AuMOfFBupfoccZQyVD4ET64fvNUDmvLjpKAsiMR7lbGS4Tw99Yx7Y4qxw+WNzz0HGd0xaWtDS5EJ/pi5mWsAyl2znaEp2T2nWDBqNFzfwLHorf+hNhNRWQ+f1ARgfsTyh+E+VC6aV0wMQMMrQxu7EYx1x8Fjf+H5kifEyB8h4zhuv0Z8awIjamDRx7zqgGbCp2wb6lEU0yr0VFaZOOt3Y5A8FMVhElYL/PT6oF4LZTY01W/tQ96s24n9uBskYqOQTRRtVu2VMcZSfUj2WMnrbKQji5RgddUM/wciN//nAfg+8JYwrTIacemrQQ2lhWMt+0Q01MP89B4wTI9HOObwabIb1LUKY/F8XA4xhIVomUaAcPdLk9lk+tF26RWzAsfun2bxXZgvG9iZ17H1r9ccECw1V2iYqL27jnJEjq/NSrSMD2Xr9S7RuK8OBDdBKQ5hVh96y5Fqq7TwC9CwwIGvGUhcwWOYPvL/S4PtUv4h7zqVmuzd+xKJlfxjK6T4kBx6tYS+H4fjhhexdIIGxjkZdbPJDRsQ5HwCPB4Hl4=", + "broken_nonce": { + "Slot": 452017971, + "Key": "G5A8tKCkcUuJwQZrUm2f1mhES7THF8WomYgoKcFBrpZZ", + "Lamports": 1056640, + "Data": "AQAAAAEAAACleBHulXexizu4EdoCCSAaMWb+RGazbal+i2Yy37HLvno71c3ykAmOW0yPmVf5iZIQuLvpBk9ysybMIS6rR2+KiBMAAAAAAAA=", + "Owner": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "Executable": false, + "RentEpoch": 18446744073709551615, + "IsDummy": false + }, + "expected_nonce": { + "Slot": 452017971, + "Key": "G5A8tKCkcUuJwQZrUm2f1mhES7THF8WomYgoKcFBrpZZ", + "Lamports": 1056640, + "Data": "AQAAAAEAAACleBHulXexizu4EdoCCSAaMWb+RGazbal+i2Yy37HLvoAtZNIsih5PPbubP8zhS/S6KgsrtkorYM2y0Ik7woF4iBMAAAAAAAA=", + "Owner": [ + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0 + ], + "Executable": false, + "RentEpoch": 18446744073709551615, + "IsDummy": false + }, + "source": "Read-only replay of testnet slot 452018181 from snapshot 452018180. Transaction 92 successfully advances this nonce. Changing only this account data reproduces the captured footer bank hash; the following four block hashes also match." +} diff --git a/pkg/rewards/delayed_commission_test.go b/pkg/rewards/delayed_commission_test.go new file mode 100644 index 000000000..df8dac58c --- /dev/null +++ b/pkg/rewards/delayed_commission_test.go @@ -0,0 +1,72 @@ +package rewards + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/epochstakes" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestDelayedCommissionUsesPersistedHistoryInAgaveOrder(t *testing.T) { + const epoch = 1058 + key := solana.MustPublicKeyFromBase58("73zB51M61WsAUZZuKYGStsn1bBhBoEEoQmXLznJphJBo") + current := &sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4, V4: sealevel.VoteState4{InflationRewardsCommissionBps: 5000}} + for _, tc := range []struct { + name string + older, newer *uint16 + want uint16 + }{ + {"epoch 1058 history predates current zero", new(uint16(10000)), new(uint16(8000)), 10000}, + {"explicit historical zero", new(uint16(0)), new(uint16(8000)), 0}, + {"new voter falls back one generation", nil, new(uint16(1234)), 1234}, + {"voter absent from both generations", nil, nil, 5000}, + } { + t.Run(tc.name, func(t *testing.T) { + for i, bps := range []*uint16{tc.older, tc.newer} { + e := uint64(epoch + i) + global.ClearEpochStakes(e) + t.Cleanup(func() { global.ClearEpochStakes(e) }) + votes := map[solana.PublicKey]*epochstakes.VoteAccount{} + if bps != nil { + votes[key] = &epochstakes.VoteAccount{InflationRewardsCommissionBps: bps} + } + cache := epochstakes.NewEpochStakesCache() + cache.PutEpoch(e, map[solana.PublicKey]uint64{key: 42}, votes, 42) + raw, err := cache.SerializeEpoch(e) + require.NoError(t, err) + _, err = global.DeserializeAndLoadEpochStakes(raw) + require.NoError(t, err) + } + if tc.want == 10000 { + current.V4.InflationRewardsCommissionBps = 0 + } else { + current.V4.InflationRewardsCommissionBps = 5000 + } + got, err := buildDelayedCommissionVoteStateCache(map[solana.PublicKey]*sealevel.VoteStateVersions{key: current}, epoch) + require.NoError(t, err) + require.Equal(t, tc.want, got[key].InflationRewardsCommissionBps()) + if tc.want == 10000 { + // Actual lost voting reward at slot 451964256, epoch 1058 -> 1059. + split := voteCommissionSplit(got[key], 350150707774, true, true) + require.Equal(t, uint64(350150707774), split.VoterPortion) + require.Zero(t, split.StakerPortion) + require.Zero(t, current.V4.InflationRewardsCommissionBps, "selection must not mutate live vote state") + } + }) + } +} + +func TestDelayedCommissionRejectsUnknownHistoricalRate(t *testing.T) { + const epoch = 91020 + key := solana.PublicKey{21} + for _, missing := range []*epochstakes.VoteAccount{nil, {}} { + global.PutEpochStakes(epoch, map[solana.PublicKey]uint64{key: 1}, map[solana.PublicKey]*epochstakes.VoteAccount{key: missing}, 1) + global.PutEpochStakes(epoch+1, map[solana.PublicKey]uint64{key: 1}, map[solana.PublicKey]*epochstakes.VoteAccount{key: {InflationRewardsCommissionBps: new(uint16(1234))}}, 1) + _, err := buildDelayedCommissionVoteStateCache(map[solana.PublicKey]*sealevel.VoteStateVersions{key: {Type: sealevel.VoteStateVersionV4}}, epoch) + require.ErrorContains(t, err, "missing historical inflation commission") + } + t.Cleanup(func() { global.ClearEpochStakes(epoch); global.ClearEpochStakes(epoch + 1) }) +} diff --git a/pkg/rewards/inflation.go b/pkg/rewards/inflation.go index c4ccebdb3..44b55402f 100644 --- a/pkg/rewards/inflation.go +++ b/pkg/rewards/inflation.go @@ -1,10 +1,6 @@ package rewards -import ( - "math" - - bin "github.com/gagliardetto/binary" -) +import bin "github.com/gagliardetto/binary" type Inflation struct { Initial float64 @@ -48,7 +44,7 @@ func (inflation *Inflation) UnmarshalWithDecoder(decoder *bin.Decoder) error { } func (inflation *Inflation) Total(year float64) float64 { - tapered := inflation.Initial * (math.Pow(1.0-inflation.Taper, year)) + tapered := inflation.Initial * inflationPow(1.0-inflation.Taper, year) if tapered > inflation.Terminal { return tapered } else { diff --git a/pkg/rewards/inflation_pow.go b/pkg/rewards/inflation_pow.go new file mode 100644 index 000000000..68b5d9c1b --- /dev/null +++ b/pkg/rewards/inflation_pow.go @@ -0,0 +1,18 @@ +package rewards + +/* +#cgo LDFLAGS: -lm +#include +*/ +import "C" + +// inflationPow uses the platform libm, like Rust's f64::powf in +// solana-inflation 3.2.0, Inflation::total, and Firedancer d2f7c311 +// src/flamenco/rewards/fd_rewards.c:218. Go's math.Pow can differ by one +// ULP, enough to change an integer reward budget and therefore the bank hash. +// At testnet epoch 1059 (slot 451964256), pow(0.85, 4.9528778573584482) +// must be 0x3fdc9d8deed6b724; math.Pow returned 0x3fdc9d8deed6b723. +// This is called only when calculating epoch inflation, outside the hot path. +func inflationPow(base, exponent float64) float64 { + return float64(C.pow(C.double(base), C.double(exponent))) +} diff --git a/pkg/rewards/inflation_pow_test.go b/pkg/rewards/inflation_pow_test.go new file mode 100644 index 000000000..5a1a087cd --- /dev/null +++ b/pkg/rewards/inflation_pow_test.go @@ -0,0 +1,19 @@ +package rewards + +import ( + "math" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestInflationMatchesAgaveLibmAtTestnetEpoch1059(t *testing.T) { + // Testnet slot 451964256. Rust f64::powf/libm returns the last bit below; + // Go math.Pow rounded down, producing 392970281850804 instead of ...805. + year := math.Float64frombits(0x4013cfbf3689bf94) + require.Equal(t, uint64(0x3fdc9d8deed6b724), math.Float64bits(inflationPow(0.85, year))) + inflation := Inflation{Initial: 0.15, Terminal: 0.015, Taper: 0.15} + require.Equal(t, uint64(0x3fb12b5528e73aaf), math.Float64bits(inflation.Validator(year))) + require.Equal(t, 0.15, inflation.Validator(0)) + require.Equal(t, 0.015, inflation.Validator(100)) +} diff --git a/pkg/rewards/rewards.go b/pkg/rewards/rewards.go index 94bab4471..a8df079b4 100644 --- a/pkg/rewards/rewards.go +++ b/pkg/rewards/rewards.go @@ -13,6 +13,7 @@ import ( "github.com/Overclock-Validator/mithril/pkg/accounts" "github.com/Overclock-Validator/mithril/pkg/accountsdb" + "github.com/Overclock-Validator/mithril/pkg/epochstakes" "github.com/Overclock-Validator/mithril/pkg/features" "github.com/Overclock-Validator/mithril/pkg/global" "github.com/Overclock-Validator/mithril/pkg/mlog" @@ -451,50 +452,42 @@ func minimumStakeDelegation(slotCtx *sealevel.SlotCtx) uint64 { return 1 } -func selectCommissionVoteState( - votePubkey solana.PublicKey, - distributionVoteStates map[solana.PublicKey]*sealevel.VoteStateVersions, - rewardedEpochVoteStates map[solana.PublicKey]*sealevel.VoteStateVersions, - snapshotEpochVoteStates map[solana.PublicKey]*sealevel.VoteStateVersions, -) *sealevel.VoteStateVersions { - if snapshotEpochVoteStates != nil { - if voteState := snapshotEpochVoteStates[votePubkey]; voteState != nil { - return voteState - } - } - - if rewardedEpochVoteStates != nil { - if voteState := rewardedEpochVoteStates[votePubkey]; voteState != nil { - return voteState - } - } - - return distributionVoteStates[votePubkey] -} - func buildDelayedCommissionVoteStateCache( distributionVoteStates map[solana.PublicKey]*sealevel.VoteStateVersions, rewardedEpoch uint64, -) map[solana.PublicKey]*sealevel.VoteStateVersions { +) (map[solana.PublicKey]*sealevel.VoteStateVersions, error) { if len(distributionVoteStates) == 0 { - return nil - } - - rewardedEpochVoteStates := global.EpochVoteStateSnapshot(rewardedEpoch) - var snapshotEpochVoteStates map[solana.PublicKey]*sealevel.VoteStateVersions - if rewardedEpoch > 0 { - snapshotEpochVoteStates = global.EpochVoteStateSnapshot(rewardedEpoch - 1) + return nil, nil } + // Agave v4.4.0-beta.0 bank.rs:1826-1846 and + // partitioned_epoch_rewards/calculation.rs:737-754 use epoch-stakes keys + // E then E+1 when rewarding E. These capture commissions at the starts of + // E-1 and E respectively. The persisted snapshots also survive restarts. + older, _ := global.EpochStakesSnapshot(rewardedEpoch) + newer, _ := global.EpochStakesSnapshot(rewardedEpoch + 1) commissionVoteStates := make(map[solana.PublicKey]*sealevel.VoteStateVersions, len(distributionVoteStates)) - for votePubkey := range distributionVoteStates { - voteState := selectCommissionVoteState(votePubkey, distributionVoteStates, rewardedEpochVoteStates, snapshotEpochVoteStates) - if voteState != nil { - commissionVoteStates[votePubkey] = voteState + for votePubkey, current := range distributionVoteStates { + commissionVoteStates[votePubkey] = current + for _, snapshot := range []epochstakes.Snapshot{older, newer} { + historical, exists := snapshot.VoteAccounts[votePubkey] + if !exists { + continue + } + // A missing field in an old checkpoint is not an absent vote + // account. Falling back here silently applies a newer commission. + if historical == nil || historical.InflationRewardsCommissionBps == nil { + return nil, fmt.Errorf("missing historical inflation commission for vote account %s in epoch stakes %d; restore this generation from retained snapshot metadata", votePubkey, snapshot.Epoch) + } + commissionVoteStates[votePubkey] = &sealevel.VoteStateVersions{ + Type: sealevel.VoteStateVersionV4, + V4: sealevel.VoteState4{InflationRewardsCommissionBps: *historical.InflationRewardsCommissionBps}, + } + break } } - return commissionVoteStates + return commissionVoteStates, nil } func CalculateRewardPartitionForPubkey(pubkey solana.PublicKey, blockhash [32]byte, numPartitions uint64) uint64 { @@ -970,7 +963,11 @@ func CalculateRewardsStreaming( pv := PointValue{Rewards: pointValue.Rewards, Points: totalPoints} var commissionVoteStates map[solana.PublicKey]*sealevel.VoteStateVersions if f.IsActive(features.DelayCommissionUpdates) { - commissionVoteStates = buildDelayedCommissionVoteStateCache(voteCache, rewardedEpoch) + commissionVoteStates, err = buildDelayedCommissionVoteStateCache(voteCache, rewardedEpoch) + if err != nil { + CleanupPointsSpoolFile(pointsPath) + return nil, err + } } tempWriter, err := NewTempSpoolWriter(spoolDir, slot) diff --git a/pkg/sealevel/vote_state.go b/pkg/sealevel/vote_state.go index 1bafa7bd0..0bc8633d2 100644 --- a/pkg/sealevel/vote_state.go +++ b/pkg/sealevel/vote_state.go @@ -1924,3 +1924,20 @@ func (vs *VoteStateVersions) BlockRevenueCollector() solana.PublicKey { } return vs.NodePubkey() } + +// InflationRewardsCommissionBps preserves the full commission value when +// capturing historical epoch stakes. Reward calculation applies the cap. +func (vs *VoteStateVersions) InflationRewardsCommissionBps() uint16 { + switch vs.Type { + case VoteStateVersionV4: + return vs.V4.InflationRewardsCommissionBps + case VoteStateVersionCurrent: + return uint16(vs.Current.Commission) * 100 + case VoteStateVersionV1_14_11: + return uint16(vs.V1_14_11.Commission) * 100 + case VoteStateVersionV0_23_5: + return uint16(vs.V0_23_5.Commission) * 100 + default: + panic("invalid vote state version") + } +} diff --git a/pkg/snapshot/manifest_decoder.go b/pkg/snapshot/manifest_decoder.go index 174ac5e1a..c8e6d9cfd 100644 --- a/pkg/snapshot/manifest_decoder.go +++ b/pkg/snapshot/manifest_decoder.go @@ -44,15 +44,16 @@ type RentCollector struct { } type VoteAccount struct { - Lamports uint64 - NodePubkey solana.PublicKey - BlsPubkeyCompressed *[48]byte - BlockRevenueCollector *solana.PublicKey - LastTimestampTs int64 - LastTimestampSlot uint64 - Owner solana.PublicKey - Executable byte - RentEpoch uint64 + Lamports uint64 + NodePubkey solana.PublicKey + BlsPubkeyCompressed *[48]byte + BlockRevenueCollector *solana.PublicKey + InflationRewardsCommissionBps *uint16 + LastTimestampTs int64 + LastTimestampSlot uint64 + Owner solana.PublicKey + Executable byte + RentEpoch uint64 } type VoteAccountsPair struct { @@ -430,6 +431,8 @@ func (voteAcct *VoteAccount) UnmarshalWithDecoder(decoder *bin.Decoder) error { collector := voteState.BlockRevenueCollector() voteAcct.BlockRevenueCollector = &collector + commission := voteState.InflationRewardsCommissionBps() + voteAcct.InflationRewardsCommissionBps = &commission voteAcct.LastTimestampTs = voteTimestamp.Timestamp voteAcct.LastTimestampSlot = voteTimestamp.Slot } diff --git a/pkg/snapshot/manifest_decoder_test.go b/pkg/snapshot/manifest_decoder_test.go index 22b49bd00..a6f67e96d 100644 --- a/pkg/snapshot/manifest_decoder_test.go +++ b/pkg/snapshot/manifest_decoder_test.go @@ -71,3 +71,40 @@ func TestManifestSeedPreservesFeeCollectors(t *testing.T) { require.Equal(t, destination, *got.VoteAccounts[vote].BlockRevenueCollector) } } + +func TestManifestCommissionSurvivesDecodeAndSeed(t *testing.T) { + for _, version := range []uint32{sealevel.VoteStateVersionV0_23_5, sealevel.VoteStateVersionV1_14_11, sealevel.VoteStateVersionCurrent, sealevel.VoteStateVersionV4} { + for _, bps := range []uint16{0, 1200, 10000, 1234} { + want := bps + vote := &sealevel.VoteStateVersions{Type: version} + if version == sealevel.VoteStateVersionV4 { + vote.V4.InflationRewardsCommissionBps = bps + } else { + want = (bps / 100) * 100 + vote.V0_23_5.Commission = byte(bps / 100) + vote.V1_14_11.Commission = byte(bps / 100) + vote.Current.Commission = byte(bps / 100) + } + var wire, account bytes.Buffer + require.NoError(t, vote.MarshalWithEncoder(bin.NewBinEncoder(&wire))) + enc := bin.NewBinEncoder(&account) + require.NoError(t, enc.WriteUint64(42, bin.LE)) + require.NoError(t, enc.WriteUint64(uint64(wire.Len()), bin.LE)) + require.NoError(t, enc.WriteBytes(wire.Bytes(), false)) + require.NoError(t, enc.WriteBytes(make([]byte, 32), false)) + require.NoError(t, enc.WriteByte(0)) + require.NoError(t, enc.WriteUint64(0, bin.LE)) + var decoded VoteAccount + require.NoError(t, decoded.UnmarshalWithDecoder(bin.NewBinDecoder(account.Bytes()))) + require.NotNil(t, decoded.InflationRewardsCommissionBps) + require.Equal(t, want, *decoded.InflationRewardsCommissionBps) + key := solana.PublicKey{31} + seeds := convertVersionedEpochStakesToPersisted([]VersionedEpochStakesPair{{Epoch: 1058, Val: VersionedEpochStakes{TotalStake: 42, Stakes: Stake{VoteAccounts: []VoteAccountsPair{{Key: key, Stake: 42, Value: decoded}}}}}}) + cache := epochstakes.NewEpochStakesCache() + _, err := cache.DeserializeAndLoadEpoch([]byte(seeds[1058])) + require.NoError(t, err) + got, _ := cache.Snapshot(1058) + require.Equal(t, want, *got.VoteAccounts[key].InflationRewardsCommissionBps) + } + } +} diff --git a/pkg/snapshot/manifest_seed.go b/pkg/snapshot/manifest_seed.go index fe2a66b69..d7b34f3fc 100644 --- a/pkg/snapshot/manifest_seed.go +++ b/pkg/snapshot/manifest_seed.go @@ -130,15 +130,16 @@ func convertVersionedEpochStakesToPersisted(stakes []VersionedEpochStakesPair) m collector = va.Value.BlockRevenueCollector.String() } persisted.VoteAccts[pkStr] = &epochstakes.VoteAccountJSON{ - BlockRevenueCollector: collector, - Lamports: va.Value.Lamports, - NodePubkey: base58.Encode(va.Value.NodePubkey[:]), - BlsPubkeyCompressed: bls, - LastTimestampTs: va.Value.LastTimestampTs, - LastTimestampSlot: va.Value.LastTimestampSlot, - Owner: base58.Encode(va.Value.Owner[:]), - Executable: va.Value.Executable, - RentEpoch: va.Value.RentEpoch, + BlockRevenueCollector: collector, + InflationRewardsCommissionBps: va.Value.InflationRewardsCommissionBps, + Lamports: va.Value.Lamports, + NodePubkey: base58.Encode(va.Value.NodePubkey[:]), + BlsPubkeyCompressed: bls, + LastTimestampTs: va.Value.LastTimestampTs, + LastTimestampSlot: va.Value.LastTimestampSlot, + Owner: base58.Encode(va.Value.Owner[:]), + Executable: va.Value.Executable, + RentEpoch: va.Value.RentEpoch, } } From 3639917c6255461a2ac0201fe477c11d5125b4fa Mon Sep 17 00:00:00 2001 From: smcio Date: Wed, 14 Oct 2026 09:33:24 +0000 Subject: [PATCH 2/2] fix a regression to preserve delayed commission history for inactive voters --- pkg/replay/epoch.go | 31 +++- pkg/replay/epoch_commission_history_test.go | 196 ++++++++++++++++++++ 2 files changed, 221 insertions(+), 6 deletions(-) create mode 100644 pkg/replay/epoch_commission_history_test.go diff --git a/pkg/replay/epoch.go b/pkg/replay/epoch.go index 6260d47aa..ebc500cc7 100644 --- a/pkg/replay/epoch.go +++ b/pkg/replay/epoch.go @@ -396,6 +396,7 @@ func updateEpochStakesAndRefreshVoteCache(leaderScheduleEpoch uint64, b *block.B if rebuildErr != nil { mlog.Log.Errorf("failed to rebuild vote cache at epoch boundary: %v", rebuildErr) } + vatActive := f != nil && f.IsActive(features.ValidatorAdmissionTicket) // An older checkpoint can have cached this boundary's stakes without its // commission fields. Recompute that new generation from the same durable @@ -407,6 +408,17 @@ func updateEpochStakesAndRefreshVoteCache(leaderScheduleEpoch uint64, b *block.B break } } + // Before VAT, zero-effective-stake voters still supply historical + // commissions. A previous attempt may have persisted only effective voters; + // this boundary's generation can be rebuilt from the same durable parent. + if hasEpochStakes && commissionsComplete && !vatActive { + for votePk := range voteMetadata { + if _, exists := cachedEpochStakes.VoteAccounts[votePk]; !exists { + commissionsComplete = false + break + } + } + } if hasEpochStakes && commissionsComplete { mlog.Log.Infof("already had EpochStakes for epoch %d", leaderScheduleEpoch) // A restart can retain this generation from a previously attempted @@ -423,7 +435,7 @@ func updateEpochStakesAndRefreshVoteCache(leaderScheduleEpoch uint64, b *block.B voteCache := global.VoteCache() effectiveStakes := scanResult.EffectiveStakes totalEffectiveStake := scanResult.TotalEffectiveStake - if f != nil && f.IsActive(features.ValidatorAdmissionTicket) { + if vatActive { minimumBalance, err := minimumVoteAccountBalanceForVAT(f, epochSchedule, slot) if err != nil { panic(err) @@ -432,11 +444,16 @@ func updateEpochStakesAndRefreshVoteCache(leaderScheduleEpoch uint64, b *block.B mlog.Log.FileOnlyf("VAT epoch stakes: admitted=%d/%d minimum_vote_balance=%d", len(effectiveStakes), len(scanResult.EffectiveStakes), minimumBalance) } epochStakes := make(map[solana.PublicKey]uint64, len(effectiveStakes)) - epochVoteAccounts := make(map[solana.PublicKey]*epochstakes.VoteAccount, len(effectiveStakes)) - for votePk, stake := range effectiveStakes { + epochVoteAccounts := make(map[solana.PublicKey]*epochstakes.VoteAccount, len(voteMetadata)) + // Agave v4.4.0-beta.0 runtime/src/stakes.rs:890-900 retains vote-account + // metadata when effective stake becomes zero. Only VAT filters those voters + // (vote/src/vote_account.rs:220-234). Preserve the pre-VAT commission history + // separately from positive execution/leader weights: when rewarding E, key E + // holds the rate from the start of E-1, even if stake was inactive then. + for votePk, meta := range voteMetadata { + stake, hasEffectiveStake := effectiveStakes[votePk] voteAcct, exists := voteCache[votePk] - meta, hasMeta := voteMetadata[votePk] - if exists && hasMeta { + if exists && (!vatActive || hasEffectiveStake) { lastTimestamp := voteAcct.LastTimestamp() var lastTimestampTs int64 var lastTimestampSlot uint64 @@ -448,7 +465,9 @@ func updateEpochStakesAndRefreshVoteCache(leaderScheduleEpoch uint64, b *block.B if meta.Executable { executable = 1 } - epochStakes[votePk] = stake + if hasEffectiveStake { + epochStakes[votePk] = stake + } collector := voteAcct.BlockRevenueCollector() commission := voteAcct.InflationRewardsCommissionBps() epochVoteAccounts[votePk] = &epochstakes.VoteAccount{ diff --git a/pkg/replay/epoch_commission_history_test.go b/pkg/replay/epoch_commission_history_test.go new file mode 100644 index 000000000..51956808b --- /dev/null +++ b/pkg/replay/epoch_commission_history_test.go @@ -0,0 +1,196 @@ +package replay + +import ( + "math" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/rewards" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +func TestEpochBoundaryDelayedCommissionForReactivatedStake(t *testing.T) { + // Synthetic epoch E=40, distributed at slot 41*54,000. At the start of + // E-1 the voter charges 10%, but its delegation has fully cooled down. + // During E-1 it reactivates and raises commission to 20%. Before VAT, + // delayed commission must still split E's 1,000 lamports as 100 / 900. + // Agave v4.4.0-beta.0 runtime/src/stakes.rs:890-900 retains vote accounts + // with zero delegated stake; calculation.rs:737-754 selects their history. + const rewardedEpoch, slotsPerEpoch = uint64(40), uint64(54_000) + const firstBoundary = (rewardedEpoch - 1) * slotsPerEpoch + const distributionSlot = (rewardedEpoch + 1) * slotsPerEpoch + voter, baseVoter := solana.PublicKey{71}, solana.PublicKey{72} + stakeKey := solana.PublicKey{0xa0, 1} // First delegation in epochBoundaryStakeDB. + + for _, tc := range []struct { + name string + active bool + delayDisabled bool + reload bool + retry bool + vat bool + wantVoter uint64 + }{ + {name: "reactivated", wantVoter: 100}, + {name: "reactivated after checkpoint reload", reload: true, wantVoter: 100}, + {name: "retry previously incomplete boundary", retry: true, wantVoter: 100}, + {name: "continuously active control", active: true, wantVoter: 100}, + {name: "delay disabled control", delayDisabled: true, wantVoter: 200}, + {name: "VAT filtered control", vat: true, reload: true, wantVoter: 200}, + } { + t.Run(tc.name, func(t *testing.T) { + for e := rewardedEpoch; e <= rewardedEpoch+2; e++ { + global.ClearEpochStakes(e) + t.Cleanup(func() { global.ClearEpochStakes(e) }) + } + f := features.NewFeaturesDefault() + if !tc.delayDisabled { + f.EnableFeature(features.DelayCommissionUpdates, 0) + } + if tc.vat { + f.EnableFeature(features.ValidatorAdmissionTicket, 0) + previousRent := sealevel.SysvarCache.Rent.Sysvar + rent := sealevel.NewDefaultRentSysvar() + sealevel.SysvarCache.Rent.Sysvar = &rent + t.Cleanup(func() { sealevel.SysvarCache.Rent.Sysvar = previousRent }) + } + delegation := sealevel.Delegation{ + VoterPubkey: voter, StakeLamports: 1_000, + ActivationEpoch: math.MaxUint64, DeactivationEpoch: rewardedEpoch - 3, + } + if tc.active { + delegation.DeactivationEpoch = math.MaxUint64 + } + db := epochBoundaryStakeDB(t, firstBoundary-1, []sealevel.Delegation{ + delegation, + {VoterPubkey: baseVoter, StakeLamports: 1_000_000, ActivationEpoch: math.MaxUint64, DeactivationEpoch: math.MaxUint64}, + }) + store := func(slot uint64, accts ...*accounts.Account) { + t.Helper() + done := make(chan struct{}) + require.NoError(t, db.StoreAccounts(accts, slot, func() { close(done) })) + <-done + } + setVote := func(slot uint64, key solana.PublicKey, commission uint16, credits []sealevel.EpochCredits) { + t.Helper() + acct, err := db.GetAccount(slot, key) + require.NoError(t, err) + acct = acct.Clone() + vs := &sealevel.VoteStateVersions{Type: sealevel.VoteStateVersionV4, V4: sealevel.VoteState4{ + NodePubkey: key, BlsPubkeyCompressed: &[48]byte{key[0]}, + InflationRewardsCommissionBps: commission, EpochCredits: credits, + }} + acct.Data, err = sealevel.MarshalVersionedVoteState(vs) + require.NoError(t, err) + store(slot, acct) + } + setVote(firstBoundary-1, voter, 1_000, nil) + setVote(firstBoundary-1, baseVoter, 0, nil) + schedule := &sealevel.SysvarEpochSchedule{SlotsPerEpoch: slotsPerEpoch, LeaderScheduleSlotOffset: slotsPerEpoch} + history := sealevel.SysvarStakeHistory{} + boundary := func(epoch uint64) *BoundaryStakeScanResult { + t.Helper() + slot := epoch * slotsPerEpoch + scan := scanStakesForEpochBoundary(db, slot-1, epoch-1, epoch, &history, schedule, f) + b := &block.Block{Slot: slot} + updateEpochStakesAndRefreshVoteCache(epoch+1, b, db, slot-1, scan, f, schedule) + // Retained zero-effective-stake metadata must not change any + // execution stake weights, totals, or leader-selection inputs. + require.Equal(t, scan.EffectiveStakes, b.EpochStakesPerVoteAcct) + require.Equal(t, scan.TotalEffectiveStake, b.TotalEpochStake) + history.Update(epoch-1, sealevel.StakeHistoryEntry{ + Effective: scan.StakeHistoryEffective, Activating: scan.StakeHistoryActivating, + Deactivating: scan.StakeHistoryDeactivating, + }) + return scan + } + firstScan := boundary(rewardedEpoch - 1) + require.Equal(t, uint64(1_000), firstScan.VoteAcctStakes[voter]) + if !tc.active { + require.NotContains(t, firstScan.EffectiveStakes, voter) + } + if tc.retry { + // An earlier PR build persisted only effective voters. Retrying + // this same boundary may recover the omitted voter from its parent. + snapshot, _ := global.EpochStakesSnapshot(rewardedEpoch) + votes := global.EpochStakesVoteAccts(rewardedEpoch) + delete(votes, voter) + global.PutEpochStakes(rewardedEpoch, snapshot.Stakes, votes, snapshot.TotalStake) + b := &block.Block{Slot: firstBoundary} + updateEpochStakesAndRefreshVoteCache(rewardedEpoch, b, db, firstBoundary-1, firstScan, f, schedule) + } + complete, _ := global.EpochStakesSnapshot(rewardedEpoch) + updateEpochStakesAndRefreshVoteCache(rewardedEpoch, &block.Block{Slot: firstBoundary}, db, firstBoundary-1, firstScan, f, schedule) + again, _ := global.EpochStakesSnapshot(rewardedEpoch) + require.Equal(t, complete.Generation, again.Generation, "complete history must stay immutable on retry") + + if !tc.active { + acct, err := db.GetAccount(firstBoundary, stakeKey) + require.NoError(t, err) + acct = acct.Clone() + ss, err := sealevel.UnmarshalStakeState(acct.Data) + require.NoError(t, err) + ss.Stake.Stake.Delegation.ActivationEpoch = rewardedEpoch - 1 + ss.Stake.Stake.Delegation.DeactivationEpoch = math.MaxUint64 + acct.Data, err = sealevel.MarshalStakeStake(ss) + require.NoError(t, err) + store(firstBoundary, acct) + } + setVote(firstBoundary, voter, 2_000, nil) + secondScan := boundary(rewardedEpoch) + require.Equal(t, uint64(1_000), secondScan.EffectiveStakes[voter]) + if tc.reload { + for e := rewardedEpoch; e <= rewardedEpoch+1; e++ { + data, err := global.SerializeEpochStakes(e) + require.NoError(t, err) + global.ClearEpochStakes(e) + _, err = global.DeserializeAndLoadEpochStakes(data) + require.NoError(t, err) + } + } + setVote(distributionSlot-1, voter, 2_000, []sealevel.EpochCredits{{Epoch: rewardedEpoch, Credits: 1}}) + boundary(rewardedEpoch + 1) + result, err := rewards.CalculateRewardsStreaming(db, distributionSlot, &history, nil, + global.VoteCacheSnapshot(), rewards.PointValue{Rewards: 1_000}, rewardedEpoch, + [32]byte{1}, &sealevel.SlotCtx{Features: f}, f, schedule, rewards.RewardCalculationMode{}) + require.NoError(t, err) + require.Len(t, result.ValidatorRewards, 1) + require.Equal(t, tc.wantVoter, result.ValidatorRewards[voter].Load()) + require.Equal(t, 1_000-tc.wantVoter, result.TotalStakerRewards) + + votes, voteParents, paidVote := rewards.DistributeVotingRewards(db, result.ValidatorRewards, distributionSlot, nil) + require.Equal(t, tc.wantVoter, paidVote) + require.Len(t, votes, 1) + require.Equal(t, tc.wantVoter, votes[0].Lamports-voteParents[0].Lamports) + var paidStake uint64 + for partition := uint64(0); partition < result.NumPartitions; partition++ { + stakes, parents, paid, burned := rewards.DistributeStakingRewardsFromSpool( + db, result.SpoolDir, result.SpoolSlot, partition, distributionSlot+1+partition, nil) + require.Zero(t, burned) + paidStake += paid + for i, acct := range stakes { + require.Equal(t, stakeKey, acct.Key) + require.Equal(t, 1_000-tc.wantVoter, acct.Lamports-parents[i].Lamports) + ss, err := sealevel.UnmarshalStakeState(acct.Data) + require.NoError(t, err) + require.Equal(t, uint64(1), ss.Stake.Stake.CreditsObserved) + require.Equal(t, 2_000-tc.wantVoter, ss.Stake.Stake.Delegation.StakeLamports) + } + } + require.Equal(t, 1_000-tc.wantVoter, paidStake) + require.Equal(t, uint64(1_000), paidVote+paidStake) + older, _ := global.EpochStakesSnapshot(rewardedEpoch) + if tc.vat { + require.NotContains(t, older.VoteAccounts, voter) + } else if !tc.delayDisabled { + require.Contains(t, older.VoteAccounts, voter) + require.Equal(t, uint16(1_000), *older.VoteAccounts[voter].InflationRewardsCommissionBps) + } + }) + } +}