From ef0c587aa717bce193554004f3d122417f772da7 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 03:58:47 +0000 Subject: [PATCH 01/25] blockprod: resolve address lookup tables for v0 transactions --- pkg/blockprod/bank.go | 32 +- pkg/blockprod/bank_lookup_test.go | 139 +++++++ pkg/blockprod/scheduler/scheduler.go | 3 + .../scheduler/scheduler_lookup_test.go | 73 ++++ pkg/blockprod/testenv_lookup.go | 111 +++++ pkg/costmodel/costmodel_test.go | 43 ++ pkg/costmodel/transaction_cost.go | 23 +- pkg/forge/forge.go | 3 + pkg/replay/address_lookup.go | 96 +++++ pkg/replay/address_lookup_test.go | 378 ++++++++++++++++++ pkg/replay/transaction_preparation.go | 5 + 11 files changed, 899 insertions(+), 7 deletions(-) create mode 100644 pkg/blockprod/bank_lookup_test.go create mode 100644 pkg/blockprod/scheduler/scheduler_lookup_test.go create mode 100644 pkg/blockprod/testenv_lookup.go create mode 100644 pkg/replay/address_lookup_test.go diff --git a/pkg/blockprod/bank.go b/pkg/blockprod/bank.go index 4f1d809b..5cd101f3 100644 --- a/pkg/blockprod/bank.go +++ b/pkg/blockprod/bank.go @@ -152,6 +152,7 @@ const ( ForgeDroppedExecution ForgeDroppedAlreadyProcessed ForgeDroppedAccountLocks + ForgeDroppedLookup ) func (r ForgeResult) String() string { @@ -172,6 +173,8 @@ func (r ForgeResult) String() string { return "dropped_already_processed" case ForgeDroppedAccountLocks: return "dropped_account_locks" + case ForgeDroppedLookup: + return "dropped_lookup" default: return "unknown" } @@ -246,8 +249,29 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre ancestorAlreadyProcessed, ancestorStatusErr = b.ancestorStatuses.ContainsMessage(tx.Message.RecentBlockhash, messageHash) } + // For a v0 transaction with address-table lookups, execTx is a resolved copy, + // while tx stays unresolved for entry encoding. + execTx := tx var cost costmodel.TransactionCost if !ancestorAlreadyProcessed && ancestorStatusErr == nil { + if replay.HasAddressTableLookups(tx) { + // Tables are read from the parent state, which this slot does not + // change, so resolve outside the bank lock. + resolved, txErr := replay.ResolveAddressTableLookups(b.slotCtx, tx) + if txErr != nil { + b.RebateSchedule(wireSize) + return ForgeDroppedLookup, costmodel.ExceedNone + } + // A loaded address can repeat a static key or another loaded + // address, or take the account count past the lock limit. + if txErr := replay.ValidateAccountLocks(resolved, lockFeats); txErr != nil { + b.RebateSchedule(wireSize) + return ForgeDroppedAccountLocks, costmodel.ExceedNone + } + execTx = resolved + // Static preparation, if any, was done on the unresolved message. + prepared = nil + } feats := b.slotCtx.Features if feats == nil { f := features.NewFeaturesDefault() @@ -256,7 +280,7 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre if prepared != nil { cost = prepared.Cost() } else { - cost, err = costmodel.EstimateTransactionCost(tx, feats) + cost, err = costmodel.EstimateTransactionCost(execTx, feats) } if err != nil { b.RebateSchedule(wireSize) @@ -294,19 +318,19 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre if reason := b.reserveEntryBytesLocked(wireSize); reason != costmodel.ExceedNone { return ForgeDroppedCost, reason } - if err := b.preparer.PayerCanFund(b.slotCtx, tx, prepared); err != nil { + if err := b.preparer.PayerCanFund(b.slotCtx, execTx, prepared); err != nil { return ForgeDroppedExecution, costmodel.ExceedNone } output := b.preparer.LoadAndExecute(replay.LoadAndExecuteTransactionInput{ SlotCtx: b.slotCtx, - Transaction: tx, + Transaction: execTx, LeanResult: true, SkipTimingMetrics: true, Arena: b.borrowedAccounts, }, prepared) if output.ProcessingResult.TransactionError != nil { - feeInfo, err := replay.ApplyFailedTransaction(b.slotCtx, tx, output) + feeInfo, err := replay.ApplyFailedTransaction(b.slotCtx, execTx, output) if err != nil { return ForgeDroppedExecution, costmodel.ExceedNone } diff --git a/pkg/blockprod/bank_lookup_test.go b/pkg/blockprod/bank_lookup_test.go new file mode 100644 index 00000000..8df5b9f7 --- /dev/null +++ b/pkg/blockprod/bank_lookup_test.go @@ -0,0 +1,139 @@ +package blockprod + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +var lookupTestTableKey = solana.PublicKey{0x7A, 0xB1, 0xE0} + +// withLookupSlotHashes installs SlotHashes for the test env's slot, which +// lookup resolution needs to classify deactivating tables. +func withLookupSlotHashes(t *testing.T, env *TestEnv) { + t.Helper() + prev := sealevel.SysvarCache.SlotHashes.Sysvar + slot := env.SlotCtx.Slot + slotHashes := sealevel.SysvarSlotHashes{{Slot: slot - 1}, {Slot: slot - 2}} + sealevel.SysvarCache.SlotHashes.Sysvar = &slotHashes + t.Cleanup(func() { sealevel.SysvarCache.SlotHashes.Sysvar = prev }) +} + +// addLookupTable puts table in the test bank's parent state, which is where +// lookups are resolved from. +func addLookupTable(t *testing.T, env *TestEnv, table LookupTable) { + t.Helper() + acct, err := table.Account() + require.NoError(t, err) + parent, ok := env.SlotCtx.UnrootedRead.(ParentAccounts) + if !ok { + parent = ParentAccounts{} + env.SlotCtx.UnrootedRead = parent + } + parent[table.Key] = acct +} + +func lookupTransfer(t *testing.T, lamports uint64) (*solana.Transaction, []byte) { + t.Helper() + wire, err := SignedV0LookupTransferWire(lamports, lookupTestTableKey, 0) + require.NoError(t, err) + tx, err := solana.TransactionFromBytes(wire) + require.NoError(t, err) + return tx, wire +} + +func lamportsOf(t *testing.T, env *TestEnv, key solana.PublicKey) uint64 { + t.Helper() + acct, err := env.SlotCtx.GetAccount(key) + require.NoError(t, err) + return acct.Lamports +} + +func TestWorkingBankForgesV0TransferThroughLookupTable(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + withLookupSlotHashes(t, env) + addLookupTable(t, env, ActiveLookupTable(lookupTestTableKey, txfixture.DestPubkey())) + + const amount = 1_234 + tx, wire := lookupTransfer(t, amount) + payerBefore := lamportsOf(t, env, txfixture.PayerPubkey()) + destBefore := lamportsOf(t, env, txfixture.DestPubkey()) + + result, reason := env.Bank.ForgeTransaction(tx, len(wire)) + require.Equal(t, ForgeAccepted, result) + require.Equal(t, costmodel.ExceedNone, reason) + + assert.Equal(t, destBefore+amount, lamportsOf(t, env, txfixture.DestPubkey())) + assert.Equal(t, payerBefore-amount-5_000, lamportsOf(t, env, txfixture.PayerPubkey())) + assert.Equal(t, uint64(5_000), env.Bank.TxFeeAccumulator().TotalFees) + + // The entry carries the original, unresolved transaction and its exact wire. + forged := env.Bank.ForgedTransactions() + require.Len(t, forged, 1) + assert.Same(t, tx, forged[0]) + assert.False(t, forged[0].Message.IsResolved()) + forgedWire, err := forged[0].MarshalBinary() + require.NoError(t, err) + assert.Equal(t, wire, forgedWire) + + // The looked-up destination is write-locked in block cost accounting. + assert.NotZero(t, env.Bank.CostTracker().WritableAccountCost(txfixture.DestPubkey())) +} + +func TestWorkingBankDropsUnresolvableLookups(t *testing.T) { + deactivated := ActiveLookupTable(lookupTestTableKey, txfixture.DestPubkey()) + deactivated.DeactivationSlot = 30 // no longer in SlotHashes + appendedThisSlot := ActiveLookupTable(lookupTestTableKey, txfixture.DestPubkey()) + appendedThisSlot.LastExtendedSlot = 42 // the test env's slot + appendedThisSlot.LastExtendedStartIndex = 0 + + cases := []struct { + name string + table *LookupTable + }{ + {"missing table", nil}, + {"deactivated table", &deactivated}, + {"address appended this slot", &appendedThisSlot}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + withLookupSlotHashes(t, env) + if tc.table != nil { + addLookupTable(t, env, *tc.table) + } + tx, wire := lookupTransfer(t, 1) + assertForgeLookupDropped(t, env, tx, wire, ForgeDroppedLookup) + }) + } +} + +// The account lock rules apply to the resolved keys too: a table that loads +// the transaction's own payer repeats a static key. +func TestWorkingBankDropsLookupThatRepeatsAStaticKey(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + withLookupSlotHashes(t, env) + addLookupTable(t, env, ActiveLookupTable(lookupTestTableKey, txfixture.PayerPubkey())) + tx, wire := lookupTransfer(t, 1) + assertForgeLookupDropped(t, env, tx, wire, ForgeDroppedAccountLocks) +} + +func assertForgeLookupDropped(t *testing.T, env *TestEnv, tx *solana.Transaction, wire []byte, want ForgeResult) { + t.Helper() + payerBefore := lamportsOf(t, env, txfixture.PayerPubkey()) + result, reason := env.Bank.ForgeTransaction(tx, len(wire)) + assert.Equal(t, want, result) + assert.Equal(t, costmodel.ExceedNone, reason) + assert.Equal(t, payerBefore, lamportsOf(t, env, txfixture.PayerPubkey())) + assert.Empty(t, env.Bank.ForgedTransactions()) + assert.Zero(t, env.Bank.TxFeeAccumulator().TotalFees) + assert.False(t, tx.Message.IsResolved(), "the scheduler's transaction must stay unresolved") +} diff --git a/pkg/blockprod/scheduler/scheduler.go b/pkg/blockprod/scheduler/scheduler.go index 60d63ba3..2985a70c 100644 --- a/pkg/blockprod/scheduler/scheduler.go +++ b/pkg/blockprod/scheduler/scheduler.go @@ -32,6 +32,7 @@ type Stats struct { DroppedExpired uint64 DroppedAlreadyProcessed uint64 DroppedAccountLocks uint64 + DroppedLookup uint64 Accepted uint64 DroppedNoBank uint64 DroppedCost uint64 @@ -365,6 +366,8 @@ func (s *Scheduler) recordForge(result blockprod.ForgeResult, reason costmodel.E s.stats.DroppedAlreadyProcessed++ case blockprod.ForgeDroppedAccountLocks: s.stats.DroppedAccountLocks++ + case blockprod.ForgeDroppedLookup: + s.stats.DroppedLookup++ default: s.stats.DroppedNoBank++ } diff --git a/pkg/blockprod/scheduler/scheduler_lookup_test.go b/pkg/blockprod/scheduler/scheduler_lookup_test.go new file mode 100644 index 00000000..6a5605d1 --- /dev/null +++ b/pkg/blockprod/scheduler/scheduler_lookup_test.go @@ -0,0 +1,73 @@ +package scheduler + +import ( + "context" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/blockprod" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/tpu/packet" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/require" +) + +var schedulerLookupTableKey = solana.PublicKey{0x7A, 0xB1, 0xE1} + +// lookupEnv is a test bank at slot whose lookup table had its only address +// appended in slot 42. +func lookupEnv(t *testing.T, slot uint64) *blockprod.TestEnv { + t.Helper() + env := blockprod.NewTestEnv(blockprod.TestEnvConfig{}) + t.Cleanup(env.Close) + env.SlotCtx.Slot = slot + table := blockprod.ActiveLookupTable(schedulerLookupTableKey, txfixture.DestPubkey()) + table.LastExtendedSlot = 42 + acct, err := table.Account() + require.NoError(t, err) + env.SlotCtx.UnrootedRead = blockprod.ParentAccounts{table.Key: acct} + return env +} + +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for !cond() { + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(2 * time.Millisecond) + } +} + +// A transaction whose lookup does not resolve is dropped, not retried, even +// though it would resolve in a later slot. +func TestSchedulerDropsFailedLookupWithoutRetry(t *testing.T) { + prev := sealevel.SysvarCache.SlotHashes.Sysvar + slotHashes := sealevel.SysvarSlotHashes{{Slot: 42}, {Slot: 41}} + sealevel.SysvarCache.SlotHashes.Sysvar = &slotHashes + t.Cleanup(func() { sealevel.SysvarCache.SlotHashes.Sysvar = prev }) + + controller := blockprod.NewController() + sched := New(controller) + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + sched.Start(ctx) + t.Cleanup(sched.Stop) + wire, err := blockprod.SignedV0LookupTransferWire(1, schedulerLookupTableKey, 0) + require.NoError(t, err) + sched.Receive(packet.Owned(wire)) + waitFor(t, "buffering", func() bool { return sched.Buffered() == 1 }) + + controller.SetWorkingBank(lookupEnv(t, 42).Bank) + waitFor(t, "lookup failure", func() bool { return sched.Stats().DroppedLookup == 1 }) + + next := lookupEnv(t, 43) + controller.SetWorkingBank(next.Bank) + time.Sleep(50 * time.Millisecond) + require.Zero(t, sched.Stats().Accepted) + require.Empty(t, next.Bank.ForgedTransactions()) + require.Zero(t, sched.Buffered()) + require.Equal(t, uint64(1), sched.Stats().DroppedLookup) +} diff --git a/pkg/blockprod/testenv_lookup.go b/pkg/blockprod/testenv_lookup.go new file mode 100644 index 00000000..cae7f3e2 --- /dev/null +++ b/pkg/blockprod/testenv_lookup.go @@ -0,0 +1,111 @@ +package blockprod + +import ( + "bytes" + "encoding/binary" + "fmt" + "math" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" +) + +// ParentAccounts serves accounts from a test bank's parent state. Install it +// as the bank's SlotCtx.UnrootedRead for accounts the leader reads from its +// parent rather than from the slot, such as address lookup tables. +type ParentAccounts map[solana.PublicKey]*accounts.Account + +func (p ParentAccounts) GetAccount(_ uint64, pubkey solana.PublicKey) (*accounts.Account, error) { + if acct, ok := p[pubkey]; ok { + return acct.Clone(), nil + } + return nil, fmt.Errorf("account %s not found in parent state", pubkey) +} + +// LookupTable describes an address lookup table account for tests. +type LookupTable struct { + Key solana.PublicKey + // DeactivationSlot is math.MaxUint64 for an active table. + DeactivationSlot uint64 + // LastExtendedSlot and LastExtendedStartIndex record the most recent + // extension: addresses from LastExtendedStartIndex on were appended in + // LastExtendedSlot. + LastExtendedSlot uint64 + LastExtendedStartIndex uint8 + Addresses []solana.PublicKey +} + +// ActiveLookupTable is an active table last extended in slot 1. +func ActiveLookupTable(key solana.PublicKey, addrs ...solana.PublicKey) LookupTable { + return LookupTable{Key: key, DeactivationSlot: math.MaxUint64, LastExtendedSlot: 1, Addresses: addrs} +} + +// Account encodes the table the way the address lookup table program stores +// it, with the fixture payer as authority. +func (lt LookupTable) Account() (*accounts.Account, error) { + var buf bytes.Buffer + enc := bin.NewBinEncoder(&buf) + if err := enc.WriteUint32(sealevel.AddressLookupTableProgramStateLookupTable, bin.LE); err != nil { + return nil, err + } + authority := txfixture.PayerPubkey() + meta := sealevel.LookupTableMeta{ + DeactivationSlot: lt.DeactivationSlot, + LastExtendedSlot: lt.LastExtendedSlot, + LastExtendedSlotStartIndex: lt.LastExtendedStartIndex, + Authority: &authority, + } + if err := meta.MarshalWithEncoder(enc); err != nil { + return nil, err + } + for _, addr := range lt.Addresses { + if err := enc.WriteBytes(addr[:], false); err != nil { + return nil, err + } + } + return &accounts.Account{ + Key: lt.Key, + Lamports: 1_000_000_000, + Owner: addresses.AddressLookupTableAddr, + Data: buf.Bytes(), + RentEpoch: math.MaxUint64, + }, nil +} + +// SignedV0LookupTransferWire is a signed v0 transfer of lamports from the +// fixture payer to entry destIndex of table, loaded as a writable lookup. Its +// static keys are [payer, system program]. +func SignedV0LookupTransferWire(lamports uint64, table solana.PublicKey, destIndex uint8) ([]byte, error) { + data := make([]byte, 12) + binary.LittleEndian.PutUint32(data, 2) // SystemInstruction::Transfer + binary.LittleEndian.PutUint64(data[4:], lamports) + msg := solana.Message{ + Header: solana.MessageHeader{NumRequiredSignatures: 1, NumReadonlyUnsignedAccounts: 1}, + AccountKeys: solana.PublicKeySlice{txfixture.PayerPubkey(), solana.SystemProgramID}, + RecentBlockhash: txfixture.TestBlockhash(), + Instructions: []solana.CompiledInstruction{{ + ProgramIDIndex: 1, + Accounts: []uint16{0, 2}, + Data: data, + }}, + AddressTableLookups: solana.MessageAddressTableLookupSlice{{AccountKey: table, WritableIndexes: []uint8{destIndex}}}, + } + if _, err := msg.SetVersion(solana.MessageVersionV0); err != nil { + return nil, err + } + tx := &solana.Transaction{Message: msg} + payerKey := txfixture.PayerPrivateKey() + if _, err := tx.Sign(func(key solana.PublicKey) *solana.PrivateKey { + if key.Equals(txfixture.PayerPubkey()) { + return &payerKey + } + return nil + }); err != nil { + return nil, err + } + return tx.MarshalBinary() +} diff --git a/pkg/costmodel/costmodel_test.go b/pkg/costmodel/costmodel_test.go index 82160999..241c39ba 100644 --- a/pkg/costmodel/costmodel_test.go +++ b/pkg/costmodel/costmodel_test.go @@ -94,6 +94,49 @@ func TestWritableAccountsUsesUnsignedWritableRange(t *testing.T) { assert.Equal(t, []solana.PublicKey{{1}, {3}, {4}}, writableAccounts(tx)) } +// v0LookupTx has static keys {1} (writable signer), {2} (writable), {3} +// (readonly), and loads two writable and one readonly address from a table. +func v0LookupTx(t *testing.T) *solana.Transaction { + t.Helper() + msg := solana.Message{ + Header: solana.MessageHeader{ + NumRequiredSignatures: 1, + NumReadonlyUnsignedAccounts: 1, + }, + AccountKeys: solana.PublicKeySlice{{1}, {2}, {3}}, + Instructions: []solana.CompiledInstruction{{ProgramIDIndex: 2, Accounts: []uint16{0, 1, 3, 4, 5}}}, + AddressTableLookups: solana.MessageAddressTableLookupSlice{{ + AccountKey: solana.PublicKey{9}, + WritableIndexes: []uint8{0, 1}, + ReadonlyIndexes: []uint8{2}, + }}, + } + _, err := msg.SetVersion(solana.MessageVersionV0) + require.NoError(t, err) + return &solana.Transaction{Signatures: []solana.Signature{{}}, Message: msg} +} + +func TestWritableAccountsIncludesWritableLookups(t *testing.T) { + tx := v0LookupTx(t) + // Before resolution only the static keys are known. + assert.Equal(t, []solana.PublicKey{{1}, {2}}, writableAccounts(tx)) + + require.NoError(t, tx.Message.SetAddressTables(map[solana.PublicKey]solana.PublicKeySlice{ + {9}: {{0xA}, {0xB}, {0xC}}, + })) + require.NoError(t, tx.Message.ResolveLookups()) + require.Equal(t, solana.PublicKeySlice{{1}, {2}, {3}, {0xA}, {0xB}, {0xC}}, tx.Message.AccountKeys) + + // The header still describes the static keys only: {3} stays readonly, + // the two writable lookups are added and the readonly one is not. + assert.Equal(t, []solana.PublicKey{{1}, {2}, {0xA}, {0xB}}, writableAccounts(tx)) + + cost, err := EstimateTransactionCost(tx, features.NewFeaturesDefault()) + require.NoError(t, err) + assert.Equal(t, uint64(4*WriteLockUnits), cost.WriteLockCost) + assert.Equal(t, []solana.PublicKey{{1}, {2}, {0xA}, {0xB}}, cost.WritableAccounts) +} + func TestEstimateTransactionCostV1UsesInlineLimits(t *testing.T) { msg := solana.Message{ Header: solana.MessageHeader{ diff --git a/pkg/costmodel/transaction_cost.go b/pkg/costmodel/transaction_cost.go index 3a804b03..038c6ac2 100644 --- a/pkg/costmodel/transaction_cost.go +++ b/pkg/costmodel/transaction_cost.go @@ -134,6 +134,10 @@ func countWriteLocks(tx *solana.Transaction) uint64 { return uint64(len(writableAccounts(tx))) } +// writableAccounts returns the accounts tx write-locks. For a v0 message with +// resolved address-table lookups, the header describes only the static keys, +// and the writable looked-up addresses follow them; an unresolved message only +// contributes its static keys, so cost it after resolution. func writableAccounts(tx *solana.Transaction) []solana.PublicKey { if tx == nil { return nil @@ -141,8 +145,21 @@ func writableAccounts(tx *solana.Transaction) []solana.PublicKey { hdr := tx.Message.Header numSigners := int(hdr.NumRequiredSignatures) numKeys := len(tx.Message.AccountKeys) + var writableLookups []solana.PublicKey + if tx.Message.GetVersion() == solana.MessageVersionV0 && tx.Message.IsResolved() { + numStatic := numKeys - tx.Message.AddressTableLookups.NumLookups() + if numStatic < 0 { + numStatic = 0 + } + lookupEnd := numStatic + tx.Message.AddressTableLookups.NumWritableLookups() + if lookupEnd > numKeys { + lookupEnd = numKeys + } + writableLookups = tx.Message.AccountKeys[numStatic:lookupEnd] + numKeys = numStatic + } if numKeys == 0 { - return nil + return append([]solana.PublicKey(nil), writableLookups...) } numWritableSigners := numSigners - int(hdr.NumReadonlySignedAccounts) if numWritableSigners < 0 { @@ -152,7 +169,7 @@ func writableAccounts(tx *solana.Transaction) []solana.PublicKey { if numWritableNonSigners < 0 { numWritableNonSigners = 0 } - out := make([]solana.PublicKey, 0, numWritableSigners+numWritableNonSigners) + out := make([]solana.PublicKey, 0, numWritableSigners+numWritableNonSigners+len(writableLookups)) for i := 0; i < numWritableSigners && i < numKeys; i++ { out = append(out, tx.Message.AccountKeys[i]) } @@ -164,7 +181,7 @@ func writableAccounts(tx *solana.Transaction) []solana.PublicKey { for i := start; i < end; i++ { out = append(out, tx.Message.AccountKeys[i]) } - return out + return append(out, writableLookups...) } func estimateAllocDelta(instrs []sealevel.Instruction, feats *features.Features) uint64 { diff --git a/pkg/forge/forge.go b/pkg/forge/forge.go index ba994494..ad9f09ae 100644 --- a/pkg/forge/forge.go +++ b/pkg/forge/forge.go @@ -25,6 +25,7 @@ type Stats struct { DroppedExecution uint64 DroppedAlreadyProcessed uint64 DroppedAccountLocks uint64 + DroppedLookup uint64 DroppedBlockCost uint64 DroppedAccountCost uint64 DroppedAllocCost uint64 @@ -82,6 +83,8 @@ func (s *Sink) Receive(pkt packet.Packet) { s.stats.DroppedAlreadyProcessed++ case blockprod.ForgeDroppedAccountLocks: s.stats.DroppedAccountLocks++ + case blockprod.ForgeDroppedLookup: + s.stats.DroppedLookup++ default: s.stats.DroppedNoBank++ } diff --git a/pkg/replay/address_lookup.go b/pkg/replay/address_lookup.go index ce72d44f..e542ac78 100644 --- a/pkg/replay/address_lookup.go +++ b/pkg/replay/address_lookup.go @@ -63,3 +63,99 @@ func checkLookupIndexes(active solana.PublicKeySlice, lookup *solana.MessageAddr } return nil } + +// HasAddressTableLookups reports whether tx is a v0 transaction that loads +// accounts from address lookup tables. +func HasAddressTableLookups(tx *solana.Transaction) bool { + return tx != nil && tx.Message.GetVersion() == solana.MessageVersionV0 && tx.Message.AddressTableLookups.NumLookups() != 0 +} + +// ResolveAddressTableLookups returns a copy of tx whose address-table lookups +// are resolved for the slot of slotCtx, the bank the transaction executes in. +// +// Tables are read from the bank's parent state, never from writes made in the +// slot itself, and give the same addresses the slot's own state would: a +// table extended during the slot keeps the new addresses inactive until the +// slot has passed, because the address lookup table program records the +// address count at the slot's first extension as the active length; a table +// deactivated during the slot is still deactivating and usable; and a table +// created during the slot has no active address yet. Reading the parent also +// keeps the result independent of which transactions of the slot ran first. +// +// Lookups follow the Solana protocol's rules: +// +// - a missing table, or a deactivated one, fails with AddressLookupTableNotFound; +// - a table not owned by the address lookup table program fails with +// InvalidAddressLookupTableOwner, and one that does not decode with +// InvalidAddressLookupTableData; +// - an index at or past the table's active length fails with +// InvalidAddressLookupTableIndex. +// +// tx is never modified: resolution appends the loaded addresses to the account +// keys, and the original must stay unresolved for entry encoding. The copy +// serializes identically to the original, since only static keys are encoded. +// Transactions without lookups are returned as is. +func ResolveAddressTableLookups(slotCtx *sealevel.SlotCtx, tx *solana.Transaction) (*solana.Transaction, *TransactionError) { + if !HasAddressTableLookups(tx) { + return tx, nil + } + if tx.Message.IsResolved() { + // Its loaded addresses came from some other bank, which cannot be + // vouched for here. + return nil, &TransactionError{ErrorType: TransactionErrorSanitizeFailure, InstructionError: TxErrSanitizeFailure} + } + slotHashes, ok := slotHashesForAddressLookups(slotCtx) + if !ok { + return nil, &TransactionError{ErrorType: TransactionErrorSanitizeFailure, InstructionError: TxErrSanitizeFailure} + } + + tables := make(map[solana.PublicKey]solana.PublicKeySlice, len(tx.Message.AddressTableLookups)) + for i := range tx.Message.AddressTableLookups { + lookup := &tx.Message.AddressTableLookups[i] + active, txErr := activeLookupTable(parentLookupTableAccount(slotCtx, lookup.AccountKey), slotCtx.Slot, slotHashes) + if txErr == nil { + txErr = checkLookupIndexes(active, lookup) + } + if txErr != nil { + return nil, txErr + } + tables[lookup.AccountKey] = active + } + + resolved := *tx + resolved.Message = tx.Message + resolved.Message.AccountKeys = append(solana.PublicKeySlice(nil), tx.Message.AccountKeys...) + if err := resolved.Message.SetAddressTables(tables); err != nil { + return nil, &TransactionError{ErrorType: TransactionErrorInvalidAddressLookupTableIndex, InstructionError: TxErrInvalidAddressLookupTableIndex} + } + if err := resolved.Message.ResolveLookups(); err != nil { + return nil, &TransactionError{ErrorType: TransactionErrorInvalidAddressLookupTableIndex, InstructionError: TxErrInvalidAddressLookupTableIndex} + } + return &resolved, nil +} + +// parentLookupTableAccount reads a table from slotCtx's parent state: the +// unrooted chain the bank was built on, then durable AccountsDb. It returns +// nil if the table does not exist there. +func parentLookupTableAccount(slotCtx *sealevel.SlotCtx, key solana.PublicKey) *accounts.Account { + if slotCtx.UnrootedRead == nil && slotCtx.AccountsDb == nil { + return nil + } + acct, err := slotCtx.GetAccountFromAccountsDb(key) + if err != nil { + return nil + } + return acct +} + +// slotHashesForAddressLookups prefers the bank's pinned SlotHashes, as the +// leader's bank carries them, and falls back to the process-wide sysvar cache. +func slotHashesForAddressLookups(slotCtx *sealevel.SlotCtx) (sealevel.SysvarSlotHashes, bool) { + if bankSysvars := slotCtx.BankSysvars(); bankSysvars != nil { + return bankSysvars.SlotHashes() + } + if sealevel.SysvarCache.SlotHashes.Sysvar == nil { + return nil, false + } + return *sealevel.SysvarCache.SlotHashes.Sysvar, true +} diff --git a/pkg/replay/address_lookup_test.go b/pkg/replay/address_lookup_test.go new file mode 100644 index 00000000..7ac5ee4f --- /dev/null +++ b/pkg/replay/address_lookup_test.go @@ -0,0 +1,378 @@ +package replay + +import ( + "bytes" + "context" + "fmt" + "math" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" + b "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + bin "github.com/gagliardetto/binary" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// lookupTestSlot matches newCommitTestSlotCtx. +const lookupTestSlot = 42 + +func encodeLookupTable(t *testing.T, key solana.PublicKey, meta sealevel.LookupTableMeta, addrs ...solana.PublicKey) *accounts.Account { + t.Helper() + var buf bytes.Buffer + enc := bin.NewBinEncoder(&buf) + require.NoError(t, enc.WriteUint32(sealevel.AddressLookupTableProgramStateLookupTable, bin.LE)) + require.NoError(t, meta.MarshalWithEncoder(enc)) + for len(buf.Bytes()) < sealevel.AddressLookupTableMetaSize { + require.NoError(t, enc.WriteByte(0)) + } + for _, addr := range addrs { + require.NoError(t, enc.WriteBytes(addr[:], false)) + } + return &accounts.Account{Key: key, Lamports: 1_000_000, Owner: addresses.AddressLookupTableAddr, Data: buf.Bytes(), RentEpoch: math.MaxUint64} +} + +func activeTableMeta() sealevel.LookupTableMeta { + authority := txfixture.PayerPubkey() + return sealevel.LookupTableMeta{DeactivationSlot: math.MaxUint64, LastExtendedSlot: 1, Authority: &authority} +} + +func lookupTestAddresses(n int) []solana.PublicKey { + addrs := make([]solana.PublicKey, n) + for i := range addrs { + addrs[i] = solana.NewWallet().PublicKey() + } + return addrs +} + +// lookupTestTx is a v0 message with static keys [payer, system] and the given +// lookups; the instruction only has to make the message well formed. +func lookupTestTx(t *testing.T, lookups ...solana.MessageAddressTableLookup) *solana.Transaction { + t.Helper() + msg := solana.Message{ + Header: solana.MessageHeader{NumRequiredSignatures: 1, NumReadonlyUnsignedAccounts: 1}, + AccountKeys: solana.PublicKeySlice{txfixture.PayerPubkey(), solana.SystemProgramID}, + RecentBlockhash: txfixture.TestBlockhash(), + Instructions: []solana.CompiledInstruction{{ProgramIDIndex: 1, Accounts: []uint16{0}}}, + AddressTableLookups: lookups, + } + _, err := msg.SetVersion(solana.MessageVersionV0) + require.NoError(t, err) + tx := &solana.Transaction{Message: msg} + payerKey := txfixture.PayerPrivateKey() + _, err = tx.Sign(func(key solana.PublicKey) *solana.PrivateKey { + if key.Equals(txfixture.PayerPubkey()) { + return &payerKey + } + return nil + }) + require.NoError(t, err) + return tx +} + +// lookupParentState is a bank's parent state for lookup tests. It serves both +// as the slot's UnrootedRead and as replay's block account source. +type lookupParentState map[solana.PublicKey]*accounts.Account + +func (s lookupParentState) GetAccount(_ uint64, pubkey solana.PublicKey) (*accounts.Account, error) { + if acct, ok := s[pubkey]; ok { + return acct, nil + } + return nil, fmt.Errorf("account %s not found", pubkey) +} + +func (s lookupParentState) GetAccountsBatch(_ context.Context, slot uint64, pks []solana.PublicKey) ([]*accounts.Account, error) { + out := make([]*accounts.Account, len(pks)) + for i, pk := range pks { + out[i], _ = s.GetAccount(slot, pk) + } + return out, nil +} + +// newLookupTestSlotCtx is a bank at lookupTestSlot whose parent state holds +// tables. +func newLookupTestSlotCtx(t *testing.T, tables ...*accounts.Account) *sealevel.SlotCtx { + t.Helper() + slotCtx, cleanup := newCommitTestSlotCtx() + t.Cleanup(cleanup) + prev := sealevel.SysvarCache.SlotHashes.Sysvar + slotHashes := sealevel.SysvarSlotHashes{{Slot: lookupTestSlot - 1}, {Slot: lookupTestSlot - 2}} + sealevel.SysvarCache.SlotHashes.Sysvar = &slotHashes + t.Cleanup(func() { sealevel.SysvarCache.SlotHashes.Sysvar = prev }) + parent := lookupParentState{} + for _, table := range tables { + parent[table.Key] = table + } + slotCtx.UnrootedRead = parent + return slotCtx +} + +func requireLookupError(t *testing.T, want TransactionErrorType, wantErr error, resolved *solana.Transaction, got *TransactionError) { + t.Helper() + require.Nil(t, resolved) + require.NotNil(t, got) + assert.Equal(t, want, got.ErrorType, "got %s", got.ErrorType.String()) + assert.ErrorIs(t, got.InstructionError, wantErr) +} + +func TestResolveAddressTableLookupsOrdersWritableThenReadonly(t *testing.T) { + t1, t2 := solana.NewWallet().PublicKey(), solana.NewWallet().PublicKey() + a1, a2 := lookupTestAddresses(3), lookupTestAddresses(2) + slotCtx := newLookupTestSlotCtx(t, + encodeLookupTable(t, t1, activeTableMeta(), a1...), + encodeLookupTable(t, t2, activeTableMeta(), a2...)) + tx := lookupTestTx(t, + solana.MessageAddressTableLookup{AccountKey: t1, WritableIndexes: []uint8{2}, ReadonlyIndexes: []uint8{0}}, + solana.MessageAddressTableLookup{AccountKey: t2, WritableIndexes: []uint8{1}, ReadonlyIndexes: []uint8{0}}, + ) + wire, err := tx.MarshalBinary() + require.NoError(t, err) + + resolved, txErr := ResolveAddressTableLookups(slotCtx, tx) + require.Nil(t, txErr) + + static := tx.Message.AccountKeys + assert.Equal(t, solana.PublicKeySlice{static[0], static[1], a1[2], a2[1], a1[0], a2[0]}, resolved.Message.AccountKeys) + for i, want := range []bool{true, false, true, true, false, false} { + writable, err := resolved.Message.IsWritable(resolved.Message.AccountKeys[i]) + require.NoError(t, err) + assert.Equal(t, want, writable, "key %d", i) + } + + // The input is untouched and both serialize to the original wire. + assert.False(t, tx.Message.IsResolved()) + assert.Len(t, tx.Message.AccountKeys, 2) + gotWire, err := resolved.MarshalBinary() + require.NoError(t, err) + assert.Equal(t, wire, gotWire) + origWire, err := tx.MarshalBinary() + require.NoError(t, err) + assert.Equal(t, wire, origWire) +} + +func TestResolveAddressTableLookupsSameTableTwice(t *testing.T) { + table := solana.NewWallet().PublicKey() + addrs := lookupTestAddresses(2) + slotCtx := newLookupTestSlotCtx(t, encodeLookupTable(t, table, activeTableMeta(), addrs...)) + tx := lookupTestTx(t, + solana.MessageAddressTableLookup{AccountKey: table, WritableIndexes: []uint8{0}}, + solana.MessageAddressTableLookup{AccountKey: table, ReadonlyIndexes: []uint8{1}}, + ) + resolved, txErr := ResolveAddressTableLookups(slotCtx, tx) + require.Nil(t, txErr) + assert.Equal(t, addrs, []solana.PublicKey(resolved.Message.AccountKeys[2:])) +} + +func TestResolveAddressTableLookupsActivation(t *testing.T) { + cases := []struct { + name string + deactivationSlot uint64 + wantNotFound bool + }{ + {"activated", math.MaxUint64, false}, + {"deactivating in the current slot", lookupTestSlot, false}, + {"deactivating, still in SlotHashes", lookupTestSlot - 2, false}, + {"deactivated, gone from SlotHashes", lookupTestSlot - 10, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + table := solana.NewWallet().PublicKey() + meta := activeTableMeta() + meta.DeactivationSlot = tc.deactivationSlot + slotCtx := newLookupTestSlotCtx(t, encodeLookupTable(t, table, meta, lookupTestAddresses(1)...)) + tx := lookupTestTx(t, solana.MessageAddressTableLookup{AccountKey: table, ReadonlyIndexes: []uint8{0}}) + + resolved, txErr := ResolveAddressTableLookups(slotCtx, tx) + if tc.wantNotFound { + requireLookupError(t, TransactionErrorAddressLookupTableNotFound, TxErrAddressLookupTableNotFound, resolved, txErr) + return + } + require.Nil(t, txErr) + assert.Len(t, resolved.Message.AccountKeys, 3) + }) + } +} + +func TestResolveAddressTableLookupsExcludesAddressesAppendedThisSlot(t *testing.T) { + table := solana.NewWallet().PublicKey() + addrs := lookupTestAddresses(3) + extendedNow := activeTableMeta() + extendedNow.LastExtendedSlot = lookupTestSlot + extendedNow.LastExtendedSlotStartIndex = 2 + + slotCtx := newLookupTestSlotCtx(t, encodeLookupTable(t, table, extendedNow, addrs...)) + resolved, txErr := ResolveAddressTableLookups(slotCtx, lookupTestTx(t, solana.MessageAddressTableLookup{AccountKey: table, WritableIndexes: []uint8{1}})) + require.Nil(t, txErr, "addresses appended before this slot are active") + assert.Equal(t, addrs[1], resolved.Message.AccountKeys[2]) + + // Index 2 was appended in this slot, so it is not active yet. + resolved, txErr = ResolveAddressTableLookups(slotCtx, lookupTestTx(t, solana.MessageAddressTableLookup{AccountKey: table, ReadonlyIndexes: []uint8{2}})) + requireLookupError(t, TransactionErrorInvalidAddressLookupTableIndex, TxErrInvalidAddressLookupTableIndex, resolved, txErr) + + // Extended in the previous slot: every address is active. + extendedBefore := extendedNow + extendedBefore.LastExtendedSlot = lookupTestSlot - 1 + slotCtx = newLookupTestSlotCtx(t, encodeLookupTable(t, table, extendedBefore, addrs...)) + resolved, txErr = ResolveAddressTableLookups(slotCtx, lookupTestTx(t, solana.MessageAddressTableLookup{AccountKey: table, ReadonlyIndexes: []uint8{2}})) + require.Nil(t, txErr) + assert.Equal(t, addrs[2], resolved.Message.AccountKeys[2]) +} + +func TestResolveAddressTableLookupsRejectsBadTables(t *testing.T) { + table := solana.NewWallet().PublicKey() + good := encodeLookupTable(t, table, activeTableMeta(), lookupTestAddresses(2)...) + + zeroLamports := *good + zeroLamports.Lamports = 0 + wrongOwner := *good + wrongOwner.Owner = addresses.SystemProgramAddr + garbage := *good + garbage.Data = []byte{1, 2, 3} + badStart := activeTableMeta() + badStart.LastExtendedSlot = lookupTestSlot + badStart.LastExtendedSlotStartIndex = 9 + + cases := []struct { + name string + table *accounts.Account + index uint8 + want TransactionErrorType + wantErr error + }{ + {"missing table", nil, 0, TransactionErrorAddressLookupTableNotFound, TxErrAddressLookupTableNotFound}, + {"zero lamports", &zeroLamports, 0, TransactionErrorAddressLookupTableNotFound, TxErrAddressLookupTableNotFound}, + {"wrong owner", &wrongOwner, 0, TransactionErrorInvalidAddressLookupTableOwner, TxErrInvalidAddressLookupTableOwner}, + {"undecodable data", &garbage, 0, TransactionErrorInvalidAddressLookupTableData, TxErrInvalidAddressLookupTableData}, + {"active length past the end", encodeLookupTable(t, table, badStart, lookupTestAddresses(2)...), 0, TransactionErrorInvalidAddressLookupTableData, TxErrInvalidAddressLookupTableData}, + {"index out of range", good, 2, TransactionErrorInvalidAddressLookupTableIndex, TxErrInvalidAddressLookupTableIndex}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + var tables []*accounts.Account + if tc.table != nil { + tables = append(tables, tc.table) + } + slotCtx := newLookupTestSlotCtx(t, tables...) + tx := lookupTestTx(t, solana.MessageAddressTableLookup{AccountKey: table, ReadonlyIndexes: []uint8{tc.index}}) + resolved, txErr := ResolveAddressTableLookups(slotCtx, tx) + requireLookupError(t, tc.want, tc.wantErr, resolved, txErr) + }) + } +} + +func TestResolveAddressTableLookupsPassesThroughAndRefusesResolved(t *testing.T) { + slotCtx := newLookupTestSlotCtx(t) + legacy, err := solana.TransactionFromBytes(txfixture.MustSignedTransferWire(0)) + require.NoError(t, err) + got, txErr := ResolveAddressTableLookups(slotCtx, legacy) + require.Nil(t, txErr) + assert.Same(t, legacy, got) + + table := solana.NewWallet().PublicKey() + tx := lookupTestTx(t, solana.MessageAddressTableLookup{AccountKey: table, ReadonlyIndexes: []uint8{0}}) + require.NoError(t, tx.Message.SetAddressTables(map[solana.PublicKey]solana.PublicKeySlice{table: lookupTestAddresses(1)})) + require.NoError(t, tx.Message.ResolveLookups()) + got, txErr = ResolveAddressTableLookups(slotCtx, tx) + requireLookupError(t, TransactionErrorSanitizeFailure, TxErrSanitizeFailure, got, txErr) +} + +// For an active table, resolving on the leader must give the account keys +// replay resolves for the same transaction in the produced block. +func TestResolveAddressTableLookupsMatchesReplayResolution(t *testing.T) { + t1, t2 := solana.NewWallet().PublicKey(), solana.NewWallet().PublicKey() + tables := []*accounts.Account{ + encodeLookupTable(t, t1, activeTableMeta(), lookupTestAddresses(4)...), + encodeLookupTable(t, t2, activeTableMeta(), lookupTestAddresses(3)...), + } + slotCtx := newLookupTestSlotCtx(t, tables...) + wire, err := lookupTestTx(t, + solana.MessageAddressTableLookup{AccountKey: t2, WritableIndexes: []uint8{2, 0}, ReadonlyIndexes: []uint8{1}}, + solana.MessageAddressTableLookup{AccountKey: t1, WritableIndexes: []uint8{3}, ReadonlyIndexes: []uint8{0, 2}}, + ).MarshalBinary() + require.NoError(t, err) + + leaderTx, err := solana.TransactionFromBytes(wire) + require.NoError(t, err) + leaderResolved, txErr := ResolveAddressTableLookups(slotCtx, leaderTx) + require.Nil(t, txErr) + + replayTx, err := solana.TransactionFromBytes(wire) + require.NoError(t, err) + source := lookupParentState{t1: tables[0], t2: tables[1]} + slotHashes, ok := slotHashesForAddressLookups(slotCtx) + require.True(t, ok) + require.NoError(t, resolveAddrTableLookups(source, &b.Block{Slot: lookupTestSlot, Transactions: []*solana.Transaction{replayTx}}, slotHashes)) + require.True(t, replayTx.Message.IsResolved()) + + assert.Equal(t, replayTx.Message.AccountKeys, leaderResolved.Message.AccountKeys) +} + +// Resolving from the parent copy of a table is only sound because whatever +// the slot does to the table cannot change lookups in that slot. The parent +// copy has 2 addresses; the slot then extends it twice to 4. Only the slot's +// first extension records the start index, at the pre-extension count, so +// both copies must give identical results at the executing slot. This fails +// if the address lookup table program ever moved the start index on later +// extensions of the same slot. +func TestLookupTableParentAndSlotCopiesAgree(t *testing.T) { + table := solana.NewWallet().PublicKey() + addrs := lookupTestAddresses(4) + parentMeta := activeTableMeta() + parentMeta.LastExtendedSlot = lookupTestSlot - 1 + parentCopy := encodeLookupTable(t, table, parentMeta, addrs[:2]...) + slotMeta := activeTableMeta() + slotMeta.LastExtendedSlot = lookupTestSlot + slotMeta.LastExtendedSlotStartIndex = 2 + slotCopy := encodeLookupTable(t, table, slotMeta, addrs...) + slotHashes := sealevel.SysvarSlotHashes{{Slot: lookupTestSlot - 1}, {Slot: lookupTestSlot - 2}} + resolve := func(acct *accounts.Account, lookup *solana.MessageAddressTableLookup) (solana.PublicKeySlice, *TransactionError) { + active, txErr := activeLookupTable(acct, lookupTestSlot, slotHashes) + if txErr == nil { + txErr = checkLookupIndexes(active, lookup) + } + if txErr != nil { + return nil, txErr + } + return active, nil + } + + for _, idx := range []uint8{0, 1, 2, 3} { + lookup := &solana.MessageAddressTableLookup{AccountKey: table, ReadonlyIndexes: []uint8{idx}} + fromParent, parentErr := resolve(parentCopy, lookup) + fromSlot, slotErr := resolve(slotCopy, lookup) + assert.Equal(t, parentErr, slotErr, "index %d", idx) + assert.Equal(t, fromParent, fromSlot, "index %d", idx) + if idx < 2 { + require.Nil(t, parentErr, "index %d", idx) + assert.Equal(t, addrs[idx], fromParent[idx]) + } else { + require.NotNil(t, parentErr, "index %d", idx) + assert.ErrorIs(t, parentErr.InstructionError, TxErrInvalidAddressLookupTableIndex) + } + } +} + +// Writes the slot itself made to a table are not read. +func TestResolveAddressTableLookupsIgnoresSlotWrites(t *testing.T) { + table := solana.NewWallet().PublicKey() + addrs := lookupTestAddresses(2) + + // Created in the slot: absent from the parent, so not found. + slotCtx := newLookupTestSlotCtx(t) + require.NoError(t, slotCtx.SetAccount(table, encodeLookupTable(t, table, activeTableMeta(), addrs...))) + resolved, txErr := ResolveAddressTableLookups(slotCtx, lookupTestTx(t, solana.MessageAddressTableLookup{AccountKey: table, ReadonlyIndexes: []uint8{0}})) + requireLookupError(t, TransactionErrorAddressLookupTableNotFound, TxErrAddressLookupTableNotFound, resolved, txErr) + + // Closed in the slot: the parent copy is what counts. + slotCtx = newLookupTestSlotCtx(t, encodeLookupTable(t, table, activeTableMeta(), addrs...)) + closed := encodeLookupTable(t, table, activeTableMeta(), addrs...) + closed.Lamports = 0 + require.NoError(t, slotCtx.SetAccount(table, closed)) + resolved, txErr = ResolveAddressTableLookups(slotCtx, lookupTestTx(t, solana.MessageAddressTableLookup{AccountKey: table, ReadonlyIndexes: []uint8{1}})) + require.Nil(t, txErr) + assert.Equal(t, addrs[1], resolved.Message.AccountKeys[2]) +} diff --git a/pkg/replay/transaction_preparation.go b/pkg/replay/transaction_preparation.go index 303dcc3e..8baded82 100644 --- a/pkg/replay/transaction_preparation.go +++ b/pkg/replay/transaction_preparation.go @@ -78,6 +78,11 @@ func (p *TransactionPreparer) Prepare(tx *solana.Transaction) *PreparedTransacti if p == nil || tx == nil { return nil } + // Account metas and cost depend on the bank the lookups resolve against, so + // a transaction with address-table lookups is prepared at execution. + if HasAddressTableLookups(tx) { + return nil + } if tx.Message.GetVersion() == solana.MessageVersionV1 && !p.feats.IsActive(features.EnableTxV1) { return nil } From 3e38e6cc69c46369ccb2740ee83d658617fa8066 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 04:41:17 +0000 Subject: [PATCH 02/25] blockprod: add leader slot hooks and a TPU admission gate --- pkg/blockprod/bank.go | 21 +- pkg/blockprod/leader.go | 44 ++- pkg/blockprod/leader_hooks.go | 58 ++++ pkg/blockprod/leader_hooks_test.go | 324 ++++++++++++++++++++++ pkg/blockprod/leader_replay_gate_test.go | 8 +- pkg/blockprod/scheduler/scheduler.go | 14 +- pkg/blockprod/scheduler/scheduler_test.go | 33 +++ pkg/blockprod/testenv.go | 2 + pkg/forge/forge.go | 2 +- pkg/forge/forge_test.go | 15 + 10 files changed, 502 insertions(+), 19 deletions(-) create mode 100644 pkg/blockprod/leader_hooks.go create mode 100644 pkg/blockprod/leader_hooks_test.go diff --git a/pkg/blockprod/bank.go b/pkg/blockprod/bank.go index 5cd101f3..c0176670 100644 --- a/pkg/blockprod/bank.go +++ b/pkg/blockprod/bank.go @@ -2,6 +2,7 @@ package blockprod import ( "sync" + "sync/atomic" "github.com/Overclock-Validator/mithril/pkg/arena" "github.com/Overclock-Validator/mithril/pkg/costmodel" @@ -49,6 +50,9 @@ type WorkingBank struct { // either phase, so their storage can be reset for the next transaction. borrowedAccounts *arena.Arena[sealevel.BorrowedAccount] preparer *replay.TransactionPreparer + // tpuAllowed admits TPU transactions. It only moves from false to true, so + // a TPU source that observed it set never races a later close. + tpuAllowed atomic.Bool } type BankConfig struct { @@ -59,6 +63,8 @@ type BankConfig struct { EntryHash solana.Hash Sink BatchSink TransactionStatuses *replay.TransactionStatusView + // TPUGated opens the bank with TPU admission closed until AllowTPU. + TPUGated bool } func NewWorkingBank(cfg BankConfig) *WorkingBank { @@ -80,7 +86,7 @@ func NewWorkingBank(cfg BankConfig) *WorkingBank { if cfg.SlotCtx != nil { preparer = replay.NewTransactionPreparer(cfg.SlotCtx.Features) } - return &WorkingBank{ + bank := &WorkingBank{ preparer: preparer, slotCtx: cfg.SlotCtx, slot: cfg.Slot, @@ -94,6 +100,19 @@ func NewWorkingBank(cfg BankConfig) *WorkingBank { seenMessages: make(map[[32]byte]struct{}), borrowedAccounts: arena.New[sealevel.BorrowedAccount](64), } + bank.tpuAllowed.Store(!cfg.TPUGated) + return bank +} + +// TPUAllowed reports whether TPU sources may forge into this bank. Other +// producers forge directly and are not subject to the gate. +func (b *WorkingBank) TPUAllowed() bool { + return b.tpuAllowed.Load() +} + +// AllowTPU opens TPU admission. It is one-way and safe to call repeatedly. +func (b *WorkingBank) AllowTPU() { + b.tpuAllowed.Store(true) } func (b *WorkingBank) Slot() uint64 { diff --git a/pkg/blockprod/leader.go b/pkg/blockprod/leader.go index 7dc15096..84a5964e 100644 --- a/pkg/blockprod/leader.go +++ b/pkg/blockprod/leader.go @@ -100,6 +100,8 @@ type LeaderLoop struct { canSignSlot func(uint64) bool onBlock func(*b.Block) commitLeaderSlot func(replay.CommitLeaderInput) (*sealevel.SlotCtx, error) + hooks LeaderHooks + gateTPU bool currentSlot func() uint64 leaderForSlot func(uint64) (solana.PublicKey, bool) @@ -161,6 +163,10 @@ type LeaderLoopConfig struct { // uses the conservative default; tune only from measured completion times. CompletionReserve time.Duration Now func() time.Time + Hooks LeaderHooks + // GateTPU opens each working bank with TPU admission closed; Hooks must + // call WorkingBank.AllowTPU to open it. Ignored without Hooks. + GateTPU bool } func NewLeaderLoop(cfg LeaderLoopConfig) *LeaderLoop { @@ -199,6 +205,8 @@ func NewLeaderLoop(cfg LeaderLoopConfig) *LeaderLoop { slotDuration: cfg.SlotDuration, completionReserve: cfg.CompletionReserve, now: cfg.Now, + hooks: cfg.Hooks, + gateTPU: cfg.GateTPU && cfg.Hooks != nil, finishedLeaderSlots: make(map[uint64]struct{}), pendingFailures: make(map[uint64]leaderSlotFailure), } @@ -246,7 +254,7 @@ func (l *LeaderLoop) tickScheduled(scheduledAt time.Time) { if l.activeBank != nil { if canonical, reason := l.activeParentStillCanonicalLocked(); !canonical { l.rememberLeaderSlotFailureLocked(l.activeSlot, reason, activeLeaderFailureDetail(reason)) - l.abortActiveSlotLocked() + l.abortActiveSlotLocked(reason) if reason == leaderReasonAlreadyResolved || reason == leaderReasonParentReadyMissedWindow { l.failProductionWindowLocked(l.productionWindow.nextSlot, reason, activeLeaderFailureDetail(reason)) } @@ -726,7 +734,14 @@ func (l *LeaderLoop) activeParentStillCanonicalLocked() (bool, string) { return true, "" } -func (l *LeaderLoop) abortActiveSlotLocked() { +func (l *LeaderLoop) abortActiveSlotLocked(reason string) { + if l.activeBank != nil && l.hooks != nil { + l.hooks.SlotAborted(l.activeSlot, reason) + } + l.releaseActiveSlotLocked() +} + +func (l *LeaderLoop) releaseActiveSlotLocked() { if l.controller != nil { l.controller.ClearWorkingBank() } @@ -952,10 +967,13 @@ func (l *LeaderLoop) finishActiveSlotLocked() { slot := l.activeSlot if canonical, reason := l.activeParentStillCanonicalLocked(); !canonical { l.failProductionWindowLocked(slot, reason, activeLeaderFailureDetail(reason)) - l.abortActiveSlotLocked() + l.abortActiveSlotLocked(reason) return } + if l.hooks != nil { + l.hooks.SlotEnding(slot) + } // Detach first so no new TPU packet can obtain this bank. Freeze then waits // for any packet that already did, closing the footer/entry race. if l.controller != nil { @@ -964,13 +982,13 @@ func (l *LeaderLoop) finishActiveSlotLocked() { l.activeBank.Freeze() if l.activeSink == nil { l.failProductionWindowLocked(slot, leaderReasonFinalizationUnavailable, "entry broadcast sink is unavailable") - l.abortActiveSlotLocked() + l.abortActiveSlotLocked(leaderReasonFinalizationUnavailable) return } l.activeSink.Wait() if err := l.activeSink.Err(); err != nil { l.failProductionWindowLocked(slot, leaderReasonEntryBroadcastFailed, err.Error()) - l.abortActiveSlotLocked() + l.abortActiveSlotLocked(leaderReasonEntryBroadcastFailed) return } @@ -986,7 +1004,7 @@ func (l *LeaderLoop) finishActiveSlotLocked() { if l.accountsDb == nil || l.activeSess == nil { l.failProductionWindowLocked(slot, leaderReasonFinalizationUnavailable, "accounts DB or broadcast session is unavailable") - l.abortActiveSlotLocked() + l.abortActiveSlotLocked(leaderReasonFinalizationUnavailable) return } parentLastBlockhash := l.parentCtx.ParentLastBlockhash @@ -1030,26 +1048,26 @@ func (l *LeaderLoop) finishActiveSlotLocked() { FooterProducerTimeNanos: producerTimeNanos, }); err != nil { l.failProductionWindowLocked(slot, leaderReasonFreezeFailed, err.Error()) - l.abortActiveSlotLocked() + l.abortActiveSlotLocked(leaderReasonFreezeFailed) return } if canonical, reason := l.activeParentStillCanonicalLocked(); !canonical { l.failProductionWindowLocked(slot, reason, activeLeaderFailureDetail(reason)) - l.abortActiveSlotLocked() + l.abortActiveSlotLocked(reason) return } footerBankhash := solana.HashFromBytes(l.activeBank.SlotCtx().FinalBankhash) if err := l.activeSess.BroadcastFooter(footerBankhash, producerTimeNanos, footerRewards.Skip, footerRewards.Notar); err != nil { l.failProductionWindowLocked(slot, leaderReasonFooterBroadcastFailed, err.Error()) - l.abortActiveSlotLocked() + l.abortActiveSlotLocked(leaderReasonFooterBroadcastFailed) return } // Ending tick is the last-in-slot shred. Mid-slot entry batches are only // emitted when they fill a FEC set; Freeze already flushed any leftover. if err := l.activeSess.BroadcastEndingTickLast(tickHash); err != nil { l.failProductionWindowLocked(slot, leaderReasonEndingTickBroadcastFailed, err.Error()) - l.abortActiveSlotLocked() + l.abortActiveSlotLocked(leaderReasonEndingTickBroadcastFailed) return } // BroadcastEndingTickLast synchronously hands the final shred batch to the @@ -1080,7 +1098,7 @@ func (l *LeaderLoop) finishActiveSlotLocked() { l.recordLeaderSlotOutcomeLocked(slot, leaderOutcomeBroadcast, leaderReasonComplete, fmt.Sprintf("block=%s parent_slot=%d txns=%d%s", solana.Hash(producedBlock.AlpenglowBlockID).String(), producedBlock.ParentSlot, len(producedBlock.Transactions), timingDetail)) l.advanceProductionWindowLocked(slot) - l.abortActiveSlotLocked() + l.releaseActiveSlotLocked() } func (l *LeaderLoop) advanceProductionWindowLocked(slot uint64) { @@ -1237,6 +1255,7 @@ func (l *LeaderLoop) startSlotLocked(slot uint64) error { EntryHash: startEntryHash, Sink: sink, TransactionStatuses: parentCtx.TransactionStatuses, + TPUGated: l.gateTPU, }) if slot > 0 && !sameReplayParentSnapshot(parentCtx, l.parentContext(slot)) { bank.Close() @@ -1280,6 +1299,9 @@ func (l *LeaderLoop) startSlotLocked(slot uint64) error { l.activeParentID = parentID l.activeParentChainedRoot = parentChainedRoot l.activeParentGeneration = parentCtx.ReplayGeneration + if l.hooks != nil { + l.hooks.SlotStarted(l.leaderSlotInfoLocked(slot, parentSlot, bank)) + } if l.controller != nil { l.controller.SetWorkingBank(bank) } diff --git a/pkg/blockprod/leader_hooks.go b/pkg/blockprod/leader_hooks.go new file mode 100644 index 00000000..609b3f48 --- /dev/null +++ b/pkg/blockprod/leader_hooks.go @@ -0,0 +1,58 @@ +package blockprod + +import "time" + +// LeaderHooks observes the lifecycle of locally produced leader slots. +// +// Every slot passed to SlotStarted later reaches SlotEnding, SlotAborted, or +// both. SlotEnding means the bank is about to freeze for finalization; +// SlotAborted means the slot was abandoned, either before SlotEnding (the +// bank is closed and its block discarded) or after it (finalization or +// broadcast failed). A slot that fails to open never reaches any hook. +// +// The LeaderLoop calls hooks synchronously while holding its lock. They must +// return promptly and must not call back into the LeaderLoop. +type LeaderHooks interface { + // SlotStarted runs before the bank is published to TPU sources. + SlotStarted(LeaderSlotInfo) + // SlotEnding runs just before the bank freezes. Forging after the freeze + // reports ForgeDroppedNoLeader. + SlotEnding(slot uint64) + // SlotAborted reports one of the leaderReason values. + SlotAborted(slot uint64, reason string) +} + +// LeaderSlotInfo describes a leader slot whose working bank is about to be +// published. +type LeaderSlotInfo struct { + Slot uint64 + ParentSlot uint64 + Bank *WorkingBank + // WindowStartSlot and WindowEndSlot bound the ParentReady leader window + // that contains Slot. Both are zero outside a ParentReady window. + WindowStartSlot uint64 + WindowEndSlot uint64 + // Start is when the bank opened. End is the local production cutoff, at + // which the leader freezes the bank; ProtocolDeadline is the protocol + // deadline for the complete block. End and ProtocolDeadline are zero + // outside a ParentReady window, where the slot ends with the live clock. + Start time.Time + End time.Time + ProtocolDeadline time.Time +} + +func (l *LeaderLoop) leaderSlotInfoLocked(slot, parentSlot uint64, bank *WorkingBank) LeaderSlotInfo { + info := LeaderSlotInfo{ + Slot: slot, + ParentSlot: parentSlot, + Bank: bank, + Start: l.now(), + End: l.productionWindowDeadlineLocked(slot), + ProtocolDeadline: l.productionWindowProtocolDeadlineLocked(slot), + } + if l.productionWindow.active && slot >= l.productionWindow.startSlot { + info.WindowStartSlot = l.productionWindow.startSlot + info.WindowEndSlot = l.productionWindow.endSlot + } + return info +} diff --git a/pkg/blockprod/leader_hooks_test.go b/pkg/blockprod/leader_hooks_test.go new file mode 100644 index 00000000..a2dc546f --- /dev/null +++ b/pkg/blockprod/leader_hooks_test.go @@ -0,0 +1,324 @@ +package blockprod + +import ( + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/accountsdb" + "github.com/Overclock-Validator/mithril/pkg/alpenglow" + "github.com/Overclock-Validator/mithril/pkg/block" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/global" + "github.com/Overclock-Validator/mithril/pkg/replay" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/Overclock-Validator/mithril/pkg/turbine" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type hookEvent struct { + kind string + slot uint64 + reason string + info LeaderSlotInfo + // published and accepting snapshot the controller and bank when the hook ran. + published *WorkingBank + accepting bool +} + +type recordingHooks struct { + controller *Controller + bank func() *WorkingBank + events []hookEvent +} + +func (h *recordingHooks) snapshot(e hookEvent) hookEvent { + if h.controller != nil { + e.published = h.controller.WorkingBank() + } + if h.bank != nil { + if bank := h.bank(); bank != nil { + bank.mu.Lock() + e.accepting = bank.accepting + bank.mu.Unlock() + } + } + return e +} + +func (h *recordingHooks) SlotStarted(info LeaderSlotInfo) { + h.events = append(h.events, h.snapshot(hookEvent{kind: "started", slot: info.Slot, info: info})) +} + +func (h *recordingHooks) SlotEnding(slot uint64) { + h.events = append(h.events, h.snapshot(hookEvent{kind: "ending", slot: slot})) +} + +func (h *recordingHooks) SlotAborted(slot uint64, reason string) { + h.events = append(h.events, h.snapshot(hookEvent{kind: "aborted", slot: slot, reason: reason})) +} + +func (h *recordingHooks) kinds() []string { + out := make([]string, 0, len(h.events)) + for _, e := range h.events { + out = append(out, e.kind) + } + return out +} + +func TestLeaderHooksSlotStartedBeforeBankIsPublished(t *testing.T) { + global.ResetAlpenglowChainMetadata() + t.Cleanup(global.ResetAlpenglowChainMetadata) + const startSlot = uint64(212) + const slot = startSlot + 1 + ready := time.Unix(1_700_000_000, 0) + now := ready.Add(250 * time.Millisecond) + global.SetReplayFrontier(startSlot) + global.SetAlpenglowBlockID(startSlot, solana.Hash{1}) + global.SetAlpenglowChainedMerkleRoot(startSlot, solana.Hash{2}) + parent := coherentTestParentContext(startSlot, solana.Hash{3}) + controller := NewController() + hooks := &recordingHooks{controller: controller} + loop := NewLeaderLoop(LeaderLoopConfig{ + Controller: controller, + Identity: txfixture.PayerPrivateKey(), + Broadcaster: &captureBroadcaster{}, + ParentContext: func(uint64) ParentContext { return parent }, + Now: func() time.Time { return now }, + Hooks: hooks, + GateTPU: true, + }) + loop.productionWindow = leaderProductionWindow{ + active: true, startSlot: startSlot, endSlot: startSlot + 3, nextSlot: slot, readyAt: ready, + } + + require.NoError(t, loop.startSlotLocked(slot)) + + require.Equal(t, []string{"started"}, hooks.kinds()) + started := hooks.events[0] + assert.Nil(t, started.published, "SlotStarted must run before TPU sources can obtain the bank") + info := started.info + require.NotNil(t, info.Bank) + assert.Same(t, loop.activeBank, info.Bank) + assert.Same(t, info.Bank, controller.WorkingBank()) + assert.Equal(t, slot, info.Slot) + assert.Equal(t, startSlot, info.ParentSlot) + assert.Equal(t, startSlot, info.WindowStartSlot) + assert.Equal(t, startSlot+3, info.WindowEndSlot) + assert.Equal(t, now, info.Start) + assert.Equal(t, ready.Add(400*time.Millisecond), info.ProtocolDeadline) + assert.Equal(t, ready.Add(400*time.Millisecond-leaderBlockCompletionReserve), info.End) + assert.False(t, info.Bank.TPUAllowed(), "GateTPU opens the bank with TPU admission closed") + info.Bank.AllowTPU() + assert.True(t, info.Bank.TPUAllowed()) +} + +func TestLeaderGateTPUIsIgnoredWithoutHooks(t *testing.T) { + global.ResetAlpenglowChainMetadata() + t.Cleanup(global.ResetAlpenglowChainMetadata) + const slot = uint64(98) + global.SetReplayFrontier(slot - 1) + global.SetAlpenglowBlockID(slot-1, solana.Hash{1}) + global.SetAlpenglowChainedMerkleRoot(slot-1, solana.Hash{2}) + parent := coherentTestParentContext(slot-1, solana.Hash{3}) + loop := NewLeaderLoop(LeaderLoopConfig{ + Identity: txfixture.PayerPrivateKey(), + Broadcaster: &captureBroadcaster{}, + ParentContext: func(uint64) ParentContext { return parent }, + GateTPU: true, + }) + + require.NoError(t, loop.startSlotLocked(slot)) + require.True(t, loop.activeBank.TPUAllowed(), "without hooks nothing could open a gated bank") + loop.releaseActiveSlotLocked() +} + +func TestLeaderHooksSkipSlotsThatFailToOpen(t *testing.T) { + global.ResetAlpenglowChainMetadata() + t.Cleanup(global.ResetAlpenglowChainMetadata) + const slot = uint64(102) + global.SetReplayFrontier(slot - 1) + global.SetAlpenglowBlockID(slot-1, solana.Hash{1}) + global.SetAlpenglowChainedMerkleRoot(slot-1, solana.Hash{2}) + parent := coherentTestParentContext(slot-1, solana.Hash{3}) + reads := 0 + hooks := &recordingHooks{} + loop := NewLeaderLoop(LeaderLoopConfig{ + Identity: txfixture.PayerPrivateKey(), + Broadcaster: &captureBroadcaster{}, + ParentContext: func(uint64) ParentContext { + reads++ + current := parent + if reads > 1 { + current.ReplayGeneration++ + } + return current + }, + Hooks: hooks, + }) + + require.ErrorIs(t, loop.startSlotLocked(slot), errParentNotReady) + require.Nil(t, loop.activeBank) + require.Empty(t, hooks.events) +} + +// newFinishingLeaderLoop returns a loop with an active, published slot that is +// ready to finalize. commit replaces replay.CommitLeaderSlot. +func newFinishingLeaderLoop(t *testing.T, slot uint64, hooks *recordingHooks, commit func(*ParentContext, replay.CommitLeaderInput) error) (*LeaderLoop, *WorkingBank, *captureBroadcaster, *bool) { + t.Helper() + global.ResetAlpenglowChainMetadata() + t.Cleanup(global.ResetAlpenglowChainMetadata) + t.Cleanup(func() { replay.TakeLocalLeaderCommit(slot) }) + parentSlot := slot - 1 + global.SetReplayFrontier(parentSlot) + global.SetAlpenglowBlockID(parentSlot, solana.Hash{1}) + global.SetAlpenglowChainedMerkleRoot(parentSlot, solana.Hash{2}) + parent := coherentTestParentContext(parentSlot, solana.Hash{3}) + parent.PrevFeeGovernor = &sealevel.FeeRateGovernor{ + PrevLamportsPerSignature: 5000, + LamportsPerSignature: 5000, + } + broadcaster := &captureBroadcaster{} + identity := txfixture.PayerPrivateKey() + session := turbine.NewBroadcastSession(turbine.BroadcastSessionConfig{ + Leader: identity, + Slot: slot, + ParentSlot: parentSlot, + ParentBlockID: parent.ParentBlockID, + ParentChainedMerkleRoot: parent.ParentChainedMerkleRoot, + Broadcaster: broadcaster, + }) + slotCtx := &sealevel.SlotCtx{ + Slot: slot, + ParentSlot: parentSlot, + Features: features.NewFeaturesDefault(), + FeeRateGovernor: parent.PrevFeeGovernor, + } + sink := NewShredSink(session) + bank := NewWorkingBank(BankConfig{ + SlotCtx: slotCtx, + Slot: slot, + Leader: identity.PublicKey(), + Sink: sink, + }) + controller := NewController() + controller.SetWorkingBank(bank) + hooks.controller = controller + hooks.bank = func() *WorkingBank { return bank } + delivered := false + loop := NewLeaderLoop(LeaderLoopConfig{ + Controller: controller, + Identity: identity, + AccountsDb: &accountsdb.AccountsDb{}, + EpochSchedule: &sealevel.SysvarEpochSchedule{SlotsPerEpoch: 1_000}, + ParentContext: func(uint64) ParentContext { return parent }, + OnBlock: func(*block.Block) { delivered = true }, + Hooks: hooks, + }) + loop.activeSlot = slot + loop.activeBank = bank + loop.activeSess = session + loop.activeSink = sink + loop.parentCtx = parent + loop.activeParentID = parent.ParentBlockID + loop.activeParentChainedRoot = parent.ParentChainedMerkleRoot + loop.activeParentGeneration = parent.ReplayGeneration + loop.commitLeaderSlot = func(in replay.CommitLeaderInput) (*sealevel.SlotCtx, error) { + in.SlotCtx.FinalBankhash = make([]byte, 32) + return in.SlotCtx, commit(&parent, in) + } + return loop, bank, broadcaster, &delivered +} + +func TestLeaderHooksSlotEndingRunsBeforeFreeze(t *testing.T) { + const slot = uint64(106) + hooks := &recordingHooks{} + loop, bank, broadcaster, delivered := newFinishingLeaderLoop(t, slot, hooks, + func(*ParentContext, replay.CommitLeaderInput) error { return nil }) + + loop.finishActiveSlotLocked() + + require.True(t, *delivered, "the slot must complete for this test to cover the success path") + assert.Greater(t, broadcaster.count(), 0) + require.Equal(t, []string{"ending"}, hooks.kinds(), "a completed slot is never reported as aborted") + ending := hooks.events[0] + assert.Equal(t, slot, ending.slot) + assert.Same(t, bank, ending.published, "SlotEnding runs before the bank is detached") + assert.True(t, ending.accepting, "SlotEnding runs before the bank freezes") + assert.Nil(t, loop.activeBank) +} + +func TestLeaderHooksSlotAbortedAfterEndingWhenFinalizationFails(t *testing.T) { + const slot = uint64(110) + hooks := &recordingHooks{} + loop, _, broadcaster, delivered := newFinishingLeaderLoop(t, slot, hooks, + func(parent *ParentContext, _ replay.CommitLeaderInput) error { + parent.ReplayGeneration++ + return nil + }) + + loop.finishActiveSlotLocked() + + require.False(t, *delivered) + assert.Zero(t, broadcaster.count()) + require.Equal(t, []string{"ending", "aborted"}, hooks.kinds()) + aborted := hooks.events[1] + assert.Equal(t, slot, aborted.slot) + assert.Equal(t, leaderReasonParentChangedBeforeFreeze, aborted.reason) + assert.Nil(t, loop.activeBank) +} + +func TestLeaderHooksSlotAbortedWithoutEndingWhenParentChanges(t *testing.T) { + global.ResetAlpenglowChainMetadata() + t.Cleanup(global.ResetAlpenglowChainMetadata) + + const startSlot = uint64(212) + const activeSlot = startSlot + 1 + self := txfixture.PayerPubkey() + controller := NewController() + bank := NewWorkingBank(BankConfig{Slot: activeSlot}) + controller.SetWorkingBank(bank) + global.SetReplayFrontier(startSlot) + hooks := &recordingHooks{controller: controller, bank: func() *WorkingBank { return bank }} + + loop := NewLeaderLoop(LeaderLoopConfig{ + Controller: controller, + Identity: txfixture.PayerPrivateKey(), + CurrentSlot: func() uint64 { return activeSlot }, + LeaderForSlot: func(slot uint64) (solana.PublicKey, bool) { + if slot == startSlot { + return self, true + } + return solana.PublicKey{}, false + }, + ProductionParent: func(slot uint64) alpenglow.BlockProductionParent { + if slot == activeSlot { + return alpenglow.BlockProductionParent{Kind: alpenglow.BlockProductionParentMissedWindow} + } + return alpenglow.BlockProductionParent{Kind: alpenglow.BlockProductionParentNotReady} + }, + ParentContext: func(uint64) ParentContext { return ParentContext{} }, + Hooks: hooks, + }) + loop.productionWindow = leaderProductionWindow{ + active: true, + startSlot: startSlot, + endSlot: startSlot + alpenglow.LeaderWindowSlots - 1, + nextSlot: activeSlot, + } + loop.activeSlot = activeSlot + loop.activeBank = bank + loop.markLeaderSlotFinished(startSlot) + + loop.tick() + + require.Equal(t, []string{"aborted"}, hooks.kinds()) + aborted := hooks.events[0] + assert.Equal(t, activeSlot, aborted.slot) + assert.Equal(t, leaderReasonParentReadyMissedWindow, aborted.reason) + assert.Same(t, bank, aborted.published, "SlotAborted runs before the bank is detached") + assert.Nil(t, controller.WorkingBank()) + assert.Nil(t, loop.activeBank) +} diff --git a/pkg/blockprod/leader_replay_gate_test.go b/pkg/blockprod/leader_replay_gate_test.go index 951d3ec3..ea85dda4 100644 --- a/pkg/blockprod/leader_replay_gate_test.go +++ b/pkg/blockprod/leader_replay_gate_test.go @@ -286,7 +286,7 @@ func TestLeaderWindowRefreshesCorrectedParentReadyWhileWaitingForReplay(t *testi defer func() { loop.mu.Lock() defer loop.mu.Unlock() - loop.abortActiveSlotLocked() + loop.releaseActiveSlotLocked() }() loop.tick() @@ -582,7 +582,7 @@ func TestLeaderWindowRejectsParentChangeDuringLocalSetup(t *testing.T) { defer func() { loop.mu.Lock() defer loop.mu.Unlock() - loop.abortActiveSlotLocked() + loop.releaseActiveSlotLocked() }() loop.tick() @@ -706,7 +706,7 @@ func TestLeaderWindowRejectsParentChangeDuringHeaderBroadcast(t *testing.T) { defer func() { loop.mu.Lock() defer loop.mu.Unlock() - loop.abortActiveSlotLocked() + loop.releaseActiveSlotLocked() }() loop.tick() @@ -806,7 +806,7 @@ func TestLeaderWindowRecheckStartsAdjacentReadyWindowWithoutPollDelay(t *testing defer func() { loop.mu.Lock() defer loop.mu.Unlock() - loop.abortActiveSlotLocked() + loop.releaseActiveSlotLocked() }() loop.tick() diff --git a/pkg/blockprod/scheduler/scheduler.go b/pkg/blockprod/scheduler/scheduler.go index 2985a70c..076f2da5 100644 --- a/pkg/blockprod/scheduler/scheduler.go +++ b/pkg/blockprod/scheduler/scheduler.go @@ -262,7 +262,7 @@ func (s *Scheduler) drainLoop(ctx context.Context) { return } s.refreshPreparation() - bank := s.banks.WorkingBank() + bank := s.tpuBank() s.noteBank(bank) if bank == nil { if !s.wait(ctx, 5*time.Millisecond) { @@ -285,7 +285,7 @@ func (s *Scheduler) drainLoop(ctx context.Context) { } // Bank may have been cleared between the nil check and pop. - bank = s.banks.WorkingBank() + bank = s.tpuBank() s.noteBank(bank) if bank == nil { s.rebuffer(e) @@ -318,6 +318,16 @@ func (s *Scheduler) drainLoop(ctx context.Context) { } } +// tpuBank returns the working bank while it admits TPU transactions. A gated +// bank is treated like no bank, so the buffer keeps everything until it opens. +func (s *Scheduler) tpuBank() *blockprod.WorkingBank { + bank := s.banks.WorkingBank() + if bank == nil || !bank.TPUAllowed() { + return nil + } + return bank +} + // popSchedulable pops the highest-reward entry that is not marked skip for gen. func (s *Scheduler) popSchedulable(gen uint64) (picked *entry, skipped []*entry) { for { diff --git a/pkg/blockprod/scheduler/scheduler_test.go b/pkg/blockprod/scheduler/scheduler_test.go index f25b731c..b9d038ef 100644 --- a/pkg/blockprod/scheduler/scheduler_test.go +++ b/pkg/blockprod/scheduler/scheduler_test.go @@ -302,3 +302,36 @@ func TestClassifyBufferedExpired(t *testing.T) { require.Equal(t, blockprod.BufferedExpired, env.Bank.ClassifyBuffered(solana.Hash{0x01}, [32]byte{})) require.Equal(t, blockprod.BufferedKeep, env.Bank.ClassifyBuffered(txfixture.TestBlockhash(), [32]byte{0xab})) } + +func TestSchedulerHoldsGatedBankUntilAllowTPU(t *testing.T) { + env := blockprod.NewTestEnv(blockprod.TestEnvConfig{TPUGated: true}) + defer env.Close() + + sched := New(env.Controller) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + sched.Start(ctx) + defer sched.Stop() + + sched.Receive(packet.Owned(txfixture.MustSignedTransferWire(0))) + deadline := time.Now().Add(500 * time.Millisecond) + for sched.Buffered() == 0 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + require.Equal(t, 1, sched.Buffered()) + + // A published but gated bank is treated like no bank. + time.Sleep(50 * time.Millisecond) + require.Equal(t, 1, sched.Buffered()) + require.Equal(t, uint64(0), sched.Stats().Accepted) + require.Empty(t, env.Bank.ForgedTransactions()) + + env.Bank.AllowTPU() + + deadline = time.Now().Add(2 * time.Second) + for sched.Stats().Accepted == 0 && time.Now().Before(deadline) { + time.Sleep(5 * time.Millisecond) + } + require.Equal(t, uint64(1), sched.Stats().Accepted) + require.Len(t, env.Bank.ForgedTransactions(), 1) +} diff --git a/pkg/blockprod/testenv.go b/pkg/blockprod/testenv.go index 854cb909..43e34117 100644 --- a/pkg/blockprod/testenv.go +++ b/pkg/blockprod/testenv.go @@ -27,6 +27,7 @@ type TestEnvConfig struct { EntryHash solana.Hash Sink BatchSink TransactionStatuses *replay.TransactionStatusView + TPUGated bool } func NewTestEnv(cfg TestEnvConfig) *TestEnv { @@ -78,6 +79,7 @@ func NewTestEnv(cfg TestEnvConfig) *TestEnv { EntryHash: entryHash, Sink: cfg.Sink, TransactionStatuses: statuses, + TPUGated: cfg.TPUGated, }) controller := NewController() controller.SetWorkingBank(bank) diff --git a/pkg/forge/forge.go b/pkg/forge/forge.go index ad9f09ae..9dd9aa36 100644 --- a/pkg/forge/forge.go +++ b/pkg/forge/forge.go @@ -57,7 +57,7 @@ func (s *Sink) Receive(pkt packet.Packet) { s.mu.Unlock() bank := s.banks.WorkingBank() - if bank == nil { + if bank == nil || !bank.TPUAllowed() { s.mu.Lock() s.stats.DroppedNoBank++ s.mu.Unlock() diff --git a/pkg/forge/forge_test.go b/pkg/forge/forge_test.go index 7a5b542a..e14e5de2 100644 --- a/pkg/forge/forge_test.go +++ b/pkg/forge/forge_test.go @@ -32,6 +32,21 @@ func TestSinkAcceptsTransfer(t *testing.T) { assert.Equal(t, uint64(1), stats.InPackets) } +func TestSinkDropsWhileBankIsTPUGated(t *testing.T) { + env := blockprod.NewTestEnv(blockprod.TestEnvConfig{TPUGated: true}) + defer env.Close() + + sink := NewSink(env.Controller) + sink.Receive(packet.Owned(txfixture.MustSignedTransferWire(0))) + assert.Equal(t, uint64(1), sink.Stats().DroppedNoBank) + assert.Empty(t, env.Bank.ForgedTransactions()) + + env.Bank.AllowTPU() + sink.Receive(packet.Owned(txfixture.MustSignedTransferWire(0))) + assert.Equal(t, uint64(1), sink.Stats().Accepted) + assert.Len(t, env.Bank.ForgedTransactions(), 1) +} + func TestSinkCountsAlreadyProcessed(t *testing.T) { env := blockprod.NewTestEnv(blockprod.TestEnvConfig{}) defer env.Close() From e9db51c024041208a498bb9e441d5ac0b0df4d21 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 04:48:18 +0000 Subject: [PATCH 03/25] blockprod: count schedule-time entry reservations per holder --- pkg/blockprod/bank.go | 83 ++++++---- pkg/blockprod/bank_test.go | 25 +-- pkg/blockprod/entry.go | 50 +++--- pkg/blockprod/entry_reservation_test.go | 193 ++++++++++++++++++++++++ pkg/blockprod/entry_test.go | 6 +- pkg/blockprod/scheduler/scheduler.go | 5 +- 6 files changed, 297 insertions(+), 65 deletions(-) create mode 100644 pkg/blockprod/entry_reservation_test.go diff --git a/pkg/blockprod/bank.go b/pkg/blockprod/bank.go index c0176670..8b38a93f 100644 --- a/pkg/blockprod/bank.go +++ b/pkg/blockprod/bank.go @@ -203,7 +203,6 @@ func (r ForgeResult) String() string { func (b *WorkingBank) Forge(wire []byte) (ForgeResult, costmodel.ExceedReason) { tx, err := solana.TransactionFromBytes(wire) if err != nil { - b.RebateSchedule(len(wire)) return ForgeDroppedParse, costmodel.ExceedNone } return b.ForgeTransaction(tx, len(wire)) @@ -212,21 +211,28 @@ func (b *WorkingBank) Forge(wire []byte) (ForgeResult, costmodel.ExceedReason) { // ForgeTransaction executes and commits a parsed transaction. The caller must // keep it immutable: accepted transactions are retained for entry publication. func (b *WorkingBank) ForgeTransaction(tx *solana.Transaction, wireSize int) (ForgeResult, costmodel.ExceedReason) { - return b.forgeTransaction(tx, wireSize, nil) + return b.forgeTransaction(tx, wireSize, nil, EntryReservation{}) } // ForgePreparedTransaction reuses static work from owned, immutable TPU bytes. // A different bank feature snapshot falls back to the ordinary execution path. func (b *WorkingBank) ForgePreparedTransaction(tx *solana.Transaction, wireSize int, prepared *replay.PreparedTransaction) (ForgeResult, costmodel.ExceedReason) { + return b.ForgeScheduledTransaction(tx, wireSize, prepared, EntryReservation{}) +} + +// ForgeScheduledTransaction forges a transaction whose entry bytes were +// reserved by PrepareSchedule. It consumes res whatever the result; prepared +// may be nil. +func (b *WorkingBank) ForgeScheduledTransaction(tx *solana.Transaction, wireSize int, prepared *replay.PreparedTransaction, res EntryReservation) (ForgeResult, costmodel.ExceedReason) { if b.slotCtx == nil || !b.preparer.Matches(prepared, tx, b.slotCtx.Features) { prepared = nil } - return b.forgeTransaction(tx, wireSize, prepared) + return b.forgeTransaction(tx, wireSize, prepared, res) } -func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, prepared *replay.PreparedTransaction) (ForgeResult, costmodel.ExceedReason) { +func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, prepared *replay.PreparedTransaction, res EntryReservation) (ForgeResult, costmodel.ExceedReason) { if tx == nil { - b.RebateSchedule(wireSize) + b.RebateSchedule(res) return ForgeDroppedParse, costmodel.ExceedNone } // Validate the full wire before execution can charge fees or publish @@ -234,7 +240,7 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre // fallible serialization step after commit, including on the prepared path. serializedSize, err := serializedTransactionSize(tx) if err != nil { - b.RebateSchedule(wireSize) + b.RebateSchedule(res) return ForgeDroppedParse, costmodel.ExceedNone } var messageHash [32]byte @@ -244,7 +250,7 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre messageHash, err = replay.TransactionMessageHash(tx) } if err != nil { - b.RebateSchedule(wireSize) + b.RebateSchedule(res) return ForgeDroppedParse, costmodel.ExceedNone } // A transaction that repeats an account or exceeds the account lock limit @@ -255,7 +261,7 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre lockFeats = b.slotCtx.Features } if txErr := replay.ValidateAccountLocks(tx, lockFeats); txErr != nil { - b.RebateSchedule(wireSize) + b.RebateSchedule(res) return ForgeDroppedAccountLocks, costmodel.ExceedNone } ancestorAlreadyProcessed := false @@ -278,13 +284,13 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre // change, so resolve outside the bank lock. resolved, txErr := replay.ResolveAddressTableLookups(b.slotCtx, tx) if txErr != nil { - b.RebateSchedule(wireSize) + b.RebateSchedule(res) return ForgeDroppedLookup, costmodel.ExceedNone } // A loaded address can repeat a static key or another loaded // address, or take the account count past the lock limit. if txErr := replay.ValidateAccountLocks(resolved, lockFeats); txErr != nil { - b.RebateSchedule(wireSize) + b.RebateSchedule(res) return ForgeDroppedAccountLocks, costmodel.ExceedNone } execTx = resolved @@ -302,7 +308,7 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre cost, err = costmodel.EstimateTransactionCost(execTx, feats) } if err != nil { - b.RebateSchedule(wireSize) + b.RebateSchedule(res) return ForgeDroppedParse, costmodel.ExceedNone } cost.WireSize = wireSize @@ -310,12 +316,9 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre b.mu.Lock() defer b.mu.Unlock() - included := false - defer func() { - if !included { - b.entries.rebateReserved(wireSize) - } - }() + // The reservation only held budget while the lock was released. The lock + // is now held until append, so admission below is checked exactly. + b.entries.release(res) if !b.accepting { return ForgeDroppedNoLeader, costmodel.ExceedNone } @@ -334,7 +337,7 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre if reason := b.costs.WouldExceed(cost); reason != costmodel.ExceedNone { return ForgeDroppedCost, reason } - if reason := b.reserveEntryBytesLocked(wireSize); reason != costmodel.ExceedNone { + if reason := b.admitEntryBytesLocked(serializedSize); reason != costmodel.ExceedNone { return ForgeDroppedCost, reason } if err := b.preparer.PayerCanFund(b.slotCtx, execTx, prepared); err != nil { @@ -376,7 +379,6 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre b.entryHash = b.entries.CurrentEntryHash() b.sink.OnEntryBatch(flushed, batchBytes) } - included = true return ForgeAccepted, costmodel.ExceedNone } @@ -421,32 +423,47 @@ func (b *WorkingBank) ClassifyBuffered(blockhash solana.Hash, messageHash [32]by // PrepareSchedule reserves entry bytes at schedule time. If the next // transaction would not fit in the current FEC set, the batch is // closed and shredded first. If the tx would then exceed the remaining -// slot entry-byte budget, it is not reserved. A failed/unincludable -// forge rebates the reservation. -func (b *WorkingBank) PrepareSchedule(wireSize int) costmodel.ExceedReason { +// slot entry-byte budget, nothing is reserved. Pass the reservation to +// ForgeScheduledTransaction, or to RebateSchedule if it is not forged. +func (b *WorkingBank) PrepareSchedule(wireSize int) (EntryReservation, costmodel.ExceedReason) { + return b.PrepareScheduleN([]int{wireSize}) +} + +// PrepareScheduleN is PrepareSchedule for transactions forged together, such +// as a bundle. The reservation covers any split into batches. +func (b *WorkingBank) PrepareScheduleN(wireSizes []int) (EntryReservation, costmodel.ExceedReason) { b.mu.Lock() defer b.mu.Unlock() - if !b.accepting { - return costmodel.ExceedNone + if !b.accepting || len(wireSizes) == 0 { + return EntryReservation{}, costmodel.ExceedNone } - return b.reserveEntryBytesLocked(wireSize) + if b.entries.wouldOverflowBatch(wireSizes[0]) { + b.flushEntriesLocked() + } + res, ok := b.entries.reserve(wireSizes) + if !ok { + return EntryReservation{}, costmodel.ExceedBatchBytes + } + return res, costmodel.ExceedNone } // RebateSchedule releases a schedule-time entry-byte reservation. -func (b *WorkingBank) RebateSchedule(wireSize int) { +func (b *WorkingBank) RebateSchedule(res EntryReservation) { + if res.bytes == 0 { + return + } b.mu.Lock() defer b.mu.Unlock() - b.entries.rebateReserved(wireSize) + b.entries.release(res) } -func (b *WorkingBank) reserveEntryBytesLocked(wireSize int) costmodel.ExceedReason { - if b.entries.ReservedBytes() > 0 { - return costmodel.ExceedNone - } - if b.entries.wouldOverflowBatch(wireSize) { +// admitEntryBytesLocked checks the slot entry-byte budget for a transaction +// appended before the bank lock is released, closing a full batch first. +func (b *WorkingBank) admitEntryBytesLocked(serializedSize int) costmodel.ExceedReason { + if b.entries.wouldOverflowBatch(serializedSize) { b.flushEntriesLocked() } - if !b.entries.reserve(wireSize) { + if b.entries.wouldExceedSlot(serializedSize) { return costmodel.ExceedBatchBytes } return costmodel.ExceedNone diff --git a/pkg/blockprod/bank_test.go b/pkg/blockprod/bank_test.go index 1d5e119f..9630e687 100644 --- a/pkg/blockprod/bank_test.go +++ b/pkg/blockprod/bank_test.go @@ -803,7 +803,8 @@ func TestPrepareScheduleClosesFullFECBatch(t *testing.T) { require.Empty(t, sink.batches) require.Equal(t, 1, env.Bank.EntryBuilder().PendingCount()) - require.Equal(t, costmodel.ExceedNone, env.Bank.PrepareSchedule(200)) + _, reason = env.Bank.PrepareSchedule(200) + require.Equal(t, costmodel.ExceedNone, reason) require.Len(t, sink.batches, 1) require.Zero(t, env.Bank.EntryBuilder().PendingCount()) } @@ -820,11 +821,14 @@ func TestPrepareScheduleReservesAndRebatesWhenNotIncluded(t *testing.T) { require.Greater(t, included, 0) require.Zero(t, env.Bank.EntryBuilder().ReservedBytes()) - require.Equal(t, costmodel.ExceedNone, env.Bank.PrepareSchedule(len(wire))) + res, reason := env.Bank.PrepareSchedule(len(wire)) + require.Equal(t, costmodel.ExceedNone, reason) require.Greater(t, env.Bank.EntryBytes(), included) require.Greater(t, env.Bank.EntryBuilder().ReservedBytes(), 0) - result, reason = env.Bank.Forge(wire) + tx, err := solana.TransactionFromBytes(wire) + require.NoError(t, err) + result, reason = env.Bank.ForgeScheduledTransaction(tx, len(wire), nil, res) require.Equal(t, ForgeDroppedAlreadyProcessed, result) require.Equal(t, costmodel.ExceedNone, reason) require.Equal(t, included, env.Bank.EntryBytes()) @@ -839,7 +843,8 @@ func TestPrepareScheduleRejectsSlotEntryBudget(t *testing.T) { defer env.Close() wire := txfixture.MustSignedTransferWire(0) - require.Equal(t, costmodel.ExceedBatchBytes, env.Bank.PrepareSchedule(len(wire))) + _, reason := env.Bank.PrepareSchedule(len(wire)) + require.Equal(t, costmodel.ExceedBatchBytes, reason) require.Zero(t, env.Bank.EntryBytes()) result, reason := env.Bank.Forge(wire) assert.Equal(t, ForgeDroppedCost, result) @@ -938,15 +943,13 @@ func TestWorkingBankSerializationFailureDoesNotCommit(t *testing.T) { require.NoError(t, err) payerBalance, destBalance := payer.Lamports, dest.Lamports hash := env.Bank.EntryHash() - require.Equal(t, costmodel.ExceedNone, env.Bank.PrepareSchedule(len(wire))) + res, reason := env.Bank.PrepareSchedule(len(wire)) + require.Equal(t, costmodel.ExceedNone, reason) require.Positive(t, env.Bank.EntryBuilder().ReservedBytes()) - var result ForgeResult - var reason costmodel.ExceedReason - if mode == "prepared" { - result, reason = env.Bank.ForgePreparedTransaction(tx, len(wire), prepared) - } else { - result, reason = env.Bank.ForgeTransaction(tx, len(wire)) + if mode != "prepared" { + prepared = nil } + result, reason := env.Bank.ForgeScheduledTransaction(tx, len(wire), prepared, res) require.Equal(t, ForgeDroppedParse, result) require.Equal(t, costmodel.ExceedNone, reason) payer, err = env.SlotCtx.GetAccount(txfixture.PayerPubkey()) diff --git a/pkg/blockprod/entry.go b/pkg/blockprod/entry.go index 75ad95a1..9809e08c 100644 --- a/pkg/blockprod/entry.go +++ b/pkg/blockprod/entry.go @@ -12,6 +12,18 @@ import ( // entry count, num_hashes, hash, and transaction count. const entryBatchOverheadBytes = 8 + 8 + 32 + 8 +// EntryReservation is slot entry-byte budget held from schedule time, while +// the bank lock is released, until its transactions are forged or it is +// rebated. The zero value holds nothing. +type EntryReservation struct { + bytes int +} + +// Bytes is the entry-byte budget the reservation holds. +func (r EntryReservation) Bytes() int { + return r.bytes +} + // EntryBuilder accumulates forged transactions into Alpenglow-style entry batches. type EntryBuilder struct { limits costmodel.Limits @@ -47,8 +59,8 @@ func (b *EntryBuilder) ReservedBytes() int { return b.reservedBytes } -// SlotBytes is flushed entry bytes plus the current pending entry and any -// in-flight schedule reservation. +// SlotBytes is flushed entry bytes plus the current pending entry and all +// outstanding schedule reservations. func (b *EntryBuilder) SlotBytes() int { pending := 0 if len(b.pendingTxns) > 0 { @@ -76,25 +88,30 @@ func (b *EntryBuilder) admitBytes(nextWire int) int { return nextWire } -func (b *EntryBuilder) reserve(wireSize int) bool { - if b.wouldExceedSlot(wireSize) { - return false +// reservationBytes bounds what appending transactions of these sizes can add +// to the slot however they are split into batches, since each may start one. +// Holders interleave, so a reservation cannot rely on the current batch. +func reservationBytes(wireSizes []int) int { + n := 0 + for _, size := range wireSizes { + n += entryBatchOverheadBytes + size } - b.reservedBytes += b.admitBytes(wireSize) - return true + return n } -func (b *EntryBuilder) rebateReserved(wireSize int) { - b.consumeReserved(wireSize) +func (b *EntryBuilder) reserve(wireSizes []int) (EntryReservation, bool) { + n := reservationBytes(wireSizes) + if maxEntry := int(b.limits.MaxEntryBytes); maxEntry > 0 && b.SlotBytes()+n > maxEntry { + return EntryReservation{}, false + } + b.reservedBytes += n + return EntryReservation{bytes: n}, true } -func (b *EntryBuilder) consumeReserved(wireSize int) { - n := b.admitBytes(wireSize) - if b.reservedBytes < n { - b.reservedBytes = 0 - return - } - b.reservedBytes -= n +// release returns a reservation. Freeze and Close drop every reservation, so a +// holder may release after its bytes are already gone. +func (b *EntryBuilder) release(r EntryReservation) { + b.reservedBytes -= min(r.bytes, b.reservedBytes) } func (b *EntryBuilder) dropReservation() { @@ -132,7 +149,6 @@ func (b *EntryBuilder) appendSerialized(tx solana.Transaction, wireSize, seriali wireSize = serializedSize } - b.consumeReserved(wireSize) if b.wouldOverflowBatch(serializedSize) { flushed, batchBytes := b.flushLocked() b.pendingTxns = append(b.pendingTxns[:0], tx) diff --git a/pkg/blockprod/entry_reservation_test.go b/pkg/blockprod/entry_reservation_test.go new file mode 100644 index 00000000..a533943b --- /dev/null +++ b/pkg/blockprod/entry_reservation_test.go @@ -0,0 +1,193 @@ +package blockprod + +import ( + "sync" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func scheduledTransfer(t *testing.T, seq uint64) (*solana.Transaction, int) { + t.Helper() + wire := txfixture.MustSignedTransferWire(seq) + tx, err := solana.TransactionFromBytes(wire) + require.NoError(t, err) + return tx, len(wire) +} + +func TestPrepareScheduleNReservesForEveryBatchSplit(t *testing.T) { + sizes := []int{200, 300, 400} + want := 3*entryBatchOverheadBytes + 200 + 300 + 400 + + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + res, reason := env.Bank.PrepareScheduleN(sizes) + require.Equal(t, costmodel.ExceedNone, reason) + require.Equal(t, want, res.Bytes()) + require.Equal(t, want, env.Bank.EntryBuilder().ReservedBytes()) + require.Equal(t, want, env.Bank.EntryBytes()) + env.Bank.RebateSchedule(res) + require.Zero(t, env.Bank.EntryBuilder().ReservedBytes()) + require.Zero(t, env.Bank.EntryBytes()) + + limits := costmodel.DefaultLimits() + limits.MaxEntryBytes = uint64(want) + exact := NewTestEnv(TestEnvConfig{Limits: limits}) + defer exact.Close() + res, reason = exact.Bank.PrepareScheduleN(sizes) + require.Equal(t, costmodel.ExceedNone, reason) + require.Equal(t, want, res.Bytes()) + + limits.MaxEntryBytes = uint64(want - 1) + short := NewTestEnv(TestEnvConfig{Limits: limits}) + defer short.Close() + res, reason = short.Bank.PrepareScheduleN(sizes) + require.Equal(t, costmodel.ExceedBatchBytes, reason) + require.Zero(t, res.Bytes()) + require.Zero(t, short.Bank.EntryBuilder().ReservedBytes()) +} + +func TestScheduleReservationsAreCounted(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + txA, sizeA := scheduledTransfer(t, 0) + txB, sizeB := scheduledTransfer(t, 1) + + resA, reason := env.Bank.PrepareSchedule(sizeA) + require.Equal(t, costmodel.ExceedNone, reason) + resB, reason := env.Bank.PrepareSchedule(sizeB) + require.Equal(t, costmodel.ExceedNone, reason) + require.Positive(t, resB.Bytes(), "a second outstanding reservation must hold its own bytes") + require.Equal(t, resA.Bytes()+resB.Bytes(), env.Bank.EntryBuilder().ReservedBytes()) + + result, _ := env.Bank.ForgeScheduledTransaction(txA, sizeA, nil, resA) + require.Equal(t, ForgeAccepted, result) + require.Equal(t, resB.Bytes(), env.Bank.EntryBuilder().ReservedBytes(), "forging A releases only A's reservation") + + result, _ = env.Bank.ForgeScheduledTransaction(txB, sizeB, nil, resB) + require.Equal(t, ForgeAccepted, result) + require.Zero(t, env.Bank.EntryBuilder().ReservedBytes()) + require.Len(t, env.Bank.ForgedTransactions(), 2) + // Reservations assume every transaction starts a batch; the appended + // transactions share one, and only their actual bytes remain. + require.Equal(t, entryBatchOverheadBytes+sizeA+sizeB, env.Bank.EntryBytes()) +} + +func TestUnscheduledForgeLeavesOtherReservationsIntact(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + txA, sizeA := scheduledTransfer(t, 0) + + resA, reason := env.Bank.PrepareSchedule(sizeA) + require.Equal(t, costmodel.ExceedNone, reason) + + result, _ := env.Bank.Forge(txfixture.MustSignedTransferWire(1)) + require.Equal(t, ForgeAccepted, result) + require.Equal(t, resA.Bytes(), env.Bank.EntryBuilder().ReservedBytes()) + + result, _ = env.Bank.ForgeScheduledTransaction(txA, sizeA, nil, resA) + require.Equal(t, ForgeAccepted, result) + require.Zero(t, env.Bank.EntryBuilder().ReservedBytes()) +} + +func TestUnscheduledForgeCannotUseReservedEntryBytes(t *testing.T) { + txA, size := scheduledTransfer(t, 0) + limits := costmodel.DefaultLimits() + // Room for one transaction in its own batch, not two. + limits.MaxEntryBytes = uint64(2*(entryBatchOverheadBytes+size) - 1) + env := NewTestEnv(TestEnvConfig{Limits: limits}) + defer env.Close() + + resA, reason := env.Bank.PrepareSchedule(size) + require.Equal(t, costmodel.ExceedNone, reason) + + result, reason := env.Bank.Forge(txfixture.MustSignedTransferWire(1)) + require.Equal(t, ForgeDroppedCost, result) + require.Equal(t, costmodel.ExceedBatchBytes, reason) + require.Equal(t, resA.Bytes(), env.Bank.EntryBuilder().ReservedBytes()) + + result, _ = env.Bank.ForgeScheduledTransaction(txA, size, nil, resA) + require.Equal(t, ForgeAccepted, result) + require.Equal(t, []*solana.Transaction{txA}, env.Bank.ForgedTransactions()) + require.LessOrEqual(t, env.Bank.EntryBytes(), int(limits.MaxEntryBytes)) +} + +func TestScheduledForgeReleasesReservationOnEveryResult(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + tx, size := scheduledTransfer(t, 0) + + result, _ := env.Bank.ForgeTransaction(tx, size) + require.Equal(t, ForgeAccepted, result) + res, reason := env.Bank.PrepareSchedule(size) + require.Equal(t, costmodel.ExceedNone, reason) + result, _ = env.Bank.ForgeScheduledTransaction(tx, size, nil, res) + require.Equal(t, ForgeDroppedAlreadyProcessed, result) + require.Zero(t, env.Bank.EntryBuilder().ReservedBytes()) + + res, reason = env.Bank.PrepareSchedule(size) + require.Equal(t, costmodel.ExceedNone, reason) + result, _ = env.Bank.ForgeScheduledTransaction(nil, size, nil, res) + require.Equal(t, ForgeDroppedParse, result) + require.Zero(t, env.Bank.EntryBuilder().ReservedBytes()) + + res, reason = env.Bank.PrepareSchedule(size) + require.Equal(t, costmodel.ExceedNone, reason) + env.Bank.Close() + result, _ = env.Bank.ForgeScheduledTransaction(tx, size, nil, res) + require.Equal(t, ForgeDroppedNoLeader, result) + require.Zero(t, env.Bank.EntryBuilder().ReservedBytes()) +} + +func TestConcurrentForgersRespectEntryByteBudget(t *testing.T) { + const perForger = 64 + _, size := scheduledTransfer(t, 0) + limits := costmodel.DefaultLimits() + limits.MaxBatchBytes = uint64(entryBatchOverheadBytes + 4*size) + limits.MaxEntryBytes = uint64(perForger * (entryBatchOverheadBytes + size)) + env := NewTestEnv(TestEnvConfig{Limits: limits}) + defer env.Close() + + scheduled := make([]*solana.Transaction, perForger) + unscheduled := make([][]byte, perForger) + for i := range perForger { + scheduled[i], _ = scheduledTransfer(t, uint64(i)) + unscheduled[i] = txfixture.MustSignedTransferWire(uint64(perForger + i)) + } + + var accepted [2]int + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for _, tx := range scheduled { + res, reason := env.Bank.PrepareSchedule(size) + if reason != costmodel.ExceedNone { + continue + } + if result, _ := env.Bank.ForgeScheduledTransaction(tx, size, nil, res); result == ForgeAccepted { + accepted[0]++ + } + } + }() + go func() { + defer wg.Done() + for _, wire := range unscheduled { + if result, _ := env.Bank.Forge(wire); result == ForgeAccepted { + accepted[1]++ + } + } + }() + wg.Wait() + + assert.Zero(t, env.Bank.EntryBuilder().ReservedBytes()) + assert.Positive(t, accepted[0]) + assert.Positive(t, accepted[1]) + assert.Less(t, accepted[0]+accepted[1], 2*perForger, "the budget must have been binding") + assert.Len(t, env.Bank.ForgedTransactions(), accepted[0]+accepted[1]) + assert.LessOrEqual(t, env.Bank.EntryBytes(), int(limits.MaxEntryBytes)) +} diff --git a/pkg/blockprod/entry_test.go b/pkg/blockprod/entry_test.go index a22ec5e4..104ece7c 100644 --- a/pkg/blockprod/entry_test.go +++ b/pkg/blockprod/entry_test.go @@ -42,11 +42,13 @@ func TestEntryBuilderReservesAndRebatesSlotBytes(t *testing.T) { wire, err := tx.MarshalBinary() require.NoError(t, err) - require.True(t, builder.reserve(len(wire))) + res, ok := builder.reserve([]int{len(wire)}) + require.True(t, ok) + require.Equal(t, entryBatchOverheadBytes+len(wire), res.Bytes()) require.Equal(t, entryBatchOverheadBytes+len(wire), builder.SlotBytes()) require.Equal(t, builder.SlotBytes(), builder.ReservedBytes()) - builder.rebateReserved(len(wire)) + builder.release(res) require.Zero(t, builder.SlotBytes()) require.Zero(t, builder.ReservedBytes()) } diff --git a/pkg/blockprod/scheduler/scheduler.go b/pkg/blockprod/scheduler/scheduler.go index 076f2da5..59febede 100644 --- a/pkg/blockprod/scheduler/scheduler.go +++ b/pkg/blockprod/scheduler/scheduler.go @@ -294,7 +294,8 @@ func (s *Scheduler) drainLoop(ctx context.Context) { // Close a full FEC set, or refuse, before execute. Slot entry bytes are // a schedule-time budget (shred-safe / SIMD-0525), like Firedancer pack. - if reason := bank.PrepareSchedule(e.wireSize); reason != costmodel.ExceedNone { + res, reason := bank.PrepareSchedule(e.wireSize) + if reason != costmodel.ExceedNone { e.skipGen = s.bankGen s.rebuffer(e) s.recordForge(blockprod.ForgeDroppedCost, reason) @@ -304,7 +305,7 @@ func (s *Scheduler) drainLoop(ctx context.Context) { // Receive owns the wire and its decoded transaction for the entire queue // lifetime. Execution modifies transaction-local account clones, not // this immutable message, so reuse the decoded transaction across banks. - result, reason := bank.ForgePreparedTransaction(e.tx, e.wireSize, e.prepared) + result, reason := bank.ForgeScheduledTransaction(e.tx, e.wireSize, e.prepared, res) switch result { case blockprod.ForgeDroppedNoLeader: s.rebuffer(e) From efcda0af66816dd3de3b51a35d6d2ecf1f9f2fa1 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 04:53:36 +0000 Subject: [PATCH 04/25] costmodel: add aggregate fit check and actual-usage costs for bundles --- pkg/costmodel/costmodel_test.go | 135 ++++++++++++++++++++++++++++++ pkg/costmodel/tracker.go | 31 +++++++ pkg/costmodel/transaction_cost.go | 9 ++ 3 files changed, 175 insertions(+) diff --git a/pkg/costmodel/costmodel_test.go b/pkg/costmodel/costmodel_test.go index 241c39ba..75cc54ad 100644 --- a/pkg/costmodel/costmodel_test.go +++ b/pkg/costmodel/costmodel_test.go @@ -1,6 +1,7 @@ package costmodel import ( + "math/rand/v2" "testing" "github.com/Overclock-Validator/mithril/pkg/addresses" @@ -327,3 +328,137 @@ func TestPackEntryBytesMaxChargesEveryFECSetInBatch(t *testing.T) { assert.Equal(t, want, PackEntryBytesMax(shreds+DataShredsPerFECSet-1, MaxMicroblockBytes)) assert.Zero(t, PackEntryBytesMax(shreds-DataShredsPerFECSet, MaxMicroblockBytes)) } + +func execCost(units, alloc uint64, writable ...solana.PublicKey) TransactionCost { + return TransactionCost{ProgramsExecutionCost: units, AllocatedAccountsDataSize: alloc, WritableAccounts: writable} +} + +func TestCostTrackerWouldExceedAllChecksCombinedTotals(t *testing.T) { + a, b := solana.PublicKey{1}, solana.PublicKey{2} + limits := Limits{BlockCost: 1000, WritableAccountCost: 600, AllocatedDataSizeDelta: 100} + + tracker := NewCostTracker(limits) + tracker.Record(execCost(300, 0, a)) + assert.Equal(t, ExceedNone, tracker.WouldExceedAll(nil)) + assert.Equal(t, ExceedNone, tracker.WouldExceedAll([]TransactionCost{execCost(300, 0, a), execCost(400, 0, b)}), + "exactly at the block limit is allowed") + + // Each cost fits alone; together they pass a limit, counting recorded usage. + bundle := []TransactionCost{execCost(400, 0, b), execCost(400, 0, b)} + for _, cost := range bundle { + require.Equal(t, ExceedNone, tracker.WouldExceed(cost)) + } + assert.Equal(t, ExceedBlockCost, tracker.WouldExceedAll(bundle)) + + shared := []TransactionCost{execCost(200, 0, a), execCost(200, 0, a)} + disjoint := []TransactionCost{execCost(200, 0, a), execCost(200, 0, b)} + assert.Equal(t, ExceedWritableAccountCost, tracker.WouldExceedAll(shared)) + assert.Equal(t, ExceedNone, tracker.WouldExceedAll(disjoint)) + + allocated := []TransactionCost{execCost(10, 60), execCost(10, 60)} + assert.Equal(t, ExceedAllocatedDataSize, tracker.WouldExceedAll(allocated)) + assert.Equal(t, ExceedNone, tracker.WouldExceedAll(allocated[:1])) + + // Checking never records. + assert.Equal(t, uint64(300), tracker.BlockCost()) + assert.Equal(t, uint64(300), tracker.WritableAccountCost(a)) + assert.Zero(t, tracker.WritableAccountCost(b)) + assert.Zero(t, tracker.AllocatedDataSizeDelta()) +} + +func TestCostTrackerWouldExceedAllMatchesSequentialRecording(t *testing.T) { + rng := rand.New(rand.NewPCG(1, 2)) + keys := []solana.PublicKey{{1}, {2}, {3}} + randomCost := func() TransactionCost { + var writable []solana.PublicKey + for _, key := range keys { + if rng.IntN(2) == 0 { + writable = append(writable, key) + } + } + return execCost(50+rng.Uint64N(550), rng.Uint64N(80), writable...) + } + var fitting, breaching int + for trial := range 2000 { + limits := Limits{ + BlockCost: 500 + rng.Uint64N(2500), + WritableAccountCost: 300 + rng.Uint64N(1200), + AllocatedDataSizeDelta: rng.Uint64N(200), + } + aggregate, sequential := NewCostTracker(limits), NewCostTracker(limits) + for range rng.IntN(3) { + cost := randomCost() + if aggregate.WouldExceed(cost) == ExceedNone { + aggregate.Record(cost) + sequential.Record(cost) + } + } + bundle := make([]TransactionCost, 1+rng.IntN(5)) + for i := range bundle { + bundle[i] = randomCost() + } + + fits := true + for _, cost := range bundle { + if sequential.WouldExceed(cost) != ExceedNone { + fits = false + break + } + sequential.Record(cost) + } + require.Equalf(t, fits, aggregate.WouldExceedAll(bundle) == ExceedNone, "trial %d", trial) + if fits { + fitting++ + } else { + breaching++ + } + } + require.Greater(t, fitting, 200) + require.Greater(t, breaching, 200) +} + +func TestWithActualUsageRecordsLikeRecordAndRebate(t *testing.T) { + a, b := solana.PublicKey{1}, solana.PublicKey{2} + requested := TransactionCost{ + SignatureCost: 720, + WriteLockCost: 600, + DataBytesCost: 3, + ProgramsExecutionCost: 200_000, + LoadedAccountsDataSizeCost: 16_000, + AllocatedAccountsDataSize: 10, + WritableAccounts: []solana.PublicKey{a, b}, + } + for _, usage := range [][2]uint64{{1_000, 8}, {200_000, 16_000}, {250_000, 20_000}} { + rebated, direct := NewCostTracker(DefaultLimits()), NewCostTracker(DefaultLimits()) + rebated.Record(requested) + rebated.Rebate(requested, usage[0], usage[1]) + actual := requested.WithActualUsage(usage[0], usage[1]) + direct.Record(actual) + + assert.Equal(t, rebated.BlockCost(), direct.BlockCost()) + assert.Equal(t, rebated.WritableAccountCost(a), direct.WritableAccountCost(a)) + assert.Equal(t, rebated.WritableAccountCost(b), direct.WritableAccountCost(b)) + assert.Equal(t, rebated.AllocatedDataSizeDelta(), direct.AllocatedDataSizeDelta()) + assert.Equal(t, uint64(720+600+3)+usage[0]+usage[1], actual.Sum()) + } + assert.Equal(t, uint64(200_000), requested.ProgramsExecutionCost, "the requested cost is not modified") +} + +func TestWouldExceedAllAdmitsOnActualUsage(t *testing.T) { + a := solana.PublicKey{1} + tracker := NewCostTracker(Limits{BlockCost: 1_000_000, WritableAccountCost: 600_000, AllocatedDataSizeDelta: 100}) + requested := TransactionCost{SignatureCost: 720, ProgramsExecutionCost: 400_000, WritableAccounts: []solana.PublicKey{a}} + bundle := []TransactionCost{requested, requested, requested} + + // Requested units exceed both the account and block limits... + require.Equal(t, ExceedBlockCost, tracker.WouldExceedAll(bundle)) + // ...but what execution consumed fits. + actual := make([]TransactionCost, len(bundle)) + for i, cost := range bundle { + actual[i] = cost.WithActualUsage(50_000, 0) + } + require.Equal(t, ExceedNone, tracker.WouldExceedAll(actual)) + // Actual usage still has to fit. + actual[1] = bundle[1].WithActualUsage(560_000, 0) + require.Equal(t, ExceedWritableAccountCost, tracker.WouldExceedAll(actual)) +} diff --git a/pkg/costmodel/tracker.go b/pkg/costmodel/tracker.go index 8b7cbd8d..57ea79c1 100644 --- a/pkg/costmodel/tracker.go +++ b/pkg/costmodel/tracker.go @@ -80,6 +80,37 @@ func (t *CostTracker) WouldExceed(cost TransactionCost) ExceedReason { return ExceedNone } +// WouldExceedAll returns whether admitting every cost together, such as a +// bundle, would breach a limit. Usage only grows as costs are recorded, so the +// combined total breaches a limit exactly when recording them one by one would. +func (t *CostTracker) WouldExceedAll(costs []TransactionCost) ExceedReason { + var blockCost, allocated uint64 + var accounts map[solana.PublicKey]uint64 + for _, cost := range costs { + sum := cost.Sum() + blockCost += sum + allocated += cost.AllocatedAccountsDataSize + for _, pk := range cost.WritableAccounts { + if accounts == nil { + accounts = make(map[solana.PublicKey]uint64, len(cost.WritableAccounts)) + } + accounts[pk] += sum + } + } + if t.blockCost+blockCost > t.limits.BlockCost { + return ExceedBlockCost + } + for pk, add := range accounts { + if t.perWritableAccountCost[pk]+add > t.limits.WritableAccountCost { + return ExceedWritableAccountCost + } + } + if t.allocatedDataSizeDelta+allocated > t.limits.AllocatedDataSizeDelta { + return ExceedAllocatedDataSize + } + return ExceedNone +} + // Record commits cost after a transaction is forged into the working bank. func (t *CostTracker) Record(cost TransactionCost) { sum := cost.Sum() diff --git a/pkg/costmodel/transaction_cost.go b/pkg/costmodel/transaction_cost.go index 038c6ac2..56246367 100644 --- a/pkg/costmodel/transaction_cost.go +++ b/pkg/costmodel/transaction_cost.go @@ -33,6 +33,15 @@ func (c TransactionCost) Sum() uint64 { c.LoadedAccountsDataSizeCost } +// WithActualUsage replaces the requested execution and loaded-accounts cost +// with what execution consumed. Recording it is the same as recording c and +// rebating it, so a check made after execution can admit on actual usage. +func (c TransactionCost) WithActualUsage(actualExec, actualLoaded uint64) TransactionCost { + c.ProgramsExecutionCost = actualExec + c.LoadedAccountsDataSizeCost = actualLoaded + return c +} + // EstimateTransactionCost applies the protocol cost model to a parsed transaction. func EstimateTransactionCost(tx *solana.Transaction, feats *features.Features) (TransactionCost, error) { if tx == nil { From 933848082ae4874494d22182e3ba2843855350a4 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 05:27:35 +0000 Subject: [PATCH 05/25] gossip: copy-on-write contact updates, TPU socket and client ID setters --- pkg/gossip/client.go | 84 ++++++- pkg/gossip/contact_info.go | 151 +++++++++++- pkg/gossip/contact_update_test.go | 371 ++++++++++++++++++++++++++++++ 3 files changed, 593 insertions(+), 13 deletions(-) create mode 100644 pkg/gossip/contact_update_test.go diff --git a/pkg/gossip/client.go b/pkg/gossip/client.go index 67c205c6..01d36eb5 100644 --- a/pkg/gossip/client.go +++ b/pkg/gossip/client.go @@ -50,10 +50,14 @@ type Client struct { alpenglow *net.UDPAddr identity ed25519.PrivateKey pubkey Pubkey - pendingTPUQUIC *net.UDPAddr contactMu sync.RWMutex - contact *ContactInfo + // contact is replaced on every update and never modified once published, + // so readers may use it after releasing contactMu. + contact *ContactInfo + // pendingContact holds updates made before Run initializes contact. + pendingContact []func(*ContactInfo) error + contactChanged chan struct{} identityConflictMu sync.Mutex identityConflictGossip contactEndpoint @@ -205,6 +209,7 @@ func NewClient(cfg Config) (*Client, error) { identity: identity, pubkey: pubkey, identityConflictCh: make(chan error, 1), + contactChanged: make(chan struct{}, 1), peers: make(map[udpAddrKey]knownPeer), repairPeers: make(map[Pubkey]RepairPeer), tvuPeers: make(map[Pubkey]TVUPeer), @@ -313,6 +318,10 @@ func (c *Client) Run(ctx context.Context) error { if err := c.pushContact(conn); err != nil { return err } + case <-c.contactChanged: + if err := c.pushContact(conn); err != nil { + return err + } case <-pingTicker.C: _ = c.sendPing(conn, c.entrypoint) // Pull peers' CRDS tables so discovery isn't purely passive (fixes @@ -380,13 +389,68 @@ func (c *Client) RepairPeers() []RepairPeer { // SetTPUQUIC updates the TPU QUIC endpoints published in this client's CRDS contact. // It may be called before Run has initialized the contact record. func (c *Client) SetTPUQUIC(addr *net.UDPAddr) error { + addr = cloneUDPAddr(addr) + return c.UpdateContact(func(contact *ContactInfo) error { + return contact.SetTPUQUIC(addr) + }) +} + +// SetTPUSockets publishes exactly the given TPU endpoints; see +// ContactInfo.SetTPUSockets. +func (c *Client) SetTPUSockets(tpu TPUSockets) error { + tpu = TPUSockets{ + UDP: cloneUDPAddr(tpu.UDP), + QUIC: cloneUDPAddr(tpu.QUIC), + ForwardsUDP: cloneUDPAddr(tpu.ForwardsUDP), + ForwardsQUIC: cloneUDPAddr(tpu.ForwardsQUIC), + } + return c.UpdateContact(func(contact *ContactInfo) error { + return contact.SetTPUSockets(tpu) + }) +} + +// SetClientID sets the client ID advertised in this client's contact. +func (c *Client) SetClientID(id uint16) error { + return c.UpdateContact(func(contact *ContactInfo) error { + contact.SetClientID(id) + return nil + }) +} + +// UpdateContact applies fn to a copy of this client's contact and publishes +// the copy, so a concurrent push never sees a partial update, then pushes it +// without waiting for the next push interval. Nothing changes if fn fails or +// leaves the contact in a form peers reject. Updates made before Run +// initializes the contact are applied in order when it does. fn runs with the +// contact lock held and must not call back into the client. +func (c *Client) UpdateContact(fn func(*ContactInfo) error) error { c.contactMu.Lock() defer c.contactMu.Unlock() if c.contact == nil { - c.pendingTPUQUIC = cloneUDPAddr(addr) + c.pendingContact = append(c.pendingContact, fn) return nil } - return c.contact.SetTPUQUIC(addr) + next, err := updatedContact(c.contact, fn) + if err != nil { + return err + } + c.contact = next + select { + case c.contactChanged <- struct{}{}: + default: + } + return nil +} + +func updatedContact(current *ContactInfo, fn func(*ContactInfo) error) (*ContactInfo, error) { + next := current.CloneWithWallclock(current.Wallclock) + if err := fn(next); err != nil { + return nil, err + } + if err := next.validateEntries(); err != nil { + return nil, err + } + return next, nil } func (c *Client) initializeContact(localGossipAddr *net.UDPAddr) error { @@ -445,14 +509,16 @@ func (c *Client) initializeContact(localGossipAddr *net.UDPAddr) error { } } c.contactMu.Lock() - if c.pendingTPUQUIC != nil { - if err := contact.SetTPUQUIC(c.pendingTPUQUIC); err != nil { - c.contactMu.Unlock() - return fmt.Errorf("set pending TPU QUIC gossip socket: %w", err) + defer c.contactMu.Unlock() + for _, fn := range c.pendingContact { + next, err := updatedContact(contact, fn) + if err != nil { + return fmt.Errorf("apply pending gossip contact update: %w", err) } + contact = next } + c.pendingContact = nil c.contact = contact - c.contactMu.Unlock() return nil } diff --git a/pkg/gossip/contact_info.go b/pkg/gossip/contact_info.go index baed0d39..3d87f005 100644 --- a/pkg/gossip/contact_info.go +++ b/pkg/gossip/contact_info.go @@ -30,6 +30,8 @@ const ( socketTagGossip = 0 socketTagServeRepairQuic = 1 socketTagServeRepair = 4 + socketTagTPU = 5 + socketTagTPUForwards = 6 socketTagTPUQUICForwards = 7 socketTagTPUQUIC = 8 socketTagTPUVote = 9 @@ -78,6 +80,15 @@ type ContactInfo struct { Extensions [][]byte } +// TPUSockets are the TPU endpoints a contact publishes. A nil endpoint is not +// published. +type TPUSockets struct { + UDP *net.UDPAddr + QUIC *net.UDPAddr + ForwardsUDP *net.UDPAddr + ForwardsQUIC *net.UDPAddr +} + type contactEndpoint struct { ip [16]byte port int @@ -191,16 +202,61 @@ func (c *ContactInfo) SetTPUQUIC(addr *net.UDPAddr) error { return c.SetSocket(socketTagTPUQUIC, addr) } -func (c *ContactInfo) SetSocket(key uint8, addr *net.UDPAddr) error { +// SetTPUSockets publishes exactly the given TPU endpoints and removes the ones +// left nil. Nothing changes if an endpoint is invalid. +func (c *ContactInfo) SetTPUSockets(tpu TPUSockets) error { + sockets := [...]struct { + key uint8 + addr *net.UDPAddr + }{ + {socketTagTPU, tpu.UDP}, + {socketTagTPUQUIC, tpu.QUIC}, + {socketTagTPUForwards, tpu.ForwardsUDP}, + {socketTagTPUQUICForwards, tpu.ForwardsQUIC}, + } + for _, socket := range sockets { + if socket.addr == nil { + continue + } + if _, err := socketIP(socket.key, socket.addr); err != nil { + return err + } + } + for _, socket := range sockets { + if socket.addr == nil { + c.RemoveSocket(socket.key) + continue + } + if err := c.SetSocket(socket.key, socket.addr); err != nil { + return err + } + } + return nil +} + +// SetClientID sets the client ID advertised in the contact's version. +func (c *ContactInfo) SetClientID(id uint16) { + c.Version.Client = id +} + +func socketIP(key uint8, addr *net.UDPAddr) (net.IP, error) { if addr == nil { - return fmt.Errorf("nil socket for key %d", key) + return nil, fmt.Errorf("nil socket for key %d", key) } if addr.Port <= 0 || addr.Port > 0xffff { - return fmt.Errorf("invalid port %d for key %d", addr.Port, key) + return nil, fmt.Errorf("invalid port %d for key %d", addr.Port, key) } ip := normalizedIP(addr.IP) if ip == nil || ip.IsUnspecified() { - return fmt.Errorf("invalid IP %q for key %d", addr.IP.String(), key) + return nil, fmt.Errorf("invalid IP %q for key %d", addr.IP.String(), key) + } + return ip, nil +} + +func (c *ContactInfo) SetSocket(key uint8, addr *net.UDPAddr) error { + ip, err := socketIP(key, addr) + if err != nil { + return err } addrIndex := -1 for idx, existing := range c.Addrs { @@ -230,6 +286,93 @@ func (c *ContactInfo) SetSocket(key uint8, addr *net.UDPAddr) error { } return c.Sockets[i].Port < c.Sockets[j].Port }) + // The replaced socket may have been the last one on its address. + c.pruneUnusedAddrs() + return nil +} + +// RemoveSocket stops publishing the socket for key, and its IP address if no +// other socket uses it. +func (c *ContactInfo) RemoveSocket(key uint8) { + filtered := c.Sockets[:0] + for _, entry := range c.Sockets { + if entry.Key != key { + filtered = append(filtered, entry) + } + } + c.Sockets = filtered + switch key { + case socketTagGossip: + c.GossipAddr = nil + case socketTagServeRepair: + c.ServeRepairAddr = nil + case socketTagTVU: + c.TVUAddr = nil + case socketTagTPUVote: + c.TPUVoteAddr = nil + case socketTagTPUVoteQuic: + c.TPUVoteQuicAddr = nil + case socketTagAlpenglow: + c.AlpenglowAddr = nil + } + c.pruneUnusedAddrs() +} + +// pruneUnusedAddrs drops IP addresses no socket references. Peers reject a +// contact that lists an unused address. +func (c *ContactInfo) pruneUnusedAddrs() { + used := make([]bool, len(c.Addrs)) + for _, entry := range c.Sockets { + if int(entry.Index) < len(used) { + used[entry.Index] = true + } + } + remap := make([]uint8, len(c.Addrs)) + kept := c.Addrs[:0] + for idx, ip := range c.Addrs { + if used[idx] { + remap[idx] = uint8(len(kept)) + kept = append(kept, ip) + } + } + c.Addrs = kept + for idx := range c.Sockets { + c.Sockets[idx].Index = remap[c.Sockets[idx].Index] + } +} + +// validateEntries applies the checks peers run on a contact's address and +// socket lists before accepting it: unique IPv4 addresses, unique socket +// keys, every socket on a listed address, and every address in use. +func (c *ContactInfo) validateEntries() error { + seen := make(map[[4]byte]struct{}, len(c.Addrs)) + for _, ip := range c.Addrs { + v4 := ip.To4() + if v4 == nil { + return fmt.Errorf("contact address %s is not IPv4", ip) + } + if _, dup := seen[[4]byte(v4)]; dup { + return fmt.Errorf("duplicate contact address %s", ip) + } + seen[[4]byte(v4)] = struct{}{} + } + var keys [256]bool + used := make([]bool, len(c.Addrs)) + for _, entry := range c.Sockets { + if keys[entry.Key] { + return fmt.Errorf("duplicate socket key %d", entry.Key) + } + keys[entry.Key] = true + if int(entry.Index) >= len(c.Addrs) { + return fmt.Errorf("socket key %d uses address index %d of %d", entry.Key, entry.Index, len(c.Addrs)) + } + used[entry.Index] = true + } + for idx, inUse := range used { + if !inUse { + return fmt.Errorf("contact address %s is unused", c.Addrs[idx]) + } + } return nil } diff --git a/pkg/gossip/contact_update_test.go b/pkg/gossip/contact_update_test.go new file mode 100644 index 00000000..4153aeb1 --- /dev/null +++ b/pkg/gossip/contact_update_test.go @@ -0,0 +1,371 @@ +package gossip + +import ( + "bytes" + "context" + "crypto/ed25519" + "crypto/rand" + "net" + "sync" + "testing" + "time" +) + +var ( + localIP = net.ParseIP("203.0.113.10") + relayerIP = net.ParseIP("198.51.100.7") +) + +func udp(ip net.IP, port int) *net.UDPAddr { + return &net.UDPAddr{IP: ip, Port: port} +} + +func newTestContact(t *testing.T) *ContactInfo { + t.Helper() + _, priv, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + pubkey, err := pubkeyFromPrivateKey(priv) + if err != nil { + t.Fatalf("pubkeyFromPrivateKey: %v", err) + } + contact, err := NewContactInfo(pubkey, 1234, udp(localIP, 8001), udp(localIP, 8002)) + if err != nil { + t.Fatalf("NewContactInfo: %v", err) + } + if err := contact.SetTPUQUIC(udp(localIP, 8009)); err != nil { + t.Fatalf("SetTPUQUIC: %v", err) + } + return contact +} + +func socketAddr(contact *ContactInfo, key uint8) *net.UDPAddr { + for _, entry := range contact.Sockets { + if entry.Key == key { + return udp(contact.Addrs[entry.Index], int(entry.Port)) + } + } + return nil +} + +func encodeContact(t *testing.T, contact *ContactInfo) []byte { + t.Helper() + var e encoder + if err := contact.encode(&e); err != nil { + t.Fatalf("encode: %v", err) + } + return e.bytes() +} + +// requireAcceptedByPeers checks the contact survives a wire round trip and +// passes the entry checks peers apply before accepting it. +func requireAcceptedByPeers(t *testing.T, contact *ContactInfo) *ContactInfo { + t.Helper() + if err := contact.validateEntries(); err != nil { + t.Fatalf("contact entries rejected: %v (addrs=%v sockets=%v)", err, contact.Addrs, contact.Sockets) + } + decoded, err := decodeContactInfo(newDecoder(encodeContact(t, contact))) + if err != nil { + t.Fatalf("decodeContactInfo: %v", err) + } + if err := decoded.validateEntries(); err != nil { + t.Fatalf("decoded contact entries rejected: %v", err) + } + return decoded +} + +func TestContactInfoPrunesAddressesNoSocketUses(t *testing.T) { + contact := newTestContact(t) + if err := contact.SetSocket(socketTagTPUQUIC, udp(relayerIP, 9000)); err != nil { + t.Fatalf("SetSocket: %v", err) + } + if len(contact.Addrs) != 2 { + t.Fatalf("addrs = %v, want local and relayer", contact.Addrs) + } + // Moving the only socket on the relayer address back drops that address. + if err := contact.SetSocket(socketTagTPUQUIC, udp(localIP, 8009)); err != nil { + t.Fatalf("SetSocket: %v", err) + } + if len(contact.Addrs) != 1 || !contact.Addrs[0].Equal(localIP) { + t.Fatalf("addrs = %v, want only the local address", contact.Addrs) + } + requireAcceptedByPeers(t, contact) + + if err := contact.SetSocket(socketTagTPUQUIC, udp(relayerIP, 9000)); err != nil { + t.Fatalf("SetSocket: %v", err) + } + contact.RemoveSocket(socketTagTPUQUIC) + if len(contact.Addrs) != 1 || socketAddr(contact, socketTagTPUQUIC) != nil { + t.Fatalf("addrs = %v sockets = %v, want the relayer address and socket removed", contact.Addrs, contact.Sockets) + } + requireAcceptedByPeers(t, contact) + + contact.RemoveSocket(socketTagTVU) + if contact.TVUAddr != nil || socketAddr(contact, socketTagTVU) != nil { + t.Fatalf("TVU socket still published") + } + requireAcceptedByPeers(t, contact) +} + +func TestContactInfoTPUSocketsRelayerAndBack(t *testing.T) { + contact := newTestContact(t) + outset := contact.Outset + local := TPUSockets{QUIC: udp(localIP, 8009), ForwardsQUIC: udp(localIP, 8009)} + relayer := TPUSockets{ + UDP: udp(relayerIP, 11222), + QUIC: udp(relayerIP, 11228), + ForwardsUDP: udp(relayerIP, 11223), + ForwardsQUIC: udp(relayerIP, 11229), + } + keys := []struct { + key uint8 + want func(TPUSockets) *net.UDPAddr + }{ + {socketTagTPU, func(s TPUSockets) *net.UDPAddr { return s.UDP }}, + {socketTagTPUQUIC, func(s TPUSockets) *net.UDPAddr { return s.QUIC }}, + {socketTagTPUForwards, func(s TPUSockets) *net.UDPAddr { return s.ForwardsUDP }}, + {socketTagTPUQUICForwards, func(s TPUSockets) *net.UDPAddr { return s.ForwardsQUIC }}, + } + for step, tpu := range []TPUSockets{relayer, local, relayer, local} { + if err := contact.SetTPUSockets(tpu); err != nil { + t.Fatalf("step %d: SetTPUSockets: %v", step, err) + } + decoded := requireAcceptedByPeers(t, contact) + for _, k := range keys { + want := k.want(tpu) + for _, c := range []*ContactInfo{contact, decoded} { + got := socketAddr(c, k.key) + if (want == nil) != (got == nil) || (want != nil && got.String() != want.String()) { + t.Fatalf("step %d: socket %d = %v, want %v", step, k.key, got, want) + } + } + } + if got := socketAddr(contact, socketTagGossip); got.String() != "203.0.113.10:8001" { + t.Fatalf("step %d: gossip socket = %v", step, got) + } + } + if len(contact.Addrs) != 1 { + t.Fatalf("addrs = %v, want only the local address after reverting", contact.Addrs) + } + if contact.Outset != outset { + t.Fatalf("outset changed") + } +} + +func TestContactInfoSetTPUSocketsRejectsInvalidWithoutChange(t *testing.T) { + contact := newTestContact(t) + before := encodeContact(t, contact) + err := contact.SetTPUSockets(TPUSockets{QUIC: udp(relayerIP, 9000), ForwardsQUIC: udp(relayerIP, 0)}) + if err == nil { + t.Fatalf("expected an invalid port to be rejected") + } + if !bytes.Equal(before, encodeContact(t, contact)) { + t.Fatalf("contact changed after a rejected update") + } +} + +func TestContactInfoValidateEntriesMatchesPeerChecks(t *testing.T) { + valid := func() *ContactInfo { + return &ContactInfo{ + Addrs: []net.IP{localIP, relayerIP}, + Sockets: []SocketEntry{{Key: socketTagGossip, Index: 0, Port: 8001}, {Key: socketTagTPUQUIC, Index: 1, Port: 9000}}, + } + } + if err := valid().validateEntries(); err != nil { + t.Fatalf("valid entries rejected: %v", err) + } + cases := map[string]func(*ContactInfo){ + "duplicate address": func(c *ContactInfo) { c.Addrs[1] = localIP }, + "IPv6 address": func(c *ContactInfo) { c.Addrs[1] = net.ParseIP("2001:db8::1") }, + "duplicate key": func(c *ContactInfo) { c.Sockets[1].Key = socketTagGossip }, + "index past end": func(c *ContactInfo) { c.Sockets[1].Index = 2 }, + "unused address": func(c *ContactInfo) { c.Sockets[1].Index = 0 }, + } + for name, corrupt := range cases { + c := valid() + corrupt(c) + if err := c.validateEntries(); err == nil { + t.Errorf("%s: accepted", name) + } + } +} + +func newUpdateTestClient(t *testing.T, entrypoint string) *Client { + t.Helper() + client, err := NewClient(Config{ + Entrypoint: entrypoint, + BindAddr: "127.0.0.1:0", + TVUAddr: "127.0.0.1:8002", + AdvertisedIP: "127.0.0.1", + ShredVersion: 4321, + PushInterval: time.Hour, + PingInterval: time.Hour, + }) + if err != nil { + t.Fatalf("NewClient: %v", err) + } + return client +} + +func publishedContact(c *Client) *ContactInfo { + c.contactMu.RLock() + defer c.contactMu.RUnlock() + return c.contact +} + +func TestClientUpdateContactPublishesACopy(t *testing.T) { + client := newUpdateTestClient(t, "127.0.0.1:8000") + if err := client.initializeContact(udp(net.ParseIP("127.0.0.1"), 65400)); err != nil { + t.Fatalf("initializeContact: %v", err) + } + before := publishedContact(client) + beforeWire := encodeContact(t, before) + + if err := client.SetClientID(15); err != nil { + t.Fatalf("SetClientID: %v", err) + } + after := publishedContact(client) + if after == before { + t.Fatalf("the published contact was modified in place") + } + if !bytes.Equal(beforeWire, encodeContact(t, before)) { + t.Fatalf("the previously published contact changed") + } + if after.Version.Client != 15 || before.Version.Client != versionClientMithril { + t.Fatalf("client id after=%#x before=%#x", after.Version.Client, before.Version.Client) + } + if after.Outset != before.Outset { + t.Fatalf("outset changed: %d -> %d", before.Outset, after.Outset) + } + select { + case <-client.contactChanged: + default: + t.Fatalf("an update must request an immediate push") + } + + // An update peers would reject publishes nothing. + err := client.UpdateContact(func(contact *ContactInfo) error { + return contact.SetSocket(socketTagTPUQUIC, udp(net.ParseIP("2001:db8::1"), 9000)) + }) + if err == nil { + t.Fatalf("expected an IPv6 socket to be rejected") + } + if publishedContact(client) != after { + t.Fatalf("a rejected update was published") + } + select { + case <-client.contactChanged: + t.Fatalf("a rejected update must not request a push") + default: + } +} + +func TestClientAppliesContactUpdatesQueuedBeforeRun(t *testing.T) { + client := newUpdateTestClient(t, "127.0.0.1:8000") + if err := client.SetTPUQUIC(udp(net.ParseIP("127.0.0.1"), 8009)); err != nil { + t.Fatalf("SetTPUQUIC: %v", err) + } + if err := client.SetTPUSockets(TPUSockets{QUIC: udp(relayerIP, 11228), ForwardsQUIC: udp(relayerIP, 11229)}); err != nil { + t.Fatalf("SetTPUSockets: %v", err) + } + if err := client.SetClientID(15); err != nil { + t.Fatalf("SetClientID: %v", err) + } + if err := client.initializeContact(udp(net.ParseIP("127.0.0.1"), 65400)); err != nil { + t.Fatalf("initializeContact: %v", err) + } + contact := publishedContact(client) + if got := socketAddr(contact, socketTagTPUQUIC); got.String() != "198.51.100.7:11228" { + t.Fatalf("TPU QUIC = %v; queued updates must apply in order", got) + } + if contact.Version.Client != 15 { + t.Fatalf("client id = %#x", contact.Version.Client) + } + requireAcceptedByPeers(t, contact) +} + +func TestClientContactUpdatesDoNotRaceWithPushes(t *testing.T) { + client := newUpdateTestClient(t, "127.0.0.1:8000") + if err := client.initializeContact(udp(net.ParseIP("127.0.0.1"), 65400)); err != nil { + t.Fatalf("initializeContact: %v", err) + } + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + for i := range 500 { + if err := client.SetTPUQUIC(udp(relayerIP, 9000+i%2)); err != nil { + t.Errorf("SetTPUQUIC: %v", err) + return + } + } + }() + go func() { + defer wg.Done() + for range 500 { + if _, err := client.currentContactValue(); err != nil { + t.Errorf("currentContactValue: %v", err) + return + } + } + }() + wg.Wait() +} + +func TestClientPushesContactUpdatesImmediately(t *testing.T) { + peer, err := net.ListenUDP("udp", udp(net.ParseIP("127.0.0.1"), 0)) + if err != nil { + t.Fatalf("ListenUDP: %v", err) + } + defer peer.Close() + client := newUpdateTestClient(t, peer.LocalAddr().String()) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- client.Run(ctx) }() + defer func() { + cancel() + <-done + }() + + // nextPushedClientID returns the client ID in the next contact pushed to + // the peer, ignoring pings. + nextPushedClientID := func() uint16 { + t.Helper() + buf := make([]byte, packetDataSize) + if err := peer.SetReadDeadline(time.Now().Add(2 * time.Second)); err != nil { + t.Fatalf("SetReadDeadline: %v", err) + } + for { + n, _, err := peer.ReadFromUDP(buf) + if err != nil { + t.Fatalf("no contact push arrived: %v", err) + } + d := newDecoder(buf[:n]) + if kind, err := d.variant(); err != nil || kind != protocolPushMessage { + continue + } + if _, err := d.read(32); err != nil { + t.Fatalf("push sender: %v", err) + } + if count, err := d.u64(); err != nil || count != 1 { + t.Fatalf("push value count = %d: %v", count, err) + } + value, err := decodeCrdsValue(d) + if err != nil || value.ContactInfo == nil { + t.Fatalf("pushed value is not a contact: %v", err) + } + return value.ContactInfo.Version.Client + } + } + if got := nextPushedClientID(); got != versionClientMithril { + t.Fatalf("initial client id = %#x", got) + } + if err := client.SetClientID(15); err != nil { + t.Fatalf("SetClientID: %v", err) + } + // The push interval is an hour, so this push comes from the update. + if got := nextPushedClientID(); got != 15 { + t.Fatalf("pushed client id = %#x, want 15", got) + } +} From a4613d94ad1d09689e7718746f9121d3096ca65c Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 05:29:34 +0000 Subject: [PATCH 06/25] blockprod: admit standalone Harmonic transactions on actual usage --- pkg/blockprod/bank.go | 41 +++++++-- pkg/blockprod/bank_standalone_test.go | 126 ++++++++++++++++++++++++++ 2 files changed, 161 insertions(+), 6 deletions(-) create mode 100644 pkg/blockprod/bank_standalone_test.go diff --git a/pkg/blockprod/bank.go b/pkg/blockprod/bank.go index 8b38a93f..ffeb87cc 100644 --- a/pkg/blockprod/bank.go +++ b/pkg/blockprod/bank.go @@ -211,7 +211,7 @@ func (b *WorkingBank) Forge(wire []byte) (ForgeResult, costmodel.ExceedReason) { // ForgeTransaction executes and commits a parsed transaction. The caller must // keep it immutable: accepted transactions are retained for entry publication. func (b *WorkingBank) ForgeTransaction(tx *solana.Transaction, wireSize int) (ForgeResult, costmodel.ExceedReason) { - return b.forgeTransaction(tx, wireSize, nil, EntryReservation{}) + return b.forgeTransaction(tx, wireSize, nil, EntryReservation{}, tpuForge) } // ForgePreparedTransaction reuses static work from owned, immutable TPU bytes. @@ -227,10 +227,30 @@ func (b *WorkingBank) ForgeScheduledTransaction(tx *solana.Transaction, wireSize if b.slotCtx == nil || !b.preparer.Matches(prepared, tx, b.slotCtx.Features) { prepared = nil } - return b.forgeTransaction(tx, wireSize, prepared, res) + return b.forgeTransaction(tx, wireSize, prepared, res, tpuForge) } -func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, prepared *replay.PreparedTransaction, res EntryReservation) (ForgeResult, costmodel.ExceedReason) { +// ForgeStandaloneTransaction forges a Harmonic block transaction that is not +// revert protected. Unlike TPU forging, block limits are checked against the +// units execution consumed, so a transaction may request more than the slot +// has left as long as what it uses fits. +func (b *WorkingBank) ForgeStandaloneTransaction(tx *solana.Transaction, wireSize int) (ForgeResult, costmodel.ExceedReason) { + return b.forgeTransaction(tx, wireSize, nil, EntryReservation{}, standaloneForge) +} + +// forgePolicy selects how a forge path checks block limits. +type forgePolicy struct { + // admitOnActual checks block limits after execution against actual usage + // instead of before it against requested units. + admitOnActual bool +} + +var ( + tpuForge = forgePolicy{} + standaloneForge = forgePolicy{admitOnActual: true} +) + +func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, prepared *replay.PreparedTransaction, res EntryReservation, policy forgePolicy) (ForgeResult, costmodel.ExceedReason) { if tx == nil { b.RebateSchedule(res) return ForgeDroppedParse, costmodel.ExceedNone @@ -334,8 +354,10 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre return ForgeDroppedNoLeader, costmodel.ExceedNone } - if reason := b.costs.WouldExceed(cost); reason != costmodel.ExceedNone { - return ForgeDroppedCost, reason + if !policy.admitOnActual { + if reason := b.costs.WouldExceed(cost); reason != costmodel.ExceedNone { + return ForgeDroppedCost, reason + } } if reason := b.admitEntryBytesLocked(serializedSize); reason != costmodel.ExceedNone { return ForgeDroppedCost, reason @@ -351,6 +373,14 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre SkipTimingMetrics: true, Arena: b.borrowedAccounts, }, prepared) + // Execution only touched transaction-local accounts; nothing is published + // until the apply below, so a transaction refused here leaves no trace. + execCU, loadedCost := actualExecutionUsage(output) + if policy.admitOnActual { + if reason := b.costs.WouldExceed(cost.WithActualUsage(execCU, loadedCost)); reason != costmodel.ExceedNone { + return ForgeDroppedCost, reason + } + } if output.ProcessingResult.TransactionError != nil { feeInfo, err := replay.ApplyFailedTransaction(b.slotCtx, execTx, output) if err != nil { @@ -373,7 +403,6 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre b.numSigs += uint64(tx.Message.Header.NumRequiredSignatures) b.costs.Record(cost) - execCU, loadedCost := actualExecutionUsage(output) b.costs.Rebate(cost, execCU, loadedCost) if flushed, batchBytes, didFlush := b.entries.appendSerialized(*tx, wireSize, serializedSize); didFlush { b.entryHash = b.entries.CurrentEntryHash() diff --git a/pkg/blockprod/bank_standalone_test.go b/pkg/blockprod/bank_standalone_test.go new file mode 100644 index 00000000..27df63a6 --- /dev/null +++ b/pkg/blockprod/bank_standalone_test.go @@ -0,0 +1,126 @@ +package blockprod + +import ( + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/features" + "github.com/Overclock-Validator/mithril/pkg/replay" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/programs/system" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func requireExecutionError(t *testing.T, env *TestEnv, tx *solana.Transaction, want replay.TransactionErrorType) { + t.Helper() + preview := replay.LoadAndExecuteTransaction(replay.LoadAndExecuteTransactionInput{ + SlotCtx: env.SlotCtx, + Transaction: tx, + LeanResult: true, + }) + require.NotNil(t, preview.ProcessingResult.TransactionError) + require.Equal(t, want, preview.ProcessingResult.TransactionError.ErrorType) +} + +func requireNothingForged(t *testing.T, env *TestEnv, payerBefore uint64) { + t.Helper() + assert.Equal(t, payerBefore, lamportsOf(t, env, txfixture.PayerPubkey())) + assert.Empty(t, env.Bank.ForgedTransactions()) + assert.Empty(t, env.Bank.seenMessages) + assert.Zero(t, env.Bank.TxFeeAccumulator().TotalFees) + assert.Zero(t, env.Bank.CostTracker().BlockCost()) + assert.Zero(t, env.Bank.EntryBuilder().PendingCount()) +} + +func TestStandaloneDropsUnprocessableTransactions(t *testing.T) { + t.Run("unknown blockhash", func(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + tx, err := solana.NewTransaction( + []solana.Instruction{system.NewTransferInstruction(1, txfixture.PayerPubkey(), txfixture.DestPubkey()).Build()}, + solana.Hash{0x42}, + solana.TransactionPayer(txfixture.PayerPubkey()), + ) + require.NoError(t, err) + payerKey := txfixture.PayerPrivateKey() + _, err = tx.Sign(func(solana.PublicKey) *solana.PrivateKey { return &payerKey }) + require.NoError(t, err) + requireExecutionError(t, env, tx, replay.TransactionErrorBlockhashNotFound) + payerBefore := lamportsOf(t, env, txfixture.PayerPubkey()) + + result, _ := env.Bank.ForgeStandaloneTransaction(tx, 0) + require.Equal(t, ForgeDroppedExecution, result) + requireNothingForged(t, env, payerBefore) + }) + t.Run("fee payer no-op", func(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + env.SlotCtx.Features.EnableFeature(features.RelaxFeePayerConstraint, 0) + bank := NewWorkingBank(BankConfig{SlotCtx: env.SlotCtx, Slot: env.SlotCtx.Slot, + TransactionStatuses: replay.NewTransactionStatusCache().View()}) + env.Bank = bank + setPayerLamports(t, env, 1) + tx := mustSignedTransfer(t, 1) + + result, _ := bank.ForgeStandaloneTransaction(tx, 0) + require.Equal(t, ForgeDroppedExecution, result) + requireNothingForged(t, env, 1) + }) +} + +// standaloneTransferUsage forges a transfer on an unlimited bank and returns +// the block cost it records, which is its actual usage. +func standaloneTransferUsage(t *testing.T) uint64 { + t.Helper() + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + result, _ := env.Bank.ForgeStandaloneTransaction(mustSignedTransfer(t, 1), 0) + require.Equal(t, ForgeAccepted, result) + return env.Bank.CostTracker().BlockCost() +} + +func TestStandaloneAdmitsOnActualUsage(t *testing.T) { + actual := standaloneTransferUsage(t) + requested, err := costmodel.EstimateTransactionCost(mustSignedTransfer(t, 1), features.NewFeaturesDefault()) + require.NoError(t, err) + require.Greater(t, requested.Sum(), 10*actual, "the transfer requests far more than it uses") + + limits := costmodel.DefaultLimits() + limits.BlockCost = actual + + tpu := NewTestEnv(TestEnvConfig{Limits: limits}) + defer tpu.Close() + payerBefore := lamportsOf(t, tpu, txfixture.PayerPubkey()) + result, reason := tpu.Bank.ForgeTransaction(mustSignedTransfer(t, 1), 0) + require.Equal(t, ForgeDroppedCost, result, "TPU forging admits on requested units") + require.Equal(t, costmodel.ExceedBlockCost, reason) + requireNothingForged(t, tpu, payerBefore) + + env := NewTestEnv(TestEnvConfig{Limits: limits}) + defer env.Close() + result, reason = env.Bank.ForgeStandaloneTransaction(mustSignedTransfer(t, 1), 0) + require.Equal(t, ForgeAccepted, result, "actual usage exactly at the limit fits") + require.Equal(t, costmodel.ExceedNone, reason) + assert.Equal(t, actual, env.Bank.CostTracker().BlockCost()) + assert.Len(t, env.Bank.ForgedTransactions(), 1) +} + +func TestStandaloneRefusesWhenActualUsageDoesNotFit(t *testing.T) { + actual := standaloneTransferUsage(t) + limits := costmodel.DefaultLimits() + limits.BlockCost = actual - 1 + env := NewTestEnv(TestEnvConfig{Limits: limits}) + defer env.Close() + payerBefore := lamportsOf(t, env, txfixture.PayerPubkey()) + destBefore := lamportsOf(t, env, txfixture.DestPubkey()) + tx := mustSignedTransfer(t, 1) + + result, reason := env.Bank.ForgeStandaloneTransaction(tx, 0) + require.Equal(t, ForgeDroppedCost, result) + require.Equal(t, costmodel.ExceedBlockCost, reason) + requireNothingForged(t, env, payerBefore) + assert.Equal(t, destBefore, lamportsOf(t, env, txfixture.DestPubkey()), "an executed but refused transaction publishes nothing") + assert.Empty(t, env.SlotCtx.ModifiedAccts) +} From 5d741534ac033b9c36669eeef86993934849f2ea Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 05:46:54 +0000 Subject: [PATCH 07/25] tpu: add Submit for packets from outside the QUIC listener --- pkg/tpu/engine.go | 11 ++++ pkg/tpu/engine_test.go | 49 +++++++++++++++ pkg/tpu/packet/packet.go | 35 ++++++++++- pkg/tpu/packet/packet_test.go | 26 ++++++++ pkg/tpu/pipeline/pipeline.go | 51 ++++++++++++++- pkg/tpu/pipeline/submit_test.go | 107 ++++++++++++++++++++++++++++++++ 6 files changed, 277 insertions(+), 2 deletions(-) create mode 100644 pkg/tpu/packet/packet_test.go create mode 100644 pkg/tpu/pipeline/submit_test.go diff --git a/pkg/tpu/engine.go b/pkg/tpu/engine.go index b8ed2e17..111a475b 100644 --- a/pkg/tpu/engine.go +++ b/pkg/tpu/engine.go @@ -8,6 +8,7 @@ import ( "runtime" "time" + "github.com/Overclock-Validator/mithril/pkg/tpu/packet" "github.com/Overclock-Validator/mithril/pkg/tpu/pipeline" "github.com/Overclock-Validator/mithril/pkg/tpu/quicserver" "github.com/Overclock-Validator/mithril/pkg/tpu/sink" @@ -136,6 +137,16 @@ func (t *TPU) AdvertisedQUICAddr() (*net.UDPAddr, error) { return &net.UDPAddr{IP: ip, Port: local.Port}, nil } +// Submit copies a transaction from outside the QUIC listener, such as a +// relayer, into the ingress pipeline. It never blocks, and returns false if +// the packet was dropped because ingress is full or the TPU has stopped. +func (t *TPU) Submit(wire []byte, src packet.Source) bool { + if t == nil || t.pipeline == nil { + return false + } + return t.pipeline.Submit(packet.OwnedFrom(wire, src)) +} + func (t *TPU) Stats() Stats { return Stats{ QUIC: t.server.Stats, diff --git a/pkg/tpu/engine_test.go b/pkg/tpu/engine_test.go index 90eb7650..e61e46ca 100644 --- a/pkg/tpu/engine_test.go +++ b/pkg/tpu/engine_test.go @@ -6,6 +6,11 @@ import ( "crypto/rand" "testing" "time" + + "github.com/Overclock-Validator/mithril/pkg/tpu/packet" + "github.com/Overclock-Validator/mithril/pkg/tpu/pipeline" + "github.com/Overclock-Validator/mithril/pkg/tpu/sink" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" ) func TestTPUStartStop(t *testing.T) { @@ -37,3 +42,47 @@ func TestTPUStartStop(t *testing.T) { t.Fatalf("Stop: %v", err) } } + +func TestTPUSubmitFeedsPipelineUntilStopped(t *testing.T) { + _, identity, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatalf("GenerateKey: %v", err) + } + recv := &sink.Noop{} + svc, err := Start(context.Background(), Config{ + Name: "test-tpu", + ListenAddr: "127.0.0.1:0", + Identity: identity, + Pipeline: pipeline.Config{SigverifyWorkers: 1, Sink: recv}, + }) + if err != nil { + t.Fatalf("Start: %v", err) + } + + if !svc.Submit(txfixture.MustSignedTransferWire(0), packet.SourceRelayer) { + t.Fatalf("Submit refused a packet on a running TPU") + } + deadline := time.Now().Add(2 * time.Second) + for recv.Snapshot().InPackets == 0 { + if time.Now().After(deadline) { + t.Fatalf("submitted packet never reached the sink: %+v", svc.Stats().Pipeline) + } + time.Sleep(5 * time.Millisecond) + } + if got := svc.Stats().Pipeline.Submitted[packet.SourceRelayer].Enqueued; got != 1 { + t.Fatalf("relayer enqueued = %d", got) + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + if err := svc.Stop(ctx); err != nil { + t.Fatalf("Stop: %v", err) + } + if svc.Submit(txfixture.MustSignedTransferWire(1), packet.SourceRelayer) { + t.Fatalf("Submit accepted a packet after Stop") + } + var stopped *TPU + if stopped.Submit(txfixture.MustSignedTransferWire(2), packet.SourceRelayer) { + t.Fatalf("Submit on a nil TPU must refuse") + } +} diff --git a/pkg/tpu/packet/packet.go b/pkg/tpu/packet/packet.go index afdca0ac..363f9751 100644 --- a/pkg/tpu/packet/packet.go +++ b/pkg/tpu/packet/packet.go @@ -5,19 +5,48 @@ package packet // larger buffer is for SIMD-0385 v1 transactions. const DataSize = 4096 +// Source identifies where a packet entered the TPU. +type Source uint8 + +const ( + // SourceQUIC is this validator's own TPU QUIC listener. + SourceQUIC Source = iota + // SourceRelayer is a relayer forwarding packets to this validator. + SourceRelayer + // NumSources is the number of sources. + NumSources +) + +func (s Source) String() string { + switch s { + case SourceQUIC: + return "quic" + case SourceRelayer: + return "relayer" + default: + return "unknown" + } +} + // Packet is a received TPU transaction payload moving through the pipeline. type Packet struct { buf []byte len int pool *Pool slot uint32 + src Source } // Owned returns a packet that owns its wire bytes. func Owned(wire []byte) Packet { + return OwnedFrom(wire, SourceQUIC) +} + +// OwnedFrom returns a packet from src that owns its wire bytes. +func OwnedFrom(wire []byte, src Source) Packet { out := make([]byte, len(wire)) copy(out, wire) - return Packet{buf: out, len: len(wire)} + return Packet{buf: out, len: len(wire), src: src} } // FromPool returns a packet backed by a pooled buffer. Call Release when done. @@ -36,6 +65,10 @@ func (p Packet) Len() int { return p.len } +func (p Packet) Source() Source { + return p.src +} + func (p Packet) Release() { if p.pool != nil { p.pool.Release(p.slot) diff --git a/pkg/tpu/packet/packet_test.go b/pkg/tpu/packet/packet_test.go new file mode 100644 index 00000000..52415404 --- /dev/null +++ b/pkg/tpu/packet/packet_test.go @@ -0,0 +1,26 @@ +package packet + +import "testing" + +func TestOwnedFromCopiesWireAndRecordsSource(t *testing.T) { + wire := []byte{1, 2, 3} + pkt := OwnedFrom(wire, SourceRelayer) + wire[0] = 9 + if got := pkt.Data(); len(got) != 3 || got[0] != 1 { + t.Fatalf("data = %v, want an independent copy", got) + } + if pkt.Source() != SourceRelayer || pkt.Source().String() != "relayer" { + t.Fatalf("source = %v", pkt.Source()) + } + if Owned(wire).Source() != SourceQUIC { + t.Fatalf("Owned must default to the QUIC source") + } + pool := NewPool(1) + buf, slot, ok := pool.Acquire() + if !ok { + t.Fatalf("Acquire failed") + } + if FromPool(pool, buf, 0, slot).Source() != SourceQUIC { + t.Fatalf("pooled packets come from the QUIC listener") + } +} diff --git a/pkg/tpu/pipeline/pipeline.go b/pkg/tpu/pipeline/pipeline.go index 44e7220f..cfddafc8 100644 --- a/pkg/tpu/pipeline/pipeline.go +++ b/pkg/tpu/pipeline/pipeline.go @@ -50,12 +50,22 @@ func (c Config) normalized() Config { // Stats aggregates per-stage counters. type Stats struct { - Ingress IngressStats + Ingress IngressStats + // Submitted counts Submit calls by packet source. Accepted packets are + // also counted in Ingress. + Submitted [packet.NumSources]SubmitStats Dedup dedup.Stats Sigverify SigverifyStats Sink sink.Stats } +// SubmitStats counts packets handed to Submit from one source. +type SubmitStats struct { + Enqueued uint64 + DroppedFull uint64 + DroppedClosed uint64 +} + type IngressStats struct { EnqueuedPackets uint64 EnqueuedBytes uint64 @@ -77,6 +87,10 @@ type Pipeline struct { cfg Config stopOnce sync.Once wg sync.WaitGroup + + // closeMu orders Submit against Stop closing ingress. + closeMu sync.RWMutex + closed bool } // Start launches the pipeline stages and returns the ingress send channel for QUIC. @@ -127,6 +141,14 @@ func (p *Pipeline) Stats() Stats { DroppedSigverify: atomic.LoadUint64(&p.stats.Sigverify.DroppedSigverify), }, } + for src := range stats.Submitted { + counts := &p.stats.Submitted[src] + stats.Submitted[src] = SubmitStats{ + Enqueued: atomic.LoadUint64(&counts.Enqueued), + DroppedFull: atomic.LoadUint64(&counts.DroppedFull), + DroppedClosed: atomic.LoadUint64(&counts.DroppedClosed), + } + } if noop, ok := p.cfg.Sink.(*sink.Noop); ok { stats.Sink = noop.Snapshot() } @@ -137,9 +159,36 @@ func (p *Pipeline) IngressStats() *IngressStats { return &p.stats.Ingress } +// Submit hands a packet from outside the QUIC listener to ingress without +// blocking, and takes ownership of it. It returns false, releasing the +// packet, if ingress is full or the pipeline has stopped. +func (p *Pipeline) Submit(pkt packet.Packet) bool { + src := pkt.Source() + if src >= packet.NumSources { + src = packet.SourceQUIC + } + counts := &p.stats.Submitted[src] + p.closeMu.RLock() + defer p.closeMu.RUnlock() + if p.closed { + atomic.AddUint64(&counts.DroppedClosed, 1) + pkt.Release() + return false + } + if !TryEnqueue(p.ingress, pkt, &p.stats.Ingress) { + atomic.AddUint64(&counts.DroppedFull, 1) + return false + } + atomic.AddUint64(&counts.Enqueued, 1) + return true +} + func (p *Pipeline) Stop() { p.stopOnce.Do(func() { + p.closeMu.Lock() + p.closed = true close(p.ingress) + p.closeMu.Unlock() p.wg.Wait() }) } diff --git a/pkg/tpu/pipeline/submit_test.go b/pkg/tpu/pipeline/submit_test.go new file mode 100644 index 00000000..59d422ae --- /dev/null +++ b/pkg/tpu/pipeline/submit_test.go @@ -0,0 +1,107 @@ +package pipeline + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/tpu/packet" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/stretchr/testify/require" +) + +// sourceSink records the source of every packet that reaches the sink. A +// non-nil hold blocks Receive until it is closed. +type sourceSink struct { + mu sync.Mutex + sources []packet.Source + hold chan struct{} +} + +func (s *sourceSink) Receive(pkt packet.Packet) { + if s.hold != nil { + <-s.hold + } + s.mu.Lock() + s.sources = append(s.sources, pkt.Source()) + s.mu.Unlock() + pkt.Release() +} + +func (s *sourceSink) received() []packet.Source { + s.mu.Lock() + defer s.mu.Unlock() + return append([]packet.Source(nil), s.sources...) +} + +func TestSubmitDeliversPacketWithItsSource(t *testing.T) { + recv := &sourceSink{} + p, _ := Start(context.Background(), Config{SigverifyWorkers: 1, Sink: recv}) + defer p.Stop() + + require.True(t, p.Submit(packet.OwnedFrom(mustValidTestWire(t), packet.SourceRelayer))) + require.Eventually(t, func() bool { return len(recv.received()) == 1 }, 2*time.Second, 5*time.Millisecond) + require.Equal(t, []packet.Source{packet.SourceRelayer}, recv.received(), "the source survives dedup and sigverify") + + stats := p.Stats() + require.Equal(t, SubmitStats{Enqueued: 1}, stats.Submitted[packet.SourceRelayer]) + require.Equal(t, SubmitStats{}, stats.Submitted[packet.SourceQUIC]) + require.Equal(t, uint64(1), stats.Ingress.EnqueuedPackets) +} + +func TestSubmitAfterStopReleasesAndReportsDrop(t *testing.T) { + p, _ := Start(context.Background(), Config{SigverifyWorkers: 1}) + p.Stop() + + pool := packet.NewPool(1) + buf, slot, ok := pool.Acquire() + require.True(t, ok) + copy(buf, mustValidTestWire(t)) + require.False(t, p.Submit(packet.FromPool(pool, buf, 1, slot))) + _, _, ok = pool.Acquire() + require.True(t, ok, "a dropped packet is released") + require.Equal(t, uint64(1), p.Stats().Submitted[packet.SourceQUIC].DroppedClosed) +} + +func TestSubmitConcurrentWithStopNeverPanics(t *testing.T) { + wire := mustValidTestWire(t) + for range 20 { + p, _ := Start(context.Background(), Config{SigverifyWorkers: 1}) + var wg sync.WaitGroup + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + for range 200 { + p.Submit(packet.OwnedFrom(wire, packet.SourceRelayer)) + } + }() + } + p.Stop() + wg.Wait() + stats := p.Stats().Submitted[packet.SourceRelayer] + require.Equal(t, uint64(4*200), stats.Enqueued+stats.DroppedFull+stats.DroppedClosed) + } +} + +func TestSubmitDropsWhenIngressIsFull(t *testing.T) { + recv := &sourceSink{hold: make(chan struct{})} + p, _ := Start(context.Background(), Config{SigverifyWorkers: 1, IngressCap: 1, DedupOutCap: 1, VerifiedCap: 1, Sink: recv}) + var dropped bool + // Distinct transactions pass dedup and back up behind the held sink until + // ingress itself is full. + for seq := range uint64(64) { + if !p.Submit(packet.OwnedFrom(txfixture.MustSignedTransferWire(seq), packet.SourceRelayer)) { + dropped = true + break + } + time.Sleep(time.Millisecond) + } + close(recv.hold) + p.Stop() + require.True(t, dropped, "a full ingress must refuse without blocking") + stats := p.Stats() + require.NotZero(t, stats.Submitted[packet.SourceRelayer].DroppedFull) + require.NotZero(t, stats.Ingress.DroppedFull) +} From 5502eed5a0a7ab8947d95879ef252af0674fe5aa Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 05:47:01 +0000 Subject: [PATCH 08/25] version: export Commit and Semver --- cmd/mithril/node/node.go | 18 +--------------- pkg/mlog/mlog.go | 2 +- pkg/version/version.go | 35 +++++++++++++++++++++++++++++++ pkg/version/version_test.go | 41 +++++++++++++++++++++++++++++++++++++ 4 files changed, 78 insertions(+), 18 deletions(-) create mode 100644 pkg/version/version_test.go diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index 574e3041..f68eb2d0 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -3125,23 +3125,7 @@ func getVersion() string { // getCommit returns the git commit hash, preferring ldflags but falling back to // runtime/debug.BuildInfo for dev builds. func getCommit() string { - // If set via ldflags (release builds), use that - if version.GitCommit != "" && version.GitCommit != "unknown" { - return version.GitCommit - } - // Fallback to runtime/debug for dev builds (go build without ldflags) - if info, ok := debug.ReadBuildInfo(); ok { - for _, setting := range info.Settings { - if setting.Key == "vcs.revision" { - // Return short hash (8 chars) for consistency - if len(setting.Value) > 8 { - return setting.Value[:8] - } - return setting.Value - } - } - } - return "unknown" + return version.Commit() } // getBranch returns the git branch name, preferring ldflags but falling back to diff --git a/pkg/mlog/mlog.go b/pkg/mlog/mlog.go index 92bfad21..7bdb0305 100644 --- a/pkg/mlog/mlog.go +++ b/pkg/mlog/mlog.go @@ -117,7 +117,7 @@ func Initialize(cfg LogConfig, runID string) error { if len(shortRunID) > 8 { shortRunID = shortRunID[:8] } - shortCommit := version.GitCommit + shortCommit := version.Commit() if len(shortCommit) > 7 { shortCommit = shortCommit[:7] } diff --git a/pkg/version/version.go b/pkg/version/version.go index 18120c72..565324f3 100644 --- a/pkg/version/version.go +++ b/pkg/version/version.go @@ -1,5 +1,10 @@ package version +import ( + "runtime/debug" + "strings" +) + // These variables are set at build time via ldflags: // go build -ldflags "-X github.com/Overclock-Validator/mithril/pkg/version.Version=v1.0.0 // -X github.com/Overclock-Validator/mithril/pkg/version.GitCommit=abc123 @@ -17,3 +22,33 @@ var ( // BuildDate is the build timestamp BuildDate = "unknown" ) + +// Commit returns the git commit the binary was built from: GitCommit when set +// at build time, otherwise the module's VCS revision shortened to 8 +// characters, otherwise "unknown". +func Commit() string { + if GitCommit != "" && GitCommit != "unknown" { + return GitCommit + } + if info, ok := debug.ReadBuildInfo(); ok { + for _, setting := range info.Settings { + if setting.Key == "vcs.revision" && setting.Value != "" { + if len(setting.Value) > 8 { + return setting.Value[:8] + } + return setting.Value + } + } + } + return "unknown" +} + +// Semver returns Version as a bare semantic version, without a leading "v". +// A build whose Version is not a release, such as "dev", reports "0.0.0-dev". +func Semver() string { + v := strings.TrimPrefix(Version, "v") + if v == "" || v[0] < '0' || v[0] > '9' { + return "0.0.0-dev" + } + return v +} diff --git a/pkg/version/version_test.go b/pkg/version/version_test.go new file mode 100644 index 00000000..445958f4 --- /dev/null +++ b/pkg/version/version_test.go @@ -0,0 +1,41 @@ +package version + +import "testing" + +func TestCommitPrefersBuildTimeValue(t *testing.T) { + saved := GitCommit + t.Cleanup(func() { GitCommit = saved }) + + GitCommit = "abc1234" + if got := Commit(); got != "abc1234" { + t.Fatalf("Commit() = %q, want the build-time value", got) + } + for _, unset := range []string{"", "unknown"} { + GitCommit = unset + got := Commit() + if got == "" || (len(got) > 8 && got != "unknown") { + t.Fatalf("Commit() with GitCommit=%q = %q, want a VCS revision of at most 8 characters or \"unknown\"", unset, got) + } + } +} + +func TestSemver(t *testing.T) { + saved := Version + t.Cleanup(func() { Version = saved }) + + for version, want := range map[string]string{ + "v1.2.3": "1.2.3", + "1.2.3": "1.2.3", + "v2.0.0-rc.1": "2.0.0-rc.1", + "dev": "0.0.0-dev", + "": "0.0.0-dev", + "unknown": "0.0.0-dev", + "v": "0.0.0-dev", + "v10.20.30+ab": "10.20.30+ab", + } { + Version = version + if got := Semver(); got != want { + t.Errorf("Semver() with Version=%q = %q, want %q", version, got, want) + } + } +} From 105b29eeaaaf98ce8802396a191e8fb84b7f2cb2 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:00:41 +0000 Subject: [PATCH 09/25] gossip: add MithrilHarmonic client ID 15 --- pkg/gossip/contact_info.go | 8 ++++++++ pkg/gossip/contact_update_test.go | 12 ++++++------ 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/pkg/gossip/contact_info.go b/pkg/gossip/contact_info.go index 3d87f005..31b8fb64 100644 --- a/pkg/gossip/contact_info.go +++ b/pkg/gossip/contact_info.go @@ -44,6 +44,14 @@ const ( versionClientMithril = 0x4d49 ) +const ( + // ClientIDMithril is the client ID mithril advertises by default. + ClientIDMithril uint16 = versionClientMithril + // ClientIDMithrilHarmonic is the client ID mithril advertises while it is + // connected to a Harmonic block engine (validator client-ID registry: 15). + ClientIDMithrilHarmonic uint16 = 15 +) + type Pubkey [32]byte type Signature [64]byte type Hash [32]byte diff --git a/pkg/gossip/contact_update_test.go b/pkg/gossip/contact_update_test.go index 4153aeb1..35d9c07f 100644 --- a/pkg/gossip/contact_update_test.go +++ b/pkg/gossip/contact_update_test.go @@ -222,7 +222,7 @@ func TestClientUpdateContactPublishesACopy(t *testing.T) { before := publishedContact(client) beforeWire := encodeContact(t, before) - if err := client.SetClientID(15); err != nil { + if err := client.SetClientID(ClientIDMithrilHarmonic); err != nil { t.Fatalf("SetClientID: %v", err) } after := publishedContact(client) @@ -232,7 +232,7 @@ func TestClientUpdateContactPublishesACopy(t *testing.T) { if !bytes.Equal(beforeWire, encodeContact(t, before)) { t.Fatalf("the previously published contact changed") } - if after.Version.Client != 15 || before.Version.Client != versionClientMithril { + if after.Version.Client != ClientIDMithrilHarmonic || before.Version.Client != versionClientMithril { t.Fatalf("client id after=%#x before=%#x", after.Version.Client, before.Version.Client) } if after.Outset != before.Outset { @@ -269,7 +269,7 @@ func TestClientAppliesContactUpdatesQueuedBeforeRun(t *testing.T) { if err := client.SetTPUSockets(TPUSockets{QUIC: udp(relayerIP, 11228), ForwardsQUIC: udp(relayerIP, 11229)}); err != nil { t.Fatalf("SetTPUSockets: %v", err) } - if err := client.SetClientID(15); err != nil { + if err := client.SetClientID(ClientIDMithrilHarmonic); err != nil { t.Fatalf("SetClientID: %v", err) } if err := client.initializeContact(udp(net.ParseIP("127.0.0.1"), 65400)); err != nil { @@ -279,7 +279,7 @@ func TestClientAppliesContactUpdatesQueuedBeforeRun(t *testing.T) { if got := socketAddr(contact, socketTagTPUQUIC); got.String() != "198.51.100.7:11228" { t.Fatalf("TPU QUIC = %v; queued updates must apply in order", got) } - if contact.Version.Client != 15 { + if contact.Version.Client != ClientIDMithrilHarmonic { t.Fatalf("client id = %#x", contact.Version.Client) } requireAcceptedByPeers(t, contact) @@ -361,11 +361,11 @@ func TestClientPushesContactUpdatesImmediately(t *testing.T) { if got := nextPushedClientID(); got != versionClientMithril { t.Fatalf("initial client id = %#x", got) } - if err := client.SetClientID(15); err != nil { + if err := client.SetClientID(ClientIDMithrilHarmonic); err != nil { t.Fatalf("SetClientID: %v", err) } // The push interval is an hour, so this push comes from the update. - if got := nextPushedClientID(); got != 15 { + if got := nextPushedClientID(); got != ClientIDMithrilHarmonic { t.Fatalf("pushed client id = %#x, want 15", got) } } From 16305bb7a3e8d1dea3a0ff952171591db1c11022 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:02:04 +0000 Subject: [PATCH 10/25] blockprod: forge revert-protected bundles all or nothing --- pkg/blockprod/bank.go | 114 ++++++----- pkg/blockprod/bank_bundle.go | 174 +++++++++++++++++ pkg/blockprod/bank_bundle_test.go | 302 ++++++++++++++++++++++++++++++ pkg/blockprod/entry.go | 28 +++ pkg/replay/commit.go | 32 +++- pkg/replay/commit_test.go | 26 +++ pkg/sealevel/scratch_slot_ctx.go | 46 +++++ 7 files changed, 677 insertions(+), 45 deletions(-) create mode 100644 pkg/blockprod/bank_bundle.go create mode 100644 pkg/blockprod/bank_bundle_test.go create mode 100644 pkg/sealevel/scratch_slot_ctx.go diff --git a/pkg/blockprod/bank.go b/pkg/blockprod/bank.go index ffeb87cc..ff97dec0 100644 --- a/pkg/blockprod/bank.go +++ b/pkg/blockprod/bank.go @@ -250,28 +250,44 @@ var ( standaloneForge = forgePolicy{admitOnActual: true} ) -func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, prepared *replay.PreparedTransaction, res EntryReservation, policy forgePolicy) (ForgeResult, costmodel.ExceedReason) { +// forgeCandidate is a transaction's forge work that needs no bank lock. +type forgeCandidate struct { + // execTx is what executes: tx, or for a v0 transaction with address-table + // lookups, a resolved copy. tx itself stays unresolved for entry encoding. + execTx *solana.Transaction + // prepared is the static preparation, if any, still valid for execTx. + prepared *replay.PreparedTransaction + serializedSize int + messageHash [32]byte + ancestorAlreadyProcessed bool + ancestorStatusErr error + cost costmodel.TransactionCost +} + +// prepareForge does the lock-free part of forging tx: full-wire validation, +// the message hash, the account lock rules, the ancestor status lookup, +// address-table lookups and the cost estimate. Unless it returns +// ForgeAccepted, tx cannot be forged. +func (b *WorkingBank) prepareForge(tx *solana.Transaction, wireSize int, prepared *replay.PreparedTransaction) (forgeCandidate, ForgeResult) { + c := forgeCandidate{execTx: tx, prepared: prepared} if tx == nil { - b.RebateSchedule(res) - return ForgeDroppedParse, costmodel.ExceedNone + return c, ForgeDroppedParse } // Validate the full wire before execution can charge fees or publish // account changes. Entry append then reuses this measured size and has no // fallible serialization step after commit, including on the prepared path. - serializedSize, err := serializedTransactionSize(tx) + var err error + c.serializedSize, err = serializedTransactionSize(tx) if err != nil { - b.RebateSchedule(res) - return ForgeDroppedParse, costmodel.ExceedNone + return c, ForgeDroppedParse } - var messageHash [32]byte if prepared != nil { - messageHash = prepared.MessageHash() + c.messageHash = prepared.MessageHash() } else { - messageHash, err = replay.TransactionMessageHash(tx) + c.messageHash, err = replay.TransactionMessageHash(tx) } if err != nil { - b.RebateSchedule(res) - return ForgeDroppedParse, costmodel.ExceedNone + return c, ForgeDroppedParse } // A transaction that repeats an account or exceeds the account lock limit // can never be processed, and a block containing one is invalid. The check @@ -281,58 +297,76 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre lockFeats = b.slotCtx.Features } if txErr := replay.ValidateAccountLocks(tx, lockFeats); txErr != nil { - b.RebateSchedule(res) - return ForgeDroppedAccountLocks, costmodel.ExceedNone + return c, ForgeDroppedAccountLocks } - ancestorAlreadyProcessed := false - var ancestorStatusErr error if b.ancestorStatuses == nil { - ancestorStatusErr = &replay.IncompleteTransactionStatusCoverageError{} + c.ancestorStatusErr = &replay.IncompleteTransactionStatusCoverageError{} } else { // The view is immutable and internally synchronizes its lazy per-blockhash // index. Do that potentially large one-time lookup outside the bank lock. - ancestorAlreadyProcessed, ancestorStatusErr = b.ancestorStatuses.ContainsMessage(tx.Message.RecentBlockhash, messageHash) + c.ancestorAlreadyProcessed, c.ancestorStatusErr = b.ancestorStatuses.ContainsMessage(tx.Message.RecentBlockhash, c.messageHash) } - // For a v0 transaction with address-table lookups, execTx is a resolved copy, - // while tx stays unresolved for entry encoding. - execTx := tx - var cost costmodel.TransactionCost - if !ancestorAlreadyProcessed && ancestorStatusErr == nil { + if !c.ancestorAlreadyProcessed && c.ancestorStatusErr == nil { if replay.HasAddressTableLookups(tx) { // Tables are read from the parent state, which this slot does not // change, so resolve outside the bank lock. resolved, txErr := replay.ResolveAddressTableLookups(b.slotCtx, tx) if txErr != nil { - b.RebateSchedule(res) - return ForgeDroppedLookup, costmodel.ExceedNone + return c, ForgeDroppedLookup } // A loaded address can repeat a static key or another loaded // address, or take the account count past the lock limit. if txErr := replay.ValidateAccountLocks(resolved, lockFeats); txErr != nil { - b.RebateSchedule(res) - return ForgeDroppedAccountLocks, costmodel.ExceedNone + return c, ForgeDroppedAccountLocks } - execTx = resolved + c.execTx = resolved // Static preparation, if any, was done on the unresolved message. - prepared = nil + c.prepared = nil } feats := b.slotCtx.Features if feats == nil { f := features.NewFeaturesDefault() feats = f } - if prepared != nil { - cost = prepared.Cost() + if c.prepared != nil { + c.cost = c.prepared.Cost() } else { - cost, err = costmodel.EstimateTransactionCost(execTx, feats) + c.cost, err = costmodel.EstimateTransactionCost(c.execTx, feats) } if err != nil { - b.RebateSchedule(res) - return ForgeDroppedParse, costmodel.ExceedNone + return c, ForgeDroppedParse } - cost.WireSize = wireSize + c.cost.WireSize = wireSize } + return c, ForgeAccepted +} + +// statusLocked rejects a candidate this bank or its ancestors already +// processed. +func (b *WorkingBank) statusLocked(c forgeCandidate) ForgeResult { + if _, alreadyProcessed := b.seenMessages[c.messageHash]; alreadyProcessed { + return ForgeDroppedAlreadyProcessed + } + if c.ancestorAlreadyProcessed { + return ForgeDroppedAlreadyProcessed + } + if c.ancestorStatusErr != nil { + // A live leader is never constructed with incomplete coverage. Preserve a + // fail-closed result for isolated callers that bypass LeaderLoop. + return ForgeDroppedNoLeader + } + return ForgeAccepted +} + +func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, prepared *replay.PreparedTransaction, res EntryReservation, policy forgePolicy) (ForgeResult, costmodel.ExceedReason) { + c, result := b.prepareForge(tx, wireSize, prepared) + if result != ForgeAccepted { + b.RebateSchedule(res) + return result, costmodel.ExceedNone + } + cost, serializedSize, messageHash := c.cost, c.serializedSize, c.messageHash + execTx, prepared := c.execTx, c.prepared b.mu.Lock() defer b.mu.Unlock() @@ -342,16 +376,8 @@ func (b *WorkingBank) forgeTransaction(tx *solana.Transaction, wireSize int, pre if !b.accepting { return ForgeDroppedNoLeader, costmodel.ExceedNone } - if _, alreadyProcessed := b.seenMessages[messageHash]; alreadyProcessed { - return ForgeDroppedAlreadyProcessed, costmodel.ExceedNone - } - if ancestorAlreadyProcessed { - return ForgeDroppedAlreadyProcessed, costmodel.ExceedNone - } - if ancestorStatusErr != nil { - // A live leader is never constructed with incomplete coverage. Preserve a - // fail-closed result for isolated callers that bypass LeaderLoop. - return ForgeDroppedNoLeader, costmodel.ExceedNone + if result := b.statusLocked(c); result != ForgeAccepted { + return result, costmodel.ExceedNone } if !policy.admitOnActual { diff --git a/pkg/blockprod/bank_bundle.go b/pkg/blockprod/bank_bundle.go new file mode 100644 index 00000000..062e3289 --- /dev/null +++ b/pkg/blockprod/bank_bundle.go @@ -0,0 +1,174 @@ +package blockprod + +import ( + "errors" + "fmt" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/replay" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" +) + +// BundleInput is a revert-protected bundle: its transactions land in order, +// in this bank, or none of them do. +type BundleInput struct { + Transactions []*solana.Transaction + // Reservation is entry-byte budget reserved by PrepareScheduleN, if any. + // It is consumed whatever the result. + Reservation EntryReservation + // AdmitOnActual checks block limits after execution against actual usage, + // as for Harmonic block bundles, instead of before it against requested + // units. + AdmitOnActual bool +} + +// BundleResult reports a bundle forge. Unless Result is ForgeAccepted, +// nothing from the bundle landed: no state change, fee, nonce advance, cost or +// entry. +type BundleResult struct { + Result ForgeResult + Reason costmodel.ExceedReason + // Index is the transaction that stopped the bundle, or -1. + Index int + // Err describes why that transaction failed, when it was executed. + Err error +} + +// ForgeBundle forges a bundle all or nothing. Transactions execute in order +// against a scratch view of the bank, each seeing the writes of the ones +// before it, and are only applied once all have succeeded. Any failure, +// including a fees-only failure, discards the whole bundle. +func (b *WorkingBank) ForgeBundle(in BundleInput) BundleResult { + txs := in.Transactions + if len(txs) == 0 { + b.RebateSchedule(in.Reservation) + return BundleResult{Result: ForgeDroppedParse, Index: -1} + } + candidates := make([]forgeCandidate, len(txs)) + seen := make(map[[32]byte]struct{}, len(txs)) + for idx, tx := range txs { + c, result := b.prepareForge(tx, 0, nil) + if result != ForgeAccepted { + b.RebateSchedule(in.Reservation) + return BundleResult{Result: result, Index: idx} + } + if _, dup := seen[c.messageHash]; dup { + b.RebateSchedule(in.Reservation) + return BundleResult{Result: ForgeDroppedAlreadyProcessed, Index: idx} + } + seen[c.messageHash] = struct{}{} + candidates[idx] = c + } + + b.mu.Lock() + defer b.mu.Unlock() + // The lock is held from here through append, so admission is exact. + b.entries.release(in.Reservation) + if !b.accepting { + return BundleResult{Result: ForgeDroppedNoLeader, Index: -1} + } + requested := make([]costmodel.TransactionCost, len(candidates)) + sizes := make([]int, len(candidates)) + for idx, c := range candidates { + if result := b.statusLocked(c); result != ForgeAccepted { + return BundleResult{Result: result, Index: idx} + } + requested[idx] = c.cost + sizes[idx] = c.serializedSize + } + if !in.AdmitOnActual { + if reason := b.costs.WouldExceedAll(requested); reason != costmodel.ExceedNone { + return BundleResult{Result: ForgeDroppedCost, Reason: reason, Index: -1} + } + } + if b.entries.wouldOverflowBatch(sizes[0]) { + b.flushEntriesLocked() + } + if b.entries.wouldExceedSlotN(sizes) { + return BundleResult{Result: ForgeDroppedCost, Reason: costmodel.ExceedBatchBytes, Index: -1} + } + + scratch := sealevel.NewScratchSlotCtx(b.slotCtx) + committed := false + defer func() { + if !committed { + b.evictProgramCacheAdds(scratch) + } + }() + outputs := make([]replay.LoadAndExecuteTransactionOutput, len(txs)) + actual := make([]costmodel.TransactionCost, len(txs)) + for idx, c := range candidates { + if err := b.preparer.PayerCanFund(scratch, c.execTx, nil); err != nil { + return BundleResult{Result: ForgeDroppedExecution, Index: idx, Err: err} + } + // Every output must outlive the next execution, so none may use the + // bank's per-transaction arena. + output := b.preparer.LoadAndExecute(replay.LoadAndExecuteTransactionInput{ + SlotCtx: scratch, + Transaction: c.execTx, + LeanResult: true, + SkipTimingMetrics: true, + }, nil) + if txErr := output.ProcessingResult.TransactionError; txErr != nil { + return BundleResult{Result: ForgeDroppedExecution, Index: idx, Err: transactionErr(txErr)} + } + if output.ExecCtx == nil { + return BundleResult{Result: ForgeDroppedExecution, Index: idx, Err: errors.New("missing execution context")} + } + txAccounts := &output.ExecCtx.TransactionContext.Accounts + if err := accounts.SetTransactionAccounts(scratch.Accounts, txAccounts.Accounts, txAccounts.Touched); err != nil { + return BundleResult{Result: ForgeDroppedExecution, Index: idx, Err: err} + } + execCU, loadedCost := actualExecutionUsage(output) + actual[idx] = candidates[idx].cost.WithActualUsage(execCU, loadedCost) + outputs[idx] = output + } + if in.AdmitOnActual { + if reason := b.costs.WouldExceedAll(actual); reason != costmodel.ExceedNone { + return BundleResult{Result: ForgeDroppedCost, Reason: reason, Index: -1} + } + } + if err := replay.ApplySuccessfulTransactions(b.slotCtx, outputs); err != nil { + return BundleResult{Result: ForgeDroppedExecution, Index: -1, Err: err} + } + committed = true + + if b.seenMessages == nil { + b.seenMessages = make(map[[32]byte]struct{}) + } + for idx, tx := range txs { + b.seenMessages[candidates[idx].messageHash] = struct{}{} + if outputs[idx].FeeInfo != nil { + b.txFees.Add(outputs[idx].FeeInfo) + } + b.forgedTxs = append(b.forgedTxs, tx) + b.numSigs += uint64(tx.Message.Header.NumRequiredSignatures) + // Equivalent to recording the requested cost and rebating it. + b.costs.Record(actual[idx]) + if flushed, batchBytes, didFlush := b.entries.appendSerialized(*tx, sizes[idx], sizes[idx]); didFlush { + b.entryHash = b.entries.CurrentEntryHash() + b.sink.OnEntryBatch(flushed, batchBytes) + } + } + return BundleResult{Result: ForgeAccepted, Index: -1} +} + +// evictProgramCacheAdds drops programs a discarded scratch execution loaded +// into the shared program cache. Eviction only forces a reload from account +// data, so it is always safe. +func (b *WorkingBank) evictProgramCacheAdds(scratch *sealevel.SlotCtx) { + for _, key := range scratch.TakeProgramCacheAdds() { + if scratch.AccountsDb != nil { + scratch.AccountsDb.RemoveProgramFromCache(key) + } + } +} + +func transactionErr(txErr *replay.TransactionError) error { + if txErr.InstructionError != nil { + return fmt.Errorf("%s: %w", txErr.ErrorType, txErr.InstructionError) + } + return errors.New(txErr.ErrorType.String()) +} diff --git a/pkg/blockprod/bank_bundle_test.go b/pkg/blockprod/bank_bundle_test.go new file mode 100644 index 00000000..08f60237 --- /dev/null +++ b/pkg/blockprod/bank_bundle_test.go @@ -0,0 +1,302 @@ +package blockprod + +import ( + "math/rand/v2" + "sync" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/replay" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" + computebudget "github.com/gagliardetto/solana-go/programs/compute-budget" + "github.com/gagliardetto/solana-go/programs/system" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func signedBundleTx(t *testing.T, signers []solana.PrivateKey, instructions ...solana.Instruction) *solana.Transaction { + t.Helper() + tx, err := solana.NewTransaction(instructions, txfixture.TestBlockhash(), solana.TransactionPayer(txfixture.PayerPubkey())) + require.NoError(t, err) + keys := append([]solana.PrivateKey{txfixture.PayerPrivateKey()}, signers...) + _, err = tx.Sign(func(pub solana.PublicKey) *solana.PrivateKey { + for i := range keys { + if keys[i].PublicKey() == pub { + return &keys[i] + } + } + return nil + }) + require.NoError(t, err) + return tx +} + +func transferIx(lamports uint64, from, to solana.PublicKey) solana.Instruction { + return system.NewTransferInstruction(lamports, from, to).Build() +} + +// requireNothingLanded checks a refused bundle left no trace in the bank. +func requireNothingLanded(t *testing.T, env *TestEnv, payerBefore, destBefore uint64) { + t.Helper() + assert.Equal(t, payerBefore, lamportsOf(t, env, txfixture.PayerPubkey())) + assert.Equal(t, destBefore, lamportsOf(t, env, txfixture.DestPubkey())) + assert.Empty(t, env.Bank.ForgedTransactions()) + assert.Empty(t, env.Bank.seenMessages) + assert.Zero(t, env.Bank.TxFeeAccumulator().TotalFees) + assert.Zero(t, env.Bank.NumSignatures()) + assert.Zero(t, env.Bank.CostTracker().BlockCost()) + assert.Zero(t, env.Bank.EntryBuilder().PendingCount()) + assert.Empty(t, env.SlotCtx.ModifiedAccts) +} + +func TestForgeBundleLaterTransactionsSeeEarlierWrites(t *testing.T) { + funded, err := solana.NewRandomPrivateKey() + require.NoError(t, err) + fund := signedBundleTx(t, nil, transferIx(2_000_000, txfixture.PayerPubkey(), funded.PublicKey())) + spend := signedBundleTx(t, []solana.PrivateKey{funded}, transferIx(500_000, funded.PublicKey(), txfixture.DestPubkey())) + + alone := NewTestEnv(TestEnvConfig{}) + defer alone.Close() + destAlone := lamportsOf(t, alone, txfixture.DestPubkey()) + alone.Bank.ForgeTransaction(spend, 0) + require.Equal(t, destAlone, lamportsOf(t, alone, txfixture.DestPubkey()), "alone, the second transaction fails: it depends on the first") + + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + payerBefore := lamportsOf(t, env, txfixture.PayerPubkey()) + destBefore := lamportsOf(t, env, txfixture.DestPubkey()) + + got := env.Bank.ForgeBundle(BundleInput{Transactions: []*solana.Transaction{fund, spend}}) + require.Equal(t, ForgeAccepted, got.Result, "%+v", got) + assert.Equal(t, -1, got.Index) + assert.Equal(t, uint64(1_500_000), lamportsOf(t, env, funded.PublicKey())) + assert.Equal(t, destBefore+500_000, lamportsOf(t, env, txfixture.DestPubkey())) + // One signature on the first transaction, two on the second. + assert.Equal(t, payerBefore-2_000_000-15_000, lamportsOf(t, env, txfixture.PayerPubkey())) + assert.Equal(t, uint64(15_000), env.Bank.TxFeeAccumulator().TotalFees) + assert.Equal(t, uint64(3), env.Bank.NumSignatures()) + assert.Equal(t, []*solana.Transaction{fund, spend}, env.Bank.ForgedTransactions()) + assert.Equal(t, 2, env.Bank.EntryBuilder().PendingCount()) + assert.NotZero(t, env.Bank.CostTracker().BlockCost()) + + again := env.Bank.ForgeBundle(BundleInput{Transactions: []*solana.Transaction{fund}}) + assert.Equal(t, ForgeDroppedAlreadyProcessed, again.Result, "landed transactions are recorded") +} + +func TestForgeBundleFailureLandsNothing(t *testing.T) { + first := mustSignBankTestTransaction(t, transferIx(1, txfixture.PayerPubkey(), txfixture.DestPubkey())) + cases := map[string]*solana.Transaction{ + "instruction error": mustSignBankTestTransaction(t, transferIx(20_000_000_000, txfixture.PayerPubkey(), txfixture.DestPubkey())), + "fees-only load failure": mustSignBankTestTransaction(t, + computebudget.NewSetLoadedAccountsDataSizeLimitInstruction(1).Build(), + transferIx(2, txfixture.PayerPubkey(), txfixture.DestPubkey())), + "rent-state failure": mustSignBankTestTransaction(t, transferIx(1, txfixture.PayerPubkey(), solana.PublicKey{0xEE})), + } + for name, failing := range cases { + t.Run(name, func(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + payerBefore := lamportsOf(t, env, txfixture.PayerPubkey()) + destBefore := lamportsOf(t, env, txfixture.DestPubkey()) + + got := env.Bank.ForgeBundle(BundleInput{Transactions: []*solana.Transaction{first, failing}}) + require.Equal(t, ForgeDroppedExecution, got.Result) + assert.Equal(t, 1, got.Index) + assert.Error(t, got.Err) + requireNothingLanded(t, env, payerBefore, destBefore) + + // The first transaction was never recorded, so it can still land. + result, _ := env.Bank.ForgeTransaction(first, 0) + assert.Equal(t, ForgeAccepted, result) + }) + } +} + +func TestForgeBundleRefusesProcessedAndDuplicateTransactions(t *testing.T) { + a := mustSignBankTestTransaction(t, transferIx(1, txfixture.PayerPubkey(), txfixture.DestPubkey())) + b := mustSignBankTestTransaction(t, transferIx(2, txfixture.PayerPubkey(), txfixture.DestPubkey())) + + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + payerBefore := lamportsOf(t, env, txfixture.PayerPubkey()) + destBefore := lamportsOf(t, env, txfixture.DestPubkey()) + got := env.Bank.ForgeBundle(BundleInput{Transactions: []*solana.Transaction{a, b, a}}) + require.Equal(t, ForgeDroppedAlreadyProcessed, got.Result) + assert.Equal(t, 2, got.Index) + requireNothingLanded(t, env, payerBefore, destBefore) + + result, _ := env.Bank.ForgeTransaction(a, 0) + require.Equal(t, ForgeAccepted, result) + got = env.Bank.ForgeBundle(BundleInput{Transactions: []*solana.Transaction{b, a}}) + require.Equal(t, ForgeDroppedAlreadyProcessed, got.Result) + assert.Equal(t, 1, got.Index) + assert.Equal(t, []*solana.Transaction{a}, env.Bank.ForgedTransactions()) + + assert.Equal(t, ForgeDroppedParse, env.Bank.ForgeBundle(BundleInput{}).Result) + env.Bank.Close() + assert.Equal(t, ForgeDroppedNoLeader, env.Bank.ForgeBundle(BundleInput{Transactions: []*solana.Transaction{b}}).Result) +} + +func TestForgeBundleCostAdmission(t *testing.T) { + actual := standaloneTransferUsage(t) + bundle := func(t *testing.T) []*solana.Transaction { + return []*solana.Transaction{mustSignedTransfer(t, 1), mustSignedTransfer(t, 2)} + } + withBlockCost := func(blockCost uint64) *TestEnv { + limits := costmodel.DefaultLimits() + limits.BlockCost = blockCost + return NewTestEnv(TestEnvConfig{Limits: limits}) + } + + t.Run("requested units are checked before execution", func(t *testing.T) { + env := withBlockCost(2 * actual) + defer env.Close() + payerBefore := lamportsOf(t, env, txfixture.PayerPubkey()) + destBefore := lamportsOf(t, env, txfixture.DestPubkey()) + got := env.Bank.ForgeBundle(BundleInput{Transactions: bundle(t)}) + require.Equal(t, ForgeDroppedCost, got.Result) + assert.Equal(t, costmodel.ExceedBlockCost, got.Reason) + requireNothingLanded(t, env, payerBefore, destBefore) + }) + t.Run("actual usage that fits is admitted", func(t *testing.T) { + env := withBlockCost(2 * actual) + defer env.Close() + got := env.Bank.ForgeBundle(BundleInput{Transactions: bundle(t), AdmitOnActual: true}) + require.Equal(t, ForgeAccepted, got.Result, "%+v", got) + assert.Equal(t, 2*actual, env.Bank.CostTracker().BlockCost()) + }) + t.Run("actual usage that does not fit lands nothing", func(t *testing.T) { + env := withBlockCost(2*actual - 1) + defer env.Close() + payerBefore := lamportsOf(t, env, txfixture.PayerPubkey()) + destBefore := lamportsOf(t, env, txfixture.DestPubkey()) + got := env.Bank.ForgeBundle(BundleInput{Transactions: bundle(t), AdmitOnActual: true}) + require.Equal(t, ForgeDroppedCost, got.Result) + assert.Equal(t, costmodel.ExceedBlockCost, got.Reason) + requireNothingLanded(t, env, payerBefore, destBefore) + }) +} + +func TestForgeBundleEntryBytes(t *testing.T) { + txs := []*solana.Transaction{mustSignedTransfer(t, 1), mustSignedTransfer(t, 2), mustSignedTransfer(t, 3)} + size, err := serializedTransactionSize(txs[0]) + require.NoError(t, err) + + t.Run("a bundle past the slot budget is refused before execution", func(t *testing.T) { + limits := costmodel.DefaultLimits() + limits.MaxEntryBytes = uint64(entryBatchOverheadBytes + 3*size - 1) + env := NewTestEnv(TestEnvConfig{Limits: limits}) + defer env.Close() + payerBefore := lamportsOf(t, env, txfixture.PayerPubkey()) + destBefore := lamportsOf(t, env, txfixture.DestPubkey()) + got := env.Bank.ForgeBundle(BundleInput{Transactions: txs}) + require.Equal(t, ForgeDroppedCost, got.Result) + assert.Equal(t, costmodel.ExceedBatchBytes, got.Reason) + requireNothingLanded(t, env, payerBefore, destBefore) + + limits.MaxEntryBytes++ + exact := NewTestEnv(TestEnvConfig{Limits: limits}) + defer exact.Close() + assert.Equal(t, ForgeAccepted, exact.Bank.ForgeBundle(BundleInput{Transactions: txs}).Result, "exactly at the budget fits") + }) + + t.Run("batches split mid-bundle keep the bundle in order", func(t *testing.T) { + sink := &captureSink{} + limits := costmodel.DefaultLimits() + limits.MaxBatchBytes = uint64(entryBatchOverheadBytes + size + size/2) + env := NewTestEnv(TestEnvConfig{Limits: limits, Sink: sink}) + defer env.Close() + require.Equal(t, ForgeAccepted, env.Bank.ForgeBundle(BundleInput{Transactions: txs}).Result) + env.Bank.FlushEntries() + var entered []solana.Signature + for _, batch := range sink.batches { + for _, entry := range batch { + for _, tx := range entry.Txns { + entered = append(entered, tx.Signatures[0]) + } + } + } + require.Len(t, sink.batches, 3, "each transaction filled its own batch") + assert.Equal(t, []solana.Signature{txs[0].Signatures[0], txs[1].Signatures[0], txs[2].Signatures[0]}, entered) + }) +} + +func TestForgeBundleMatchesReplay(t *testing.T) { + funded, err := solana.NewRandomPrivateKey() + require.NoError(t, err) + txs := []*solana.Transaction{ + signedBundleTx(t, nil, transferIx(2_000_000, txfixture.PayerPubkey(), funded.PublicKey())), + signedBundleTx(t, []solana.PrivateKey{funded}, transferIx(500_000, funded.PublicKey(), txfixture.DestPubkey())), + mustSignedTransfer(t, 7), + } + leader := NewTestEnv(TestEnvConfig{}) + defer leader.Close() + require.Equal(t, ForgeAccepted, leader.Bank.ForgeBundle(BundleInput{Transactions: txs}).Result) + + validator := NewTestEnv(TestEnvConfig{}) + defer validator.Close() + var totalFee uint64 + for _, tx := range txs { + feeInfo, _, err := replay.ProcessTransaction(validator.SlotCtx, &sync.WaitGroup{}, tx, nil, nil, nil, false) + require.NoError(t, err) + require.NotNil(t, feeInfo) + totalFee += feeInfo.TotalFee + } + assert.Equal(t, totalFee, leader.Bank.TxFeeAccumulator().TotalFees) + for _, key := range []solana.PublicKey{txfixture.PayerPubkey(), txfixture.DestPubkey(), funded.PublicKey()} { + assert.Equal(t, lamportsOf(t, validator, key), lamportsOf(t, leader, key), "%s", key) + } +} + +func TestEntryBuilderWouldExceedSlotNMatchesAppends(t *testing.T) { + rng := rand.New(rand.NewPCG(3, 4)) + for trial := range 500 { + limits := costmodel.DefaultLimits() + limits.MaxBatchBytes = uint64(300 + rng.IntN(2000)) + prefix := make([]int, rng.IntN(4)) + sizes := make([]int, 1+rng.IntN(5)) + for i := range prefix { + prefix[i] = 100 + rng.IntN(600) + } + for i := range sizes { + sizes[i] = 100 + rng.IntN(600) + } + builder := NewEntryBuilder(limits, solana.Hash{}) + for _, size := range prefix { + builder.appendSerialized(solana.Transaction{}, size, size) + } + probe := *builder + probe.pendingTxns = append([]solana.Transaction(nil), builder.pendingTxns...) + for _, size := range sizes { + probe.appendSerialized(solana.Transaction{}, size, size) + } + after := probe.SlotBytes() + + builder.limits.MaxEntryBytes = uint64(after) + require.Falsef(t, builder.wouldExceedSlotN(sizes), "trial %d: exactly at the budget", trial) + builder.limits.MaxEntryBytes = uint64(after - 1) + require.Truef(t, builder.wouldExceedSlotN(sizes), "trial %d: one byte over", trial) + } +} + +func TestScratchSlotCtxLeavesBaseUnchanged(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + scratch := sealevel.NewScratchSlotCtx(env.SlotCtx) + assert.True(t, scratch.DeferVoteCachePublication) + assert.True(t, scratch.TrackProgramCacheAdds) + assert.Same(t, env.SlotCtx.Features, scratch.Features) + assert.Same(t, env.SlotCtx.BankSysvars(), scratch.BankSysvars()) + + payer, err := scratch.GetAccount(txfixture.PayerPubkey()) + require.NoError(t, err, "reads fall through to the bank") + before := payer.Lamports + payer.Lamports = 1 + require.NoError(t, scratch.SetAccount(txfixture.PayerPubkey(), payer)) + got, err := scratch.GetAccount(txfixture.PayerPubkey()) + require.NoError(t, err) + assert.Equal(t, uint64(1), got.Lamports) + assert.Equal(t, before, lamportsOf(t, env, txfixture.PayerPubkey()), "writes stay in the scratch view") +} diff --git a/pkg/blockprod/entry.go b/pkg/blockprod/entry.go index 9809e08c..811edd5e 100644 --- a/pkg/blockprod/entry.go +++ b/pkg/blockprod/entry.go @@ -81,6 +81,34 @@ func (b *EntryBuilder) wouldExceedSlot(nextWire int) bool { return b.SlotBytes()+b.admitBytes(nextWire) > maxEntry } +// wouldExceedSlotN reports whether appending transactions of these +// serialized sizes in order, before the bank lock is released, would pass the +// slot entry-byte budget. It follows the batching appendSerialized does. +func (b *EntryBuilder) wouldExceedSlotN(sizes []int) bool { + maxEntry := int(b.limits.MaxEntryBytes) + if maxEntry <= 0 { + return false + } + total := b.SlotBytes() + pending, pendingBytes := len(b.pendingTxns), b.pendingSerializedBytes + for _, size := range sizes { + switch { + case pending > 0 && entryBatchOverheadBytes+pendingBytes+size > int(b.limits.MaxBatchBytes): + // The pending batch is already counted; the transaction starts a new one. + total += entryBatchOverheadBytes + size + pending, pendingBytes = 1, size + case pending == 0: + total += entryBatchOverheadBytes + size + pending, pendingBytes = 1, size + default: + total += size + pending++ + pendingBytes += size + } + } + return total > maxEntry +} + func (b *EntryBuilder) admitBytes(nextWire int) int { if b.wouldOverflowBatch(nextWire) || len(b.pendingTxns) == 0 { return entryBatchOverheadBytes + nextWire diff --git a/pkg/replay/commit.go b/pkg/replay/commit.go index 63a5c9f1..2659065a 100644 --- a/pkg/replay/commit.go +++ b/pkg/replay/commit.go @@ -63,10 +63,40 @@ func applySuccessfulTransactionState(slotCtx *sealevel.SlotCtx, execCtx *sealeve // LoadAndExecuteTransaction result into slotCtx. Divergence checks are // skipped; callers must only pass outputs with no TransactionError. func ApplySuccessfulTransaction(slotCtx *sealevel.SlotCtx, output LoadAndExecuteTransactionOutput) error { + if err := checkSuccessfulTransaction(slotCtx, output); err != nil { + return err + } + return applySuccessfulTransactionState(slotCtx, output.ExecCtx, output.ExecutionResult) +} + +// ApplySuccessfulTransactions commits successful results in order. It checks +// every result before applying any, so either all of them land or none do. +func ApplySuccessfulTransactions(slotCtx *sealevel.SlotCtx, outputs []LoadAndExecuteTransactionOutput) error { + for idx, output := range outputs { + if err := checkSuccessfulTransaction(slotCtx, output); err != nil { + return fmt.Errorf("transaction %d: %w", idx, err) + } + } + for idx, output := range outputs { + // The checks above cover every applySuccessfulTransactionState error. + if err := applySuccessfulTransactionState(slotCtx, output.ExecCtx, output.ExecutionResult); err != nil { + panic(fmt.Sprintf("transaction %d failed to apply after its checks passed: %v", idx, err)) + } + } + return nil +} + +func checkSuccessfulTransaction(slotCtx *sealevel.SlotCtx, output LoadAndExecuteTransactionOutput) error { if output.ProcessingResult.TransactionError != nil { return fmt.Errorf("cannot apply failed transaction: %s", output.ProcessingResult.TransactionError.ErrorType.String()) } - return applySuccessfulTransactionState(slotCtx, output.ExecCtx, output.ExecutionResult) + if output.ExecCtx == nil { + return fmt.Errorf("missing execution context") + } + if output.ExecutionResult == nil && !accountsDeltaHashRemoved(slotCtx) { + return fmt.Errorf("missing execution result while accounts delta hash is enabled") + } + return nil } // includableFailure reports whether a failed transaction was processed, so a diff --git a/pkg/replay/commit_test.go b/pkg/replay/commit_test.go index 14c3dd4b..8187463a 100644 --- a/pkg/replay/commit_test.go +++ b/pkg/replay/commit_test.go @@ -861,3 +861,29 @@ func newCommitTestSlotCtx() (*sealevel.SlotCtx, func()) { sealevel.SysvarCache.Rent.Sysvar = prevRent } } + +func TestApplySuccessfulTransactionsChecksEveryResultFirst(t *testing.T) { + slotCtx, cleanup := newCommitTestSlotCtx() + defer cleanup() + tx, err := solana.TransactionFromBytes(txfixture.MustSignedTransferWire(0)) + require.NoError(t, err) + valid := LoadAndExecuteTransaction(LoadAndExecuteTransactionInput{SlotCtx: slotCtx, Transaction: tx, LeanResult: true}) + require.Nil(t, valid.ProcessingResult.TransactionError) + failed := LoadAndExecuteTransactionOutput{ProcessingResult: TransactionProcessingResult{ + TransactionError: &TransactionError{ErrorType: TransactionErrorInstructionError}, + }} + payerBefore, err := slotCtx.GetAccount(txfixture.PayerPubkey()) + require.NoError(t, err) + + err = ApplySuccessfulTransactions(slotCtx, []LoadAndExecuteTransactionOutput{valid, failed}) + require.ErrorContains(t, err, "transaction 1") + payer, err := slotCtx.GetAccount(txfixture.PayerPubkey()) + require.NoError(t, err) + assert.Equal(t, payerBefore.Lamports, payer.Lamports, "nothing is applied when any result fails its check") + assert.Empty(t, slotCtx.ModifiedAccts) + + require.NoError(t, ApplySuccessfulTransactions(slotCtx, []LoadAndExecuteTransactionOutput{valid})) + payer, err = slotCtx.GetAccount(txfixture.PayerPubkey()) + require.NoError(t, err) + assert.Less(t, payer.Lamports, payerBefore.Lamports) +} diff --git a/pkg/sealevel/scratch_slot_ctx.go b/pkg/sealevel/scratch_slot_ctx.go new file mode 100644 index 00000000..f190056f --- /dev/null +++ b/pkg/sealevel/scratch_slot_ctx.go @@ -0,0 +1,46 @@ +package sealevel + +import ( + "sync" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/gagliardetto/solana-go" +) + +// NewScratchSlotCtx returns a slot context that executes against base's state +// without changing it. Account writes go to an overlay on base's accounts; +// vote-cache changes are deferred and program-cache insertions are recorded, +// so a caller that discards the scratch context can evict them. Execution +// only reads the remaining fields, which are shared with base. +func NewScratchSlotCtx(base *SlotCtx) *SlotCtx { + scratch := &SlotCtx{ + Accounts: accounts.NewOverlayAccounts(base.Accounts), + ParentAccts: base.ParentAccts, + AccountsDb: base.AccountsDb, + UnrootedRead: base.UnrootedRead, + FeeRateGovernor: base.FeeRateGovernor, + Slot: base.Slot, + ParentSlot: base.ParentSlot, + Epoch: base.Epoch, + AcctMapsMu: &sync.Mutex{}, + ModifiedAccts: make(map[solana.PublicKey]bool), + WritableAccts: make(map[solana.PublicKey]bool), + NumSignatures: base.NumSignatures, + Blockhash: base.Blockhash, + LastBlockhash: base.LastBlockhash, + SlotBank: base.SlotBank, + Features: base.Features, + VoteTimestampMu: &sync.Mutex{}, + VoteTimestamps: make(map[solana.PublicKey]BlockTimestamp), + VoteAccts: base.VoteAccts, + TotalEpochStake: base.TotalEpochStake, + Replay: base.Replay, + LatestEvictedBlockhash: base.LatestEvictedBlockhash, + SerializedParameterArena: base.SerializedParameterArena, + TraceCtx: base.TraceCtx, + DeferVoteCachePublication: true, + TrackProgramCacheAdds: true, + } + scratch.bankSysvars.Store(base.BankSysvars()) + return scratch +} From 48ce626ee650613fcd528ef2872f0014f61c6219 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 15:03:44 +0000 Subject: [PATCH 11/25] blockprod: add nonce test for rp bundles --- pkg/blockprod/bank_bundle_test.go | 101 ++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) diff --git a/pkg/blockprod/bank_bundle_test.go b/pkg/blockprod/bank_bundle_test.go index 08f60237..aff6457d 100644 --- a/pkg/blockprod/bank_bundle_test.go +++ b/pkg/blockprod/bank_bundle_test.go @@ -1,10 +1,13 @@ package blockprod import ( + "math" "math/rand/v2" "sync" "testing" + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/Overclock-Validator/mithril/pkg/costmodel" "github.com/Overclock-Validator/mithril/pkg/replay" "github.com/Overclock-Validator/mithril/pkg/sealevel" @@ -300,3 +303,101 @@ func TestScratchSlotCtxLeavesBaseUnchanged(t *testing.T) { assert.Equal(t, uint64(1), got.Lamports) assert.Equal(t, before, lamportsOf(t, env, txfixture.PayerPubkey()), "writes stay in the scratch view") } + +// installNonce puts an initialized nonce account, with the payer as its +// authority, in the bank and returns its durable nonce. +func installNonce(t *testing.T, env *TestEnv, key solana.PublicKey) solana.Hash { + t.Helper() + durableNonce := solana.Hash{0xAA} + nonceState := sealevel.NonceStateVersions{ + Type: sealevel.NonceVersionCurrent, + Current: sealevel.NonceData{ + IsInitialized: true, + Authority: txfixture.PayerPubkey(), + DurableNonce: durableNonce, + FeeCalculator: sealevel.FeeCalculator{LamportsPerSignature: 5000}, + }, + } + data, err := nonceState.Marshal() + require.NoError(t, err) + require.NoError(t, env.SlotCtx.SetAccount(key, &accounts.Account{ + Key: key, Lamports: 1_500_000, Owner: addresses.SystemProgramAddr, Data: data, RentEpoch: math.MaxUint64, + })) + return durableNonce +} + +func nonceTx(t *testing.T, nonce solana.PublicKey, durableNonce solana.Hash, instructions ...solana.Instruction) *solana.Transaction { + t.Helper() + all := append([]solana.Instruction{ + system.NewAdvanceNonceAccountInstruction(nonce, solana.SysVarRecentBlockHashesPubkey, txfixture.PayerPubkey()).Build(), + }, instructions...) + tx, err := solana.NewTransaction(all, durableNonce, solana.TransactionPayer(txfixture.PayerPubkey())) + require.NoError(t, err) + payerKey := txfixture.PayerPrivateKey() + _, err = tx.Sign(func(solana.PublicKey) *solana.PrivateKey { return &payerKey }) + require.NoError(t, err) + return tx +} + +func TestForgeBundleFailureNeitherChargesFeesNorAdvancesNonce(t *testing.T) { + nonceKey := solana.PublicKey{0xD9} + // A one-byte loaded-data limit makes this a fees-only failure: outside a + // bundle it lands, charging the fee and advancing the nonce. + failing := func(t *testing.T, durableNonce solana.Hash) *solana.Transaction { + return nonceTx(t, nonceKey, durableNonce, + computebudget.NewSetLoadedAccountsDataSizeLimitInstruction(1).Build(), + transferIx(1, txfixture.PayerPubkey(), txfixture.DestPubkey())) + } + + control := NewTestEnv(TestEnvConfig{}) + defer control.Close() + controlNonce := installNonce(t, control, nonceKey) + nonceBefore, err := control.SlotCtx.GetAccount(nonceKey) + require.NoError(t, err) + payerBefore := lamportsOf(t, control, txfixture.PayerPubkey()) + result, _ := control.Bank.ForgeTransaction(failing(t, controlNonce), 0) + require.Equal(t, ForgeAccepted, result, "the control lands as a fees-only failure") + nonceAfter, err := control.SlotCtx.GetAccount(nonceKey) + require.NoError(t, err) + require.NotEqual(t, nonceBefore.Data, nonceAfter.Data, "outside a bundle the nonce advances") + require.Equal(t, payerBefore-5_000, lamportsOf(t, control, txfixture.PayerPubkey()), "outside a bundle the fee is charged") + + for name, tc := range map[string]struct { + bundle func(t *testing.T, durableNonce solana.Hash) []*solana.Transaction + failingAt int + }{ + "alone": {func(t *testing.T, durableNonce solana.Hash) []*solana.Transaction { + return []*solana.Transaction{failing(t, durableNonce)} + }, 0}, + "after a successful transaction": {func(t *testing.T, durableNonce solana.Hash) []*solana.Transaction { + return []*solana.Transaction{mustSignedTransfer(t, 3), failing(t, durableNonce)} + }, 1}, + // The durable-nonce transaction succeeds and advances the nonce in the + // scratch view; the bundle then fails, so that advance is discarded. + "successful, before a failing transaction": {func(t *testing.T, durableNonce solana.Hash) []*solana.Transaction { + return []*solana.Transaction{ + nonceTx(t, nonceKey, durableNonce, transferIx(4, txfixture.PayerPubkey(), txfixture.DestPubkey())), + mustSignBankTestTransaction(t, transferIx(20_000_000_000, txfixture.PayerPubkey(), txfixture.DestPubkey())), + } + }, 1}, + } { + t.Run(name, func(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + durableNonce := installNonce(t, env, nonceKey) + nonceBefore, err := env.SlotCtx.GetAccount(nonceKey) + require.NoError(t, err) + payerBefore := lamportsOf(t, env, txfixture.PayerPubkey()) + destBefore := lamportsOf(t, env, txfixture.DestPubkey()) + + got := env.Bank.ForgeBundle(BundleInput{Transactions: tc.bundle(t, durableNonce)}) + require.Equal(t, ForgeDroppedExecution, got.Result, "%+v", got) + require.Equal(t, tc.failingAt, got.Index, "%v", got.Err) + nonceAfter, err := env.SlotCtx.GetAccount(nonceKey) + require.NoError(t, err) + assert.Equal(t, nonceBefore.Data, nonceAfter.Data, "the nonce is not advanced") + assert.Equal(t, nonceBefore.Lamports, nonceAfter.Lamports) + requireNothingLanded(t, env, payerBefore, destBefore) + }) + } +} From 42014314f9bdaa967d5fab54b317cef7db7f31a2 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 16:21:05 +0000 Subject: [PATCH 12/25] gossip: gofmt Client struct --- pkg/gossip/client.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/gossip/client.go b/pkg/gossip/client.go index 01d36eb5..280dc095 100644 --- a/pkg/gossip/client.go +++ b/pkg/gossip/client.go @@ -44,12 +44,12 @@ type Config struct { type Client struct { cfg Config - entrypoint *net.UDPAddr - bindAddr *net.UDPAddr - tvuAddr *net.UDPAddr - alpenglow *net.UDPAddr - identity ed25519.PrivateKey - pubkey Pubkey + entrypoint *net.UDPAddr + bindAddr *net.UDPAddr + tvuAddr *net.UDPAddr + alpenglow *net.UDPAddr + identity ed25519.PrivateKey + pubkey Pubkey contactMu sync.RWMutex // contact is replaced on every update and never modified once published, From 68efea1093468e16e6094cef8afae7611d02c232 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 16:47:47 +0000 Subject: [PATCH 13/25] harmonic: vendor validator protos, add endpoint connection and token auth --- pkg/harmonic/auth.go | 160 ++++ pkg/harmonic/auth_test.go | 289 ++++++ pkg/harmonic/conn.go | 166 ++++ pkg/harmonic/conn_test.go | 103 ++ pkg/harmonic/mock_test.go | 136 +++ pkg/harmonic/proto/auth/auth.pb.go | 540 +++++++++++ pkg/harmonic/proto/auth/auth_grpc.pb.go | 210 +++++ pkg/harmonic/proto/block/block.pb.go | 197 ++++ .../proto/blockengine/block_engine.pb.go | 881 ++++++++++++++++++ .../proto/blockengine/block_engine_grpc.pb.go | 468 ++++++++++ pkg/harmonic/proto/bundle/bundle.pb.go | 203 ++++ pkg/harmonic/proto/gen.sh | 34 + pkg/harmonic/proto/packet/packet.pb.go | 375 ++++++++ pkg/harmonic/proto/relayer/relayer.pb.go | 337 +++++++ pkg/harmonic/proto/relayer/relayer_grpc.pb.go | 173 ++++ pkg/harmonic/proto/shared/shared.pb.go | 237 +++++ pkg/harmonic/proto/upstream/LICENSE | 201 ++++ pkg/harmonic/proto/upstream/README.md | 4 + pkg/harmonic/proto/upstream/auth.proto | 72 ++ pkg/harmonic/proto/upstream/block.proto | 24 + .../proto/upstream/block_engine.proto | 127 +++ pkg/harmonic/proto/upstream/bundle.proto | 23 + pkg/harmonic/proto/upstream/packet.proto | 47 + pkg/harmonic/proto/upstream/relayer.proto | 42 + pkg/harmonic/proto/upstream/shared.proto | 26 + pkg/harmonic/tokens.go | 142 +++ 26 files changed, 5217 insertions(+) create mode 100644 pkg/harmonic/auth.go create mode 100644 pkg/harmonic/auth_test.go create mode 100644 pkg/harmonic/conn.go create mode 100644 pkg/harmonic/conn_test.go create mode 100644 pkg/harmonic/mock_test.go create mode 100644 pkg/harmonic/proto/auth/auth.pb.go create mode 100644 pkg/harmonic/proto/auth/auth_grpc.pb.go create mode 100644 pkg/harmonic/proto/block/block.pb.go create mode 100644 pkg/harmonic/proto/blockengine/block_engine.pb.go create mode 100644 pkg/harmonic/proto/blockengine/block_engine_grpc.pb.go create mode 100644 pkg/harmonic/proto/bundle/bundle.pb.go create mode 100755 pkg/harmonic/proto/gen.sh create mode 100644 pkg/harmonic/proto/packet/packet.pb.go create mode 100644 pkg/harmonic/proto/relayer/relayer.pb.go create mode 100644 pkg/harmonic/proto/relayer/relayer_grpc.pb.go create mode 100644 pkg/harmonic/proto/shared/shared.pb.go create mode 100644 pkg/harmonic/proto/upstream/LICENSE create mode 100644 pkg/harmonic/proto/upstream/README.md create mode 100644 pkg/harmonic/proto/upstream/auth.proto create mode 100644 pkg/harmonic/proto/upstream/block.proto create mode 100644 pkg/harmonic/proto/upstream/block_engine.proto create mode 100644 pkg/harmonic/proto/upstream/bundle.proto create mode 100644 pkg/harmonic/proto/upstream/packet.proto create mode 100644 pkg/harmonic/proto/upstream/relayer.proto create mode 100644 pkg/harmonic/proto/upstream/shared.proto create mode 100644 pkg/harmonic/tokens.go diff --git a/pkg/harmonic/auth.go b/pkg/harmonic/auth.go new file mode 100644 index 00000000..74b52d62 --- /dev/null +++ b/pkg/harmonic/auth.go @@ -0,0 +1,160 @@ +package harmonic + +import ( + "context" + "crypto/ed25519" + "errors" + "fmt" + "strings" + "sync/atomic" + "time" + + authpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/auth" + "github.com/gagliardetto/solana-go" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/status" +) + +const ( + authChallengeBytes = 9 + maxTokenBytes = 1024 + authRPCTimeout = 5 * time.Second + authServicePrefix = "/auth.AuthService/" +) + +// Token is a validated auth token. +type Token struct { + Value string + Expiry time.Time +} + +// Tokens are one endpoint's access and refresh tokens. +type Tokens struct { + Access Token + Refresh Token +} + +// AuthMessage returns the exact bytes a validator signs to authenticate: +// base58(pubkey), "-", then the 9-byte challenge verbatim. The whole string +// is also what GenerateAuthTokens takes as its challenge. +func AuthMessage(pubkey ed25519.PublicKey, challenge string) ([]byte, error) { + if len(pubkey) != ed25519.PublicKeySize { + return nil, fmt.Errorf("auth pubkey has %d bytes", len(pubkey)) + } + if len(challenge) != authChallengeBytes { + return nil, fmt.Errorf("auth challenge has %d bytes, want %d", len(challenge), authChallengeBytes) + } + return []byte(solana.PublicKeyFromBytes(pubkey).String() + "-" + challenge), nil +} + +// Authenticate runs the challenge-response handshake and returns validated +// tokens. +func Authenticate(ctx context.Context, client authpb.AuthServiceClient, identity ed25519.PrivateKey) (Tokens, error) { + pubkey, ok := identity.Public().(ed25519.PublicKey) + if !ok || len(identity) != ed25519.PrivateKeySize { + return Tokens{}, errors.New("invalid identity key") + } + challengeCtx, cancel := context.WithTimeout(ctx, authRPCTimeout) + defer cancel() + challenge, err := client.GenerateAuthChallenge(challengeCtx, &authpb.GenerateAuthChallengeRequest{ + Role: authpb.Role_VALIDATOR, + Pubkey: pubkey, + }) + if err != nil { + return Tokens{}, fmt.Errorf("generate auth challenge: %w", err) + } + message, err := AuthMessage(pubkey, challenge.GetChallenge()) + if err != nil { + return Tokens{}, err + } + tokensCtx, cancel := context.WithTimeout(ctx, authRPCTimeout) + defer cancel() + resp, err := client.GenerateAuthTokens(tokensCtx, &authpb.GenerateAuthTokensRequest{ + Challenge: string(message), + ClientPubkey: pubkey, + SignedChallenge: ed25519.Sign(identity, message), + }) + if err != nil { + return Tokens{}, fmt.Errorf("generate auth tokens: %w", err) + } + access, err := validateToken(resp.GetAccessToken(), "access") + if err != nil { + return Tokens{}, err + } + refresh, err := validateToken(resp.GetRefreshToken(), "refresh") + if err != nil { + return Tokens{}, err + } + return Tokens{Access: access, Refresh: refresh}, nil +} + +// RefreshAccess exchanges a refresh token for a new access token. +func RefreshAccess(ctx context.Context, client authpb.AuthServiceClient, refresh string) (Token, error) { + ctx, cancel := context.WithTimeout(ctx, authRPCTimeout) + defer cancel() + resp, err := client.RefreshAccessToken(ctx, &authpb.RefreshAccessTokenRequest{RefreshToken: refresh}) + if err != nil { + return Token{}, fmt.Errorf("refresh access token: %w", err) + } + return validateToken(resp.GetAccessToken(), "access") +} + +func validateToken(token *authpb.Token, name string) (Token, error) { + switch { + case token == nil: + return Token{}, fmt.Errorf("missing %s token", name) + case token.GetValue() == "": + return Token{}, fmt.Errorf("empty %s token", name) + case len(token.GetValue()) > maxTokenBytes: + return Token{}, fmt.Errorf("%s token has %d bytes, limit %d", name, len(token.GetValue()), maxTokenBytes) + case token.GetExpiresAtUtc() == nil: + return Token{}, fmt.Errorf("%s token has no expiry", name) + } + return Token{Value: token.GetValue(), Expiry: token.GetExpiresAtUtc().AsTime()}, nil +} + +// Bearer attaches the current access token to every RPC except the auth +// service's own. With no token, RPCs fail with UNAUTHENTICATED rather than +// going out without one. +type Bearer struct { + value atomic.Pointer[string] +} + +// Set rotates the token. RPCs started afterwards use it; open streams keep +// the token they started with. +func (b *Bearer) Set(token string) { + header := "Bearer " + token + b.value.Store(&header) +} + +// Clear drops the token. +func (b *Bearer) Clear() { + b.value.Store(nil) +} + +func (b *Bearer) GetRequestMetadata(ctx context.Context, _ ...string) (map[string]string, error) { + if info, ok := credentials.RequestInfoFromContext(ctx); ok && strings.HasPrefix(info.Method, authServicePrefix) { + return nil, nil + } + header := b.value.Load() + if header == nil { + return nil, status.Error(codes.Unauthenticated, "no Harmonic access token") + } + return map[string]string{"authorization": *header}, nil +} + +// RequireTransportSecurity is false so plaintext development endpoints work. +func (b *Bearer) RequireTransportSecurity() bool { + return false +} + +// IsAuthError reports a status that requires discarding tokens and +// authenticating again. +func IsAuthError(err error) bool { + switch status.Code(err) { + case codes.Unauthenticated, codes.PermissionDenied: + return true + } + return false +} diff --git a/pkg/harmonic/auth_test.go b/pkg/harmonic/auth_test.go new file mode 100644 index 00000000..55230da1 --- /dev/null +++ b/pkg/harmonic/auth_test.go @@ -0,0 +1,289 @@ +package harmonic + +import ( + "context" + "crypto/ed25519" + "encoding/hex" + "math/big" + "strings" + "testing" + "time" + + authpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/auth" + relayerpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/relayer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// base58 is a plain big-integer Bitcoin-alphabet encoder, independent of the +// one the code under test uses. +func base58(b []byte) string { + const alphabet = "123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz" + n := new(big.Int).SetBytes(b) + var out []byte + for radix, mod := big.NewInt(58), new(big.Int); n.Sign() > 0; { + n.DivMod(n, radix, mod) + out = append([]byte{alphabet[mod.Int64()]}, out...) + } + for _, c := range b { + if c != 0 { + break + } + out = append([]byte{'1'}, out...) + } + return string(out) +} + +// rfc8032Key is the RFC 8032 section 7.1 test 1 key. +func rfc8032Key(t *testing.T) ed25519.PrivateKey { + t.Helper() + seed, err := hex.DecodeString("9d61b19deffd5a60ba844af492ec2cc44449c5697b326919703bac031cae7f60") + require.NoError(t, err) + key := ed25519.NewKeyFromSeed(seed) + require.Equal(t, "d75a980182b10ab7d54bfed3c964073a0ee172f3daa62325af021a68f707511a", hex.EncodeToString(key.Public().(ed25519.PublicKey))) + return key +} + +func TestAuthMessageIsBase58PubkeyDashChallenge(t *testing.T) { + pubkey := rfc8032Key(t).Public().(ed25519.PublicKey) + message, err := AuthMessage(pubkey, "abcdefghi") + require.NoError(t, err) + // The same bytes the reference client signs: format!("{}-{}", pubkey, challenge). + assert.Equal(t, base58(pubkey)+"-abcdefghi", string(message)) + // The 9 challenge bytes are used verbatim, even when not printable. + message, err = AuthMessage(pubkey, "\x00\xff-é\n123") + require.NoError(t, err) + assert.Equal(t, base58(pubkey)+"-\x00\xff-é\n123", string(message)) + + for _, challenge := range []string{"", "abcdefgh", "abcdefghij", "ééééé"} { + _, err := AuthMessage(pubkey, challenge) + assert.Errorf(t, err, "challenge of %d bytes", len(challenge)) + } + _, err = AuthMessage(pubkey[:31], "abcdefghi") + assert.Error(t, err) +} + +func TestAuthenticateHandshakeAndBearer(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + auth := newMockAuth(func() time.Time { return now }) + relayer := &mockRelayer{} + cfg := serveBufconn(t, func(s *grpc.Server) { + authpb.RegisterAuthServiceServer(s, auth) + relayerpb.RegisterRelayerServer(s, relayer) + }) + identity := rfc8032Key(t) + tokens := NewTokenSource(nil, identity) + cfg.Credentials = tokens.Bearer() + conn, err := Dial(context.Background(), cfg) + require.NoError(t, err) + defer conn.Close() + tokens.client = authpb.NewAuthServiceClient(conn) + relayerClient := relayerpb.NewRelayerClient(conn) + + _, err = relayerClient.GetTpuConfigs(context.Background(), &relayerpb.GetTpuConfigsRequest{}) + require.Equal(t, codes.Unauthenticated, status.Code(err), "no RPC goes out without a token") + + require.NoError(t, tokens.Authenticate(context.Background())) + got := tokens.Tokens() + assert.Equal(t, "access-1", got.Access.Value) + assert.True(t, now.Add(10*time.Minute).Equal(got.Access.Expiry)) + assert.Equal(t, "refresh-2", got.Refresh.Value) + assert.Empty(t, auth.authHeaders, "auth RPCs carry no authorization header") + + _, err = relayerClient.GetTpuConfigs(context.Background(), &relayerpb.GetTpuConfigsRequest{}) + require.NoError(t, err) + assert.Equal(t, []string{"Bearer access-1"}, relayer.last().Get("authorization")) + assert.True(t, strings.HasPrefix(relayer.last().Get("user-agent")[0], "mithril/")) + + tokens.Invalidate() + _, err = relayerClient.GetTpuConfigs(context.Background(), &relayerpb.GetTpuConfigsRequest{}) + assert.Equal(t, codes.Unauthenticated, status.Code(err)) + assert.True(t, IsAuthError(err)) + assert.True(t, IsAuthError(status.Error(codes.PermissionDenied, ""))) + assert.False(t, IsAuthError(status.Error(codes.Unavailable, ""))) +} + +func TestAuthenticateRejectsBadResponses(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + valid := func() *authpb.Token { + return &authpb.Token{Value: "token", ExpiresAtUtc: timestamppb.New(now.Add(time.Hour))} + } + cases := map[string]func(*mockAuth){ + "short challenge": func(m *mockAuth) { m.challenge = "abcdefgh" }, + "long challenge": func(m *mockAuth) { m.challenge = "abcdefghij" }, + "missing access token": func(m *mockAuth) { + m.tokens = func() *authpb.GenerateAuthTokensResponse { + return &authpb.GenerateAuthTokensResponse{RefreshToken: valid()} + } + }, + "missing refresh token": func(m *mockAuth) { + m.tokens = func() *authpb.GenerateAuthTokensResponse { + return &authpb.GenerateAuthTokensResponse{AccessToken: valid()} + } + }, + "empty token": func(m *mockAuth) { + m.tokens = func() *authpb.GenerateAuthTokensResponse { + return &authpb.GenerateAuthTokensResponse{AccessToken: &authpb.Token{ExpiresAtUtc: timestamppb.New(now.Add(time.Hour))}, RefreshToken: valid()} + } + }, + "oversized token": func(m *mockAuth) { + m.tokens = func() *authpb.GenerateAuthTokensResponse { + big := valid() + big.Value = strings.Repeat("x", maxTokenBytes+1) + return &authpb.GenerateAuthTokensResponse{AccessToken: valid(), RefreshToken: big} + } + }, + "no expiry": func(m *mockAuth) { + m.tokens = func() *authpb.GenerateAuthTokensResponse { + return &authpb.GenerateAuthTokensResponse{AccessToken: &authpb.Token{Value: "token"}, RefreshToken: valid()} + } + }, + "already expired": func(m *mockAuth) { + m.tokens = func() *authpb.GenerateAuthTokensResponse { + expired := valid() + expired.ExpiresAtUtc = timestamppb.New(now) + return &authpb.GenerateAuthTokensResponse{AccessToken: expired, RefreshToken: valid()} + } + }, + } + for name, configure := range cases { + t.Run(name, func(t *testing.T) { + auth := newMockAuth(func() time.Time { return now }) + configure(auth) + cfg := serveBufconn(t, func(s *grpc.Server) { authpb.RegisterAuthServiceServer(s, auth) }) + conn, err := Dial(context.Background(), cfg) + require.NoError(t, err) + defer conn.Close() + tokens := NewTokenSource(authpb.NewAuthServiceClient(conn), rfc8032Key(t)) + tokens.now = func() time.Time { return now } + tokens.bearer.Set("stale") + + require.Error(t, tokens.Authenticate(context.Background())) + _, err = tokens.Bearer().GetRequestMetadata(context.Background()) + assert.Equal(t, codes.Unauthenticated, status.Code(err), "a failed handshake leaves no token") + if strings.Contains(name, "challenge") { + assert.Zero(t, auth.callCount("tokens"), "a bad challenge is never signed") + } + }) + } +} + +// fakeTimers hands the test every timer TokenSource.Run starts. +type fakeTimers struct { + waits chan time.Duration + fires chan chan time.Time +} + +func newFakeTimers() *fakeTimers { + return &fakeTimers{waits: make(chan time.Duration, 8), fires: make(chan chan time.Time, 8)} +} + +func (f *fakeTimers) after(d time.Duration) <-chan time.Time { + c := make(chan time.Time, 1) + f.waits <- d + f.fires <- c + return c +} + +// next returns the next timer's duration and a function that fires it. +func (f *fakeTimers) next(t *testing.T) (time.Duration, func()) { + t.Helper() + select { + case d := <-f.waits: + c := <-f.fires + return d, func() { c <- time.Time{} } + case <-time.After(2 * time.Second): + t.Fatal("no timer started") + return 0, nil + } +} + +func startTokenSource(t *testing.T, auth *mockAuth, now time.Time) (*TokenSource, *fakeTimers, <-chan error) { + t.Helper() + cfg := serveBufconn(t, func(s *grpc.Server) { authpb.RegisterAuthServiceServer(s, auth) }) + conn, err := Dial(context.Background(), cfg) + require.NoError(t, err) + t.Cleanup(func() { conn.Close() }) + tokens := NewTokenSource(authpb.NewAuthServiceClient(conn), rfc8032Key(t)) + timers := newFakeTimers() + tokens.now = func() time.Time { return now } + tokens.after = timers.after + require.NoError(t, tokens.Authenticate(context.Background())) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- tokens.Run(ctx) }() + t.Cleanup(func() { + cancel() + <-done + }) + return tokens, timers, done +} + +func bearerValue(t *testing.T, tokens *TokenSource) string { + t.Helper() + md, err := tokens.Bearer().GetRequestMetadata(context.Background()) + require.NoError(t, err) + return md["authorization"] +} + +func TestTokenSourceRefreshesAtHalfLifeAndFallsBackToReauth(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + auth := newMockAuth(func() time.Time { return now }) + tokens, timers, _ := startTokenSource(t, auth, now) + + wait, fire := timers.next(t) + assert.Equal(t, 5*time.Minute, wait, "half of the access token's 10 minutes") + fire() + wait, fire = timers.next(t) + assert.Equal(t, 1, auth.callCount("refresh")) + assert.Equal(t, "Bearer access-3", bearerValue(t, tokens)) + assert.Equal(t, 5*time.Minute, wait) + + auth.mu.Lock() + auth.refreshErr = status.Error(codes.Unauthenticated, "refresh token revoked") + auth.mu.Unlock() + fire() + timers.next(t) + assert.Equal(t, 2, auth.callCount("refresh")) + assert.Equal(t, 2, auth.callCount("tokens"), "a failed refresh authenticates again") + assert.Equal(t, "Bearer access-4", bearerValue(t, tokens)) + + tokens.Invalidate() + timers.next(t) + assert.Equal(t, 3, auth.callCount("tokens"), "an auth error authenticates again at once") + assert.Equal(t, "Bearer access-6", bearerValue(t, tokens)) +} + +func TestTokenSourceReauthenticatesAtRefreshTokenHalfLife(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + auth := newMockAuth(func() time.Time { return now }) + auth.accessTTL, auth.refreshTTL = time.Hour, 20*time.Minute + _, timers, _ := startTokenSource(t, auth, now) + + wait, fire := timers.next(t) + assert.Equal(t, 10*time.Minute, wait, "half of the refresh token's 20 minutes comes first") + fire() + timers.next(t) + assert.Zero(t, auth.callCount("refresh")) + assert.Equal(t, 2, auth.callCount("tokens")) +} + +func TestTokenSourceDiscardsRefreshThatOutlivesInvalidation(t *testing.T) { + now := time.Unix(1_800_000_000, 0) + auth := newMockAuth(func() time.Time { return now }) + auth.refreshGate = make(chan struct{}) + tokens, timers, _ := startTokenSource(t, auth, now) + + _, fire := timers.next(t) + fire() + require.Eventually(t, func() bool { return auth.callCount("refresh") == 1 }, 2*time.Second, time.Millisecond) + tokens.Invalidate() + close(auth.refreshGate) + timers.next(t) + assert.Equal(t, 2, auth.callCount("tokens"), "the stale refresh falls back to authenticating again") + assert.Equal(t, "Bearer access-4", bearerValue(t, tokens), "the refreshed token (access-3) is never used") +} diff --git a/pkg/harmonic/conn.go b/pkg/harmonic/conn.go new file mode 100644 index 00000000..3bbc439d --- /dev/null +++ b/pkg/harmonic/conn.go @@ -0,0 +1,166 @@ +// Package harmonic connects a validator to a Harmonic block engine and relayer. +package harmonic + +import ( + "context" + "crypto/tls" + "fmt" + "net" + "net/url" + "time" + + "github.com/Overclock-Validator/mithril/pkg/mlog" + "github.com/Overclock-Validator/mithril/pkg/version" + "google.golang.org/grpc" + "google.golang.org/grpc/connectivity" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" +) + +const ( + connectTimeout = 10 * time.Second + // Fixed HTTP/2 windows disable BDP estimation, which restarts at 64 KiB + // and would stall a bursty block stream. + streamWindowBytes = 16 << 20 + connWindowBytes = 64 << 20 + maxRecvMessageBytes = 64 << 20 + readBufferBytes = 1 << 20 + socketRecvBufferBytes = 8 << 20 + // HTTP/2 PINGs run even while idle; an unanswered PING fails the + // connection. + defaultPingInterval = 10 * time.Second + pingTimeout = 10 * time.Second + tcpKeepAliveIdle = 60 * time.Second + tcpKeepAliveEvery = 10 * time.Second + tcpKeepAliveProbes = 3 +) + +// Endpoint is a block engine or relayer URL. +type Endpoint struct { + // Target is host:port. + Target string + TLS bool + // ServerName is the TLS server name: the URL host unless overridden. + ServerName string +} + +// ParseEndpoint parses an http:// or https:// endpoint URL. A non-empty +// serverName overrides the TLS server name. +func ParseEndpoint(raw, serverName string) (Endpoint, error) { + u, err := url.Parse(raw) + if err != nil { + return Endpoint{}, fmt.Errorf("parse endpoint %q: %w", raw, err) + } + var ep Endpoint + switch u.Scheme { + case "https": + ep.TLS = true + case "http": + default: + return Endpoint{}, fmt.Errorf("endpoint %q: scheme must be http or https", raw) + } + host := u.Hostname() + if host == "" { + return Endpoint{}, fmt.Errorf("endpoint %q has no host", raw) + } + port := u.Port() + if port == "" { + port = "80" + if ep.TLS { + port = "443" + } + } + ep.Target = net.JoinHostPort(host, port) + ep.ServerName = host + if serverName != "" { + ep.ServerName = serverName + } + return ep, nil +} + +// DialConfig configures a connection to one endpoint. +type DialConfig struct { + Endpoint Endpoint + // InsecureSkipVerify disables TLS certificate verification. Development + // only. + InsecureSkipVerify bool + // Credentials, if set, attaches per-RPC metadata such as a Bearer. + Credentials credentials.PerRPCCredentials + // PingInterval defaults to 10s. + PingInterval time.Duration + // Dialer replaces the TCP dialer, for tests. + Dialer func(ctx context.Context, addr string) (net.Conn, error) +} + +// Dial opens a connection and waits until it is ready, for at most 10s. +func Dial(ctx context.Context, cfg DialConfig) (*grpc.ClientConn, error) { + ping := cfg.PingInterval + if ping <= 0 { + ping = defaultPingInterval + } + transport := insecure.NewCredentials() + if cfg.Endpoint.TLS { + if cfg.InsecureSkipVerify { + mlog.Log.Infof("WARNING: Harmonic endpoint %s: TLS certificate verification is disabled", cfg.Endpoint.Target) + } + transport = credentials.NewTLS(&tls.Config{ + ServerName: cfg.Endpoint.ServerName, + MinVersion: tls.VersionTLS13, + InsecureSkipVerify: cfg.InsecureSkipVerify, //nolint:gosec // explicit development switch + }) + } + dialer := cfg.Dialer + if dialer == nil { + dialer = dialTCP + } + opts := []grpc.DialOption{ + grpc.WithTransportCredentials(transport), + grpc.WithContextDialer(dialer), + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: ping, + Timeout: pingTimeout, + PermitWithoutStream: true, + }), + grpc.WithInitialWindowSize(streamWindowBytes), + grpc.WithInitialConnWindowSize(connWindowBytes), + grpc.WithReadBufferSize(readBufferBytes), + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxRecvMessageBytes)), + grpc.WithUserAgent("mithril/" + version.Semver()), + } + if cfg.Credentials != nil { + opts = append(opts, grpc.WithPerRPCCredentials(cfg.Credentials)) + } + conn, err := grpc.NewClient("passthrough:///"+cfg.Endpoint.Target, opts...) + if err != nil { + return nil, fmt.Errorf("dial %s: %w", cfg.Endpoint.Target, err) + } + ctx, cancel := context.WithTimeout(ctx, connectTimeout) + defer cancel() + conn.Connect() + for state := conn.GetState(); state != connectivity.Ready; state = conn.GetState() { + if !conn.WaitForStateChange(ctx, state) { + conn.Close() + return nil, fmt.Errorf("connect %s: %w (last state %s)", cfg.Endpoint.Target, ctx.Err(), state) + } + } + return conn, nil +} + +func dialTCP(ctx context.Context, addr string) (net.Conn, error) { + d := net.Dialer{KeepAliveConfig: net.KeepAliveConfig{ + Enable: true, + Idle: tcpKeepAliveIdle, + Interval: tcpKeepAliveEvery, + Count: tcpKeepAliveProbes, + }} + conn, err := d.DialContext(ctx, "tcp", addr) + if err != nil { + return nil, err + } + if tcp, ok := conn.(*net.TCPConn); ok { + _ = tcp.SetNoDelay(true) + _ = tcp.SetReadBuffer(socketRecvBufferBytes) + } + return conn, nil +} diff --git a/pkg/harmonic/conn_test.go b/pkg/harmonic/conn_test.go new file mode 100644 index 00000000..3e128126 --- /dev/null +++ b/pkg/harmonic/conn_test.go @@ -0,0 +1,103 @@ +package harmonic + +import ( + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "math/big" + "net" + "testing" + "time" + + authpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/auth" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" +) + +func TestParseEndpoint(t *testing.T) { + cases := map[string]struct { + raw, sni string + want Endpoint + }{ + "https default port": {"https://be.example.com", "", Endpoint{Target: "be.example.com:443", TLS: true, ServerName: "be.example.com"}}, + "https explicit port": {"https://be.example.com:8443/", "", Endpoint{Target: "be.example.com:8443", TLS: true, ServerName: "be.example.com"}}, + "server name override": {"https://10.0.0.1:443", "be.example.com", Endpoint{Target: "10.0.0.1:443", TLS: true, ServerName: "be.example.com"}}, + "plaintext": {"http://127.0.0.1:5999", "", Endpoint{Target: "127.0.0.1:5999", ServerName: "127.0.0.1"}}, + "plaintext default": {"http://relayer.local", "", Endpoint{Target: "relayer.local:80", ServerName: "relayer.local"}}, + } + for name, tc := range cases { + got, err := ParseEndpoint(tc.raw, tc.sni) + require.NoError(t, err, name) + assert.Equal(t, tc.want, got, name) + } + for _, raw := range []string{"grpc://be.example.com", "be.example.com:443", "https://", "://bad"} { + _, err := ParseEndpoint(raw, "") + assert.Errorf(t, err, "%q", raw) + } +} + +func selfSignedTLS(t *testing.T, host string) credentials.TransportCredentials { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + require.NoError(t, err) + template := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: host}, + DNSNames: []string{host}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + der, err := x509.CreateCertificate(rand.Reader, template, template, &key.PublicKey, key) + require.NoError(t, err) + return credentials.NewServerTLSFromCert(&tls.Certificate{Certificate: [][]byte{der}, PrivateKey: key}) +} + +func TestDialTLSVerifiesCertificateUnlessDisabled(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + srv := grpc.NewServer(grpc.Creds(selfSignedTLS(t, "be.test"))) + auth := newMockAuth(time.Now) + authpb.RegisterAuthServiceServer(srv, auth) + go func() { _ = srv.Serve(lis) }() + defer srv.Stop() + + ep, err := ParseEndpoint("https://"+lis.Addr().String(), "be.test") + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + _, err = Dial(ctx, DialConfig{Endpoint: ep}) + require.Error(t, err, "an untrusted certificate is refused by default") + + conn, err := Dial(context.Background(), DialConfig{Endpoint: ep, InsecureSkipVerify: true}) + require.NoError(t, err) + defer conn.Close() + _, err = authpb.NewAuthServiceClient(conn).GenerateAuthChallenge(context.Background(), &authpb.GenerateAuthChallengeRequest{ + Role: authpb.Role_VALIDATOR, Pubkey: make([]byte, 32), + }) + require.NoError(t, err) +} + +func TestDialFailsWithinTheConnectDeadline(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + addr := lis.Addr().String() + require.NoError(t, lis.Close()) + ep, err := ParseEndpoint("http://"+addr, "") + require.NoError(t, err) + + ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond) + defer cancel() + start := time.Now() + _, err = Dial(ctx, DialConfig{Endpoint: ep}) + require.Error(t, err) + assert.Less(t, time.Since(start), 5*time.Second) +} diff --git a/pkg/harmonic/mock_test.go b/pkg/harmonic/mock_test.go new file mode 100644 index 00000000..2ceeba02 --- /dev/null +++ b/pkg/harmonic/mock_test.go @@ -0,0 +1,136 @@ +package harmonic + +import ( + "context" + "crypto/ed25519" + "net" + "sync" + "testing" + "time" + + authpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/auth" + relayerpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/relayer" + "github.com/gagliardetto/solana-go" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/types/known/timestamppb" +) + +// mockAuth is an AuthService that checks the handshake the way the block +// engine does and issues numbered tokens. +type mockAuth struct { + authpb.UnimplementedAuthServiceServer + now func() time.Time + challenge string + accessTTL, refreshTTL time.Duration + // tokens, if set, replaces the GenerateAuthTokens response. + tokens func() *authpb.GenerateAuthTokensResponse + // refreshGate, if set, holds RefreshAccessToken until it is closed. + refreshGate chan struct{} + + mu sync.Mutex + refreshErr error + issued int + calls map[string]int + authHeaders []string +} + +func newMockAuth(now func() time.Time) *mockAuth { + return &mockAuth{now: now, challenge: "abcdefghi", accessTTL: 10 * time.Minute, refreshTTL: time.Hour, calls: map[string]int{}} +} + +func (m *mockAuth) record(ctx context.Context, method string) { + m.mu.Lock() + defer m.mu.Unlock() + m.calls[method]++ + md, _ := metadata.FromIncomingContext(ctx) + m.authHeaders = append(m.authHeaders, md.Get("authorization")...) +} + +func (m *mockAuth) callCount(method string) int { + m.mu.Lock() + defer m.mu.Unlock() + return m.calls[method] +} + +func (m *mockAuth) token(prefix string, ttl time.Duration) *authpb.Token { + m.issued++ + return &authpb.Token{Value: prefix + "-" + string(rune('0'+m.issued)), ExpiresAtUtc: timestamppb.New(m.now().Add(ttl))} +} + +func (m *mockAuth) GenerateAuthChallenge(ctx context.Context, req *authpb.GenerateAuthChallengeRequest) (*authpb.GenerateAuthChallengeResponse, error) { + m.record(ctx, "challenge") + if req.GetRole() != authpb.Role_VALIDATOR || len(req.GetPubkey()) != ed25519.PublicKeySize { + return nil, status.Error(codes.InvalidArgument, "bad challenge request") + } + return &authpb.GenerateAuthChallengeResponse{Challenge: m.challenge}, nil +} + +func (m *mockAuth) GenerateAuthTokens(ctx context.Context, req *authpb.GenerateAuthTokensRequest) (*authpb.GenerateAuthTokensResponse, error) { + m.record(ctx, "tokens") + want := solana.PublicKeyFromBytes(req.GetClientPubkey()).String() + "-" + m.challenge + if req.GetChallenge() != want || !ed25519.Verify(req.GetClientPubkey(), []byte(req.GetChallenge()), req.GetSignedChallenge()) { + return nil, status.Error(codes.Unauthenticated, "bad signature") + } + if m.tokens != nil { + return m.tokens(), nil + } + m.mu.Lock() + defer m.mu.Unlock() + return &authpb.GenerateAuthTokensResponse{ + AccessToken: m.token("access", m.accessTTL), + RefreshToken: m.token("refresh", m.refreshTTL), + }, nil +} + +func (m *mockAuth) RefreshAccessToken(ctx context.Context, req *authpb.RefreshAccessTokenRequest) (*authpb.RefreshAccessTokenResponse, error) { + m.record(ctx, "refresh") + if m.refreshGate != nil { + <-m.refreshGate + } + m.mu.Lock() + defer m.mu.Unlock() + if m.refreshErr != nil { + return nil, m.refreshErr + } + return &authpb.RefreshAccessTokenResponse{AccessToken: m.token("access", m.accessTTL)}, nil +} + +// mockRelayer records the metadata of each GetTpuConfigs call. +type mockRelayer struct { + relayerpb.UnimplementedRelayerServer + mu sync.Mutex + md []metadata.MD +} + +func (r *mockRelayer) GetTpuConfigs(ctx context.Context, _ *relayerpb.GetTpuConfigsRequest) (*relayerpb.GetTpuConfigsResponse, error) { + md, _ := metadata.FromIncomingContext(ctx) + r.mu.Lock() + r.md = append(r.md, md) + r.mu.Unlock() + return &relayerpb.GetTpuConfigsResponse{}, nil +} + +func (r *mockRelayer) last() metadata.MD { + r.mu.Lock() + defer r.mu.Unlock() + return r.md[len(r.md)-1] +} + +// serveBufconn starts a gRPC server on an in-memory listener and returns a +// dial config that reaches it. +func serveBufconn(t *testing.T, register func(*grpc.Server)) DialConfig { + t.Helper() + lis := bufconn.Listen(1 << 20) + srv := grpc.NewServer() + register(srv) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + return DialConfig{ + Endpoint: Endpoint{Target: "bufnet"}, + Dialer: func(ctx context.Context, _ string) (net.Conn, error) { return lis.DialContext(ctx) }, + } +} diff --git a/pkg/harmonic/proto/auth/auth.pb.go b/pkg/harmonic/proto/auth/auth.pb.go new file mode 100644 index 00000000..659902a9 --- /dev/null +++ b/pkg/harmonic/proto/auth/auth.pb.go @@ -0,0 +1,540 @@ +// Authentication service used by the Harmonic validator to obtain access tokens +// for the block engine and relayer. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.21.12 +// source: auth.proto + +package auth + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Role the client is authenticating as. +type Role int32 + +const ( + Role_ROLE_UNSPECIFIED Role = 0 + // Harmonic validator role. + Role_VALIDATOR Role = 2 +) + +// Enum value maps for Role. +var ( + Role_name = map[int32]string{ + 0: "ROLE_UNSPECIFIED", + 2: "VALIDATOR", + } + Role_value = map[string]int32{ + "ROLE_UNSPECIFIED": 0, + "VALIDATOR": 2, + } +) + +func (x Role) Enum() *Role { + p := new(Role) + *p = x + return p +} + +func (x Role) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Role) Descriptor() protoreflect.EnumDescriptor { + return file_auth_proto_enumTypes[0].Descriptor() +} + +func (Role) Type() protoreflect.EnumType { + return &file_auth_proto_enumTypes[0] +} + +func (x Role) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Role.Descriptor instead. +func (Role) EnumDescriptor() ([]byte, []int) { + return file_auth_proto_rawDescGZIP(), []int{0} +} + +type GenerateAuthChallengeRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Role the client is attempting to generate tokens for. + Role Role `protobuf:"varint,1,opt,name=role,proto3,enum=auth.Role" json:"role,omitempty"` + // Validator's 32-byte public key. + Pubkey []byte `protobuf:"bytes,2,opt,name=pubkey,proto3" json:"pubkey,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GenerateAuthChallengeRequest) Reset() { + *x = GenerateAuthChallengeRequest{} + mi := &file_auth_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GenerateAuthChallengeRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GenerateAuthChallengeRequest) ProtoMessage() {} + +func (x *GenerateAuthChallengeRequest) ProtoReflect() protoreflect.Message { + mi := &file_auth_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GenerateAuthChallengeRequest.ProtoReflect.Descriptor instead. +func (*GenerateAuthChallengeRequest) Descriptor() ([]byte, []int) { + return file_auth_proto_rawDescGZIP(), []int{0} +} + +func (x *GenerateAuthChallengeRequest) GetRole() Role { + if x != nil { + return x.Role + } + return Role_ROLE_UNSPECIFIED +} + +func (x *GenerateAuthChallengeRequest) GetPubkey() []byte { + if x != nil { + return x.Pubkey + } + return nil +} + +type GenerateAuthChallengeResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Challenge string to be signed by the validator. + Challenge string `protobuf:"bytes,1,opt,name=challenge,proto3" json:"challenge,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GenerateAuthChallengeResponse) Reset() { + *x = GenerateAuthChallengeResponse{} + mi := &file_auth_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GenerateAuthChallengeResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GenerateAuthChallengeResponse) ProtoMessage() {} + +func (x *GenerateAuthChallengeResponse) ProtoReflect() protoreflect.Message { + mi := &file_auth_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GenerateAuthChallengeResponse.ProtoReflect.Descriptor instead. +func (*GenerateAuthChallengeResponse) Descriptor() ([]byte, []int) { + return file_auth_proto_rawDescGZIP(), []int{1} +} + +func (x *GenerateAuthChallengeResponse) GetChallenge() string { + if x != nil { + return x.Challenge + } + return "" +} + +type GenerateAuthTokensRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The challenge string returned by GenerateAuthChallenge. + Challenge string `protobuf:"bytes,1,opt,name=challenge,proto3" json:"challenge,omitempty"` + // Validator's 32-byte public key (must match the pubkey from GenerateAuthChallenge). + ClientPubkey []byte `protobuf:"bytes,2,opt,name=client_pubkey,json=clientPubkey,proto3" json:"client_pubkey,omitempty"` + // 64-byte signature of the challenge signed by the validator's private key. + // The validator is expected to sign the challenge prepended with its pubkey: + // sign(pubkey, challenge). + SignedChallenge []byte `protobuf:"bytes,3,opt,name=signed_challenge,json=signedChallenge,proto3" json:"signed_challenge,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GenerateAuthTokensRequest) Reset() { + *x = GenerateAuthTokensRequest{} + mi := &file_auth_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GenerateAuthTokensRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GenerateAuthTokensRequest) ProtoMessage() {} + +func (x *GenerateAuthTokensRequest) ProtoReflect() protoreflect.Message { + mi := &file_auth_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GenerateAuthTokensRequest.ProtoReflect.Descriptor instead. +func (*GenerateAuthTokensRequest) Descriptor() ([]byte, []int) { + return file_auth_proto_rawDescGZIP(), []int{2} +} + +func (x *GenerateAuthTokensRequest) GetChallenge() string { + if x != nil { + return x.Challenge + } + return "" +} + +func (x *GenerateAuthTokensRequest) GetClientPubkey() []byte { + if x != nil { + return x.ClientPubkey + } + return nil +} + +func (x *GenerateAuthTokensRequest) GetSignedChallenge() []byte { + if x != nil { + return x.SignedChallenge + } + return nil +} + +// An auth token with an expiration timestamp. +type Token struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The token string, used as a Bearer token in the Authorization header. + Value string `protobuf:"bytes,1,opt,name=value,proto3" json:"value,omitempty"` + // When this token expires. + ExpiresAtUtc *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=expires_at_utc,json=expiresAtUtc,proto3" json:"expires_at_utc,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Token) Reset() { + *x = Token{} + mi := &file_auth_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Token) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Token) ProtoMessage() {} + +func (x *Token) ProtoReflect() protoreflect.Message { + mi := &file_auth_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Token.ProtoReflect.Descriptor instead. +func (*Token) Descriptor() ([]byte, []int) { + return file_auth_proto_rawDescGZIP(), []int{3} +} + +func (x *Token) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +func (x *Token) GetExpiresAtUtc() *timestamppb.Timestamp { + if x != nil { + return x.ExpiresAtUtc + } + return nil +} + +type GenerateAuthTokensResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Short-lived token used in the Authorization header for API calls. + AccessToken *Token `protobuf:"bytes,1,opt,name=access_token,json=accessToken,proto3" json:"access_token,omitempty"` + // Longer-lived token used to obtain a fresh access_token via RefreshAccessToken. + RefreshToken *Token `protobuf:"bytes,2,opt,name=refresh_token,json=refreshToken,proto3" json:"refresh_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GenerateAuthTokensResponse) Reset() { + *x = GenerateAuthTokensResponse{} + mi := &file_auth_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GenerateAuthTokensResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GenerateAuthTokensResponse) ProtoMessage() {} + +func (x *GenerateAuthTokensResponse) ProtoReflect() protoreflect.Message { + mi := &file_auth_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GenerateAuthTokensResponse.ProtoReflect.Descriptor instead. +func (*GenerateAuthTokensResponse) Descriptor() ([]byte, []int) { + return file_auth_proto_rawDescGZIP(), []int{4} +} + +func (x *GenerateAuthTokensResponse) GetAccessToken() *Token { + if x != nil { + return x.AccessToken + } + return nil +} + +func (x *GenerateAuthTokensResponse) GetRefreshToken() *Token { + if x != nil { + return x.RefreshToken + } + return nil +} + +type RefreshAccessTokenRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Non-expired refresh token obtained from GenerateAuthTokens. + RefreshToken string `protobuf:"bytes,1,opt,name=refresh_token,json=refreshToken,proto3" json:"refresh_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RefreshAccessTokenRequest) Reset() { + *x = RefreshAccessTokenRequest{} + mi := &file_auth_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RefreshAccessTokenRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RefreshAccessTokenRequest) ProtoMessage() {} + +func (x *RefreshAccessTokenRequest) ProtoReflect() protoreflect.Message { + mi := &file_auth_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RefreshAccessTokenRequest.ProtoReflect.Descriptor instead. +func (*RefreshAccessTokenRequest) Descriptor() ([]byte, []int) { + return file_auth_proto_rawDescGZIP(), []int{5} +} + +func (x *RefreshAccessTokenRequest) GetRefreshToken() string { + if x != nil { + return x.RefreshToken + } + return "" +} + +type RefreshAccessTokenResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Freshly issued access token. + AccessToken *Token `protobuf:"bytes,1,opt,name=access_token,json=accessToken,proto3" json:"access_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *RefreshAccessTokenResponse) Reset() { + *x = RefreshAccessTokenResponse{} + mi := &file_auth_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *RefreshAccessTokenResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*RefreshAccessTokenResponse) ProtoMessage() {} + +func (x *RefreshAccessTokenResponse) ProtoReflect() protoreflect.Message { + mi := &file_auth_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use RefreshAccessTokenResponse.ProtoReflect.Descriptor instead. +func (*RefreshAccessTokenResponse) Descriptor() ([]byte, []int) { + return file_auth_proto_rawDescGZIP(), []int{6} +} + +func (x *RefreshAccessTokenResponse) GetAccessToken() *Token { + if x != nil { + return x.AccessToken + } + return nil +} + +var File_auth_proto protoreflect.FileDescriptor + +const file_auth_proto_rawDesc = "" + + "\n" + + "\n" + + "auth.proto\x12\x04auth\x1a\x1fgoogle/protobuf/timestamp.proto\"V\n" + + "\x1cGenerateAuthChallengeRequest\x12\x1e\n" + + "\x04role\x18\x01 \x01(\x0e2\n" + + ".auth.RoleR\x04role\x12\x16\n" + + "\x06pubkey\x18\x02 \x01(\fR\x06pubkey\"=\n" + + "\x1dGenerateAuthChallengeResponse\x12\x1c\n" + + "\tchallenge\x18\x01 \x01(\tR\tchallenge\"\x89\x01\n" + + "\x19GenerateAuthTokensRequest\x12\x1c\n" + + "\tchallenge\x18\x01 \x01(\tR\tchallenge\x12#\n" + + "\rclient_pubkey\x18\x02 \x01(\fR\fclientPubkey\x12)\n" + + "\x10signed_challenge\x18\x03 \x01(\fR\x0fsignedChallenge\"_\n" + + "\x05Token\x12\x14\n" + + "\x05value\x18\x01 \x01(\tR\x05value\x12@\n" + + "\x0eexpires_at_utc\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\fexpiresAtUtc\"~\n" + + "\x1aGenerateAuthTokensResponse\x12.\n" + + "\faccess_token\x18\x01 \x01(\v2\v.auth.TokenR\vaccessToken\x120\n" + + "\rrefresh_token\x18\x02 \x01(\v2\v.auth.TokenR\frefreshToken\"@\n" + + "\x19RefreshAccessTokenRequest\x12#\n" + + "\rrefresh_token\x18\x01 \x01(\tR\frefreshToken\"L\n" + + "\x1aRefreshAccessTokenResponse\x12.\n" + + "\faccess_token\x18\x01 \x01(\v2\v.auth.TokenR\vaccessToken*+\n" + + "\x04Role\x12\x14\n" + + "\x10ROLE_UNSPECIFIED\x10\x00\x12\r\n" + + "\tVALIDATOR\x10\x022\xa7\x02\n" + + "\vAuthService\x12b\n" + + "\x15GenerateAuthChallenge\x12\".auth.GenerateAuthChallengeRequest\x1a#.auth.GenerateAuthChallengeResponse\"\x00\x12Y\n" + + "\x12GenerateAuthTokens\x12\x1f.auth.GenerateAuthTokensRequest\x1a .auth.GenerateAuthTokensResponse\"\x00\x12Y\n" + + "\x12RefreshAccessToken\x12\x1f.auth.RefreshAccessTokenRequest\x1a .auth.RefreshAccessTokenResponse\"\x00b\x06proto3" + +var ( + file_auth_proto_rawDescOnce sync.Once + file_auth_proto_rawDescData []byte +) + +func file_auth_proto_rawDescGZIP() []byte { + file_auth_proto_rawDescOnce.Do(func() { + file_auth_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_auth_proto_rawDesc), len(file_auth_proto_rawDesc))) + }) + return file_auth_proto_rawDescData +} + +var file_auth_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_auth_proto_msgTypes = make([]protoimpl.MessageInfo, 7) +var file_auth_proto_goTypes = []any{ + (Role)(0), // 0: auth.Role + (*GenerateAuthChallengeRequest)(nil), // 1: auth.GenerateAuthChallengeRequest + (*GenerateAuthChallengeResponse)(nil), // 2: auth.GenerateAuthChallengeResponse + (*GenerateAuthTokensRequest)(nil), // 3: auth.GenerateAuthTokensRequest + (*Token)(nil), // 4: auth.Token + (*GenerateAuthTokensResponse)(nil), // 5: auth.GenerateAuthTokensResponse + (*RefreshAccessTokenRequest)(nil), // 6: auth.RefreshAccessTokenRequest + (*RefreshAccessTokenResponse)(nil), // 7: auth.RefreshAccessTokenResponse + (*timestamppb.Timestamp)(nil), // 8: google.protobuf.Timestamp +} +var file_auth_proto_depIdxs = []int32{ + 0, // 0: auth.GenerateAuthChallengeRequest.role:type_name -> auth.Role + 8, // 1: auth.Token.expires_at_utc:type_name -> google.protobuf.Timestamp + 4, // 2: auth.GenerateAuthTokensResponse.access_token:type_name -> auth.Token + 4, // 3: auth.GenerateAuthTokensResponse.refresh_token:type_name -> auth.Token + 4, // 4: auth.RefreshAccessTokenResponse.access_token:type_name -> auth.Token + 1, // 5: auth.AuthService.GenerateAuthChallenge:input_type -> auth.GenerateAuthChallengeRequest + 3, // 6: auth.AuthService.GenerateAuthTokens:input_type -> auth.GenerateAuthTokensRequest + 6, // 7: auth.AuthService.RefreshAccessToken:input_type -> auth.RefreshAccessTokenRequest + 2, // 8: auth.AuthService.GenerateAuthChallenge:output_type -> auth.GenerateAuthChallengeResponse + 5, // 9: auth.AuthService.GenerateAuthTokens:output_type -> auth.GenerateAuthTokensResponse + 7, // 10: auth.AuthService.RefreshAccessToken:output_type -> auth.RefreshAccessTokenResponse + 8, // [8:11] is the sub-list for method output_type + 5, // [5:8] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_auth_proto_init() } +func file_auth_proto_init() { + if File_auth_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_auth_proto_rawDesc), len(file_auth_proto_rawDesc)), + NumEnums: 1, + NumMessages: 7, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_auth_proto_goTypes, + DependencyIndexes: file_auth_proto_depIdxs, + EnumInfos: file_auth_proto_enumTypes, + MessageInfos: file_auth_proto_msgTypes, + }.Build() + File_auth_proto = out.File + file_auth_proto_goTypes = nil + file_auth_proto_depIdxs = nil +} diff --git a/pkg/harmonic/proto/auth/auth_grpc.pb.go b/pkg/harmonic/proto/auth/auth_grpc.pb.go new file mode 100644 index 00000000..ea999e3a --- /dev/null +++ b/pkg/harmonic/proto/auth/auth_grpc.pb.go @@ -0,0 +1,210 @@ +// Authentication service used by the Harmonic validator to obtain access tokens +// for the block engine and relayer. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc v3.21.12 +// source: auth.proto + +package auth + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + AuthService_GenerateAuthChallenge_FullMethodName = "/auth.AuthService/GenerateAuthChallenge" + AuthService_GenerateAuthTokens_FullMethodName = "/auth.AuthService/GenerateAuthTokens" + AuthService_RefreshAccessToken_FullMethodName = "/auth.AuthService/RefreshAccessToken" +) + +// AuthServiceClient is the client API for AuthService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Issues auth tokens to the Harmonic validator for API access. +type AuthServiceClient interface { + // Returns a challenge; the validator must sign it to prove key ownership. + GenerateAuthChallenge(ctx context.Context, in *GenerateAuthChallengeRequest, opts ...grpc.CallOption) (*GenerateAuthChallengeResponse, error) + // Exchanges a signed challenge for an access/refresh token pair. + GenerateAuthTokens(ctx context.Context, in *GenerateAuthTokensRequest, opts ...grpc.CallOption) (*GenerateAuthTokensResponse, error) + // Refreshes an access token using a non-expired refresh token. + RefreshAccessToken(ctx context.Context, in *RefreshAccessTokenRequest, opts ...grpc.CallOption) (*RefreshAccessTokenResponse, error) +} + +type authServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewAuthServiceClient(cc grpc.ClientConnInterface) AuthServiceClient { + return &authServiceClient{cc} +} + +func (c *authServiceClient) GenerateAuthChallenge(ctx context.Context, in *GenerateAuthChallengeRequest, opts ...grpc.CallOption) (*GenerateAuthChallengeResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GenerateAuthChallengeResponse) + err := c.cc.Invoke(ctx, AuthService_GenerateAuthChallenge_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *authServiceClient) GenerateAuthTokens(ctx context.Context, in *GenerateAuthTokensRequest, opts ...grpc.CallOption) (*GenerateAuthTokensResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GenerateAuthTokensResponse) + err := c.cc.Invoke(ctx, AuthService_GenerateAuthTokens_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *authServiceClient) RefreshAccessToken(ctx context.Context, in *RefreshAccessTokenRequest, opts ...grpc.CallOption) (*RefreshAccessTokenResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(RefreshAccessTokenResponse) + err := c.cc.Invoke(ctx, AuthService_RefreshAccessToken_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// AuthServiceServer is the server API for AuthService service. +// All implementations must embed UnimplementedAuthServiceServer +// for forward compatibility. +// +// Issues auth tokens to the Harmonic validator for API access. +type AuthServiceServer interface { + // Returns a challenge; the validator must sign it to prove key ownership. + GenerateAuthChallenge(context.Context, *GenerateAuthChallengeRequest) (*GenerateAuthChallengeResponse, error) + // Exchanges a signed challenge for an access/refresh token pair. + GenerateAuthTokens(context.Context, *GenerateAuthTokensRequest) (*GenerateAuthTokensResponse, error) + // Refreshes an access token using a non-expired refresh token. + RefreshAccessToken(context.Context, *RefreshAccessTokenRequest) (*RefreshAccessTokenResponse, error) + mustEmbedUnimplementedAuthServiceServer() +} + +// UnimplementedAuthServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedAuthServiceServer struct{} + +func (UnimplementedAuthServiceServer) GenerateAuthChallenge(context.Context, *GenerateAuthChallengeRequest) (*GenerateAuthChallengeResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GenerateAuthChallenge not implemented") +} +func (UnimplementedAuthServiceServer) GenerateAuthTokens(context.Context, *GenerateAuthTokensRequest) (*GenerateAuthTokensResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GenerateAuthTokens not implemented") +} +func (UnimplementedAuthServiceServer) RefreshAccessToken(context.Context, *RefreshAccessTokenRequest) (*RefreshAccessTokenResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method RefreshAccessToken not implemented") +} +func (UnimplementedAuthServiceServer) mustEmbedUnimplementedAuthServiceServer() {} +func (UnimplementedAuthServiceServer) testEmbeddedByValue() {} + +// UnsafeAuthServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to AuthServiceServer will +// result in compilation errors. +type UnsafeAuthServiceServer interface { + mustEmbedUnimplementedAuthServiceServer() +} + +func RegisterAuthServiceServer(s grpc.ServiceRegistrar, srv AuthServiceServer) { + // If the following call pancis, it indicates UnimplementedAuthServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&AuthService_ServiceDesc, srv) +} + +func _AuthService_GenerateAuthChallenge_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GenerateAuthChallengeRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AuthServiceServer).GenerateAuthChallenge(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AuthService_GenerateAuthChallenge_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AuthServiceServer).GenerateAuthChallenge(ctx, req.(*GenerateAuthChallengeRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AuthService_GenerateAuthTokens_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GenerateAuthTokensRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AuthServiceServer).GenerateAuthTokens(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AuthService_GenerateAuthTokens_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AuthServiceServer).GenerateAuthTokens(ctx, req.(*GenerateAuthTokensRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _AuthService_RefreshAccessToken_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(RefreshAccessTokenRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(AuthServiceServer).RefreshAccessToken(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: AuthService_RefreshAccessToken_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(AuthServiceServer).RefreshAccessToken(ctx, req.(*RefreshAccessTokenRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// AuthService_ServiceDesc is the grpc.ServiceDesc for AuthService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var AuthService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "auth.AuthService", + HandlerType: (*AuthServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GenerateAuthChallenge", + Handler: _AuthService_GenerateAuthChallenge_Handler, + }, + { + MethodName: "GenerateAuthTokens", + Handler: _AuthService_GenerateAuthTokens_Handler, + }, + { + MethodName: "RefreshAccessToken", + Handler: _AuthService_RefreshAccessToken_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "auth.proto", +} diff --git a/pkg/harmonic/proto/block/block.pb.go b/pkg/harmonic/proto/block/block.pb.go new file mode 100644 index 00000000..d3764ba6 --- /dev/null +++ b/pkg/harmonic/proto/block/block.pb.go @@ -0,0 +1,197 @@ +// Harmonic block stream received by the validator from the block engine. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.21.12 +// source: block.proto + +package block + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// An ordered list of transactions targeting a given slot. +type Block struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Slot target for these transactions. + Slot uint64 `protobuf:"varint,1,opt,name=slot,proto3" json:"slot,omitempty"` // Field numbers 2-14 left free for future block-level metadata. + // Transactions in execution order. + Transactions []*Transaction `protobuf:"bytes,15,rep,name=transactions,proto3" json:"transactions,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Block) Reset() { + *x = Block{} + mi := &file_block_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Block) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Block) ProtoMessage() {} + +func (x *Block) ProtoReflect() protoreflect.Message { + mi := &file_block_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Block.ProtoReflect.Descriptor instead. +func (*Block) Descriptor() ([]byte, []int) { + return file_block_proto_rawDescGZIP(), []int{0} +} + +func (x *Block) GetSlot() uint64 { + if x != nil { + return x.Slot + } + return 0 +} + +func (x *Block) GetTransactions() []*Transaction { + if x != nil { + return x.Transactions + } + return nil +} + +// A single transaction; field numbers match builder-protos `block.Transaction`. +type Transaction struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Raw transaction bytes. + Transaction []byte `protobuf:"bytes,1,opt,name=transaction,proto3" json:"transaction,omitempty"` + // Bundle uuid; 0 == standalone transaction, nonzero == revert protected bundle. + // Consecutive transactions sharing the same bundle_id form one bundle, of any length. + BundleId uint64 `protobuf:"varint,3,opt,name=bundle_id,json=bundleId,proto3" json:"bundle_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Transaction) Reset() { + *x = Transaction{} + mi := &file_block_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Transaction) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Transaction) ProtoMessage() {} + +func (x *Transaction) ProtoReflect() protoreflect.Message { + mi := &file_block_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Transaction.ProtoReflect.Descriptor instead. +func (*Transaction) Descriptor() ([]byte, []int) { + return file_block_proto_rawDescGZIP(), []int{1} +} + +func (x *Transaction) GetTransaction() []byte { + if x != nil { + return x.Transaction + } + return nil +} + +func (x *Transaction) GetBundleId() uint64 { + if x != nil { + return x.BundleId + } + return 0 +} + +var File_block_proto protoreflect.FileDescriptor + +const file_block_proto_rawDesc = "" + + "\n" + + "\vblock.proto\x12\x05block\"S\n" + + "\x05Block\x12\x12\n" + + "\x04slot\x18\x01 \x01(\x04R\x04slot\x126\n" + + "\ftransactions\x18\x0f \x03(\v2\x12.block.TransactionR\ftransactions\"X\n" + + "\vTransaction\x12 \n" + + "\vtransaction\x18\x01 \x01(\fR\vtransaction\x12\x1b\n" + + "\tbundle_id\x18\x03 \x01(\x04R\bbundleIdJ\x04\b\x02\x10\x03J\x04\b\x04\x10\x05b\x06proto3" + +var ( + file_block_proto_rawDescOnce sync.Once + file_block_proto_rawDescData []byte +) + +func file_block_proto_rawDescGZIP() []byte { + file_block_proto_rawDescOnce.Do(func() { + file_block_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_block_proto_rawDesc), len(file_block_proto_rawDesc))) + }) + return file_block_proto_rawDescData +} + +var file_block_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_block_proto_goTypes = []any{ + (*Block)(nil), // 0: block.Block + (*Transaction)(nil), // 1: block.Transaction +} +var file_block_proto_depIdxs = []int32{ + 1, // 0: block.Block.transactions:type_name -> block.Transaction + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_block_proto_init() } +func file_block_proto_init() { + if File_block_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_block_proto_rawDesc), len(file_block_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_block_proto_goTypes, + DependencyIndexes: file_block_proto_depIdxs, + MessageInfos: file_block_proto_msgTypes, + }.Build() + File_block_proto = out.File + file_block_proto_goTypes = nil + file_block_proto_depIdxs = nil +} diff --git a/pkg/harmonic/proto/blockengine/block_engine.pb.go b/pkg/harmonic/proto/blockengine/block_engine.pb.go new file mode 100644 index 00000000..c8b14761 --- /dev/null +++ b/pkg/harmonic/proto/blockengine/block_engine.pb.go @@ -0,0 +1,881 @@ +// Block engine interface for the Harmonic validator. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.21.12 +// source: block_engine.proto + +package blockengine + +import ( + block "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/block" + bundle "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/bundle" + packet "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/packet" + shared "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/shared" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Scheduling strategy the block engine runs for this validator session. +type SchedulingStrategy int32 + +const ( + SchedulingStrategy_SCHEDULING_STRATEGY_UNSPECIFIED SchedulingStrategy = 0 + // First-in, first-out: continuous arrival-order sequencing. + SchedulingStrategy_SCHEDULING_STRATEGY_FIFO SchedulingStrategy = 1 + // Frequent batch auction: discrete batches ordered by priority fees and tips. + SchedulingStrategy_SCHEDULING_STRATEGY_FBA SchedulingStrategy = 2 + // Maximum revenue strategy: continuous streaming with revenue-prioritizing selection. + SchedulingStrategy_SCHEDULING_STRATEGY_MREV SchedulingStrategy = 3 +) + +// Enum value maps for SchedulingStrategy. +var ( + SchedulingStrategy_name = map[int32]string{ + 0: "SCHEDULING_STRATEGY_UNSPECIFIED", + 1: "SCHEDULING_STRATEGY_FIFO", + 2: "SCHEDULING_STRATEGY_FBA", + 3: "SCHEDULING_STRATEGY_MREV", + } + SchedulingStrategy_value = map[string]int32{ + "SCHEDULING_STRATEGY_UNSPECIFIED": 0, + "SCHEDULING_STRATEGY_FIFO": 1, + "SCHEDULING_STRATEGY_FBA": 2, + "SCHEDULING_STRATEGY_MREV": 3, + } +) + +func (x SchedulingStrategy) Enum() *SchedulingStrategy { + p := new(SchedulingStrategy) + *p = x + return p +} + +func (x SchedulingStrategy) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SchedulingStrategy) Descriptor() protoreflect.EnumDescriptor { + return file_block_engine_proto_enumTypes[0].Descriptor() +} + +func (SchedulingStrategy) Type() protoreflect.EnumType { + return &file_block_engine_proto_enumTypes[0] +} + +func (x SchedulingStrategy) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use SchedulingStrategy.Descriptor instead. +func (SchedulingStrategy) EnumDescriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{0} +} + +// Request to subscribe to the block engine's packet stream. +type SubscribePacketsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubscribePacketsRequest) Reset() { + *x = SubscribePacketsRequest{} + mi := &file_block_engine_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubscribePacketsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubscribePacketsRequest) ProtoMessage() {} + +func (x *SubscribePacketsRequest) ProtoReflect() protoreflect.Message { + mi := &file_block_engine_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubscribePacketsRequest.ProtoReflect.Descriptor instead. +func (*SubscribePacketsRequest) Descriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{0} +} + +// A batch of packets forwarded by the block engine. +type SubscribePacketsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Timestamp of when the batch was sent. + Header *shared.Header `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` + // Transaction packet batch. + Batch *packet.PacketBatch `protobuf:"bytes,2,opt,name=batch,proto3" json:"batch,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubscribePacketsResponse) Reset() { + *x = SubscribePacketsResponse{} + mi := &file_block_engine_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubscribePacketsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubscribePacketsResponse) ProtoMessage() {} + +func (x *SubscribePacketsResponse) ProtoReflect() protoreflect.Message { + mi := &file_block_engine_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubscribePacketsResponse.ProtoReflect.Descriptor instead. +func (*SubscribePacketsResponse) Descriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{1} +} + +func (x *SubscribePacketsResponse) GetHeader() *shared.Header { + if x != nil { + return x.Header + } + return nil +} + +func (x *SubscribePacketsResponse) GetBatch() *packet.PacketBatch { + if x != nil { + return x.Batch + } + return nil +} + +// Request to subscribe to the block engine's bundle stream. +type SubscribeBundlesRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubscribeBundlesRequest) Reset() { + *x = SubscribeBundlesRequest{} + mi := &file_block_engine_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubscribeBundlesRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubscribeBundlesRequest) ProtoMessage() {} + +func (x *SubscribeBundlesRequest) ProtoReflect() protoreflect.Message { + mi := &file_block_engine_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubscribeBundlesRequest.ProtoReflect.Descriptor instead. +func (*SubscribeBundlesRequest) Descriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{2} +} + +// Request to subscribe to the block engine's block stream. +type SubscribeBlocksRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"` + CommitHash string `protobuf:"bytes,2,opt,name=commit_hash,json=commitHash,proto3" json:"commit_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubscribeBlocksRequest) Reset() { + *x = SubscribeBlocksRequest{} + mi := &file_block_engine_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubscribeBlocksRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubscribeBlocksRequest) ProtoMessage() {} + +func (x *SubscribeBlocksRequest) ProtoReflect() protoreflect.Message { + mi := &file_block_engine_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubscribeBlocksRequest.ProtoReflect.Descriptor instead. +func (*SubscribeBlocksRequest) Descriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{3} +} + +func (x *SubscribeBlocksRequest) GetVersion() string { + if x != nil { + return x.Version + } + return "" +} + +func (x *SubscribeBlocksRequest) GetCommitHash() string { + if x != nil { + return x.CommitHash + } + return "" +} + +// One or more bundles forwarded by the block engine. +type SubscribeBundlesResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Bundles with their server-assigned UUIDs. + Bundles []*bundle.BundleUuid `protobuf:"bytes,1,rep,name=bundles,proto3" json:"bundles,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubscribeBundlesResponse) Reset() { + *x = SubscribeBundlesResponse{} + mi := &file_block_engine_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubscribeBundlesResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubscribeBundlesResponse) ProtoMessage() {} + +func (x *SubscribeBundlesResponse) ProtoReflect() protoreflect.Message { + mi := &file_block_engine_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubscribeBundlesResponse.ProtoReflect.Descriptor instead. +func (*SubscribeBundlesResponse) Descriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{4} +} + +func (x *SubscribeBundlesResponse) GetBundles() []*bundle.BundleUuid { + if x != nil { + return x.Bundles + } + return nil +} + +// Request for the block builder's fee configuration. +type BlockBuilderFeeInfoRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BlockBuilderFeeInfoRequest) Reset() { + *x = BlockBuilderFeeInfoRequest{} + mi := &file_block_engine_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BlockBuilderFeeInfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BlockBuilderFeeInfoRequest) ProtoMessage() {} + +func (x *BlockBuilderFeeInfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_block_engine_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BlockBuilderFeeInfoRequest.ProtoReflect.Descriptor instead. +func (*BlockBuilderFeeInfoRequest) Descriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{5} +} + +// Fee configuration for the block builder. +type BlockBuilderFeeInfoResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Block builder's tip-receiving pubkey. + Pubkey string `protobuf:"bytes,1,opt,name=pubkey,proto3" json:"pubkey,omitempty"` + // Tip commission percentage (0-100). + Commission uint64 `protobuf:"varint,2,opt,name=commission,proto3" json:"commission,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BlockBuilderFeeInfoResponse) Reset() { + *x = BlockBuilderFeeInfoResponse{} + mi := &file_block_engine_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BlockBuilderFeeInfoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BlockBuilderFeeInfoResponse) ProtoMessage() {} + +func (x *BlockBuilderFeeInfoResponse) ProtoReflect() protoreflect.Message { + mi := &file_block_engine_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BlockBuilderFeeInfoResponse.ProtoReflect.Descriptor instead. +func (*BlockBuilderFeeInfoResponse) Descriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{6} +} + +func (x *BlockBuilderFeeInfoResponse) GetPubkey() string { + if x != nil { + return x.Pubkey + } + return "" +} + +func (x *BlockBuilderFeeInfoResponse) GetCommission() uint64 { + if x != nil { + return x.Commission + } + return 0 +} + +// Notifies the block engine of the validator's upcoming leader slot. +type SubmitLeaderWindowInfoRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // When the leader request is sent. + StartTimestamp *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=start_timestamp,json=startTimestamp,proto3" json:"start_timestamp,omitempty"` + // The leader slot number. + Slot uint64 `protobuf:"varint,2,opt,name=slot,proto3" json:"slot,omitempty"` + // When the leader slot ends. + EndTimestamp *timestamppb.Timestamp `protobuf:"bytes,3,opt,name=end_timestamp,json=endTimestamp,proto3" json:"end_timestamp,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubmitLeaderWindowInfoRequest) Reset() { + *x = SubmitLeaderWindowInfoRequest{} + mi := &file_block_engine_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubmitLeaderWindowInfoRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubmitLeaderWindowInfoRequest) ProtoMessage() {} + +func (x *SubmitLeaderWindowInfoRequest) ProtoReflect() protoreflect.Message { + mi := &file_block_engine_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubmitLeaderWindowInfoRequest.ProtoReflect.Descriptor instead. +func (*SubmitLeaderWindowInfoRequest) Descriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{7} +} + +func (x *SubmitLeaderWindowInfoRequest) GetStartTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.StartTimestamp + } + return nil +} + +func (x *SubmitLeaderWindowInfoRequest) GetSlot() uint64 { + if x != nil { + return x.Slot + } + return 0 +} + +func (x *SubmitLeaderWindowInfoRequest) GetEndTimestamp() *timestamppb.Timestamp { + if x != nil { + return x.EndTimestamp + } + return nil +} + +type SubmitLeaderWindowInfoResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubmitLeaderWindowInfoResponse) Reset() { + *x = SubmitLeaderWindowInfoResponse{} + mi := &file_block_engine_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubmitLeaderWindowInfoResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubmitLeaderWindowInfoResponse) ProtoMessage() {} + +func (x *SubmitLeaderWindowInfoResponse) ProtoReflect() protoreflect.Message { + mi := &file_block_engine_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubmitLeaderWindowInfoResponse.ProtoReflect.Descriptor instead. +func (*SubmitLeaderWindowInfoResponse) Descriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{8} +} + +// Selects the scheduling strategy for this validator session. +type SetStrategyRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Strategy SchedulingStrategy `protobuf:"varint,1,opt,name=strategy,proto3,enum=block_engine.SchedulingStrategy" json:"strategy,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SetStrategyRequest) Reset() { + *x = SetStrategyRequest{} + mi := &file_block_engine_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SetStrategyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetStrategyRequest) ProtoMessage() {} + +func (x *SetStrategyRequest) ProtoReflect() protoreflect.Message { + mi := &file_block_engine_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetStrategyRequest.ProtoReflect.Descriptor instead. +func (*SetStrategyRequest) Descriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{9} +} + +func (x *SetStrategyRequest) GetStrategy() SchedulingStrategy { + if x != nil { + return x.Strategy + } + return SchedulingStrategy_SCHEDULING_STRATEGY_UNSPECIFIED +} + +type SetStrategyResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SetStrategyResponse) Reset() { + *x = SetStrategyResponse{} + mi := &file_block_engine_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SetStrategyResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetStrategyResponse) ProtoMessage() {} + +func (x *SetStrategyResponse) ProtoReflect() protoreflect.Message { + mi := &file_block_engine_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetStrategyResponse.ProtoReflect.Descriptor instead. +func (*SetStrategyResponse) Descriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{10} +} + +// Request for available block engine endpoints. +type GetBlockEngineEndpointRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetBlockEngineEndpointRequest) Reset() { + *x = GetBlockEngineEndpointRequest{} + mi := &file_block_engine_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetBlockEngineEndpointRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBlockEngineEndpointRequest) ProtoMessage() {} + +func (x *GetBlockEngineEndpointRequest) ProtoReflect() protoreflect.Message { + mi := &file_block_engine_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBlockEngineEndpointRequest.ProtoReflect.Descriptor instead. +func (*GetBlockEngineEndpointRequest) Descriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{11} +} + +// A block engine endpoint. +type BlockEngineEndpoint struct { + state protoimpl.MessageState `protogen:"open.v1"` + // gRPC URL, e.g. https://mainnet.block-engine.jito.wtf + BlockEngineUrl string `protobuf:"bytes,1,opt,name=block_engine_url,json=blockEngineUrl,proto3" json:"block_engine_url,omitempty"` + // ShredStream receiver address (host:port), e.g. shredstream.mainnet.block-engine.jito.wtf:1002 + ShredstreamReceiverAddress string `protobuf:"bytes,2,opt,name=shredstream_receiver_address,json=shredstreamReceiverAddress,proto3" json:"shredstream_receiver_address,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BlockEngineEndpoint) Reset() { + *x = BlockEngineEndpoint{} + mi := &file_block_engine_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BlockEngineEndpoint) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BlockEngineEndpoint) ProtoMessage() {} + +func (x *BlockEngineEndpoint) ProtoReflect() protoreflect.Message { + mi := &file_block_engine_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BlockEngineEndpoint.ProtoReflect.Descriptor instead. +func (*BlockEngineEndpoint) Descriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{12} +} + +func (x *BlockEngineEndpoint) GetBlockEngineUrl() string { + if x != nil { + return x.BlockEngineUrl + } + return "" +} + +func (x *BlockEngineEndpoint) GetShredstreamReceiverAddress() string { + if x != nil { + return x.ShredstreamReceiverAddress + } + return "" +} + +// Available block engine endpoints. +type GetBlockEngineEndpointResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Anycast endpoint that routes to the nearest block engine. + GlobalEndpoint *BlockEngineEndpoint `protobuf:"bytes,1,opt,name=global_endpoint,json=globalEndpoint,proto3" json:"global_endpoint,omitempty"` + // Region-specific endpoints for direct connection. + RegionedEndpoints []*BlockEngineEndpoint `protobuf:"bytes,2,rep,name=regioned_endpoints,json=regionedEndpoints,proto3" json:"regioned_endpoints,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetBlockEngineEndpointResponse) Reset() { + *x = GetBlockEngineEndpointResponse{} + mi := &file_block_engine_proto_msgTypes[13] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetBlockEngineEndpointResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetBlockEngineEndpointResponse) ProtoMessage() {} + +func (x *GetBlockEngineEndpointResponse) ProtoReflect() protoreflect.Message { + mi := &file_block_engine_proto_msgTypes[13] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetBlockEngineEndpointResponse.ProtoReflect.Descriptor instead. +func (*GetBlockEngineEndpointResponse) Descriptor() ([]byte, []int) { + return file_block_engine_proto_rawDescGZIP(), []int{13} +} + +func (x *GetBlockEngineEndpointResponse) GetGlobalEndpoint() *BlockEngineEndpoint { + if x != nil { + return x.GlobalEndpoint + } + return nil +} + +func (x *GetBlockEngineEndpointResponse) GetRegionedEndpoints() []*BlockEngineEndpoint { + if x != nil { + return x.RegionedEndpoints + } + return nil +} + +var File_block_engine_proto protoreflect.FileDescriptor + +const file_block_engine_proto_rawDesc = "" + + "\n" + + "\x12block_engine.proto\x12\fblock_engine\x1a\fpacket.proto\x1a\fshared.proto\x1a\fbundle.proto\x1a\vblock.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x19\n" + + "\x17SubscribePacketsRequest\"m\n" + + "\x18SubscribePacketsResponse\x12&\n" + + "\x06header\x18\x01 \x01(\v2\x0e.shared.HeaderR\x06header\x12)\n" + + "\x05batch\x18\x02 \x01(\v2\x13.packet.PacketBatchR\x05batch\"\x19\n" + + "\x17SubscribeBundlesRequest\"S\n" + + "\x16SubscribeBlocksRequest\x12\x18\n" + + "\aversion\x18\x01 \x01(\tR\aversion\x12\x1f\n" + + "\vcommit_hash\x18\x02 \x01(\tR\n" + + "commitHash\"H\n" + + "\x18SubscribeBundlesResponse\x12,\n" + + "\abundles\x18\x01 \x03(\v2\x12.bundle.BundleUuidR\abundles\"\x1c\n" + + "\x1aBlockBuilderFeeInfoRequest\"U\n" + + "\x1bBlockBuilderFeeInfoResponse\x12\x16\n" + + "\x06pubkey\x18\x01 \x01(\tR\x06pubkey\x12\x1e\n" + + "\n" + + "commission\x18\x02 \x01(\x04R\n" + + "commission\"\xb9\x01\n" + + "\x1dSubmitLeaderWindowInfoRequest\x12C\n" + + "\x0fstart_timestamp\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\x0estartTimestamp\x12\x12\n" + + "\x04slot\x18\x02 \x01(\x04R\x04slot\x12?\n" + + "\rend_timestamp\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\fendTimestamp\" \n" + + "\x1eSubmitLeaderWindowInfoResponse\"R\n" + + "\x12SetStrategyRequest\x12<\n" + + "\bstrategy\x18\x01 \x01(\x0e2 .block_engine.SchedulingStrategyR\bstrategy\"\x15\n" + + "\x13SetStrategyResponse\"\x1f\n" + + "\x1dGetBlockEngineEndpointRequest\"\x81\x01\n" + + "\x13BlockEngineEndpoint\x12(\n" + + "\x10block_engine_url\x18\x01 \x01(\tR\x0eblockEngineUrl\x12@\n" + + "\x1cshredstream_receiver_address\x18\x02 \x01(\tR\x1ashredstreamReceiverAddress\"\xbe\x01\n" + + "\x1eGetBlockEngineEndpointResponse\x12J\n" + + "\x0fglobal_endpoint\x18\x01 \x01(\v2!.block_engine.BlockEngineEndpointR\x0eglobalEndpoint\x12P\n" + + "\x12regioned_endpoints\x18\x02 \x03(\v2!.block_engine.BlockEngineEndpointR\x11regionedEndpoints*\x92\x01\n" + + "\x12SchedulingStrategy\x12#\n" + + "\x1fSCHEDULING_STRATEGY_UNSPECIFIED\x10\x00\x12\x1c\n" + + "\x18SCHEDULING_STRATEGY_FIFO\x10\x01\x12\x1b\n" + + "\x17SCHEDULING_STRATEGY_FBA\x10\x02\x12\x1c\n" + + "\x18SCHEDULING_STRATEGY_MREV\x10\x032\xb3\a\n" + + "\x14BlockEngineValidator\x12e\n" + + "\x10SubscribePackets\x12%.block_engine.SubscribePacketsRequest\x1a&.block_engine.SubscribePacketsResponse\"\x000\x01\x12e\n" + + "\x10SubscribeBundles\x12%.block_engine.SubscribeBundlesRequest\x1a&.block_engine.SubscribeBundlesResponse\"\x000\x01\x12f\n" + + "\x11SubscribeBundles2\x12%.block_engine.SubscribeBundlesRequest\x1a&.block_engine.SubscribeBundlesResponse\"\x000\x01\x12c\n" + + "\x0fSubscribeBlocks\x12$.block_engine.SubscribeBlocksRequest\x1a&.block_engine.SubscribeBundlesResponse\"\x000\x01\x12J\n" + + "\x10SubscribeBlocks2\x12$.block_engine.SubscribeBlocksRequest\x1a\f.block.Block\"\x000\x01\x12o\n" + + "\x16GetBlockBuilderFeeInfo\x12(.block_engine.BlockBuilderFeeInfoRequest\x1a).block_engine.BlockBuilderFeeInfoResponse\"\x00\x12u\n" + + "\x16SubmitLeaderWindowInfo\x12+.block_engine.SubmitLeaderWindowInfoRequest\x1a,.block_engine.SubmitLeaderWindowInfoResponse\"\x00\x12v\n" + + "\x17GetBlockEngineEndpoints\x12+.block_engine.GetBlockEngineEndpointRequest\x1a,.block_engine.GetBlockEngineEndpointResponse\"\x00\x12T\n" + + "\vSetStrategy\x12 .block_engine.SetStrategyRequest\x1a!.block_engine.SetStrategyResponse\"\x00b\x06proto3" + +var ( + file_block_engine_proto_rawDescOnce sync.Once + file_block_engine_proto_rawDescData []byte +) + +func file_block_engine_proto_rawDescGZIP() []byte { + file_block_engine_proto_rawDescOnce.Do(func() { + file_block_engine_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_block_engine_proto_rawDesc), len(file_block_engine_proto_rawDesc))) + }) + return file_block_engine_proto_rawDescData +} + +var file_block_engine_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_block_engine_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_block_engine_proto_goTypes = []any{ + (SchedulingStrategy)(0), // 0: block_engine.SchedulingStrategy + (*SubscribePacketsRequest)(nil), // 1: block_engine.SubscribePacketsRequest + (*SubscribePacketsResponse)(nil), // 2: block_engine.SubscribePacketsResponse + (*SubscribeBundlesRequest)(nil), // 3: block_engine.SubscribeBundlesRequest + (*SubscribeBlocksRequest)(nil), // 4: block_engine.SubscribeBlocksRequest + (*SubscribeBundlesResponse)(nil), // 5: block_engine.SubscribeBundlesResponse + (*BlockBuilderFeeInfoRequest)(nil), // 6: block_engine.BlockBuilderFeeInfoRequest + (*BlockBuilderFeeInfoResponse)(nil), // 7: block_engine.BlockBuilderFeeInfoResponse + (*SubmitLeaderWindowInfoRequest)(nil), // 8: block_engine.SubmitLeaderWindowInfoRequest + (*SubmitLeaderWindowInfoResponse)(nil), // 9: block_engine.SubmitLeaderWindowInfoResponse + (*SetStrategyRequest)(nil), // 10: block_engine.SetStrategyRequest + (*SetStrategyResponse)(nil), // 11: block_engine.SetStrategyResponse + (*GetBlockEngineEndpointRequest)(nil), // 12: block_engine.GetBlockEngineEndpointRequest + (*BlockEngineEndpoint)(nil), // 13: block_engine.BlockEngineEndpoint + (*GetBlockEngineEndpointResponse)(nil), // 14: block_engine.GetBlockEngineEndpointResponse + (*shared.Header)(nil), // 15: shared.Header + (*packet.PacketBatch)(nil), // 16: packet.PacketBatch + (*bundle.BundleUuid)(nil), // 17: bundle.BundleUuid + (*timestamppb.Timestamp)(nil), // 18: google.protobuf.Timestamp + (*block.Block)(nil), // 19: block.Block +} +var file_block_engine_proto_depIdxs = []int32{ + 15, // 0: block_engine.SubscribePacketsResponse.header:type_name -> shared.Header + 16, // 1: block_engine.SubscribePacketsResponse.batch:type_name -> packet.PacketBatch + 17, // 2: block_engine.SubscribeBundlesResponse.bundles:type_name -> bundle.BundleUuid + 18, // 3: block_engine.SubmitLeaderWindowInfoRequest.start_timestamp:type_name -> google.protobuf.Timestamp + 18, // 4: block_engine.SubmitLeaderWindowInfoRequest.end_timestamp:type_name -> google.protobuf.Timestamp + 0, // 5: block_engine.SetStrategyRequest.strategy:type_name -> block_engine.SchedulingStrategy + 13, // 6: block_engine.GetBlockEngineEndpointResponse.global_endpoint:type_name -> block_engine.BlockEngineEndpoint + 13, // 7: block_engine.GetBlockEngineEndpointResponse.regioned_endpoints:type_name -> block_engine.BlockEngineEndpoint + 1, // 8: block_engine.BlockEngineValidator.SubscribePackets:input_type -> block_engine.SubscribePacketsRequest + 3, // 9: block_engine.BlockEngineValidator.SubscribeBundles:input_type -> block_engine.SubscribeBundlesRequest + 3, // 10: block_engine.BlockEngineValidator.SubscribeBundles2:input_type -> block_engine.SubscribeBundlesRequest + 4, // 11: block_engine.BlockEngineValidator.SubscribeBlocks:input_type -> block_engine.SubscribeBlocksRequest + 4, // 12: block_engine.BlockEngineValidator.SubscribeBlocks2:input_type -> block_engine.SubscribeBlocksRequest + 6, // 13: block_engine.BlockEngineValidator.GetBlockBuilderFeeInfo:input_type -> block_engine.BlockBuilderFeeInfoRequest + 8, // 14: block_engine.BlockEngineValidator.SubmitLeaderWindowInfo:input_type -> block_engine.SubmitLeaderWindowInfoRequest + 12, // 15: block_engine.BlockEngineValidator.GetBlockEngineEndpoints:input_type -> block_engine.GetBlockEngineEndpointRequest + 10, // 16: block_engine.BlockEngineValidator.SetStrategy:input_type -> block_engine.SetStrategyRequest + 2, // 17: block_engine.BlockEngineValidator.SubscribePackets:output_type -> block_engine.SubscribePacketsResponse + 5, // 18: block_engine.BlockEngineValidator.SubscribeBundles:output_type -> block_engine.SubscribeBundlesResponse + 5, // 19: block_engine.BlockEngineValidator.SubscribeBundles2:output_type -> block_engine.SubscribeBundlesResponse + 5, // 20: block_engine.BlockEngineValidator.SubscribeBlocks:output_type -> block_engine.SubscribeBundlesResponse + 19, // 21: block_engine.BlockEngineValidator.SubscribeBlocks2:output_type -> block.Block + 7, // 22: block_engine.BlockEngineValidator.GetBlockBuilderFeeInfo:output_type -> block_engine.BlockBuilderFeeInfoResponse + 9, // 23: block_engine.BlockEngineValidator.SubmitLeaderWindowInfo:output_type -> block_engine.SubmitLeaderWindowInfoResponse + 14, // 24: block_engine.BlockEngineValidator.GetBlockEngineEndpoints:output_type -> block_engine.GetBlockEngineEndpointResponse + 11, // 25: block_engine.BlockEngineValidator.SetStrategy:output_type -> block_engine.SetStrategyResponse + 17, // [17:26] is the sub-list for method output_type + 8, // [8:17] is the sub-list for method input_type + 8, // [8:8] is the sub-list for extension type_name + 8, // [8:8] is the sub-list for extension extendee + 0, // [0:8] is the sub-list for field type_name +} + +func init() { file_block_engine_proto_init() } +func file_block_engine_proto_init() { + if File_block_engine_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_block_engine_proto_rawDesc), len(file_block_engine_proto_rawDesc)), + NumEnums: 1, + NumMessages: 14, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_block_engine_proto_goTypes, + DependencyIndexes: file_block_engine_proto_depIdxs, + EnumInfos: file_block_engine_proto_enumTypes, + MessageInfos: file_block_engine_proto_msgTypes, + }.Build() + File_block_engine_proto = out.File + file_block_engine_proto_goTypes = nil + file_block_engine_proto_depIdxs = nil +} diff --git a/pkg/harmonic/proto/blockengine/block_engine_grpc.pb.go b/pkg/harmonic/proto/blockengine/block_engine_grpc.pb.go new file mode 100644 index 00000000..66f231ac --- /dev/null +++ b/pkg/harmonic/proto/blockengine/block_engine_grpc.pb.go @@ -0,0 +1,468 @@ +// Block engine interface for the Harmonic validator. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc v3.21.12 +// source: block_engine.proto + +package blockengine + +import ( + context "context" + block "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/block" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + BlockEngineValidator_SubscribePackets_FullMethodName = "/block_engine.BlockEngineValidator/SubscribePackets" + BlockEngineValidator_SubscribeBundles_FullMethodName = "/block_engine.BlockEngineValidator/SubscribeBundles" + BlockEngineValidator_SubscribeBundles2_FullMethodName = "/block_engine.BlockEngineValidator/SubscribeBundles2" + BlockEngineValidator_SubscribeBlocks_FullMethodName = "/block_engine.BlockEngineValidator/SubscribeBlocks" + BlockEngineValidator_SubscribeBlocks2_FullMethodName = "/block_engine.BlockEngineValidator/SubscribeBlocks2" + BlockEngineValidator_GetBlockBuilderFeeInfo_FullMethodName = "/block_engine.BlockEngineValidator/GetBlockBuilderFeeInfo" + BlockEngineValidator_SubmitLeaderWindowInfo_FullMethodName = "/block_engine.BlockEngineValidator/SubmitLeaderWindowInfo" + BlockEngineValidator_GetBlockEngineEndpoints_FullMethodName = "/block_engine.BlockEngineValidator/GetBlockEngineEndpoints" + BlockEngineValidator_SetStrategy_FullMethodName = "/block_engine.BlockEngineValidator/SetStrategy" +) + +// BlockEngineValidatorClient is the client API for BlockEngineValidator service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Service consumed by the Harmonic validator to stream packets and bundles +// from the block engine. +type BlockEngineValidatorClient interface { + // Subscribe to a stream of transaction packets. + SubscribePackets(ctx context.Context, in *SubscribePacketsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SubscribePacketsResponse], error) + // Subscribe to a stream of simulated and profitable bundles. + SubscribeBundles(ctx context.Context, in *SubscribeBundlesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SubscribeBundlesResponse], error) + // Subscribe to bundles (v2). + SubscribeBundles2(ctx context.Context, in *SubscribeBundlesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SubscribeBundlesResponse], error) + // Deprecated: use SubscribeBlocks2. + SubscribeBlocks(ctx context.Context, in *SubscribeBlocksRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SubscribeBundlesResponse], error) + // Subscribe to blocks with explicit bundle boundaries. + SubscribeBlocks2(ctx context.Context, in *SubscribeBlocksRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[block.Block], error) + // Returns the block builder's tip pubkey and commission. + GetBlockBuilderFeeInfo(ctx context.Context, in *BlockBuilderFeeInfoRequest, opts ...grpc.CallOption) (*BlockBuilderFeeInfoResponse, error) + // Report the validator's upcoming leader slot to the block engine. + SubmitLeaderWindowInfo(ctx context.Context, in *SubmitLeaderWindowInfoRequest, opts ...grpc.CallOption) (*SubmitLeaderWindowInfoResponse, error) + // Discover available block engine endpoints for optimal region selection. + GetBlockEngineEndpoints(ctx context.Context, in *GetBlockEngineEndpointRequest, opts ...grpc.CallOption) (*GetBlockEngineEndpointResponse, error) + // Set the scheduling strategy for this validator session. + SetStrategy(ctx context.Context, in *SetStrategyRequest, opts ...grpc.CallOption) (*SetStrategyResponse, error) +} + +type blockEngineValidatorClient struct { + cc grpc.ClientConnInterface +} + +func NewBlockEngineValidatorClient(cc grpc.ClientConnInterface) BlockEngineValidatorClient { + return &blockEngineValidatorClient{cc} +} + +func (c *blockEngineValidatorClient) SubscribePackets(ctx context.Context, in *SubscribePacketsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SubscribePacketsResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &BlockEngineValidator_ServiceDesc.Streams[0], BlockEngineValidator_SubscribePackets_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[SubscribePacketsRequest, SubscribePacketsResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BlockEngineValidator_SubscribePacketsClient = grpc.ServerStreamingClient[SubscribePacketsResponse] + +func (c *blockEngineValidatorClient) SubscribeBundles(ctx context.Context, in *SubscribeBundlesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SubscribeBundlesResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &BlockEngineValidator_ServiceDesc.Streams[1], BlockEngineValidator_SubscribeBundles_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[SubscribeBundlesRequest, SubscribeBundlesResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BlockEngineValidator_SubscribeBundlesClient = grpc.ServerStreamingClient[SubscribeBundlesResponse] + +func (c *blockEngineValidatorClient) SubscribeBundles2(ctx context.Context, in *SubscribeBundlesRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SubscribeBundlesResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &BlockEngineValidator_ServiceDesc.Streams[2], BlockEngineValidator_SubscribeBundles2_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[SubscribeBundlesRequest, SubscribeBundlesResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BlockEngineValidator_SubscribeBundles2Client = grpc.ServerStreamingClient[SubscribeBundlesResponse] + +func (c *blockEngineValidatorClient) SubscribeBlocks(ctx context.Context, in *SubscribeBlocksRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SubscribeBundlesResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &BlockEngineValidator_ServiceDesc.Streams[3], BlockEngineValidator_SubscribeBlocks_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[SubscribeBlocksRequest, SubscribeBundlesResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BlockEngineValidator_SubscribeBlocksClient = grpc.ServerStreamingClient[SubscribeBundlesResponse] + +func (c *blockEngineValidatorClient) SubscribeBlocks2(ctx context.Context, in *SubscribeBlocksRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[block.Block], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &BlockEngineValidator_ServiceDesc.Streams[4], BlockEngineValidator_SubscribeBlocks2_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[SubscribeBlocksRequest, block.Block]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BlockEngineValidator_SubscribeBlocks2Client = grpc.ServerStreamingClient[block.Block] + +func (c *blockEngineValidatorClient) GetBlockBuilderFeeInfo(ctx context.Context, in *BlockBuilderFeeInfoRequest, opts ...grpc.CallOption) (*BlockBuilderFeeInfoResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(BlockBuilderFeeInfoResponse) + err := c.cc.Invoke(ctx, BlockEngineValidator_GetBlockBuilderFeeInfo_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blockEngineValidatorClient) SubmitLeaderWindowInfo(ctx context.Context, in *SubmitLeaderWindowInfoRequest, opts ...grpc.CallOption) (*SubmitLeaderWindowInfoResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SubmitLeaderWindowInfoResponse) + err := c.cc.Invoke(ctx, BlockEngineValidator_SubmitLeaderWindowInfo_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blockEngineValidatorClient) GetBlockEngineEndpoints(ctx context.Context, in *GetBlockEngineEndpointRequest, opts ...grpc.CallOption) (*GetBlockEngineEndpointResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetBlockEngineEndpointResponse) + err := c.cc.Invoke(ctx, BlockEngineValidator_GetBlockEngineEndpoints_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *blockEngineValidatorClient) SetStrategy(ctx context.Context, in *SetStrategyRequest, opts ...grpc.CallOption) (*SetStrategyResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SetStrategyResponse) + err := c.cc.Invoke(ctx, BlockEngineValidator_SetStrategy_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// BlockEngineValidatorServer is the server API for BlockEngineValidator service. +// All implementations must embed UnimplementedBlockEngineValidatorServer +// for forward compatibility. +// +// Service consumed by the Harmonic validator to stream packets and bundles +// from the block engine. +type BlockEngineValidatorServer interface { + // Subscribe to a stream of transaction packets. + SubscribePackets(*SubscribePacketsRequest, grpc.ServerStreamingServer[SubscribePacketsResponse]) error + // Subscribe to a stream of simulated and profitable bundles. + SubscribeBundles(*SubscribeBundlesRequest, grpc.ServerStreamingServer[SubscribeBundlesResponse]) error + // Subscribe to bundles (v2). + SubscribeBundles2(*SubscribeBundlesRequest, grpc.ServerStreamingServer[SubscribeBundlesResponse]) error + // Deprecated: use SubscribeBlocks2. + SubscribeBlocks(*SubscribeBlocksRequest, grpc.ServerStreamingServer[SubscribeBundlesResponse]) error + // Subscribe to blocks with explicit bundle boundaries. + SubscribeBlocks2(*SubscribeBlocksRequest, grpc.ServerStreamingServer[block.Block]) error + // Returns the block builder's tip pubkey and commission. + GetBlockBuilderFeeInfo(context.Context, *BlockBuilderFeeInfoRequest) (*BlockBuilderFeeInfoResponse, error) + // Report the validator's upcoming leader slot to the block engine. + SubmitLeaderWindowInfo(context.Context, *SubmitLeaderWindowInfoRequest) (*SubmitLeaderWindowInfoResponse, error) + // Discover available block engine endpoints for optimal region selection. + GetBlockEngineEndpoints(context.Context, *GetBlockEngineEndpointRequest) (*GetBlockEngineEndpointResponse, error) + // Set the scheduling strategy for this validator session. + SetStrategy(context.Context, *SetStrategyRequest) (*SetStrategyResponse, error) + mustEmbedUnimplementedBlockEngineValidatorServer() +} + +// UnimplementedBlockEngineValidatorServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedBlockEngineValidatorServer struct{} + +func (UnimplementedBlockEngineValidatorServer) SubscribePackets(*SubscribePacketsRequest, grpc.ServerStreamingServer[SubscribePacketsResponse]) error { + return status.Errorf(codes.Unimplemented, "method SubscribePackets not implemented") +} +func (UnimplementedBlockEngineValidatorServer) SubscribeBundles(*SubscribeBundlesRequest, grpc.ServerStreamingServer[SubscribeBundlesResponse]) error { + return status.Errorf(codes.Unimplemented, "method SubscribeBundles not implemented") +} +func (UnimplementedBlockEngineValidatorServer) SubscribeBundles2(*SubscribeBundlesRequest, grpc.ServerStreamingServer[SubscribeBundlesResponse]) error { + return status.Errorf(codes.Unimplemented, "method SubscribeBundles2 not implemented") +} +func (UnimplementedBlockEngineValidatorServer) SubscribeBlocks(*SubscribeBlocksRequest, grpc.ServerStreamingServer[SubscribeBundlesResponse]) error { + return status.Errorf(codes.Unimplemented, "method SubscribeBlocks not implemented") +} +func (UnimplementedBlockEngineValidatorServer) SubscribeBlocks2(*SubscribeBlocksRequest, grpc.ServerStreamingServer[block.Block]) error { + return status.Errorf(codes.Unimplemented, "method SubscribeBlocks2 not implemented") +} +func (UnimplementedBlockEngineValidatorServer) GetBlockBuilderFeeInfo(context.Context, *BlockBuilderFeeInfoRequest) (*BlockBuilderFeeInfoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetBlockBuilderFeeInfo not implemented") +} +func (UnimplementedBlockEngineValidatorServer) SubmitLeaderWindowInfo(context.Context, *SubmitLeaderWindowInfoRequest) (*SubmitLeaderWindowInfoResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SubmitLeaderWindowInfo not implemented") +} +func (UnimplementedBlockEngineValidatorServer) GetBlockEngineEndpoints(context.Context, *GetBlockEngineEndpointRequest) (*GetBlockEngineEndpointResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetBlockEngineEndpoints not implemented") +} +func (UnimplementedBlockEngineValidatorServer) SetStrategy(context.Context, *SetStrategyRequest) (*SetStrategyResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method SetStrategy not implemented") +} +func (UnimplementedBlockEngineValidatorServer) mustEmbedUnimplementedBlockEngineValidatorServer() {} +func (UnimplementedBlockEngineValidatorServer) testEmbeddedByValue() {} + +// UnsafeBlockEngineValidatorServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to BlockEngineValidatorServer will +// result in compilation errors. +type UnsafeBlockEngineValidatorServer interface { + mustEmbedUnimplementedBlockEngineValidatorServer() +} + +func RegisterBlockEngineValidatorServer(s grpc.ServiceRegistrar, srv BlockEngineValidatorServer) { + // If the following call pancis, it indicates UnimplementedBlockEngineValidatorServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&BlockEngineValidator_ServiceDesc, srv) +} + +func _BlockEngineValidator_SubscribePackets_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(SubscribePacketsRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(BlockEngineValidatorServer).SubscribePackets(m, &grpc.GenericServerStream[SubscribePacketsRequest, SubscribePacketsResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BlockEngineValidator_SubscribePacketsServer = grpc.ServerStreamingServer[SubscribePacketsResponse] + +func _BlockEngineValidator_SubscribeBundles_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(SubscribeBundlesRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(BlockEngineValidatorServer).SubscribeBundles(m, &grpc.GenericServerStream[SubscribeBundlesRequest, SubscribeBundlesResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BlockEngineValidator_SubscribeBundlesServer = grpc.ServerStreamingServer[SubscribeBundlesResponse] + +func _BlockEngineValidator_SubscribeBundles2_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(SubscribeBundlesRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(BlockEngineValidatorServer).SubscribeBundles2(m, &grpc.GenericServerStream[SubscribeBundlesRequest, SubscribeBundlesResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BlockEngineValidator_SubscribeBundles2Server = grpc.ServerStreamingServer[SubscribeBundlesResponse] + +func _BlockEngineValidator_SubscribeBlocks_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(SubscribeBlocksRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(BlockEngineValidatorServer).SubscribeBlocks(m, &grpc.GenericServerStream[SubscribeBlocksRequest, SubscribeBundlesResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BlockEngineValidator_SubscribeBlocksServer = grpc.ServerStreamingServer[SubscribeBundlesResponse] + +func _BlockEngineValidator_SubscribeBlocks2_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(SubscribeBlocksRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(BlockEngineValidatorServer).SubscribeBlocks2(m, &grpc.GenericServerStream[SubscribeBlocksRequest, block.Block]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type BlockEngineValidator_SubscribeBlocks2Server = grpc.ServerStreamingServer[block.Block] + +func _BlockEngineValidator_GetBlockBuilderFeeInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(BlockBuilderFeeInfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlockEngineValidatorServer).GetBlockBuilderFeeInfo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BlockEngineValidator_GetBlockBuilderFeeInfo_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlockEngineValidatorServer).GetBlockBuilderFeeInfo(ctx, req.(*BlockBuilderFeeInfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _BlockEngineValidator_SubmitLeaderWindowInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SubmitLeaderWindowInfoRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlockEngineValidatorServer).SubmitLeaderWindowInfo(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BlockEngineValidator_SubmitLeaderWindowInfo_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlockEngineValidatorServer).SubmitLeaderWindowInfo(ctx, req.(*SubmitLeaderWindowInfoRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _BlockEngineValidator_GetBlockEngineEndpoints_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetBlockEngineEndpointRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlockEngineValidatorServer).GetBlockEngineEndpoints(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BlockEngineValidator_GetBlockEngineEndpoints_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlockEngineValidatorServer).GetBlockEngineEndpoints(ctx, req.(*GetBlockEngineEndpointRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _BlockEngineValidator_SetStrategy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetStrategyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BlockEngineValidatorServer).SetStrategy(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BlockEngineValidator_SetStrategy_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BlockEngineValidatorServer).SetStrategy(ctx, req.(*SetStrategyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// BlockEngineValidator_ServiceDesc is the grpc.ServiceDesc for BlockEngineValidator service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var BlockEngineValidator_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "block_engine.BlockEngineValidator", + HandlerType: (*BlockEngineValidatorServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetBlockBuilderFeeInfo", + Handler: _BlockEngineValidator_GetBlockBuilderFeeInfo_Handler, + }, + { + MethodName: "SubmitLeaderWindowInfo", + Handler: _BlockEngineValidator_SubmitLeaderWindowInfo_Handler, + }, + { + MethodName: "GetBlockEngineEndpoints", + Handler: _BlockEngineValidator_GetBlockEngineEndpoints_Handler, + }, + { + MethodName: "SetStrategy", + Handler: _BlockEngineValidator_SetStrategy_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "SubscribePackets", + Handler: _BlockEngineValidator_SubscribePackets_Handler, + ServerStreams: true, + }, + { + StreamName: "SubscribeBundles", + Handler: _BlockEngineValidator_SubscribeBundles_Handler, + ServerStreams: true, + }, + { + StreamName: "SubscribeBundles2", + Handler: _BlockEngineValidator_SubscribeBundles2_Handler, + ServerStreams: true, + }, + { + StreamName: "SubscribeBlocks", + Handler: _BlockEngineValidator_SubscribeBlocks_Handler, + ServerStreams: true, + }, + { + StreamName: "SubscribeBlocks2", + Handler: _BlockEngineValidator_SubscribeBlocks2_Handler, + ServerStreams: true, + }, + }, + Metadata: "block_engine.proto", +} diff --git a/pkg/harmonic/proto/bundle/bundle.pb.go b/pkg/harmonic/proto/bundle/bundle.pb.go new file mode 100644 index 00000000..1f94d562 --- /dev/null +++ b/pkg/harmonic/proto/bundle/bundle.pb.go @@ -0,0 +1,203 @@ +// Bundle types received by the validator from the block engine. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.21.12 +// source: bundle.proto + +package bundle + +import ( + packet "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/packet" + shared "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/shared" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// An ordered list of transactions that must execute sequentially and atomically. +type Bundle struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Timestamp of when this bundle was created. + Header *shared.Header `protobuf:"bytes,2,opt,name=header,proto3" json:"header,omitempty"` + // Transactions in execution order. + Packets []*packet.Packet `protobuf:"bytes,3,rep,name=packets,proto3" json:"packets,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Bundle) Reset() { + *x = Bundle{} + mi := &file_bundle_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Bundle) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Bundle) ProtoMessage() {} + +func (x *Bundle) ProtoReflect() protoreflect.Message { + mi := &file_bundle_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Bundle.ProtoReflect.Descriptor instead. +func (*Bundle) Descriptor() ([]byte, []int) { + return file_bundle_proto_rawDescGZIP(), []int{0} +} + +func (x *Bundle) GetHeader() *shared.Header { + if x != nil { + return x.Header + } + return nil +} + +func (x *Bundle) GetPackets() []*packet.Packet { + if x != nil { + return x.Packets + } + return nil +} + +// A Bundle paired with its server-assigned identifier. +type BundleUuid struct { + state protoimpl.MessageState `protogen:"open.v1"` + // The bundle contents. + Bundle *Bundle `protobuf:"bytes,1,opt,name=bundle,proto3" json:"bundle,omitempty"` + // Unique identifier assigned by the block engine. + Uuid string `protobuf:"bytes,2,opt,name=uuid,proto3" json:"uuid,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *BundleUuid) Reset() { + *x = BundleUuid{} + mi := &file_bundle_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *BundleUuid) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BundleUuid) ProtoMessage() {} + +func (x *BundleUuid) ProtoReflect() protoreflect.Message { + mi := &file_bundle_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BundleUuid.ProtoReflect.Descriptor instead. +func (*BundleUuid) Descriptor() ([]byte, []int) { + return file_bundle_proto_rawDescGZIP(), []int{1} +} + +func (x *BundleUuid) GetBundle() *Bundle { + if x != nil { + return x.Bundle + } + return nil +} + +func (x *BundleUuid) GetUuid() string { + if x != nil { + return x.Uuid + } + return "" +} + +var File_bundle_proto protoreflect.FileDescriptor + +const file_bundle_proto_rawDesc = "" + + "\n" + + "\fbundle.proto\x12\x06bundle\x1a\fpacket.proto\x1a\fshared.proto\"Z\n" + + "\x06Bundle\x12&\n" + + "\x06header\x18\x02 \x01(\v2\x0e.shared.HeaderR\x06header\x12(\n" + + "\apackets\x18\x03 \x03(\v2\x0e.packet.PacketR\apackets\"H\n" + + "\n" + + "BundleUuid\x12&\n" + + "\x06bundle\x18\x01 \x01(\v2\x0e.bundle.BundleR\x06bundle\x12\x12\n" + + "\x04uuid\x18\x02 \x01(\tR\x04uuidb\x06proto3" + +var ( + file_bundle_proto_rawDescOnce sync.Once + file_bundle_proto_rawDescData []byte +) + +func file_bundle_proto_rawDescGZIP() []byte { + file_bundle_proto_rawDescOnce.Do(func() { + file_bundle_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_bundle_proto_rawDesc), len(file_bundle_proto_rawDesc))) + }) + return file_bundle_proto_rawDescData +} + +var file_bundle_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_bundle_proto_goTypes = []any{ + (*Bundle)(nil), // 0: bundle.Bundle + (*BundleUuid)(nil), // 1: bundle.BundleUuid + (*shared.Header)(nil), // 2: shared.Header + (*packet.Packet)(nil), // 3: packet.Packet +} +var file_bundle_proto_depIdxs = []int32{ + 2, // 0: bundle.Bundle.header:type_name -> shared.Header + 3, // 1: bundle.Bundle.packets:type_name -> packet.Packet + 0, // 2: bundle.BundleUuid.bundle:type_name -> bundle.Bundle + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_bundle_proto_init() } +func file_bundle_proto_init() { + if File_bundle_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_bundle_proto_rawDesc), len(file_bundle_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_bundle_proto_goTypes, + DependencyIndexes: file_bundle_proto_depIdxs, + MessageInfos: file_bundle_proto_msgTypes, + }.Build() + File_bundle_proto = out.File + file_bundle_proto_goTypes = nil + file_bundle_proto_depIdxs = nil +} diff --git a/pkg/harmonic/proto/gen.sh b/pkg/harmonic/proto/gen.sh new file mode 100755 index 00000000..d139dd20 --- /dev/null +++ b/pkg/harmonic/proto/gen.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# Regenerates the Go code for the Harmonic validator protos vendored in +# upstream/. Requires protoc v3.21.12, protoc-gen-go v1.36.11 and +# protoc-gen-go-grpc v1.5.1 on PATH. The upstream files set no go_package, so +# each one is mapped to its Go package here. +set -euo pipefail +cd "$(dirname "$0")" + +module=github.com/Overclock-Validator/mithril/pkg/harmonic/proto +declare -A packages=( + [auth]=auth + [block]=block + [block_engine]=blockengine + [bundle]=bundle + [packet]=packet + [relayer]=relayer + [shared]=shared +) + +go_opts=(--go_opt=paths=source_relative) +grpc_opts=(--go-grpc_opt=paths=source_relative) +for proto in "${!packages[@]}"; do + go_opts+=("--go_opt=M${proto}.proto=${module}/${packages[$proto]}") + grpc_opts+=("--go-grpc_opt=M${proto}.proto=${module}/${packages[$proto]}") +done + +for proto in "${!packages[@]}"; do + out=${packages[$proto]} + mkdir -p "$out" + protoc -I upstream \ + --go_out="$out" "${go_opts[@]}" \ + --go-grpc_out="$out" "${grpc_opts[@]}" \ + "upstream/${proto}.proto" +done diff --git a/pkg/harmonic/proto/packet/packet.pb.go b/pkg/harmonic/proto/packet/packet.pb.go new file mode 100644 index 00000000..c72e29af --- /dev/null +++ b/pkg/harmonic/proto/packet/packet.pb.go @@ -0,0 +1,375 @@ +// Transaction packet types received by the validator from the block engine and relayer. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.21.12 +// source: packet.proto + +package packet + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// A batch of transaction packets. +type PacketBatch struct { + state protoimpl.MessageState `protogen:"open.v1"` + Packets []*Packet `protobuf:"bytes,1,rep,name=packets,proto3" json:"packets,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PacketBatch) Reset() { + *x = PacketBatch{} + mi := &file_packet_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PacketBatch) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PacketBatch) ProtoMessage() {} + +func (x *PacketBatch) ProtoReflect() protoreflect.Message { + mi := &file_packet_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PacketBatch.ProtoReflect.Descriptor instead. +func (*PacketBatch) Descriptor() ([]byte, []int) { + return file_packet_proto_rawDescGZIP(), []int{0} +} + +func (x *PacketBatch) GetPackets() []*Packet { + if x != nil { + return x.Packets + } + return nil +} + +// A single transaction packet with metadata. +type Packet struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Raw transaction bytes. + Data []byte `protobuf:"bytes,1,opt,name=data,proto3" json:"data,omitempty"` + // Packet metadata. + Meta *Meta `protobuf:"bytes,2,opt,name=meta,proto3" json:"meta,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Packet) Reset() { + *x = Packet{} + mi := &file_packet_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Packet) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Packet) ProtoMessage() {} + +func (x *Packet) ProtoReflect() protoreflect.Message { + mi := &file_packet_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Packet.ProtoReflect.Descriptor instead. +func (*Packet) Descriptor() ([]byte, []int) { + return file_packet_proto_rawDescGZIP(), []int{1} +} + +func (x *Packet) GetData() []byte { + if x != nil { + return x.Data + } + return nil +} + +func (x *Packet) GetMeta() *Meta { + if x != nil { + return x.Meta + } + return nil +} + +// Packet metadata mirroring solana_sdk::packet::Meta. +type Meta struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Number of valid bytes in the packet data. + Size uint64 `protobuf:"varint,1,opt,name=size,proto3" json:"size,omitempty"` + // Source IP address. + Addr string `protobuf:"bytes,2,opt,name=addr,proto3" json:"addr,omitempty"` + // Source port. + Port uint32 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"` + // Packet processing flags. + Flags *PacketFlags `protobuf:"bytes,4,opt,name=flags,proto3" json:"flags,omitempty"` + // Stake of the sending node (0 if unknown). + SenderStake uint64 `protobuf:"varint,5,opt,name=sender_stake,json=senderStake,proto3" json:"sender_stake,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Meta) Reset() { + *x = Meta{} + mi := &file_packet_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Meta) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Meta) ProtoMessage() {} + +func (x *Meta) ProtoReflect() protoreflect.Message { + mi := &file_packet_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Meta.ProtoReflect.Descriptor instead. +func (*Meta) Descriptor() ([]byte, []int) { + return file_packet_proto_rawDescGZIP(), []int{2} +} + +func (x *Meta) GetSize() uint64 { + if x != nil { + return x.Size + } + return 0 +} + +func (x *Meta) GetAddr() string { + if x != nil { + return x.Addr + } + return "" +} + +func (x *Meta) GetPort() uint32 { + if x != nil { + return x.Port + } + return 0 +} + +func (x *Meta) GetFlags() *PacketFlags { + if x != nil { + return x.Flags + } + return nil +} + +func (x *Meta) GetSenderStake() uint64 { + if x != nil { + return x.SenderStake + } + return 0 +} + +// Boolean flags that influence how the validator processes a packet. +type PacketFlags struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Packet should be dropped. + Discard bool `protobuf:"varint,1,opt,name=discard,proto3" json:"discard,omitempty"` + // Packet was forwarded from another node (not received directly). + Forwarded bool `protobuf:"varint,2,opt,name=forwarded,proto3" json:"forwarded,omitempty"` + // Packet is a repair request. + Repair bool `protobuf:"varint,3,opt,name=repair,proto3" json:"repair,omitempty"` + // Packet contains a simple vote transaction. + SimpleVoteTx bool `protobuf:"varint,4,opt,name=simple_vote_tx,json=simpleVoteTx,proto3" json:"simple_vote_tx,omitempty"` + // Packet is a tracer used for debugging. + TracerPacket bool `protobuf:"varint,5,opt,name=tracer_packet,json=tracerPacket,proto3" json:"tracer_packet,omitempty"` + // Packet originated from a staked node. + FromStakedNode bool `protobuf:"varint,6,opt,name=from_staked_node,json=fromStakedNode,proto3" json:"from_staked_node,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PacketFlags) Reset() { + *x = PacketFlags{} + mi := &file_packet_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PacketFlags) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PacketFlags) ProtoMessage() {} + +func (x *PacketFlags) ProtoReflect() protoreflect.Message { + mi := &file_packet_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PacketFlags.ProtoReflect.Descriptor instead. +func (*PacketFlags) Descriptor() ([]byte, []int) { + return file_packet_proto_rawDescGZIP(), []int{3} +} + +func (x *PacketFlags) GetDiscard() bool { + if x != nil { + return x.Discard + } + return false +} + +func (x *PacketFlags) GetForwarded() bool { + if x != nil { + return x.Forwarded + } + return false +} + +func (x *PacketFlags) GetRepair() bool { + if x != nil { + return x.Repair + } + return false +} + +func (x *PacketFlags) GetSimpleVoteTx() bool { + if x != nil { + return x.SimpleVoteTx + } + return false +} + +func (x *PacketFlags) GetTracerPacket() bool { + if x != nil { + return x.TracerPacket + } + return false +} + +func (x *PacketFlags) GetFromStakedNode() bool { + if x != nil { + return x.FromStakedNode + } + return false +} + +var File_packet_proto protoreflect.FileDescriptor + +const file_packet_proto_rawDesc = "" + + "\n" + + "\fpacket.proto\x12\x06packet\"7\n" + + "\vPacketBatch\x12(\n" + + "\apackets\x18\x01 \x03(\v2\x0e.packet.PacketR\apackets\">\n" + + "\x06Packet\x12\x12\n" + + "\x04data\x18\x01 \x01(\fR\x04data\x12 \n" + + "\x04meta\x18\x02 \x01(\v2\f.packet.MetaR\x04meta\"\x90\x01\n" + + "\x04Meta\x12\x12\n" + + "\x04size\x18\x01 \x01(\x04R\x04size\x12\x12\n" + + "\x04addr\x18\x02 \x01(\tR\x04addr\x12\x12\n" + + "\x04port\x18\x03 \x01(\rR\x04port\x12)\n" + + "\x05flags\x18\x04 \x01(\v2\x13.packet.PacketFlagsR\x05flags\x12!\n" + + "\fsender_stake\x18\x05 \x01(\x04R\vsenderStake\"\xd2\x01\n" + + "\vPacketFlags\x12\x18\n" + + "\adiscard\x18\x01 \x01(\bR\adiscard\x12\x1c\n" + + "\tforwarded\x18\x02 \x01(\bR\tforwarded\x12\x16\n" + + "\x06repair\x18\x03 \x01(\bR\x06repair\x12$\n" + + "\x0esimple_vote_tx\x18\x04 \x01(\bR\fsimpleVoteTx\x12#\n" + + "\rtracer_packet\x18\x05 \x01(\bR\ftracerPacket\x12(\n" + + "\x10from_staked_node\x18\x06 \x01(\bR\x0efromStakedNodeb\x06proto3" + +var ( + file_packet_proto_rawDescOnce sync.Once + file_packet_proto_rawDescData []byte +) + +func file_packet_proto_rawDescGZIP() []byte { + file_packet_proto_rawDescOnce.Do(func() { + file_packet_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_packet_proto_rawDesc), len(file_packet_proto_rawDesc))) + }) + return file_packet_proto_rawDescData +} + +var file_packet_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_packet_proto_goTypes = []any{ + (*PacketBatch)(nil), // 0: packet.PacketBatch + (*Packet)(nil), // 1: packet.Packet + (*Meta)(nil), // 2: packet.Meta + (*PacketFlags)(nil), // 3: packet.PacketFlags +} +var file_packet_proto_depIdxs = []int32{ + 1, // 0: packet.PacketBatch.packets:type_name -> packet.Packet + 2, // 1: packet.Packet.meta:type_name -> packet.Meta + 3, // 2: packet.Meta.flags:type_name -> packet.PacketFlags + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_packet_proto_init() } +func file_packet_proto_init() { + if File_packet_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_packet_proto_rawDesc), len(file_packet_proto_rawDesc)), + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_packet_proto_goTypes, + DependencyIndexes: file_packet_proto_depIdxs, + MessageInfos: file_packet_proto_msgTypes, + }.Build() + File_packet_proto = out.File + file_packet_proto_goTypes = nil + file_packet_proto_depIdxs = nil +} diff --git a/pkg/harmonic/proto/relayer/relayer.pb.go b/pkg/harmonic/proto/relayer/relayer.pb.go new file mode 100644 index 00000000..36963860 --- /dev/null +++ b/pkg/harmonic/proto/relayer/relayer.pb.go @@ -0,0 +1,337 @@ +// Relayer/TPU proxy interface for the Harmonic validator. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.21.12 +// source: relayer.proto + +package relayer + +import ( + packet "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/packet" + shared "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/shared" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Request for the relayer's TPU socket configuration. +type GetTpuConfigsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetTpuConfigsRequest) Reset() { + *x = GetTpuConfigsRequest{} + mi := &file_relayer_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetTpuConfigsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetTpuConfigsRequest) ProtoMessage() {} + +func (x *GetTpuConfigsRequest) ProtoReflect() protoreflect.Message { + mi := &file_relayer_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetTpuConfigsRequest.ProtoReflect.Descriptor instead. +func (*GetTpuConfigsRequest) Descriptor() ([]byte, []int) { + return file_relayer_proto_rawDescGZIP(), []int{0} +} + +// TPU socket addresses that the validator should advertise in gossip. +type GetTpuConfigsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Relayer's TPU socket for regular transactions. + Tpu *shared.Socket `protobuf:"bytes,1,opt,name=tpu,proto3" json:"tpu,omitempty"` + // Relayer's TPU forward socket for forwarded transactions. + TpuForward *shared.Socket `protobuf:"bytes,2,opt,name=tpu_forward,json=tpuForward,proto3" json:"tpu_forward,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetTpuConfigsResponse) Reset() { + *x = GetTpuConfigsResponse{} + mi := &file_relayer_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetTpuConfigsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetTpuConfigsResponse) ProtoMessage() {} + +func (x *GetTpuConfigsResponse) ProtoReflect() protoreflect.Message { + mi := &file_relayer_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetTpuConfigsResponse.ProtoReflect.Descriptor instead. +func (*GetTpuConfigsResponse) Descriptor() ([]byte, []int) { + return file_relayer_proto_rawDescGZIP(), []int{1} +} + +func (x *GetTpuConfigsResponse) GetTpu() *shared.Socket { + if x != nil { + return x.Tpu + } + return nil +} + +func (x *GetTpuConfigsResponse) GetTpuForward() *shared.Socket { + if x != nil { + return x.TpuForward + } + return nil +} + +// Request to subscribe to the relayer's packet stream. +type SubscribePacketsRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubscribePacketsRequest) Reset() { + *x = SubscribePacketsRequest{} + mi := &file_relayer_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubscribePacketsRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubscribePacketsRequest) ProtoMessage() {} + +func (x *SubscribePacketsRequest) ProtoReflect() protoreflect.Message { + mi := &file_relayer_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubscribePacketsRequest.ProtoReflect.Descriptor instead. +func (*SubscribePacketsRequest) Descriptor() ([]byte, []int) { + return file_relayer_proto_rawDescGZIP(), []int{2} +} + +// Multiplexed stream of packet batches and keepalive heartbeats. +type SubscribePacketsResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Timestamp of when this message was sent. + Header *shared.Header `protobuf:"bytes,1,opt,name=header,proto3" json:"header,omitempty"` + // Types that are valid to be assigned to Msg: + // + // *SubscribePacketsResponse_Heartbeat + // *SubscribePacketsResponse_Batch + Msg isSubscribePacketsResponse_Msg `protobuf_oneof:"msg"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SubscribePacketsResponse) Reset() { + *x = SubscribePacketsResponse{} + mi := &file_relayer_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SubscribePacketsResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SubscribePacketsResponse) ProtoMessage() {} + +func (x *SubscribePacketsResponse) ProtoReflect() protoreflect.Message { + mi := &file_relayer_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SubscribePacketsResponse.ProtoReflect.Descriptor instead. +func (*SubscribePacketsResponse) Descriptor() ([]byte, []int) { + return file_relayer_proto_rawDescGZIP(), []int{3} +} + +func (x *SubscribePacketsResponse) GetHeader() *shared.Header { + if x != nil { + return x.Header + } + return nil +} + +func (x *SubscribePacketsResponse) GetMsg() isSubscribePacketsResponse_Msg { + if x != nil { + return x.Msg + } + return nil +} + +func (x *SubscribePacketsResponse) GetHeartbeat() *shared.Heartbeat { + if x != nil { + if x, ok := x.Msg.(*SubscribePacketsResponse_Heartbeat); ok { + return x.Heartbeat + } + } + return nil +} + +func (x *SubscribePacketsResponse) GetBatch() *packet.PacketBatch { + if x != nil { + if x, ok := x.Msg.(*SubscribePacketsResponse_Batch); ok { + return x.Batch + } + } + return nil +} + +type isSubscribePacketsResponse_Msg interface { + isSubscribePacketsResponse_Msg() +} + +type SubscribePacketsResponse_Heartbeat struct { + // Keepalive heartbeat when no packets are available. + Heartbeat *shared.Heartbeat `protobuf:"bytes,2,opt,name=heartbeat,proto3,oneof"` +} + +type SubscribePacketsResponse_Batch struct { + // Batch of transaction packets received at the relayer's TPU ports. + Batch *packet.PacketBatch `protobuf:"bytes,3,opt,name=batch,proto3,oneof"` +} + +func (*SubscribePacketsResponse_Heartbeat) isSubscribePacketsResponse_Msg() {} + +func (*SubscribePacketsResponse_Batch) isSubscribePacketsResponse_Msg() {} + +var File_relayer_proto protoreflect.FileDescriptor + +const file_relayer_proto_rawDesc = "" + + "\n" + + "\rrelayer.proto\x12\arelayer\x1a\fpacket.proto\x1a\fshared.proto\"\x16\n" + + "\x14GetTpuConfigsRequest\"j\n" + + "\x15GetTpuConfigsResponse\x12 \n" + + "\x03tpu\x18\x01 \x01(\v2\x0e.shared.SocketR\x03tpu\x12/\n" + + "\vtpu_forward\x18\x02 \x01(\v2\x0e.shared.SocketR\n" + + "tpuForward\"\x19\n" + + "\x17SubscribePacketsRequest\"\xa9\x01\n" + + "\x18SubscribePacketsResponse\x12&\n" + + "\x06header\x18\x01 \x01(\v2\x0e.shared.HeaderR\x06header\x121\n" + + "\theartbeat\x18\x02 \x01(\v2\x11.shared.HeartbeatH\x00R\theartbeat\x12+\n" + + "\x05batch\x18\x03 \x01(\v2\x13.packet.PacketBatchH\x00R\x05batchB\x05\n" + + "\x03msg2\xb8\x01\n" + + "\aRelayer\x12P\n" + + "\rGetTpuConfigs\x12\x1d.relayer.GetTpuConfigsRequest\x1a\x1e.relayer.GetTpuConfigsResponse\"\x00\x12[\n" + + "\x10SubscribePackets\x12 .relayer.SubscribePacketsRequest\x1a!.relayer.SubscribePacketsResponse\"\x000\x01b\x06proto3" + +var ( + file_relayer_proto_rawDescOnce sync.Once + file_relayer_proto_rawDescData []byte +) + +func file_relayer_proto_rawDescGZIP() []byte { + file_relayer_proto_rawDescOnce.Do(func() { + file_relayer_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_relayer_proto_rawDesc), len(file_relayer_proto_rawDesc))) + }) + return file_relayer_proto_rawDescData +} + +var file_relayer_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_relayer_proto_goTypes = []any{ + (*GetTpuConfigsRequest)(nil), // 0: relayer.GetTpuConfigsRequest + (*GetTpuConfigsResponse)(nil), // 1: relayer.GetTpuConfigsResponse + (*SubscribePacketsRequest)(nil), // 2: relayer.SubscribePacketsRequest + (*SubscribePacketsResponse)(nil), // 3: relayer.SubscribePacketsResponse + (*shared.Socket)(nil), // 4: shared.Socket + (*shared.Header)(nil), // 5: shared.Header + (*shared.Heartbeat)(nil), // 6: shared.Heartbeat + (*packet.PacketBatch)(nil), // 7: packet.PacketBatch +} +var file_relayer_proto_depIdxs = []int32{ + 4, // 0: relayer.GetTpuConfigsResponse.tpu:type_name -> shared.Socket + 4, // 1: relayer.GetTpuConfigsResponse.tpu_forward:type_name -> shared.Socket + 5, // 2: relayer.SubscribePacketsResponse.header:type_name -> shared.Header + 6, // 3: relayer.SubscribePacketsResponse.heartbeat:type_name -> shared.Heartbeat + 7, // 4: relayer.SubscribePacketsResponse.batch:type_name -> packet.PacketBatch + 0, // 5: relayer.Relayer.GetTpuConfigs:input_type -> relayer.GetTpuConfigsRequest + 2, // 6: relayer.Relayer.SubscribePackets:input_type -> relayer.SubscribePacketsRequest + 1, // 7: relayer.Relayer.GetTpuConfigs:output_type -> relayer.GetTpuConfigsResponse + 3, // 8: relayer.Relayer.SubscribePackets:output_type -> relayer.SubscribePacketsResponse + 7, // [7:9] is the sub-list for method output_type + 5, // [5:7] is the sub-list for method input_type + 5, // [5:5] is the sub-list for extension type_name + 5, // [5:5] is the sub-list for extension extendee + 0, // [0:5] is the sub-list for field type_name +} + +func init() { file_relayer_proto_init() } +func file_relayer_proto_init() { + if File_relayer_proto != nil { + return + } + file_relayer_proto_msgTypes[3].OneofWrappers = []any{ + (*SubscribePacketsResponse_Heartbeat)(nil), + (*SubscribePacketsResponse_Batch)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_relayer_proto_rawDesc), len(file_relayer_proto_rawDesc)), + NumEnums: 0, + NumMessages: 4, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_relayer_proto_goTypes, + DependencyIndexes: file_relayer_proto_depIdxs, + MessageInfos: file_relayer_proto_msgTypes, + }.Build() + File_relayer_proto = out.File + file_relayer_proto_goTypes = nil + file_relayer_proto_depIdxs = nil +} diff --git a/pkg/harmonic/proto/relayer/relayer_grpc.pb.go b/pkg/harmonic/proto/relayer/relayer_grpc.pb.go new file mode 100644 index 00000000..f3fc309d --- /dev/null +++ b/pkg/harmonic/proto/relayer/relayer_grpc.pb.go @@ -0,0 +1,173 @@ +// Relayer/TPU proxy interface for the Harmonic validator. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc v3.21.12 +// source: relayer.proto + +package relayer + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Relayer_GetTpuConfigs_FullMethodName = "/relayer.Relayer/GetTpuConfigs" + Relayer_SubscribePackets_FullMethodName = "/relayer.Relayer/SubscribePackets" +) + +// RelayerClient is the client API for Relayer service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Relayer service providing TPU proxy for the Harmonic validator. +type RelayerClient interface { + // Fetch the relayer's TPU and TPU-forward socket addresses. + GetTpuConfigs(ctx context.Context, in *GetTpuConfigsRequest, opts ...grpc.CallOption) (*GetTpuConfigsResponse, error) + // Subscribe to a stream of packets arriving at the relayer's TPU ports. + SubscribePackets(ctx context.Context, in *SubscribePacketsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SubscribePacketsResponse], error) +} + +type relayerClient struct { + cc grpc.ClientConnInterface +} + +func NewRelayerClient(cc grpc.ClientConnInterface) RelayerClient { + return &relayerClient{cc} +} + +func (c *relayerClient) GetTpuConfigs(ctx context.Context, in *GetTpuConfigsRequest, opts ...grpc.CallOption) (*GetTpuConfigsResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(GetTpuConfigsResponse) + err := c.cc.Invoke(ctx, Relayer_GetTpuConfigs_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *relayerClient) SubscribePackets(ctx context.Context, in *SubscribePacketsRequest, opts ...grpc.CallOption) (grpc.ServerStreamingClient[SubscribePacketsResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &Relayer_ServiceDesc.Streams[0], Relayer_SubscribePackets_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[SubscribePacketsRequest, SubscribePacketsResponse]{ClientStream: stream} + if err := x.ClientStream.SendMsg(in); err != nil { + return nil, err + } + if err := x.ClientStream.CloseSend(); err != nil { + return nil, err + } + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Relayer_SubscribePacketsClient = grpc.ServerStreamingClient[SubscribePacketsResponse] + +// RelayerServer is the server API for Relayer service. +// All implementations must embed UnimplementedRelayerServer +// for forward compatibility. +// +// Relayer service providing TPU proxy for the Harmonic validator. +type RelayerServer interface { + // Fetch the relayer's TPU and TPU-forward socket addresses. + GetTpuConfigs(context.Context, *GetTpuConfigsRequest) (*GetTpuConfigsResponse, error) + // Subscribe to a stream of packets arriving at the relayer's TPU ports. + SubscribePackets(*SubscribePacketsRequest, grpc.ServerStreamingServer[SubscribePacketsResponse]) error + mustEmbedUnimplementedRelayerServer() +} + +// UnimplementedRelayerServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedRelayerServer struct{} + +func (UnimplementedRelayerServer) GetTpuConfigs(context.Context, *GetTpuConfigsRequest) (*GetTpuConfigsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetTpuConfigs not implemented") +} +func (UnimplementedRelayerServer) SubscribePackets(*SubscribePacketsRequest, grpc.ServerStreamingServer[SubscribePacketsResponse]) error { + return status.Errorf(codes.Unimplemented, "method SubscribePackets not implemented") +} +func (UnimplementedRelayerServer) mustEmbedUnimplementedRelayerServer() {} +func (UnimplementedRelayerServer) testEmbeddedByValue() {} + +// UnsafeRelayerServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to RelayerServer will +// result in compilation errors. +type UnsafeRelayerServer interface { + mustEmbedUnimplementedRelayerServer() +} + +func RegisterRelayerServer(s grpc.ServiceRegistrar, srv RelayerServer) { + // If the following call pancis, it indicates UnimplementedRelayerServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Relayer_ServiceDesc, srv) +} + +func _Relayer_GetTpuConfigs_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetTpuConfigsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(RelayerServer).GetTpuConfigs(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Relayer_GetTpuConfigs_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(RelayerServer).GetTpuConfigs(ctx, req.(*GetTpuConfigsRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Relayer_SubscribePackets_Handler(srv interface{}, stream grpc.ServerStream) error { + m := new(SubscribePacketsRequest) + if err := stream.RecvMsg(m); err != nil { + return err + } + return srv.(RelayerServer).SubscribePackets(m, &grpc.GenericServerStream[SubscribePacketsRequest, SubscribePacketsResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type Relayer_SubscribePacketsServer = grpc.ServerStreamingServer[SubscribePacketsResponse] + +// Relayer_ServiceDesc is the grpc.ServiceDesc for Relayer service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Relayer_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "relayer.Relayer", + HandlerType: (*RelayerServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetTpuConfigs", + Handler: _Relayer_GetTpuConfigs_Handler, + }, + }, + Streams: []grpc.StreamDesc{ + { + StreamName: "SubscribePackets", + Handler: _Relayer_SubscribePackets_Handler, + ServerStreams: true, + }, + }, + Metadata: "relayer.proto", +} diff --git a/pkg/harmonic/proto/shared/shared.pb.go b/pkg/harmonic/proto/shared/shared.pb.go new file mode 100644 index 00000000..6adf7957 --- /dev/null +++ b/pkg/harmonic/proto/shared/shared.pb.go @@ -0,0 +1,237 @@ +// Common types shared across Harmonic validator proto definitions. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.11 +// protoc v3.21.12 +// source: shared.proto + +package shared + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Timestamp header attached to streaming responses. +type Header struct { + state protoimpl.MessageState `protogen:"open.v1"` + // When this message was generated. + Ts *timestamppb.Timestamp `protobuf:"bytes,1,opt,name=ts,proto3" json:"ts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Header) Reset() { + *x = Header{} + mi := &file_shared_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Header) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Header) ProtoMessage() {} + +func (x *Header) ProtoReflect() protoreflect.Message { + mi := &file_shared_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Header.ProtoReflect.Descriptor instead. +func (*Header) Descriptor() ([]byte, []int) { + return file_shared_proto_rawDescGZIP(), []int{0} +} + +func (x *Header) GetTs() *timestamppb.Timestamp { + if x != nil { + return x.Ts + } + return nil +} + +// Keepalive signal sent on streaming RPCs. +type Heartbeat struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Monotonically increasing counter. + Count uint64 `protobuf:"varint,1,opt,name=count,proto3" json:"count,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Heartbeat) Reset() { + *x = Heartbeat{} + mi := &file_shared_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Heartbeat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Heartbeat) ProtoMessage() {} + +func (x *Heartbeat) ProtoReflect() protoreflect.Message { + mi := &file_shared_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. +func (*Heartbeat) Descriptor() ([]byte, []int) { + return file_shared_proto_rawDescGZIP(), []int{1} +} + +func (x *Heartbeat) GetCount() uint64 { + if x != nil { + return x.Count + } + return 0 +} + +// Network socket address. +type Socket struct { + state protoimpl.MessageState `protogen:"open.v1"` + // IPv4 or IPv6 address. + Ip string `protobuf:"bytes,1,opt,name=ip,proto3" json:"ip,omitempty"` + // Port number. + Port int64 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Socket) Reset() { + *x = Socket{} + mi := &file_shared_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Socket) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Socket) ProtoMessage() {} + +func (x *Socket) ProtoReflect() protoreflect.Message { + mi := &file_shared_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Socket.ProtoReflect.Descriptor instead. +func (*Socket) Descriptor() ([]byte, []int) { + return file_shared_proto_rawDescGZIP(), []int{2} +} + +func (x *Socket) GetIp() string { + if x != nil { + return x.Ip + } + return "" +} + +func (x *Socket) GetPort() int64 { + if x != nil { + return x.Port + } + return 0 +} + +var File_shared_proto protoreflect.FileDescriptor + +const file_shared_proto_rawDesc = "" + + "\n" + + "\fshared.proto\x12\x06shared\x1a\x1fgoogle/protobuf/timestamp.proto\"4\n" + + "\x06Header\x12*\n" + + "\x02ts\x18\x01 \x01(\v2\x1a.google.protobuf.TimestampR\x02ts\"!\n" + + "\tHeartbeat\x12\x14\n" + + "\x05count\x18\x01 \x01(\x04R\x05count\",\n" + + "\x06Socket\x12\x0e\n" + + "\x02ip\x18\x01 \x01(\tR\x02ip\x12\x12\n" + + "\x04port\x18\x02 \x01(\x03R\x04portb\x06proto3" + +var ( + file_shared_proto_rawDescOnce sync.Once + file_shared_proto_rawDescData []byte +) + +func file_shared_proto_rawDescGZIP() []byte { + file_shared_proto_rawDescOnce.Do(func() { + file_shared_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_shared_proto_rawDesc), len(file_shared_proto_rawDesc))) + }) + return file_shared_proto_rawDescData +} + +var file_shared_proto_msgTypes = make([]protoimpl.MessageInfo, 3) +var file_shared_proto_goTypes = []any{ + (*Header)(nil), // 0: shared.Header + (*Heartbeat)(nil), // 1: shared.Heartbeat + (*Socket)(nil), // 2: shared.Socket + (*timestamppb.Timestamp)(nil), // 3: google.protobuf.Timestamp +} +var file_shared_proto_depIdxs = []int32{ + 3, // 0: shared.Header.ts:type_name -> google.protobuf.Timestamp + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_shared_proto_init() } +func file_shared_proto_init() { + if File_shared_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_shared_proto_rawDesc), len(file_shared_proto_rawDesc)), + NumEnums: 0, + NumMessages: 3, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_shared_proto_goTypes, + DependencyIndexes: file_shared_proto_depIdxs, + MessageInfos: file_shared_proto_msgTypes, + }.Build() + File_shared_proto = out.File + file_shared_proto_goTypes = nil + file_shared_proto_depIdxs = nil +} diff --git a/pkg/harmonic/proto/upstream/LICENSE b/pkg/harmonic/proto/upstream/LICENSE new file mode 100644 index 00000000..0aa906a0 --- /dev/null +++ b/pkg/harmonic/proto/upstream/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (c) 2025 Meiji Labs Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/pkg/harmonic/proto/upstream/README.md b/pkg/harmonic/proto/upstream/README.md new file mode 100644 index 00000000..35ef049d --- /dev/null +++ b/pkg/harmonic/proto/upstream/README.md @@ -0,0 +1,4 @@ +Vendored from https://github.com/harmonic/validator-protos, tag `v0.2.1` +(commit `fef4a096c9ef92e0d725fc40da2fa3d80f0602cd`), under the Apache License +2.0 in `LICENSE`. The `.proto` files are unmodified; regenerate the Go code +with `../gen.sh`. diff --git a/pkg/harmonic/proto/upstream/auth.proto b/pkg/harmonic/proto/upstream/auth.proto new file mode 100644 index 00000000..5d278868 --- /dev/null +++ b/pkg/harmonic/proto/upstream/auth.proto @@ -0,0 +1,72 @@ +// Authentication service used by the Harmonic validator to obtain access tokens +// for the block engine and relayer. +syntax = "proto3"; + +package auth; + +import "google/protobuf/timestamp.proto"; + +// Role the client is authenticating as. +enum Role { + ROLE_UNSPECIFIED = 0; + // Harmonic validator role. + VALIDATOR = 2; +} + +message GenerateAuthChallengeRequest { + // Role the client is attempting to generate tokens for. + Role role = 1; + // Validator's 32-byte public key. + bytes pubkey = 2; +} + +message GenerateAuthChallengeResponse { + // Challenge string to be signed by the validator. + string challenge = 1; +} + +message GenerateAuthTokensRequest { + // The challenge string returned by GenerateAuthChallenge. + string challenge = 1; + // Validator's 32-byte public key (must match the pubkey from GenerateAuthChallenge). + bytes client_pubkey = 2; + // 64-byte signature of the challenge signed by the validator's private key. + // The validator is expected to sign the challenge prepended with its pubkey: + // sign(pubkey, challenge). + bytes signed_challenge = 3; +} + +// An auth token with an expiration timestamp. +message Token { + // The token string, used as a Bearer token in the Authorization header. + string value = 1; + // When this token expires. + google.protobuf.Timestamp expires_at_utc = 2; +} + +message GenerateAuthTokensResponse { + // Short-lived token used in the Authorization header for API calls. + Token access_token = 1; + // Longer-lived token used to obtain a fresh access_token via RefreshAccessToken. + Token refresh_token = 2; +} + +message RefreshAccessTokenRequest { + // Non-expired refresh token obtained from GenerateAuthTokens. + string refresh_token = 1; +} + +message RefreshAccessTokenResponse { + // Freshly issued access token. + Token access_token = 1; +} + +// Issues auth tokens to the Harmonic validator for API access. +service AuthService { + // Returns a challenge; the validator must sign it to prove key ownership. + rpc GenerateAuthChallenge(GenerateAuthChallengeRequest) returns (GenerateAuthChallengeResponse) {} + // Exchanges a signed challenge for an access/refresh token pair. + rpc GenerateAuthTokens(GenerateAuthTokensRequest) returns (GenerateAuthTokensResponse) {} + // Refreshes an access token using a non-expired refresh token. + rpc RefreshAccessToken(RefreshAccessTokenRequest) returns (RefreshAccessTokenResponse) {} +} diff --git a/pkg/harmonic/proto/upstream/block.proto b/pkg/harmonic/proto/upstream/block.proto new file mode 100644 index 00000000..3a8c1388 --- /dev/null +++ b/pkg/harmonic/proto/upstream/block.proto @@ -0,0 +1,24 @@ +// Harmonic block stream received by the validator from the block engine. +syntax = "proto3"; + +package block; + +// An ordered list of transactions targeting a given slot. +message Block { + // Slot target for these transactions. + uint64 slot = 1; + // Field numbers 2-14 left free for future block-level metadata. + + // Transactions in execution order. + repeated Transaction transactions = 15; +} + +// A single transaction; field numbers match builder-protos `block.Transaction`. +message Transaction { + reserved 2, 4; + // Raw transaction bytes. + bytes transaction = 1; + // Bundle uuid; 0 == standalone transaction, nonzero == revert protected bundle. + // Consecutive transactions sharing the same bundle_id form one bundle, of any length. + uint64 bundle_id = 3; +} diff --git a/pkg/harmonic/proto/upstream/block_engine.proto b/pkg/harmonic/proto/upstream/block_engine.proto new file mode 100644 index 00000000..e14d01c1 --- /dev/null +++ b/pkg/harmonic/proto/upstream/block_engine.proto @@ -0,0 +1,127 @@ +// Block engine interface for the Harmonic validator. +syntax = "proto3"; + +import "packet.proto"; +import "shared.proto"; +import "bundle.proto"; +import "block.proto"; +import "google/protobuf/timestamp.proto"; + +package block_engine; + +// Request to subscribe to the block engine's packet stream. +message SubscribePacketsRequest {} + +// A batch of packets forwarded by the block engine. +message SubscribePacketsResponse { + // Timestamp of when the batch was sent. + shared.Header header = 1; + // Transaction packet batch. + packet.PacketBatch batch = 2; +} + +// Request to subscribe to the block engine's bundle stream. +message SubscribeBundlesRequest {} + +// Request to subscribe to the block engine's block stream. +message SubscribeBlocksRequest { + string version = 1; + string commit_hash = 2; +} + +// One or more bundles forwarded by the block engine. +message SubscribeBundlesResponse { + // Bundles with their server-assigned UUIDs. + repeated bundle.BundleUuid bundles = 1; +} + +// Request for the block builder's fee configuration. +message BlockBuilderFeeInfoRequest {} + +// Fee configuration for the block builder. +message BlockBuilderFeeInfoResponse { + // Block builder's tip-receiving pubkey. + string pubkey = 1; + // Tip commission percentage (0-100). + uint64 commission = 2; +} + +// Notifies the block engine of the validator's upcoming leader slot. +message SubmitLeaderWindowInfoRequest { + // When the leader request is sent. + google.protobuf.Timestamp start_timestamp = 1; + // The leader slot number. + uint64 slot = 2; + // When the leader slot ends. + google.protobuf.Timestamp end_timestamp = 3; +} + +message SubmitLeaderWindowInfoResponse {} + +// Scheduling strategy the block engine runs for this validator session. +enum SchedulingStrategy { + SCHEDULING_STRATEGY_UNSPECIFIED = 0; + // First-in, first-out: continuous arrival-order sequencing. + SCHEDULING_STRATEGY_FIFO = 1; + // Frequent batch auction: discrete batches ordered by priority fees and tips. + SCHEDULING_STRATEGY_FBA = 2; + // Maximum revenue strategy: continuous streaming with revenue-prioritizing selection. + SCHEDULING_STRATEGY_MREV = 3; +} + +// Selects the scheduling strategy for this validator session. +message SetStrategyRequest { + SchedulingStrategy strategy = 1; +} + +message SetStrategyResponse {} + +// Request for available block engine endpoints. +message GetBlockEngineEndpointRequest {} + +// A block engine endpoint. +message BlockEngineEndpoint { + // gRPC URL, e.g. https://mainnet.block-engine.jito.wtf + string block_engine_url = 1; + // ShredStream receiver address (host:port), e.g. shredstream.mainnet.block-engine.jito.wtf:1002 + string shredstream_receiver_address = 2; +} + +// Available block engine endpoints. +message GetBlockEngineEndpointResponse { + // Anycast endpoint that routes to the nearest block engine. + BlockEngineEndpoint global_endpoint = 1; + // Region-specific endpoints for direct connection. + repeated BlockEngineEndpoint regioned_endpoints = 2; +} + +// Service consumed by the Harmonic validator to stream packets and bundles +// from the block engine. +service BlockEngineValidator { + // Subscribe to a stream of transaction packets. + rpc SubscribePackets (SubscribePacketsRequest) returns (stream SubscribePacketsResponse) {} + + // Subscribe to a stream of simulated and profitable bundles. + rpc SubscribeBundles (SubscribeBundlesRequest) returns (stream SubscribeBundlesResponse) {} + + // Subscribe to bundles (v2). + rpc SubscribeBundles2 (SubscribeBundlesRequest) returns (stream SubscribeBundlesResponse) {} + + // Deprecated: use SubscribeBlocks2. + rpc SubscribeBlocks (SubscribeBlocksRequest) returns (stream SubscribeBundlesResponse) {} + + // Subscribe to blocks with explicit bundle boundaries. + rpc SubscribeBlocks2 (SubscribeBlocksRequest) returns (stream block.Block) {} + + // Returns the block builder's tip pubkey and commission. + rpc GetBlockBuilderFeeInfo (BlockBuilderFeeInfoRequest) returns (BlockBuilderFeeInfoResponse) {} + + // Report the validator's upcoming leader slot to the block engine. + rpc SubmitLeaderWindowInfo (SubmitLeaderWindowInfoRequest) returns (SubmitLeaderWindowInfoResponse) {} + + // Discover available block engine endpoints for optimal region selection. + rpc GetBlockEngineEndpoints (GetBlockEngineEndpointRequest) returns (GetBlockEngineEndpointResponse) {} + + // Set the scheduling strategy for this validator session. + rpc SetStrategy (SetStrategyRequest) returns (SetStrategyResponse) {} +} diff --git a/pkg/harmonic/proto/upstream/bundle.proto b/pkg/harmonic/proto/upstream/bundle.proto new file mode 100644 index 00000000..01ce29d5 --- /dev/null +++ b/pkg/harmonic/proto/upstream/bundle.proto @@ -0,0 +1,23 @@ +// Bundle types received by the validator from the block engine. +syntax = "proto3"; + +import "packet.proto"; +import "shared.proto"; + +package bundle; + +// An ordered list of transactions that must execute sequentially and atomically. +message Bundle { + // Timestamp of when this bundle was created. + shared.Header header = 2; + // Transactions in execution order. + repeated packet.Packet packets = 3; +} + +// A Bundle paired with its server-assigned identifier. +message BundleUuid { + // The bundle contents. + bundle.Bundle bundle = 1; + // Unique identifier assigned by the block engine. + string uuid = 2; +} diff --git a/pkg/harmonic/proto/upstream/packet.proto b/pkg/harmonic/proto/upstream/packet.proto new file mode 100644 index 00000000..1adbb612 --- /dev/null +++ b/pkg/harmonic/proto/upstream/packet.proto @@ -0,0 +1,47 @@ +// Transaction packet types received by the validator from the block engine and relayer. +syntax = "proto3"; + +package packet; + +// A batch of transaction packets. +message PacketBatch { + repeated Packet packets = 1; +} + +// A single transaction packet with metadata. +message Packet { + // Raw transaction bytes. + bytes data = 1; + // Packet metadata. + Meta meta = 2; +} + +// Packet metadata mirroring solana_sdk::packet::Meta. +message Meta { + // Number of valid bytes in the packet data. + uint64 size = 1; + // Source IP address. + string addr = 2; + // Source port. + uint32 port = 3; + // Packet processing flags. + PacketFlags flags = 4; + // Stake of the sending node (0 if unknown). + uint64 sender_stake = 5; +} + +// Boolean flags that influence how the validator processes a packet. +message PacketFlags { + // Packet should be dropped. + bool discard = 1; + // Packet was forwarded from another node (not received directly). + bool forwarded = 2; + // Packet is a repair request. + bool repair = 3; + // Packet contains a simple vote transaction. + bool simple_vote_tx = 4; + // Packet is a tracer used for debugging. + bool tracer_packet = 5; + // Packet originated from a staked node. + bool from_staked_node = 6; +} diff --git a/pkg/harmonic/proto/upstream/relayer.proto b/pkg/harmonic/proto/upstream/relayer.proto new file mode 100644 index 00000000..74b8fb88 --- /dev/null +++ b/pkg/harmonic/proto/upstream/relayer.proto @@ -0,0 +1,42 @@ +// Relayer/TPU proxy interface for the Harmonic validator. +syntax = "proto3"; + +import "packet.proto"; +import "shared.proto"; + +package relayer; + +// Request for the relayer's TPU socket configuration. +message GetTpuConfigsRequest {} + +// TPU socket addresses that the validator should advertise in gossip. +message GetTpuConfigsResponse { + // Relayer's TPU socket for regular transactions. + shared.Socket tpu = 1; + // Relayer's TPU forward socket for forwarded transactions. + shared.Socket tpu_forward = 2; +} + +// Request to subscribe to the relayer's packet stream. +message SubscribePacketsRequest {} + +// Multiplexed stream of packet batches and keepalive heartbeats. +message SubscribePacketsResponse { + // Timestamp of when this message was sent. + shared.Header header = 1; + oneof msg { + // Keepalive heartbeat when no packets are available. + shared.Heartbeat heartbeat = 2; + // Batch of transaction packets received at the relayer's TPU ports. + packet.PacketBatch batch = 3; + } +} + +// Relayer service providing TPU proxy for the Harmonic validator. +service Relayer { + // Fetch the relayer's TPU and TPU-forward socket addresses. + rpc GetTpuConfigs (GetTpuConfigsRequest) returns (GetTpuConfigsResponse) {} + + // Subscribe to a stream of packets arriving at the relayer's TPU ports. + rpc SubscribePackets (SubscribePacketsRequest) returns (stream SubscribePacketsResponse) {} +} diff --git a/pkg/harmonic/proto/upstream/shared.proto b/pkg/harmonic/proto/upstream/shared.proto new file mode 100644 index 00000000..ca368355 --- /dev/null +++ b/pkg/harmonic/proto/upstream/shared.proto @@ -0,0 +1,26 @@ +// Common types shared across Harmonic validator proto definitions. +syntax = "proto3"; + +import "google/protobuf/timestamp.proto"; + +package shared; + +// Timestamp header attached to streaming responses. +message Header { + // When this message was generated. + google.protobuf.Timestamp ts = 1; +} + +// Keepalive signal sent on streaming RPCs. +message Heartbeat { + // Monotonically increasing counter. + uint64 count = 1; +} + +// Network socket address. +message Socket { + // IPv4 or IPv6 address. + string ip = 1; + // Port number. + int64 port = 2; +} diff --git a/pkg/harmonic/tokens.go b/pkg/harmonic/tokens.go new file mode 100644 index 00000000..e1f859d7 --- /dev/null +++ b/pkg/harmonic/tokens.go @@ -0,0 +1,142 @@ +package harmonic + +import ( + "context" + "crypto/ed25519" + "fmt" + "sync" + "sync/atomic" + "time" + + authpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/auth" +) + +// TokenSource keeps one endpoint's tokens fresh and publishes the access +// token through its Bearer. +type TokenSource struct { + client authpb.AuthServiceClient + identity ed25519.PrivateKey + bearer Bearer + now func() time.Time + after func(time.Duration) <-chan time.Time + reauth chan struct{} + // generation changes whenever the tokens are invalidated, so a refresh + // that was in flight across an invalidation is discarded. + generation atomic.Uint64 + + mu sync.Mutex + tokens Tokens +} + +func NewTokenSource(client authpb.AuthServiceClient, identity ed25519.PrivateKey) *TokenSource { + return &TokenSource{ + client: client, + identity: identity, + now: time.Now, + after: time.After, + reauth: make(chan struct{}, 1), + } +} + +// Bearer returns the credentials that carry this source's access token. +func (s *TokenSource) Bearer() *Bearer { + return &s.bearer +} + +// Tokens returns the current tokens. +func (s *TokenSource) Tokens() Tokens { + s.mu.Lock() + defer s.mu.Unlock() + return s.tokens +} + +// Authenticate replaces the tokens with a fresh pair. +func (s *TokenSource) Authenticate(ctx context.Context) error { + tokens, err := Authenticate(ctx, s.client, s.identity) + if err == nil { + err = s.checkUnexpired(tokens.Access, tokens.Refresh) + } + if err != nil { + s.bearer.Clear() + return err + } + s.mu.Lock() + s.tokens = tokens + s.mu.Unlock() + s.bearer.Set(tokens.Access.Value) + return nil +} + +// Invalidate discards the access token after an UNAUTHENTICATED or +// PERMISSION_DENIED status. Run then authenticates again. +func (s *TokenSource) Invalidate() { + s.generation.Add(1) + s.bearer.Clear() + select { + case s.reauth <- struct{}{}: + default: + } +} + +// Run refreshes the access token halfway to its expiry and authenticates +// again halfway to the refresh token's expiry, until ctx ends or +// authentication fails. A failed refresh falls back to authenticating again. +// Call Authenticate first. +func (s *TokenSource) Run(ctx context.Context) error { + for { + tokens := s.Tokens() + now := s.now() + refreshAt := halfway(now, tokens.Access.Expiry) + reauthAt := halfway(now, tokens.Refresh.Expiry) + wait, reauth := refreshAt.Sub(now), false + if !reauthAt.After(refreshAt) { + wait, reauth = reauthAt.Sub(now), true + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-s.reauth: + reauth = true + case <-s.after(wait): + } + if !reauth && s.refresh(ctx, tokens.Refresh.Value) { + continue + } + if err := s.Authenticate(ctx); err != nil { + return fmt.Errorf("authenticate: %w", err) + } + } +} + +func (s *TokenSource) refresh(ctx context.Context, refreshToken string) bool { + generation := s.generation.Load() + access, err := RefreshAccess(ctx, s.client, refreshToken) + if err == nil { + err = s.checkUnexpired(access) + } + if err != nil || s.generation.Load() != generation { + return false + } + s.mu.Lock() + s.tokens.Access = access + s.mu.Unlock() + s.bearer.Set(access.Value) + return true +} + +func (s *TokenSource) checkUnexpired(tokens ...Token) error { + now := s.now() + for _, token := range tokens { + if !token.Expiry.After(now) { + return fmt.Errorf("token expired at %s", token.Expiry.Format(time.RFC3339)) + } + } + return nil +} + +func halfway(now, expiry time.Time) time.Time { + if !expiry.After(now) { + return now + } + return now.Add(expiry.Sub(now) / 2) +} From 7c6402e7910db44bd358327e6336331294484796 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 18:20:02 +0000 Subject: [PATCH 14/25] harmonic: relayer session and TPU advertisement --- pkg/harmonic/auth_test.go | 3 +- pkg/harmonic/mock_test.go | 80 ++++++- pkg/harmonic/relayer.go | 390 +++++++++++++++++++++++++++++++++++ pkg/harmonic/relayer_test.go | 286 +++++++++++++++++++++++++ 4 files changed, 753 insertions(+), 6 deletions(-) create mode 100644 pkg/harmonic/relayer.go create mode 100644 pkg/harmonic/relayer_test.go diff --git a/pkg/harmonic/auth_test.go b/pkg/harmonic/auth_test.go index 55230da1..f5c4257a 100644 --- a/pkg/harmonic/auth_test.go +++ b/pkg/harmonic/auth_test.go @@ -70,7 +70,8 @@ func TestAuthMessageIsBase58PubkeyDashChallenge(t *testing.T) { func TestAuthenticateHandshakeAndBearer(t *testing.T) { now := time.Unix(1_800_000_000, 0) auth := newMockAuth(func() time.Time { return now }) - relayer := &mockRelayer{} + relayer := newMockRelayer(nil) + relayer.heartbeatEvery = 0 cfg := serveBufconn(t, func(s *grpc.Server) { authpb.RegisterAuthServiceServer(s, auth) relayerpb.RegisterRelayerServer(s, relayer) diff --git a/pkg/harmonic/mock_test.go b/pkg/harmonic/mock_test.go index 2ceeba02..7903b63c 100644 --- a/pkg/harmonic/mock_test.go +++ b/pkg/harmonic/mock_test.go @@ -4,12 +4,15 @@ import ( "context" "crypto/ed25519" "net" + "strings" "sync" + "sync/atomic" "testing" "time" authpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/auth" relayerpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/relayer" + sharedpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/shared" "github.com/gagliardetto/solana-go" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -99,19 +102,86 @@ func (m *mockAuth) RefreshAccessToken(ctx context.Context, req *authpb.RefreshAc return &authpb.RefreshAccessTokenResponse{AccessToken: m.token("access", m.accessTTL)}, nil } -// mockRelayer records the metadata of each GetTpuConfigs call. +// mockRelayer serves a scripted relayer: it records each GetTpuConfigs +// call's metadata, returns the configured TPU sockets, and streams heartbeats +// every heartbeatEvery (unless stalled) plus whatever the test sends. type mockRelayer struct { relayerpb.UnimplementedRelayerServer - mu sync.Mutex - md []metadata.MD + heartbeatEvery time.Duration + send chan *relayerpb.SubscribePacketsResponse + endStream chan error + stall atomic.Bool + subscriptions atomic.Int32 + + mu sync.Mutex + md []metadata.MD + config *relayerpb.GetTpuConfigsResponse +} + +func newMockRelayer(config *relayerpb.GetTpuConfigsResponse) *mockRelayer { + return &mockRelayer{ + heartbeatEvery: 20 * time.Millisecond, + send: make(chan *relayerpb.SubscribePacketsResponse, 16), + endStream: make(chan error, 1), + config: config, + } +} + +func (r *mockRelayer) setConfig(config *relayerpb.GetTpuConfigsResponse) { + r.mu.Lock() + r.config = config + r.mu.Unlock() } func (r *mockRelayer) GetTpuConfigs(ctx context.Context, _ *relayerpb.GetTpuConfigsRequest) (*relayerpb.GetTpuConfigsResponse, error) { md, _ := metadata.FromIncomingContext(ctx) r.mu.Lock() + defer r.mu.Unlock() r.md = append(r.md, md) - r.mu.Unlock() - return &relayerpb.GetTpuConfigsResponse{}, nil + if r.config == nil { + return &relayerpb.GetTpuConfigsResponse{}, nil + } + return r.config, nil +} + +func (r *mockRelayer) SubscribePackets(_ *relayerpb.SubscribePacketsRequest, stream relayerpb.Relayer_SubscribePacketsServer) error { + md, _ := metadata.FromIncomingContext(stream.Context()) + if auth := md.Get("authorization"); len(auth) != 1 || !strings.HasPrefix(auth[0], "Bearer access-") { + return status.Error(codes.Unauthenticated, "missing bearer") + } + r.subscriptions.Add(1) + if err := stream.SendHeader(metadata.MD{}); err != nil { + return err + } + var heartbeats <-chan time.Time + if r.heartbeatEvery > 0 { + ticker := time.NewTicker(r.heartbeatEvery) + defer ticker.Stop() + heartbeats = ticker.C + } + var count uint64 + for { + select { + case <-stream.Context().Done(): + return nil + case err := <-r.endStream: + return err + case msg := <-r.send: + if err := stream.Send(msg); err != nil { + return err + } + case <-heartbeats: + if r.stall.Load() { + continue + } + count++ + if err := stream.Send(&relayerpb.SubscribePacketsResponse{ + Msg: &relayerpb.SubscribePacketsResponse_Heartbeat{Heartbeat: &sharedpb.Heartbeat{Count: count}}, + }); err != nil { + return err + } + } + } } func (r *mockRelayer) last() metadata.MD { diff --git a/pkg/harmonic/relayer.go b/pkg/harmonic/relayer.go new file mode 100644 index 00000000..b2076481 --- /dev/null +++ b/pkg/harmonic/relayer.go @@ -0,0 +1,390 @@ +package harmonic + +import ( + "context" + "crypto/ed25519" + "errors" + "fmt" + "math/rand/v2" + "net" + "net/netip" + "sync" + "sync/atomic" + "time" + + "github.com/Overclock-Validator/mithril/pkg/gossip" + authpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/auth" + relayerpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/relayer" + sharedpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/shared" + "github.com/Overclock-Validator/mithril/pkg/mlog" + tpupacket "github.com/Overclock-Validator/mithril/pkg/tpu/packet" +) + +const ( + unaryRPCTimeout = 10 * time.Second + streamHeaderTimeout = 10 * time.Second + defaultConfigInterval = 60 * time.Second + defaultHeartbeatTimeout = 400 * time.Millisecond + defaultMaxBackoff = 5 * time.Second + // quicPortOffset is where a relayer's QUIC sockets sit relative to the UDP + // ports it reports. + quicPortOffset = 6 +) + +var errHeartbeatTimeout = errors.New("relayer stream went quiet") + +// PacketSink takes packets from a Harmonic stream into the TPU. It must not +// block. +type PacketSink interface { + Submit(wire []byte, src tpupacket.Source) bool +} + +// TPUConfig is a relayer's validated TPU UDP sockets. +type TPUConfig struct { + TPU netip.AddrPort + TPUForward netip.AddrPort +} + +// Sockets returns the gossip sockets that advertise the relayer: the UDP +// sockets as reported and the QUIC sockets 6 ports above them. +func (c TPUConfig) Sockets() gossip.TPUSockets { + udp := func(ap netip.AddrPort, offset uint16) *net.UDPAddr { + return net.UDPAddrFromAddrPort(netip.AddrPortFrom(ap.Addr(), ap.Port()+offset)) + } + return gossip.TPUSockets{ + UDP: udp(c.TPU, 0), + QUIC: udp(c.TPU, quicPortOffset), + ForwardsUDP: udp(c.TPUForward, 0), + ForwardsQUIC: udp(c.TPUForward, quicPortOffset), + } +} + +// ParseTPUConfig validates a GetTpuConfigs response. +func ParseTPUConfig(resp *relayerpb.GetTpuConfigsResponse) (TPUConfig, error) { + tpu, err := parseRelayerSocket(resp.GetTpu(), "tpu") + if err != nil { + return TPUConfig{}, err + } + forward, err := parseRelayerSocket(resp.GetTpuForward(), "tpu_forward") + if err != nil { + return TPUConfig{}, err + } + return TPUConfig{TPU: tpu, TPUForward: forward}, nil +} + +func parseRelayerSocket(socket *sharedpb.Socket, name string) (netip.AddrPort, error) { + if socket == nil { + return netip.AddrPort{}, fmt.Errorf("relayer %s socket missing", name) + } + addr, err := netip.ParseAddr(socket.GetIp()) + if err != nil || !addr.Is4() || addr.IsUnspecified() { + return netip.AddrPort{}, fmt.Errorf("relayer %s address %q is not a usable IPv4 address", name, socket.GetIp()) + } + // The QUIC socket at port+6 must be a valid port too. + if port := socket.GetPort(); port < 1 || port > 0xffff-quicPortOffset { + return netip.AddrPort{}, fmt.Errorf("relayer %s port %d out of range", name, port) + } + return netip.AddrPortFrom(addr, uint16(socket.GetPort())), nil +} + +// RelayerConfig configures a relayer session. +type RelayerConfig struct { + Endpoint Endpoint + InsecureSkipVerify bool + Identity ed25519.PrivateKey + Packets PacketSink + // OnStatus is called from the session goroutine with the relayer's TPU + // config whenever it becomes usable or changes, and with nil whenever it + // stops being usable. + OnStatus func(*TPUConfig) + // Zero values use the defaults: 60s, 400ms, 5s. + ConfigInterval time.Duration + HeartbeatTimeout time.Duration + MaxBackoff time.Duration + // Dialer replaces the TCP dialer, for tests. + Dialer func(ctx context.Context, addr string) (net.Conn, error) +} + +// RelayerStats counts relayer session activity. +type RelayerStats struct { + Sessions uint64 + Resets uint64 + Heartbeats uint64 + Batches uint64 + Packets uint64 + DroppedSize uint64 + DroppedFull uint64 + InvalidTPUConfigs uint64 +} + +// RelayerSession keeps an authenticated packet subscription to a relayer and +// feeds its packets into the TPU. +type RelayerSession struct { + cfg RelayerConfig + stats struct { + sessions, resets, heartbeats, batches, packets, droppedSize, droppedFull, invalidConfigs atomic.Uint64 + } + statusMu sync.Mutex + status *TPUConfig +} + +func NewRelayerSession(cfg RelayerConfig) *RelayerSession { + if cfg.ConfigInterval <= 0 { + cfg.ConfigInterval = defaultConfigInterval + } + if cfg.HeartbeatTimeout <= 0 { + cfg.HeartbeatTimeout = defaultHeartbeatTimeout + } + if cfg.MaxBackoff <= 0 { + cfg.MaxBackoff = defaultMaxBackoff + } + return &RelayerSession{cfg: cfg} +} + +func (r *RelayerSession) Stats() RelayerStats { + return RelayerStats{ + Sessions: r.stats.sessions.Load(), + Resets: r.stats.resets.Load(), + Heartbeats: r.stats.heartbeats.Load(), + Batches: r.stats.batches.Load(), + Packets: r.stats.packets.Load(), + DroppedSize: r.stats.droppedSize.Load(), + DroppedFull: r.stats.droppedFull.Load(), + InvalidTPUConfigs: r.stats.invalidConfigs.Load(), + } +} + +// Status returns the relayer's TPU config while it is usable, else nil. +func (r *RelayerSession) Status() *TPUConfig { + r.statusMu.Lock() + defer r.statusMu.Unlock() + return r.status +} + +// Run keeps a session open until ctx ends, reconnecting after jittered +// backoff whenever it fails. +func (r *RelayerSession) Run(ctx context.Context) { + for ctx.Err() == nil { + err := r.runSession(ctx) + r.setStatus(nil) + if ctx.Err() != nil { + return + } + r.stats.resets.Add(1) + mlog.Log.Infof("Harmonic relayer %s session reset: %v", r.cfg.Endpoint.Target, err) + select { + case <-ctx.Done(): + case <-time.After(jitteredBackoff(r.cfg.MaxBackoff)): + } + } +} + +// jitteredBackoff waits between half of max and max, so reconnects spread +// out but never exceed max. +func jitteredBackoff(max time.Duration) time.Duration { + return max/2 + rand.N(max/2+1) +} + +func (r *RelayerSession) setStatus(cfg *TPUConfig) { + r.statusMu.Lock() + changed := (r.status == nil) != (cfg == nil) || (cfg != nil && *r.status != *cfg) + r.status = cfg + r.statusMu.Unlock() + if changed && r.cfg.OnStatus != nil { + r.cfg.OnStatus(cfg) + } +} + +func (r *RelayerSession) runSession(ctx context.Context) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + tokens := NewTokenSource(nil, r.cfg.Identity) + conn, err := Dial(ctx, DialConfig{ + Endpoint: r.cfg.Endpoint, + InsecureSkipVerify: r.cfg.InsecureSkipVerify, + Credentials: tokens.Bearer(), + Dialer: r.cfg.Dialer, + }) + if err != nil { + return err + } + defer conn.Close() + tokens.client = authpb.NewAuthServiceClient(conn) + if err := tokens.Authenticate(ctx); err != nil { + return err + } + r.stats.sessions.Add(1) + errs := make(chan error, 2) + go func() { errs <- tokens.Run(ctx) }() + + client := relayerpb.NewRelayerClient(conn) + config, configErr := r.fetchTPUConfig(ctx, client) + if IsAuthError(configErr) { + return configErr + } + stream, err := subscribePackets(ctx, client) + if err != nil { + return err + } + beats := make(chan struct{}, 1) + go func() { errs <- r.readPackets(stream, beats) }() + + if configErr == nil { + r.setStatus(&config) + } + watchdog := time.NewTimer(r.cfg.HeartbeatTimeout) + defer watchdog.Stop() + refresh := time.NewTicker(r.cfg.ConfigInterval) + defer refresh.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-errs: + return err + case <-beats: + watchdog.Reset(r.cfg.HeartbeatTimeout) + case <-watchdog.C: + return errHeartbeatTimeout + case <-refresh.C: + config, err := r.fetchTPUConfig(ctx, client) + if IsAuthError(err) { + return err + } + if err != nil { + r.setStatus(nil) + continue + } + r.setStatus(&config) + } + } +} + +// fetchTPUConfig returns the relayer's TPU config. An invalid config makes +// the relayer unusable for advertisement but does not end the session. +func (r *RelayerSession) fetchTPUConfig(ctx context.Context, client relayerpb.RelayerClient) (TPUConfig, error) { + ctx, cancel := context.WithTimeout(ctx, unaryRPCTimeout) + defer cancel() + resp, err := client.GetTpuConfigs(ctx, &relayerpb.GetTpuConfigsRequest{}) + if err != nil { + return TPUConfig{}, fmt.Errorf("get TPU configs: %w", err) + } + config, err := ParseTPUConfig(resp) + if err != nil { + r.stats.invalidConfigs.Add(1) + mlog.Log.Infof("Harmonic relayer %s: %v", r.cfg.Endpoint.Target, err) + } + return config, err +} + +// subscribePackets opens the packet stream and waits, for at most 10s, for +// its response headers; the stream itself has no deadline. +func subscribePackets(ctx context.Context, client relayerpb.RelayerClient) (relayerpb.Relayer_SubscribePacketsClient, error) { + stream, err := client.SubscribePackets(ctx, &relayerpb.SubscribePacketsRequest{}) + if err != nil { + return nil, fmt.Errorf("subscribe packets: %w", err) + } + headers := make(chan error, 1) + go func() { + _, err := stream.Header() + headers <- err + }() + select { + case err := <-headers: + if err != nil { + return nil, fmt.Errorf("subscribe packets: %w", err) + } + return stream, nil + case <-time.After(streamHeaderTimeout): + return nil, errors.New("subscribe packets: no response headers within 10s") + } +} + +func (r *RelayerSession) readPackets(stream relayerpb.Relayer_SubscribePacketsClient, beats chan<- struct{}) error { + for { + msg, err := stream.Recv() + if err != nil { + return fmt.Errorf("packet stream: %w", err) + } + select { + case beats <- struct{}{}: + default: + } + switch m := msg.GetMsg().(type) { + case *relayerpb.SubscribePacketsResponse_Heartbeat: + r.stats.heartbeats.Add(1) + case *relayerpb.SubscribePacketsResponse_Batch: + r.stats.batches.Add(1) + for _, pkt := range m.Batch.GetPackets() { + data := pkt.GetData() + if len(data) == 0 || len(data) > tpupacket.DataSize { + r.stats.droppedSize.Add(1) + continue + } + r.stats.packets.Add(1) + if !r.cfg.Packets.Submit(data, tpupacket.SourceRelayer) { + r.stats.droppedFull.Add(1) + } + } + } + } +} + +// TPUAdvertiser publishes the TPU sockets in gossip. +type TPUAdvertiser interface { + SetTPUSockets(gossip.TPUSockets) error +} + +// TPUAdvertisement chooses the TPU sockets gossip advertises: the relayer's +// while it is usable and the block engine session is healthy, since relayed +// packets are only worth attracting while the validator can schedule them, +// and this validator's own otherwise. +type TPUAdvertisement struct { + gossip TPUAdvertiser + local gossip.TPUSockets + + mu sync.Mutex + relayer *TPUConfig + blockEngineHealth bool + advertised *TPUConfig + initialized bool +} + +func NewTPUAdvertisement(g TPUAdvertiser, local gossip.TPUSockets) *TPUAdvertisement { + return &TPUAdvertisement{gossip: g, local: local} +} + +// SetRelayer records the relayer's usable TPU config, or nil. +func (a *TPUAdvertisement) SetRelayer(cfg *TPUConfig) error { + a.mu.Lock() + defer a.mu.Unlock() + a.relayer = cfg + return a.applyLocked() +} + +// SetBlockEngineHealthy records whether the block engine session is healthy. +func (a *TPUAdvertisement) SetBlockEngineHealthy(healthy bool) error { + a.mu.Lock() + defer a.mu.Unlock() + a.blockEngineHealth = healthy + return a.applyLocked() +} + +func (a *TPUAdvertisement) applyLocked() error { + var want *TPUConfig + if a.relayer != nil && a.blockEngineHealth { + want = a.relayer + } + if a.initialized && (want == nil) == (a.advertised == nil) && (want == nil || *want == *a.advertised) { + return nil + } + sockets := a.local + if want != nil { + sockets = want.Sockets() + } + if err := a.gossip.SetTPUSockets(sockets); err != nil { + return err + } + a.advertised, a.initialized = want, true + return nil +} diff --git a/pkg/harmonic/relayer_test.go b/pkg/harmonic/relayer_test.go new file mode 100644 index 00000000..37a24b12 --- /dev/null +++ b/pkg/harmonic/relayer_test.go @@ -0,0 +1,286 @@ +package harmonic + +import ( + "context" + "errors" + "net" + "net/netip" + "sync" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/gossip" + authpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/auth" + packetpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/packet" + relayerpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/relayer" + sharedpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/shared" + tpupacket "github.com/Overclock-Validator/mithril/pkg/tpu/packet" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +func tpuConfigResponse(ip string, tpuPort, forwardPort int64) *relayerpb.GetTpuConfigsResponse { + return &relayerpb.GetTpuConfigsResponse{ + Tpu: &sharedpb.Socket{Ip: ip, Port: tpuPort}, + TpuForward: &sharedpb.Socket{Ip: ip, Port: forwardPort}, + } +} + +func TestParseTPUConfig(t *testing.T) { + got, err := ParseTPUConfig(tpuConfigResponse("198.51.100.7", 11222, 11223)) + require.NoError(t, err) + assert.Equal(t, netip.MustParseAddrPort("198.51.100.7:11222"), got.TPU) + assert.Equal(t, netip.MustParseAddrPort("198.51.100.7:11223"), got.TPUForward) + sockets := got.Sockets() + assert.Equal(t, "198.51.100.7:11222", sockets.UDP.String()) + assert.Equal(t, "198.51.100.7:11228", sockets.QUIC.String(), "QUIC is 6 ports above UDP") + assert.Equal(t, "198.51.100.7:11223", sockets.ForwardsUDP.String()) + assert.Equal(t, "198.51.100.7:11229", sockets.ForwardsQUIC.String()) + + _, err = ParseTPUConfig(tpuConfigResponse("198.51.100.7", 65529, 1)) + assert.NoError(t, err, "the highest port whose QUIC socket still fits") + + for name, resp := range map[string]*relayerpb.GetTpuConfigsResponse{ + "empty": {}, + "missing forward": {Tpu: &sharedpb.Socket{Ip: "198.51.100.7", Port: 11222}}, + "unspecified": tpuConfigResponse("0.0.0.0", 11222, 11223), + "IPv6": tpuConfigResponse("2001:db8::1", 11222, 11223), + "IPv4-mapped IPv6": tpuConfigResponse("::ffff:198.51.100.7", 11222, 11223), + "not an address": tpuConfigResponse("relayer.example.com", 11222, 11223), + "zero port": tpuConfigResponse("198.51.100.7", 0, 11223), + "negative port": tpuConfigResponse("198.51.100.7", -1, 11223), + "QUIC overflow": tpuConfigResponse("198.51.100.7", 65530, 11223), + } { + _, err := ParseTPUConfig(resp) + assert.Errorf(t, err, "%s", name) + } +} + +func TestJitteredBackoffStaysWithinBound(t *testing.T) { + for range 1000 { + d := jitteredBackoff(5 * time.Second) + require.GreaterOrEqual(t, d, 2500*time.Millisecond) + require.LessOrEqual(t, d, 5*time.Second) + } +} + +type packetRecorder struct { + mu sync.Mutex + packets [][]byte + sources []tpupacket.Source + refuse bool +} + +func (p *packetRecorder) Submit(wire []byte, src tpupacket.Source) bool { + p.mu.Lock() + defer p.mu.Unlock() + p.packets = append(p.packets, append([]byte(nil), wire...)) + p.sources = append(p.sources, src) + return !p.refuse +} + +func (p *packetRecorder) count() int { + p.mu.Lock() + defer p.mu.Unlock() + return len(p.packets) +} + +type statusRecorder struct { + mu sync.Mutex + statuses []*TPUConfig +} + +func (s *statusRecorder) record(cfg *TPUConfig) { + s.mu.Lock() + defer s.mu.Unlock() + s.statuses = append(s.statuses, cfg) +} + +func (s *statusRecorder) all() []*TPUConfig { + s.mu.Lock() + defer s.mu.Unlock() + return append([]*TPUConfig(nil), s.statuses...) +} + +func (s *statusRecorder) last() *TPUConfig { + all := s.all() + if len(all) == 0 { + return nil + } + return all[len(all)-1] +} + +type relayerHarness struct { + relayer *mockRelayer + session *RelayerSession + packets *packetRecorder + statuses *statusRecorder +} + +func startRelayerSession(t *testing.T, config *relayerpb.GetTpuConfigsResponse, configInterval time.Duration) *relayerHarness { + t.Helper() + h := &relayerHarness{relayer: newMockRelayer(config), packets: &packetRecorder{}, statuses: &statusRecorder{}} + dial := serveBufconn(t, func(s *grpc.Server) { + authpb.RegisterAuthServiceServer(s, newMockAuth(time.Now)) + relayerpb.RegisterRelayerServer(s, h.relayer) + }) + h.session = NewRelayerSession(RelayerConfig{ + Endpoint: dial.Endpoint, + Dialer: dial.Dialer, + Identity: rfc8032Key(t), + Packets: h.packets, + OnStatus: h.statuses.record, + ConfigInterval: configInterval, + HeartbeatTimeout: 150 * time.Millisecond, + MaxBackoff: 20 * time.Millisecond, + }) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + h.session.Run(ctx) + close(done) + }() + t.Cleanup(func() { + cancel() + <-done + }) + return h +} + +func batch(packets ...[]byte) *relayerpb.SubscribePacketsResponse { + pb := &packetpb.PacketBatch{} + for _, data := range packets { + pb.Packets = append(pb.Packets, &packetpb.Packet{Data: data}) + } + return &relayerpb.SubscribePacketsResponse{Msg: &relayerpb.SubscribePacketsResponse_Batch{Batch: pb}} +} + +func TestRelayerSessionDeliversPacketsAndReportsConfig(t *testing.T) { + h := startRelayerSession(t, tpuConfigResponse("198.51.100.7", 11222, 11223), time.Hour) + require.Eventually(t, func() bool { return h.session.Status() != nil }, 2*time.Second, 5*time.Millisecond) + assert.Equal(t, netip.MustParseAddrPort("198.51.100.7:11222"), h.session.Status().TPU) + assert.Equal(t, "Bearer access-1", h.relayer.last().Get("authorization")[0]) + + h.relayer.send <- batch([]byte{1, 2, 3}, nil, make([]byte, tpupacket.DataSize+1), make([]byte, tpupacket.DataSize)) + require.Eventually(t, func() bool { return h.packets.count() == 2 }, 2*time.Second, 5*time.Millisecond) + assert.Equal(t, []byte{1, 2, 3}, h.packets.packets[0]) + assert.Equal(t, []tpupacket.Source{tpupacket.SourceRelayer, tpupacket.SourceRelayer}, h.packets.sources) + + // Heartbeats keep a quiet stream alive across several watchdog periods. + time.Sleep(600 * time.Millisecond) + stats := h.session.Stats() + assert.Equal(t, uint64(1), stats.Sessions) + assert.Zero(t, stats.Resets) + assert.Equal(t, uint64(1), stats.Batches) + assert.Equal(t, uint64(2), stats.Packets) + assert.Equal(t, uint64(2), stats.DroppedSize, "empty and oversized packets are dropped one by one") + assert.NotZero(t, stats.Heartbeats) + assert.Len(t, h.statuses.all(), 1, "the status only changes once") +} + +func TestRelayerSessionResetsWhenTheStreamGoesQuiet(t *testing.T) { + h := startRelayerSession(t, tpuConfigResponse("198.51.100.7", 11222, 11223), time.Hour) + require.Eventually(t, func() bool { return h.session.Status() != nil }, 2*time.Second, 5*time.Millisecond) + + h.relayer.stall.Store(true) + require.Eventually(t, func() bool { return h.session.Status() == nil }, 2*time.Second, 5*time.Millisecond) + assert.NotZero(t, h.session.Stats().Resets) + h.relayer.stall.Store(false) + require.Eventually(t, func() bool { return h.session.Status() != nil }, 2*time.Second, 5*time.Millisecond) + assert.GreaterOrEqual(t, h.relayer.subscriptions.Load(), int32(2), "a new session subscribed again") + statuses := h.statuses.all() + require.GreaterOrEqual(t, len(statuses), 3) + assert.NotNil(t, statuses[0]) + assert.Nil(t, statuses[1], "the relayer stops being advertised while it is down") +} + +func TestRelayerSessionResetsWhenTheStreamFails(t *testing.T) { + h := startRelayerSession(t, tpuConfigResponse("198.51.100.7", 11222, 11223), time.Hour) + require.Eventually(t, func() bool { return h.session.Status() != nil }, 2*time.Second, 5*time.Millisecond) + h.relayer.endStream <- status.Error(codes.Unavailable, "relayer restarting") + require.Eventually(t, func() bool { return h.session.Stats().Sessions == 2 }, 2*time.Second, 5*time.Millisecond) + assert.Equal(t, uint64(1), h.session.Stats().Resets) +} + +func TestRelayerSessionNeverReportsAnInvalidConfig(t *testing.T) { + h := startRelayerSession(t, tpuConfigResponse("0.0.0.0", 11222, 11223), 50*time.Millisecond) + require.Eventually(t, func() bool { return h.session.Stats().InvalidTPUConfigs >= 2 }, 2*time.Second, 5*time.Millisecond) + assert.Nil(t, h.session.Status()) + assert.Empty(t, h.statuses.all()) + + // Packets still flow while the config is unusable. + h.relayer.send <- batch([]byte{9}) + require.Eventually(t, func() bool { return h.packets.count() == 1 }, 2*time.Second, 5*time.Millisecond) + + h.relayer.setConfig(tpuConfigResponse("198.51.100.8", 12000, 12001)) + require.Eventually(t, func() bool { return h.session.Status() != nil }, 2*time.Second, 5*time.Millisecond) + assert.Equal(t, netip.MustParseAddrPort("198.51.100.8:12000"), h.statuses.last().TPU) + + h.relayer.setConfig(tpuConfigResponse("198.51.100.9", 12000, 12001)) + require.Eventually(t, func() bool { + last := h.statuses.last() + return last != nil && last.TPU.Addr() == netip.MustParseAddr("198.51.100.9") + }, 2*time.Second, 5*time.Millisecond, "an address change is reported") + assert.Zero(t, h.session.Stats().Resets, "a config change does not reset the session") +} + +type advertiserRecorder struct { + calls []gossip.TPUSockets + err error +} + +func (a *advertiserRecorder) SetTPUSockets(s gossip.TPUSockets) error { + if a.err != nil { + return a.err + } + a.calls = append(a.calls, s) + return nil +} + +func TestTPUAdvertisementAdvertisesTheRelayerOnlyWhileUsable(t *testing.T) { + local := gossip.TPUSockets{QUIC: netipUDP("203.0.113.10:8009"), ForwardsQUIC: netipUDP("203.0.113.10:8009")} + relayerA, err := ParseTPUConfig(tpuConfigResponse("198.51.100.7", 11222, 11223)) + require.NoError(t, err) + relayerB, err := ParseTPUConfig(tpuConfigResponse("198.51.100.8", 11222, 11223)) + require.NoError(t, err) + gossipRec := &advertiserRecorder{} + ad := NewTPUAdvertisement(gossipRec, local) + + require.NoError(t, ad.SetBlockEngineHealthy(false)) + require.NoError(t, ad.SetRelayer(&relayerA)) + require.Len(t, gossipRec.calls, 1, "without a healthy block engine the local TPU stays advertised") + assert.Equal(t, local, gossipRec.calls[0]) + + require.NoError(t, ad.SetBlockEngineHealthy(true)) + require.Len(t, gossipRec.calls, 2) + assert.Equal(t, relayerA.Sockets(), gossipRec.calls[1]) + require.NoError(t, ad.SetBlockEngineHealthy(true)) + require.NoError(t, ad.SetRelayer(&relayerA)) + assert.Len(t, gossipRec.calls, 2, "an unchanged choice is not re-published") + + require.NoError(t, ad.SetRelayer(&relayerB)) + require.Len(t, gossipRec.calls, 3) + assert.Equal(t, relayerB.Sockets(), gossipRec.calls[2]) + + require.NoError(t, ad.SetRelayer(nil)) + require.Len(t, gossipRec.calls, 4) + assert.Equal(t, local, gossipRec.calls[3], "a lost relayer reverts to the local TPU") + require.NoError(t, ad.SetBlockEngineHealthy(false)) + assert.Len(t, gossipRec.calls, 4) + + // A failed publish is retried on the next update. + gossipRec.err = errors.New("gossip not running") + require.NoError(t, ad.SetBlockEngineHealthy(true)) + assert.Error(t, ad.SetRelayer(&relayerA)) + gossipRec.err = nil + require.NoError(t, ad.SetRelayer(&relayerA)) + require.Len(t, gossipRec.calls, 5) + assert.Equal(t, relayerA.Sockets(), gossipRec.calls[4]) +} + +func netipUDP(s string) *net.UDPAddr { + return net.UDPAddrFromAddrPort(netip.MustParseAddrPort(s)) +} From c4507410aca5df528cad7879d9414f12566f46c4 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:48:16 +0000 Subject: [PATCH 15/25] harmonic: block engine session --- pkg/harmonic/blockengine.go | 509 +++++++++++++++++++++++++++++++ pkg/harmonic/blockengine_test.go | 297 ++++++++++++++++++ pkg/harmonic/mock_test.go | 135 ++++++++ pkg/harmonic/relayer.go | 20 +- pkg/tpu/packet/packet.go | 4 + 5 files changed, 958 insertions(+), 7 deletions(-) create mode 100644 pkg/harmonic/blockengine.go create mode 100644 pkg/harmonic/blockengine_test.go diff --git a/pkg/harmonic/blockengine.go b/pkg/harmonic/blockengine.go new file mode 100644 index 00000000..078cab5a --- /dev/null +++ b/pkg/harmonic/blockengine.go @@ -0,0 +1,509 @@ +package harmonic + +import ( + "context" + "crypto/ed25519" + "errors" + "fmt" + "net" + "sync" + "sync/atomic" + "time" + + authpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/auth" + blockpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/block" + blockenginepb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/blockengine" + "github.com/Overclock-Validator/mithril/pkg/mlog" + tpupacket "github.com/Overclock-Validator/mithril/pkg/tpu/packet" + "github.com/Overclock-Validator/mithril/pkg/version" + "github.com/gagliardetto/solana-go" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/timestamppb" +) + +const ( + defaultFeeInfoInterval = 5 * time.Minute + leaderNoticeTimeout = 8 * time.Second + maxVersionBytes = 32 + maxCommitBytes = 40 + maxCommissionPercent = 100 +) + +var errStrategyChanged = errors.New("scheduling strategy changed") + +// BlockHandler receives the Harmonic block stream. +type BlockHandler interface { + // HandleBlock receives each block message in stream order, stamped with + // its arrival time. It runs on the block-stream goroutine and may block, + // which holds back the stream. + HandleBlock(block *blockpb.Block, arrival time.Time) + // StreamLost reports that the session reset after slot's leader notice + // was sent and before the slot ended, so the rest of its block is lost. + StreamLost(slot uint64) +} + +// FeeInfo is the block builder's validated tip fee configuration. +type FeeInfo struct { + Pubkey solana.PublicKey + // Commission is a percentage, 0 to 100. + Commission uint64 +} + +// ParseFeeInfo validates a GetBlockBuilderFeeInfo response. +func ParseFeeInfo(resp *blockenginepb.BlockBuilderFeeInfoResponse) (FeeInfo, error) { + pubkey, err := solana.PublicKeyFromBase58(resp.GetPubkey()) + if err != nil { + return FeeInfo{}, fmt.Errorf("block builder pubkey %q: %w", resp.GetPubkey(), err) + } + if pubkey.IsZero() { + return FeeInfo{}, errors.New("block builder pubkey is all zero") + } + if resp.GetCommission() > maxCommissionPercent { + return FeeInfo{}, fmt.Errorf("block builder commission %d%% is over 100%%", resp.GetCommission()) + } + return FeeInfo{Pubkey: pubkey, Commission: resp.GetCommission()}, nil +} + +// BlockEngineConfig configures a block engine session. +type BlockEngineConfig struct { + Endpoint Endpoint + InsecureSkipVerify bool + Identity ed25519.PrivateKey + // Strategy defaults to FBA. + Strategy blockenginepb.SchedulingStrategy + Packets PacketSink + Blocks BlockHandler + // OnHealth is called from the session goroutine whenever the session's + // health changes. + OnHealth func(bool) + // Zero values use the defaults: 5m, 5s. + FeeInfoInterval time.Duration + MaxBackoff time.Duration + // Dialer replaces the TCP dialer, for tests. + Dialer func(ctx context.Context, addr string) (net.Conn, error) +} + +// BlockEngineStats counts block engine session activity. +type BlockEngineStats struct { + Sessions uint64 + Resets uint64 + Packets uint64 + DroppedSize uint64 + DroppedFull uint64 + Blocks uint64 + NoticesSent uint64 + NoticesExpired uint64 + NoticeErrors uint64 + StreamsLost uint64 + StrategyErrors uint64 + InvalidFeeInfo uint64 +} + +type leaderNotice struct { + slot uint64 + end time.Time +} + +// BlockEngineSession keeps an authenticated session with a Harmonic block +// engine: packets into the TPU, the block stream into a BlockHandler, and +// leader notices out. +type BlockEngineSession struct { + cfg BlockEngineConfig + now func() time.Time + strategy atomic.Int32 + restart chan struct{} + notices chan struct{} + stats struct { + sessions, resets, packets, droppedSize, droppedFull, blocks, + noticesSent, noticesExpired, noticeErrors, streamsLost, strategyErrors, invalidFeeInfo atomic.Uint64 + } + + mu sync.Mutex + // fee is the last valid fee info; it outlives sessions. + fee *FeeInfo + // pending is the queued leader notice; sent is the one most recently + // sent, or in flight, on the current session. + pending, sent *leaderNotice + live sessionLiveness + healthy bool +} + +// sessionLiveness is what the current session has established. +type sessionLiveness struct { + authenticated, packets, blocks bool +} + +func NewBlockEngineSession(cfg BlockEngineConfig) *BlockEngineSession { + if cfg.Strategy == blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_UNSPECIFIED { + cfg.Strategy = blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_FBA + } + if cfg.FeeInfoInterval <= 0 { + cfg.FeeInfoInterval = defaultFeeInfoInterval + } + if cfg.MaxBackoff <= 0 { + cfg.MaxBackoff = defaultMaxBackoff + } + b := &BlockEngineSession{ + cfg: cfg, + now: time.Now, + restart: make(chan struct{}, 1), + notices: make(chan struct{}, 1), + } + b.strategy.Store(int32(cfg.Strategy)) + return b +} + +func (b *BlockEngineSession) Stats() BlockEngineStats { + return BlockEngineStats{ + Sessions: b.stats.sessions.Load(), + Resets: b.stats.resets.Load(), + Packets: b.stats.packets.Load(), + DroppedSize: b.stats.droppedSize.Load(), + DroppedFull: b.stats.droppedFull.Load(), + Blocks: b.stats.blocks.Load(), + NoticesSent: b.stats.noticesSent.Load(), + NoticesExpired: b.stats.noticesExpired.Load(), + NoticeErrors: b.stats.noticeErrors.Load(), + StreamsLost: b.stats.streamsLost.Load(), + StrategyErrors: b.stats.strategyErrors.Load(), + InvalidFeeInfo: b.stats.invalidFeeInfo.Load(), + } +} + +// FeeInfo returns the last valid fee info. +func (b *BlockEngineSession) FeeInfo() (FeeInfo, bool) { + b.mu.Lock() + defer b.mu.Unlock() + if b.fee == nil { + return FeeInfo{}, false + } + return *b.fee, true +} + +// Healthy reports whether the session is authenticated, has valid fee info, +// and has both its packet and block subscriptions live. +func (b *BlockEngineSession) Healthy() bool { + b.mu.Lock() + defer b.mu.Unlock() + return b.healthy +} + +// SetStrategy changes the scheduling strategy. The session reconnects so the +// block engine applies it. +func (b *BlockEngineSession) SetStrategy(strategy blockenginepb.SchedulingStrategy) { + if strategy == blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_UNSPECIFIED || + b.strategy.Swap(int32(strategy)) == int32(strategy) { + return + } + select { + case b.restart <- struct{}{}: + default: + } +} + +// NotifyLeaderSlot queues the leader notice for slot, which the validator +// stops recording at end. The notice is sent once the block subscription is +// live, never after end. A newer notice replaces a queued one. +func (b *BlockEngineSession) NotifyLeaderSlot(slot uint64, end time.Time) { + b.mu.Lock() + if b.pending != nil { + b.stats.noticesExpired.Add(1) + } + b.pending = &leaderNotice{slot: slot, end: end} + b.mu.Unlock() + b.signalNotice() +} + +func (b *BlockEngineSession) signalNotice() { + select { + case b.notices <- struct{}{}: + default: + } +} + +// Run keeps a session open until ctx ends, reconnecting after jittered +// backoff whenever it fails. +func (b *BlockEngineSession) Run(ctx context.Context) { + for ctx.Err() == nil { + err := b.runSession(ctx) + lost := b.endSession() + if ctx.Err() != nil { + return + } + b.stats.resets.Add(1) + if lost != nil { + b.stats.streamsLost.Add(1) + b.cfg.Blocks.StreamLost(lost.slot) + } + mlog.Log.Infof("Harmonic block engine %s session reset: %v", b.cfg.Endpoint.Target, err) + if errors.Is(err, errStrategyChanged) { + continue + } + select { + case <-ctx.Done(): + case <-time.After(jitteredBackoff(b.cfg.MaxBackoff)): + } + } +} + +// endSession clears the session's liveness and returns the notice whose slot +// the reset interrupted, if any. +func (b *BlockEngineSession) endSession() *leaderNotice { + b.mu.Lock() + sent := b.sent + b.sent = nil + b.live = sessionLiveness{} + b.mu.Unlock() + b.updateHealth() + if sent != nil && b.now().Before(sent.end) { + return sent + } + return nil +} + +func (b *BlockEngineSession) setLive(update func(*sessionLiveness)) { + b.mu.Lock() + update(&b.live) + b.mu.Unlock() + b.updateHealth() +} + +func (b *BlockEngineSession) updateHealth() { + b.mu.Lock() + healthy := b.live.authenticated && b.live.packets && b.live.blocks && b.fee != nil + changed := healthy != b.healthy + b.healthy = healthy + b.mu.Unlock() + if changed && b.cfg.OnHealth != nil { + b.cfg.OnHealth(healthy) + } +} + +func (b *BlockEngineSession) runSession(ctx context.Context) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + tokens := NewTokenSource(nil, b.cfg.Identity) + dial := DialConfig{ + Endpoint: b.cfg.Endpoint, + InsecureSkipVerify: b.cfg.InsecureSkipVerify, + Credentials: tokens.Bearer(), + Dialer: b.cfg.Dialer, + } + conn, err := Dial(ctx, dial) + if err != nil { + return err + } + defer conn.Close() + tokens.client = authpb.NewAuthServiceClient(conn) + if err := tokens.Authenticate(ctx); err != nil { + return err + } + b.stats.sessions.Add(1) + b.setLive(func(l *sessionLiveness) { l.authenticated = true }) + errs := make(chan error, 4) + go func() { errs <- tokens.Run(ctx) }() + + client := blockenginepb.NewBlockEngineValidatorClient(conn) + if err := b.setStrategy(ctx, client); err != nil { + return err + } + if err := b.refreshFeeInfo(ctx, client); err != nil { + return err + } + packets, err := client.SubscribePackets(ctx, &blockenginepb.SubscribePacketsRequest{}) + if err == nil { + err = awaitHeaders(packets) + } + if err != nil { + return fmt.Errorf("subscribe packets: %w", err) + } + b.setLive(func(l *sessionLiveness) { l.packets = true }) + go func() { errs <- b.readPackets(packets) }() + + // The block stream gets its own connection, so it never shares an HTTP/2 + // flow-control window with packets. + blockConn, err := Dial(ctx, dial) + if err != nil { + return fmt.Errorf("block connection: %w", err) + } + defer blockConn.Close() + blocks, err := blockenginepb.NewBlockEngineValidatorClient(blockConn).SubscribeBlocks2(ctx, &blockenginepb.SubscribeBlocksRequest{ + Version: truncate(version.Semver(), maxVersionBytes), + CommitHash: truncate(version.Commit(), maxCommitBytes), + }) + if err == nil { + err = awaitHeaders(blocks) + } + if err != nil { + return fmt.Errorf("subscribe blocks: %w", err) + } + b.setLive(func(l *sessionLiveness) { l.blocks = true }) + go func() { errs <- b.readBlocks(blocks) }() + go b.sendNotices(ctx, client, errs) + b.signalNotice() + + feeInfo := time.NewTicker(b.cfg.FeeInfoInterval) + defer feeInfo.Stop() + for { + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-errs: + return err + case <-b.restart: + return errStrategyChanged + case <-feeInfo.C: + if err := b.refreshFeeInfo(ctx, client); err != nil { + return err + } + } + } +} + +// setStrategy applies the scheduling strategy. Only an auth error fails the +// session; a block engine without SetStrategy is used without it. +func (b *BlockEngineSession) setStrategy(ctx context.Context, client blockenginepb.BlockEngineValidatorClient) error { + ctx, cancel := context.WithTimeout(ctx, unaryRPCTimeout) + defer cancel() + strategy := blockenginepb.SchedulingStrategy(b.strategy.Load()) + _, err := client.SetStrategy(ctx, &blockenginepb.SetStrategyRequest{Strategy: strategy}) + if err == nil { + return nil + } + if IsAuthError(err) { + return fmt.Errorf("set strategy: %w", err) + } + b.stats.strategyErrors.Add(1) + mlog.Log.Infof("WARNING: Harmonic block engine %s: set strategy %s: %v; continuing without it", b.cfg.Endpoint.Target, strategy, err) + return nil +} + +// refreshFeeInfo fetches the fee info. An invalid response or a failed call +// keeps the previous value; only an auth error fails the session. +func (b *BlockEngineSession) refreshFeeInfo(ctx context.Context, client blockenginepb.BlockEngineValidatorClient) error { + ctx, cancel := context.WithTimeout(ctx, unaryRPCTimeout) + defer cancel() + resp, err := client.GetBlockBuilderFeeInfo(ctx, &blockenginepb.BlockBuilderFeeInfoRequest{}) + if IsAuthError(err) { + return fmt.Errorf("get block builder fee info: %w", err) + } + var fee FeeInfo + if err == nil { + fee, err = ParseFeeInfo(resp) + } + if err != nil { + b.stats.invalidFeeInfo.Add(1) + mlog.Log.Infof("Harmonic block engine %s: fee info: %v; keeping the previous value", b.cfg.Endpoint.Target, err) + return nil + } + b.mu.Lock() + b.fee = &fee + b.mu.Unlock() + b.updateHealth() + return nil +} + +func (b *BlockEngineSession) readPackets(stream blockenginepb.BlockEngineValidator_SubscribePacketsClient) error { + for { + msg, err := stream.Recv() + if err != nil { + return fmt.Errorf("packet stream: %w", err) + } + for _, pkt := range msg.GetBatch().GetPackets() { + data := pkt.GetData() + if len(data) == 0 || len(data) > tpupacket.DataSize { + b.stats.droppedSize.Add(1) + continue + } + b.stats.packets.Add(1) + if !b.cfg.Packets.Submit(data, tpupacket.SourceBlockEngine) { + b.stats.droppedFull.Add(1) + } + } + } +} + +// readBlocks hands each block message to the handler. A message that fails +// to decode ends the stream, and with it the session. +func (b *BlockEngineSession) readBlocks(stream blockenginepb.BlockEngineValidator_SubscribeBlocks2Client) error { + for { + block, err := stream.Recv() + if err != nil { + return fmt.Errorf("block stream: %w", err) + } + b.stats.blocks.Add(1) + b.cfg.Blocks.HandleBlock(block, b.now()) + } +} + +// sendNotices sends queued leader notices while the block subscription is +// live, which is only while this goroutine runs. +func (b *BlockEngineSession) sendNotices(ctx context.Context, client blockenginepb.BlockEngineValidatorClient, errs chan<- error) { + for { + select { + case <-ctx.Done(): + return + case <-b.notices: + } + if ctx.Err() != nil { + // Leave the notice queued for the next session. + b.signalNotice() + return + } + notice := b.takeNotice() + if notice == nil { + continue + } + err := b.sendNotice(ctx, client, notice) + switch { + case err == nil: + b.stats.noticesSent.Add(1) + case status.Code(err) == codes.FailedPrecondition, IsAuthError(err): + // FAILED_PRECONDITION with a live subscription means the block + // engine lost it; every later notice would fail the same way. + errs <- fmt.Errorf("leader notice for slot %d: %w", notice.slot, err) + return + default: + // Not retried: the slot may be over before a retry lands. + b.stats.noticeErrors.Add(1) + mlog.Log.Infof("Harmonic block engine %s: leader notice for slot %d: %v", b.cfg.Endpoint.Target, notice.slot, err) + } + } +} + +// takeNotice dequeues the pending notice and records it as sent, unless its +// slot has already ended. +func (b *BlockEngineSession) takeNotice() *leaderNotice { + b.mu.Lock() + defer b.mu.Unlock() + notice := b.pending + b.pending = nil + if notice == nil { + return nil + } + if !b.now().Before(notice.end) { + b.stats.noticesExpired.Add(1) + return nil + } + b.sent = notice + return notice +} + +func (b *BlockEngineSession) sendNotice(ctx context.Context, client blockenginepb.BlockEngineValidatorClient, notice *leaderNotice) error { + ctx, cancel := context.WithTimeout(ctx, leaderNoticeTimeout) + defer cancel() + _, err := client.SubmitLeaderWindowInfo(ctx, &blockenginepb.SubmitLeaderWindowInfoRequest{ + Slot: notice.slot, + StartTimestamp: timestamppb.New(b.now()), + EndTimestamp: timestamppb.New(notice.end), + }) + return err +} + +func truncate(s string, maxBytes int) string { + if len(s) > maxBytes { + return s[:maxBytes] + } + return s +} diff --git a/pkg/harmonic/blockengine_test.go b/pkg/harmonic/blockengine_test.go new file mode 100644 index 00000000..10c69d74 --- /dev/null +++ b/pkg/harmonic/blockengine_test.go @@ -0,0 +1,297 @@ +package harmonic + +import ( + "context" + "sync" + "testing" + "time" + + authpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/auth" + blockpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/block" + blockenginepb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/blockengine" + packetpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/packet" + tpupacket "github.com/Overclock-Validator/mithril/pkg/tpu/packet" + "github.com/Overclock-Validator/mithril/pkg/version" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type blockRecorder struct { + mu sync.Mutex + slots []uint64 + arrivals []time.Time + lost []uint64 +} + +func (r *blockRecorder) HandleBlock(block *blockpb.Block, arrival time.Time) { + r.mu.Lock() + defer r.mu.Unlock() + r.slots = append(r.slots, block.GetSlot()) + r.arrivals = append(r.arrivals, arrival) +} + +func (r *blockRecorder) StreamLost(slot uint64) { + r.mu.Lock() + defer r.mu.Unlock() + r.lost = append(r.lost, slot) +} + +func (r *blockRecorder) received() []uint64 { + r.mu.Lock() + defer r.mu.Unlock() + return append([]uint64(nil), r.slots...) +} + +func (r *blockRecorder) lostSlots() []uint64 { + r.mu.Lock() + defer r.mu.Unlock() + return append([]uint64(nil), r.lost...) +} + +type healthRecorder struct { + mu sync.Mutex + changes []bool +} + +func (h *healthRecorder) record(healthy bool) { + h.mu.Lock() + defer h.mu.Unlock() + h.changes = append(h.changes, healthy) +} + +func (h *healthRecorder) all() []bool { + h.mu.Lock() + defer h.mu.Unlock() + return append([]bool(nil), h.changes...) +} + +type blockEngineHarness struct { + be *mockBlockEngine + session *BlockEngineSession + packets *packetRecorder + blocks *blockRecorder + health *healthRecorder + start func() +} + +// newBlockEngineHarness builds a session against a mock block engine; call +// start to run it, after configuring the mock or queueing notices. +func newBlockEngineHarness(t *testing.T, feeInfoInterval time.Duration) *blockEngineHarness { + t.Helper() + h := &blockEngineHarness{be: newMockBlockEngine(), packets: &packetRecorder{}, blocks: &blockRecorder{}, health: &healthRecorder{}} + dial := serveBufconn(t, func(s *grpc.Server) { + authpb.RegisterAuthServiceServer(s, newMockAuth(time.Now)) + blockenginepb.RegisterBlockEngineValidatorServer(s, h.be) + }) + h.session = NewBlockEngineSession(BlockEngineConfig{ + Endpoint: dial.Endpoint, + Dialer: dial.Dialer, + Identity: rfc8032Key(t), + Packets: h.packets, + Blocks: h.blocks, + OnHealth: h.health.record, + FeeInfoInterval: feeInfoInterval, + MaxBackoff: 20 * time.Millisecond, + }) + h.start = func() { + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + h.session.Run(ctx) + close(done) + }() + t.Cleanup(func() { + cancel() + <-done + }) + } + return h +} + +func eventually(t *testing.T, cond func() bool, msg string) { + t.Helper() + require.Eventually(t, cond, 2*time.Second, 5*time.Millisecond, msg) +} + +func TestParseFeeInfo(t *testing.T) { + got, err := ParseFeeInfo(&blockenginepb.BlockBuilderFeeInfoResponse{Pubkey: "11111111111111111111111111111112", Commission: 100}) + require.NoError(t, err) + assert.Equal(t, solana.MustPublicKeyFromBase58("11111111111111111111111111111112"), got.Pubkey) + assert.Equal(t, uint64(100), got.Commission, "commission is a percentage, up to 100") + for name, resp := range map[string]*blockenginepb.BlockBuilderFeeInfoResponse{ + "not base58": {Pubkey: "0OIl", Commission: 5}, + "wrong length": {Pubkey: "1111", Commission: 5}, + "all zero": {Pubkey: "11111111111111111111111111111111", Commission: 5}, + "over 100": {Pubkey: "11111111111111111111111111111112", Commission: 101}, + "basis points?": {Pubkey: "11111111111111111111111111111112", Commission: 500}, + } { + _, err := ParseFeeInfo(resp) + assert.Errorf(t, err, "%s", name) + } +} + +func TestBlockEngineSessionEstablishesAndStreams(t *testing.T) { + h := newBlockEngineHarness(t, time.Hour) + h.start() + eventually(t, h.session.Healthy, "the session becomes healthy") + assert.Equal(t, []bool{true}, h.health.all()) + fee, ok := h.session.FeeInfo() + require.True(t, ok) + assert.Equal(t, uint64(5), fee.Commission) + + h.be.mu.Lock() + assert.Equal(t, []blockenginepb.SchedulingStrategy{blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_FBA}, h.be.strategies) + require.Len(t, h.be.blockRequests, 1) + assert.Equal(t, version.Semver(), h.be.blockRequests[0].GetVersion()) + assert.Equal(t, version.Commit(), h.be.blockRequests[0].GetCommitHash()) + h.be.mu.Unlock() + + h.be.packets <- &blockenginepb.SubscribePacketsResponse{Batch: &packetpb.PacketBatch{Packets: []*packetpb.Packet{ + {Data: []byte{7}}, {Data: nil}, {Data: make([]byte, tpupacket.DataSize+1)}, + }}} + eventually(t, func() bool { return h.packets.count() == 1 }, "the valid packet is submitted") + assert.Equal(t, []tpupacket.Source{tpupacket.SourceBlockEngine}, h.packets.sources) + + h.be.blocks <- &blockpb.Block{Slot: 10} + h.be.blocks <- &blockpb.Block{Slot: 11} + eventually(t, func() bool { return len(h.blocks.received()) == 2 }, "blocks are delivered") + assert.Equal(t, []uint64{10, 11}, h.blocks.received(), "in stream order") + assert.False(t, h.blocks.arrivals[0].IsZero()) + + stats := h.session.Stats() + assert.Equal(t, uint64(1), stats.Sessions) + assert.Zero(t, stats.Resets) + assert.Equal(t, uint64(2), stats.DroppedSize) +} + +func TestBlockEngineSessionToleratesSetStrategyErrorsButNotAuthErrors(t *testing.T) { + h := newBlockEngineHarness(t, time.Hour) + h.be.strategyErrs = []error{status.Error(codes.Unimplemented, "old block engine")} + h.start() + eventually(t, h.session.Healthy, "an unsupported SetStrategy does not block the session") + assert.Equal(t, uint64(1), h.session.Stats().StrategyErrors) + assert.Zero(t, h.session.Stats().Resets) + + auth := newBlockEngineHarness(t, time.Hour) + auth.be.strategyErrs = []error{status.Error(codes.Unauthenticated, "token rejected")} + auth.start() + eventually(t, auth.session.Healthy, "an auth error re-authenticates on a new session") + assert.Equal(t, uint64(1), auth.session.Stats().Resets) + assert.Equal(t, uint64(2), auth.session.Stats().Sessions) +} + +func TestBlockEngineFeeInfoGatesHealthAndKeepsTheLastGoodValue(t *testing.T) { + h := newBlockEngineHarness(t, 30*time.Millisecond) + h.be.feeInfo = &blockenginepb.BlockBuilderFeeInfoResponse{Pubkey: "11111111111111111111111111111111", Commission: 5} + h.start() + eventually(t, func() bool { return h.session.Stats().InvalidFeeInfo >= 2 }, "fee info keeps being refreshed") + assert.False(t, h.session.Healthy(), "no valid fee info, not healthy") + _, ok := h.session.FeeInfo() + assert.False(t, ok) + + h.be.mu.Lock() + h.be.feeInfo = &blockenginepb.BlockBuilderFeeInfoResponse{Pubkey: "11111111111111111111111111111112", Commission: 7} + h.be.mu.Unlock() + eventually(t, h.session.Healthy, "valid fee info makes the session healthy") + + h.be.mu.Lock() + h.be.feeInfo = &blockenginepb.BlockBuilderFeeInfoResponse{Pubkey: "11111111111111111111111111111112", Commission: 250} + h.be.mu.Unlock() + invalid := h.session.Stats().InvalidFeeInfo + eventually(t, func() bool { return h.session.Stats().InvalidFeeInfo > invalid }, "the bad value is fetched") + fee, ok := h.session.FeeInfo() + require.True(t, ok) + assert.Equal(t, uint64(7), fee.Commission, "a bad refresh keeps the previous value") + assert.True(t, h.session.Healthy()) + assert.Zero(t, h.session.Stats().Resets) +} + +func TestBlockEngineLeaderNotices(t *testing.T) { + t.Run("queued before the block stream is live, latest wins, ended ones are dropped", func(t *testing.T) { + h := newBlockEngineHarness(t, time.Hour) + h.session.NotifyLeaderSlot(99, time.Now().Add(-time.Second)) + h.session.NotifyLeaderSlot(100, time.Now().Add(time.Hour)) + h.session.NotifyLeaderSlot(101, time.Now().Add(time.Hour)) + h.start() + eventually(t, func() bool { return h.session.Stats().NoticesSent == 1 }, "the latest notice is sent") + assert.Equal(t, []uint64{101}, h.be.noticeSlots()) + assert.Equal(t, uint64(2), h.session.Stats().NoticesExpired, "the replaced notices count as expired") + + h.be.mu.Lock() + notice := h.be.notices[0] + h.be.mu.Unlock() + assert.WithinDuration(t, time.Now(), notice.GetStartTimestamp().AsTime(), time.Second) + assert.True(t, notice.GetEndTimestamp().AsTime().After(time.Now())) + + h.session.NotifyLeaderSlot(102, time.Now().Add(-time.Millisecond)) + eventually(t, func() bool { return h.session.Stats().NoticesExpired == 3 }, "a notice past its end is dropped") + assert.Equal(t, []uint64{101}, h.be.noticeSlots(), "and never sent") + }) + t.Run("other errors are counted, not retried", func(t *testing.T) { + h := newBlockEngineHarness(t, time.Hour) + h.be.noticeErr = status.Error(codes.Unavailable, "busy") + h.start() + eventually(t, h.session.Healthy, "healthy") + h.session.NotifyLeaderSlot(200, time.Now().Add(time.Hour)) + eventually(t, func() bool { return h.session.Stats().NoticeErrors == 1 }, "the error is counted") + time.Sleep(50 * time.Millisecond) + assert.Equal(t, []uint64{200}, h.be.noticeSlots()) + assert.Zero(t, h.session.Stats().Resets) + }) + t.Run("FAILED_PRECONDITION resets the session and loses the open slot", func(t *testing.T) { + h := newBlockEngineHarness(t, time.Hour) + h.be.noticeErr = status.Error(codes.FailedPrecondition, "block subscription lost") + h.start() + eventually(t, h.session.Healthy, "healthy") + h.session.NotifyLeaderSlot(300, time.Now().Add(time.Hour)) + eventually(t, func() bool { return h.session.Stats().Resets == 1 }, "the session resets") + assert.Equal(t, []uint64{300}, h.blocks.lostSlots()) + eventually(t, func() bool { return h.session.Stats().Sessions == 2 }, "and reconnects") + }) +} + +func TestBlockEngineStreamLossMarksOnlyAnOpenAnnouncedSlotLost(t *testing.T) { + h := newBlockEngineHarness(t, time.Hour) + h.start() + eventually(t, h.session.Healthy, "healthy") + + h.session.NotifyLeaderSlot(400, time.Now().Add(time.Hour)) + eventually(t, func() bool { return h.session.Stats().NoticesSent == 1 }, "notice sent") + h.be.endBlocks <- status.Error(codes.Unavailable, "block stream reset") + eventually(t, func() bool { return h.session.Stats().Resets == 1 }, "the session resets") + assert.Equal(t, []uint64{400}, h.blocks.lostSlots()) + eventually(t, h.session.Healthy, "and recovers") + assert.Equal(t, []bool{true, false, true}, h.health.all()) + + // No notice on the new session: a reset loses nothing. + h.be.endBlocks <- status.Error(codes.Unavailable, "again") + eventually(t, func() bool { return h.session.Stats().Resets == 2 }, "second reset") + eventually(t, h.session.Healthy, "recovers again") + + // A notice whose slot has ended is not lost by a later reset. + h.session.NotifyLeaderSlot(401, time.Now().Add(100*time.Millisecond)) + eventually(t, func() bool { return h.session.Stats().NoticesSent == 2 }, "notice sent") + time.Sleep(150 * time.Millisecond) + h.be.endBlocks <- status.Error(codes.Unavailable, "after the slot") + eventually(t, func() bool { return h.session.Stats().Resets == 3 }, "third reset") + assert.Equal(t, []uint64{400}, h.blocks.lostSlots()) + assert.Equal(t, uint64(1), h.session.Stats().StreamsLost) +} + +func TestBlockEngineStrategyChangeReconnectsWithTheNewStrategy(t *testing.T) { + h := newBlockEngineHarness(t, time.Hour) + h.start() + eventually(t, h.session.Healthy, "healthy") + h.session.SetStrategy(blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_MREV) + eventually(t, func() bool { return h.session.Stats().Sessions == 2 }, "a new session") + h.be.mu.Lock() + defer h.be.mu.Unlock() + assert.Equal(t, []blockenginepb.SchedulingStrategy{ + blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_FBA, + blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_MREV, + }, h.be.strategies) +} diff --git a/pkg/harmonic/mock_test.go b/pkg/harmonic/mock_test.go index 7903b63c..ae95d5b7 100644 --- a/pkg/harmonic/mock_test.go +++ b/pkg/harmonic/mock_test.go @@ -11,6 +11,8 @@ import ( "time" authpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/auth" + blockpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/block" + blockenginepb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/blockengine" relayerpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/relayer" sharedpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/shared" "github.com/gagliardetto/solana-go" @@ -204,3 +206,136 @@ func serveBufconn(t *testing.T, register func(*grpc.Server)) DialConfig { Dialer: func(ctx context.Context, _ string) (net.Conn, error) { return lis.DialContext(ctx) }, } } + +// mockBlockEngine serves a scripted block engine. SubmitLeaderWindowInfo +// fails with FAILED_PRECONDITION while no block stream is live, as the real +// one does. +type mockBlockEngine struct { + blockenginepb.UnimplementedBlockEngineValidatorServer + packets chan *blockenginepb.SubscribePacketsResponse + blocks chan *blockpb.Block + endBlocks chan error + blocksLive atomic.Int32 + blockSubs atomic.Int32 + + mu sync.Mutex + strategyErrs []error + noticeErr error + feeInfo *blockenginepb.BlockBuilderFeeInfoResponse + strategies []blockenginepb.SchedulingStrategy + notices []*blockenginepb.SubmitLeaderWindowInfoRequest + blockRequests []*blockenginepb.SubscribeBlocksRequest +} + +func newMockBlockEngine() *mockBlockEngine { + return &mockBlockEngine{ + packets: make(chan *blockenginepb.SubscribePacketsResponse, 16), + blocks: make(chan *blockpb.Block, 16), + endBlocks: make(chan error, 1), + feeInfo: &blockenginepb.BlockBuilderFeeInfoResponse{Pubkey: "11111111111111111111111111111112", Commission: 5}, + } +} + +func requireBearer(ctx context.Context) error { + md, _ := metadata.FromIncomingContext(ctx) + if auth := md.Get("authorization"); len(auth) != 1 || !strings.HasPrefix(auth[0], "Bearer access-") { + return status.Error(codes.Unauthenticated, "missing bearer") + } + return nil +} + +func (m *mockBlockEngine) SetStrategy(ctx context.Context, req *blockenginepb.SetStrategyRequest) (*blockenginepb.SetStrategyResponse, error) { + if err := requireBearer(ctx); err != nil { + return nil, err + } + m.mu.Lock() + defer m.mu.Unlock() + m.strategies = append(m.strategies, req.GetStrategy()) + if len(m.strategyErrs) > 0 { + err := m.strategyErrs[0] + m.strategyErrs = m.strategyErrs[1:] + return nil, err + } + return &blockenginepb.SetStrategyResponse{}, nil +} + +func (m *mockBlockEngine) GetBlockBuilderFeeInfo(ctx context.Context, _ *blockenginepb.BlockBuilderFeeInfoRequest) (*blockenginepb.BlockBuilderFeeInfoResponse, error) { + if err := requireBearer(ctx); err != nil { + return nil, err + } + m.mu.Lock() + defer m.mu.Unlock() + return m.feeInfo, nil +} + +func (m *mockBlockEngine) SubmitLeaderWindowInfo(ctx context.Context, req *blockenginepb.SubmitLeaderWindowInfoRequest) (*blockenginepb.SubmitLeaderWindowInfoResponse, error) { + if err := requireBearer(ctx); err != nil { + return nil, err + } + m.mu.Lock() + defer m.mu.Unlock() + m.notices = append(m.notices, req) + if m.noticeErr != nil { + return nil, m.noticeErr + } + if m.blocksLive.Load() == 0 { + return nil, status.Error(codes.FailedPrecondition, "no block subscription") + } + return &blockenginepb.SubmitLeaderWindowInfoResponse{}, nil +} + +func (m *mockBlockEngine) SubscribePackets(_ *blockenginepb.SubscribePacketsRequest, stream blockenginepb.BlockEngineValidator_SubscribePacketsServer) error { + if err := requireBearer(stream.Context()); err != nil { + return err + } + if err := stream.SendHeader(metadata.MD{}); err != nil { + return err + } + for { + select { + case <-stream.Context().Done(): + return nil + case msg := <-m.packets: + if err := stream.Send(msg); err != nil { + return err + } + } + } +} + +func (m *mockBlockEngine) SubscribeBlocks2(req *blockenginepb.SubscribeBlocksRequest, stream blockenginepb.BlockEngineValidator_SubscribeBlocks2Server) error { + if err := requireBearer(stream.Context()); err != nil { + return err + } + m.mu.Lock() + m.blockRequests = append(m.blockRequests, req) + m.mu.Unlock() + m.blockSubs.Add(1) + m.blocksLive.Add(1) + defer m.blocksLive.Add(-1) + if err := stream.SendHeader(metadata.MD{}); err != nil { + return err + } + for { + select { + case <-stream.Context().Done(): + return nil + case err := <-m.endBlocks: + return err + case block := <-m.blocks: + if err := stream.Send(block); err != nil { + return err + } + } + } +} + +func (m *mockBlockEngine) noticeSlots() []uint64 { + m.mu.Lock() + defer m.mu.Unlock() + var slots []uint64 + for _, n := range m.notices { + slots = append(slots, n.GetSlot()) + } + return slots +} diff --git a/pkg/harmonic/relayer.go b/pkg/harmonic/relayer.go index b2076481..b1db47cc 100644 --- a/pkg/harmonic/relayer.go +++ b/pkg/harmonic/relayer.go @@ -18,6 +18,7 @@ import ( sharedpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/shared" "github.com/Overclock-Validator/mithril/pkg/mlog" tpupacket "github.com/Overclock-Validator/mithril/pkg/tpu/packet" + "google.golang.org/grpc" ) const ( @@ -277,13 +278,21 @@ func (r *RelayerSession) fetchTPUConfig(ctx context.Context, client relayerpb.Re return config, err } -// subscribePackets opens the packet stream and waits, for at most 10s, for -// its response headers; the stream itself has no deadline. func subscribePackets(ctx context.Context, client relayerpb.RelayerClient) (relayerpb.Relayer_SubscribePacketsClient, error) { stream, err := client.SubscribePackets(ctx, &relayerpb.SubscribePacketsRequest{}) + if err == nil { + err = awaitHeaders(stream) + } if err != nil { return nil, fmt.Errorf("subscribe packets: %w", err) } + return stream, nil +} + +// awaitHeaders waits, for at most 10s, for a stream's response headers, which +// is when a subscription counts as live. The stream itself has no deadline; +// on a timeout the caller cancels its context. +func awaitHeaders(stream grpc.ClientStream) error { headers := make(chan error, 1) go func() { _, err := stream.Header() @@ -291,12 +300,9 @@ func subscribePackets(ctx context.Context, client relayerpb.RelayerClient) (rela }() select { case err := <-headers: - if err != nil { - return nil, fmt.Errorf("subscribe packets: %w", err) - } - return stream, nil + return err case <-time.After(streamHeaderTimeout): - return nil, errors.New("subscribe packets: no response headers within 10s") + return errors.New("no response headers within 10s") } } diff --git a/pkg/tpu/packet/packet.go b/pkg/tpu/packet/packet.go index 363f9751..0e75e967 100644 --- a/pkg/tpu/packet/packet.go +++ b/pkg/tpu/packet/packet.go @@ -13,6 +13,8 @@ const ( SourceQUIC Source = iota // SourceRelayer is a relayer forwarding packets to this validator. SourceRelayer + // SourceBlockEngine is a block engine streaming packets to this validator. + SourceBlockEngine // NumSources is the number of sources. NumSources ) @@ -23,6 +25,8 @@ func (s Source) String() string { return "quic" case SourceRelayer: return "relayer" + case SourceBlockEngine: + return "block_engine" default: return "unknown" } From 4211804c94b8066b1ce1420535f534a72a1d9fc4 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 20:57:04 +0000 Subject: [PATCH 16/25] harmonic: block receive pipeline --- pkg/harmonic/blocks.go | 242 ++++++++++++++++++++++++++++++++++++ pkg/harmonic/blocks_test.go | 226 +++++++++++++++++++++++++++++++++ 2 files changed, 468 insertions(+) create mode 100644 pkg/harmonic/blocks.go create mode 100644 pkg/harmonic/blocks_test.go diff --git a/pkg/harmonic/blocks.go b/pkg/harmonic/blocks.go new file mode 100644 index 00000000..634bd376 --- /dev/null +++ b/pkg/harmonic/blocks.go @@ -0,0 +1,242 @@ +package harmonic + +import ( + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + blockpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/block" + "github.com/Overclock-Validator/mithril/pkg/tpu/packet" + "github.com/Overclock-Validator/mithril/pkg/tpu/sigverify" + "github.com/Overclock-Validator/mithril/pkg/txverify" + "github.com/gagliardetto/solana-go" +) + +// MaxBundleTransactions is the most transactions a block bundle may hold. +const MaxBundleTransactions = 5 + +// BlockEvent is what the block pipeline hands the executor, in stream order. +type BlockEvent interface { + isBlockEvent() +} + +// BundleEvent is a validated bundle of the current slot's block. +type BundleEvent struct { + Slot uint64 + // Seq is the bundle's 1-based position in the slot's bundle sequence. + Seq uint64 + Arrival time.Time + RevertProtected bool + Transactions []*solana.Transaction +} + +// InvalidBundleEvent is a bundle that failed validation. It holds its place +// in the sequence, and under the strict profile ends the slot's block. +type InvalidBundleEvent struct { + Slot uint64 + Seq uint64 + Arrival time.Time + Reason error +} + +// StreamLostEvent reports that the rest of slot's block was lost. +type StreamLostEvent struct { + Slot uint64 +} + +func (BundleEvent) isBlockEvent() {} +func (InvalidBundleEvent) isBlockEvent() {} +func (StreamLostEvent) isBlockEvent() {} + +// BlockPipelineStats counts block pipeline activity. +type BlockPipelineStats struct { + Messages uint64 + WrongSlot uint64 + LostSlot uint64 + Bundles uint64 + InvalidBundles uint64 +} + +// BlockPipeline turns the block stream into executor events. It admits only +// messages for the current block slot, splits each into bundles, validates +// them, and emits them in order. It is the session's BlockHandler. +type BlockPipeline struct { + events chan<- BlockEvent + txV1Enabled func() bool + current atomic.Uint64 + verifier txverify.BatchVerifier + stats struct { + messages, wrongSlot, lostSlot, bundles, invalid atomic.Uint64 + } + + mu sync.Mutex + lost map[uint64]bool + seqSlot uint64 + nextSeq uint64 +} + +// NewBlockPipeline sends events on events, blocking rather than dropping +// when the executor falls behind. txV1Enabled reports whether the current bank +// accepts v1 transactions; nil rejects them. +func NewBlockPipeline(events chan<- BlockEvent, txV1Enabled func() bool) *BlockPipeline { + return &BlockPipeline{events: events, txV1Enabled: txV1Enabled, lost: make(map[uint64]bool)} +} + +func (p *BlockPipeline) Stats() BlockPipelineStats { + return BlockPipelineStats{ + Messages: p.stats.messages.Load(), + WrongSlot: p.stats.wrongSlot.Load(), + LostSlot: p.stats.lostSlot.Load(), + Bundles: p.stats.bundles.Load(), + InvalidBundles: p.stats.invalid.Load(), + } +} + +// SetCurrentSlot makes slot the current block slot, or clears it with 0. Call +// it before sending slot's leader notice: the block engine streams the slot +// only after the notice, so none of it can arrive earlier. +func (p *BlockPipeline) SetCurrentSlot(slot uint64) { + p.current.Store(slot) + p.mu.Lock() + defer p.mu.Unlock() + for lost := range p.lost { + if lost < slot { + delete(p.lost, lost) + } + } +} + +// StreamLost marks slot lost: the rest of its block is discarded even after +// the session reconnects, since nothing says where a new stream resumes. +func (p *BlockPipeline) StreamLost(slot uint64) { + p.mu.Lock() + p.lost[slot] = true + p.mu.Unlock() + p.events <- StreamLostEvent{Slot: slot} +} + +// HandleBlock admits, segments and validates one block message. +func (p *BlockPipeline) HandleBlock(block *blockpb.Block, arrival time.Time) { + p.stats.messages.Add(1) + slot := block.GetSlot() + if current := p.current.Load(); current == 0 || slot != current { + p.stats.wrongSlot.Add(1) + return + } + p.mu.Lock() + lost := p.lost[slot] + if !lost && p.seqSlot != slot { + p.seqSlot, p.nextSeq = slot, 1 + } + p.mu.Unlock() + if lost { + p.stats.lostSlot.Add(1) + return + } + + txs := block.GetTransactions() + parsed, parseErrs := p.parse(txs) + for _, bundle := range segment(txs) { + seq := p.takeSeq() + if err := validateBundle(txs, parsed, parseErrs, bundle); err != nil { + p.stats.invalid.Add(1) + p.events <- InvalidBundleEvent{Slot: slot, Seq: seq, Arrival: arrival, Reason: err} + continue + } + p.stats.bundles.Add(1) + p.events <- BundleEvent{ + Slot: slot, + Seq: seq, + Arrival: arrival, + RevertProtected: txs[bundle.start].GetBundleId() != 0, + Transactions: append([]*solana.Transaction(nil), parsed[bundle.start:bundle.end]...), + } + } +} + +func (p *BlockPipeline) takeSeq() uint64 { + p.mu.Lock() + defer p.mu.Unlock() + seq := p.nextSeq + p.nextSeq++ + return seq +} + +// parse parses every transaction in a message and verifies all their +// signatures as one batch. errs[i] is set for a transaction that is unusable. +func (p *BlockPipeline) parse(txs []*blockpb.Transaction) ([]*solana.Transaction, []error) { + parsed := make([]*solana.Transaction, len(txs)) + errs := make([]error, len(txs)) + verify := make([]*solana.Transaction, 0, len(txs)) + verifyAt := make([]int, 0, len(txs)) + for i, t := range txs { + data := t.GetTransaction() + if len(data) == 0 || len(data) > packet.DataSize { + errs[i] = fmt.Errorf("transaction %d has %d bytes", i, len(data)) + continue + } + tx, err := sigverify.ParseTx(data) + if err != nil { + errs[i] = fmt.Errorf("transaction %d: %w", i, err) + continue + } + if tx.Message.GetVersion() == solana.MessageVersionV1 && (p.txV1Enabled == nil || !p.txV1Enabled()) { + errs[i] = fmt.Errorf("transaction %d: v1 transactions are not enabled", i) + continue + } + parsed[i] = tx + verify = append(verify, tx) + verifyAt = append(verifyAt, i) + } + verifyErrs := make([]error, len(verify)) + p.verifier.Verify(verify, verifyErrs) + for j, err := range verifyErrs { + if err != nil { + errs[verifyAt[j]] = fmt.Errorf("transaction %d: %w", verifyAt[j], err) + } + } + return parsed, errs +} + +type bundleSpan struct { + start, end int +} + +// segment splits one message into bundles: each zero bundle_id transaction is +// its own standalone bundle, and each run of equal nonzero ids is one +// revert-protected bundle. Segmentation never spans messages. +func segment(txs []*blockpb.Transaction) []bundleSpan { + var spans []bundleSpan + for i := 0; i < len(txs); { + id := txs[i].GetBundleId() + j := i + 1 + if id != 0 { + for j < len(txs) && txs[j].GetBundleId() == id { + j++ + } + } + spans = append(spans, bundleSpan{start: i, end: j}) + i = j + } + return spans +} + +func validateBundle(txs []*blockpb.Transaction, parsed []*solana.Transaction, errs []error, bundle bundleSpan) error { + if n := bundle.end - bundle.start; n == 0 || n > MaxBundleTransactions { + return fmt.Errorf("bundle has %d transactions", n) + } + seen := make(map[solana.Signature]struct{}, bundle.end-bundle.start) + for i := bundle.start; i < bundle.end; i++ { + if errs[i] != nil { + return errs[i] + } + signature := parsed[i].Signatures[0] + if _, dup := seen[signature]; dup { + return errors.New("bundle repeats a transaction signature") + } + seen[signature] = struct{}{} + } + return nil +} diff --git a/pkg/harmonic/blocks_test.go b/pkg/harmonic/blocks_test.go new file mode 100644 index 00000000..772e21f0 --- /dev/null +++ b/pkg/harmonic/blocks_test.go @@ -0,0 +1,226 @@ +package harmonic + +import ( + "testing" + "time" + + blockpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/block" + "github.com/Overclock-Validator/mithril/pkg/tpu/packet" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// wireTx is a distinct signed transfer for each seq. +func wireTx(seq uint64) []byte { + return txfixture.MustSignedTransferWire(seq) +} + +func blockMsg(slot uint64, txs ...*blockpb.Transaction) *blockpb.Block { + return &blockpb.Block{Slot: slot, Transactions: txs} +} + +func blockTx(data []byte, bundleID uint64) *blockpb.Transaction { + return &blockpb.Transaction{Transaction: data, BundleId: bundleID} +} + +// collect runs fn and returns every event it emitted. +func collect(t *testing.T, fn func(*BlockPipeline), txV1Enabled func() bool, current uint64) ([]BlockEvent, *BlockPipeline) { + t.Helper() + events := make(chan BlockEvent, 64) + p := NewBlockPipeline(events, txV1Enabled) + p.SetCurrentSlot(current) + fn(p) + close(events) + var out []BlockEvent + for e := range events { + out = append(out, e) + } + return out, p +} + +type bundleShape struct { + seq uint64 + revertProtected bool + size int +} + +func shapes(t *testing.T, events []BlockEvent) []bundleShape { + t.Helper() + var out []bundleShape + for _, e := range events { + b, ok := e.(BundleEvent) + require.Truef(t, ok, "unexpected event %#v", e) + out = append(out, bundleShape{seq: b.Seq, revertProtected: b.RevertProtected, size: len(b.Transactions)}) + } + return out +} + +func TestBlockPipelineSegmentsPerMessage(t *testing.T) { + arrival := time.Unix(1_800_000_000, 0) + ids := []uint64{0, 5, 0, 0, 9, 9, 5} + var txs []*blockpb.Transaction + for i, id := range ids { + txs = append(txs, blockTx(wireTx(uint64(i)), id)) + } + events, p := collect(t, func(p *BlockPipeline) { + p.HandleBlock(blockMsg(10, txs...), arrival) + // A run that seems to continue into the next message is a new bundle. + p.HandleBlock(blockMsg(10, blockTx(wireTx(20), 5), blockTx(wireTx(21), 5)), arrival) + }, nil, 10) + + assert.Equal(t, []bundleShape{ + {1, false, 1}, {2, true, 1}, {3, false, 1}, {4, false, 1}, {5, true, 2}, {6, true, 1}, + {7, true, 2}, + }, shapes(t, events)) + first := events[0].(BundleEvent) + assert.Equal(t, uint64(10), first.Slot) + assert.Equal(t, arrival, first.Arrival) + fifth := events[4].(BundleEvent) + assert.Equal(t, solana.SignatureFromBytes(wireTx(4)[1:65]), fifth.Transactions[0].Signatures[0], "members keep stream order") + assert.Equal(t, uint64(7), p.Stats().Bundles) +} + +func TestBlockPipelineAdmitsOnlyTheCurrentSlot(t *testing.T) { + events, p := collect(t, func(p *BlockPipeline) { + p.HandleBlock(blockMsg(9, blockTx(wireTx(1), 0)), time.Now()) + p.HandleBlock(blockMsg(11, blockTx(wireTx(2), 0)), time.Now()) + p.HandleBlock(blockMsg(10, blockTx(wireTx(3), 0)), time.Now()) + p.SetCurrentSlot(0) + p.HandleBlock(blockMsg(10, blockTx(wireTx(4), 0)), time.Now()) + p.SetCurrentSlot(11) + p.HandleBlock(blockMsg(11, blockTx(wireTx(5), 0)), time.Now()) + }, nil, 10) + + require.Len(t, events, 2) + b10, b11 := events[0].(BundleEvent), events[1].(BundleEvent) + assert.Equal(t, uint64(10), b10.Slot) + assert.Equal(t, uint64(1), b10.Seq, "messages for other slots consume no sequence numbers") + assert.Equal(t, uint64(11), b11.Slot) + assert.Equal(t, uint64(1), b11.Seq, "each slot's sequence starts at 1") + assert.Equal(t, uint64(3), p.Stats().WrongSlot) +} + +func TestBlockPipelineDiscardsALostSlotEvenAfterReconnect(t *testing.T) { + events, p := collect(t, func(p *BlockPipeline) { + p.HandleBlock(blockMsg(10, blockTx(wireTx(1), 0)), time.Now()) + p.StreamLost(10) + p.HandleBlock(blockMsg(10, blockTx(wireTx(2), 0)), time.Now()) + p.SetCurrentSlot(10) + p.HandleBlock(blockMsg(10, blockTx(wireTx(3), 0)), time.Now()) + p.SetCurrentSlot(11) + p.HandleBlock(blockMsg(11, blockTx(wireTx(4), 0)), time.Now()) + }, nil, 10) + + require.Len(t, events, 3) + assert.Equal(t, uint64(10), events[0].(BundleEvent).Slot) + assert.Equal(t, StreamLostEvent{Slot: 10}, events[1]) + assert.Equal(t, uint64(11), events[2].(BundleEvent).Slot) + assert.Equal(t, uint64(2), p.Stats().LostSlot) +} + +func TestBlockPipelineDropsInvalidBundlesWhole(t *testing.T) { + badSig := wireTx(3) + badSig[10] ^= 0xff + valid := func(seq uint64) *blockpb.Transaction { return blockTx(wireTx(seq), 7) } + cases := map[string][]*blockpb.Transaction{ + "empty payload": {valid(1), blockTx(nil, 7)}, + "oversized payload": {valid(1), blockTx(make([]byte, packet.DataSize+1), 7)}, + "unparseable": {valid(1), blockTx([]byte{1, 2, 3}, 7)}, + "bad signature": {valid(1), blockTx(badSig, 7)}, + "repeated signature": {valid(1), valid(2), valid(1)}, + "too many members": {valid(1), valid(2), valid(3), valid(4), valid(5), valid(6)}, + "v1 while not active": {valid(1), blockTx(txfixture.MustSignedV1Wire(1, 0), 7)}, + } + for name, bad := range cases { + t.Run(name, func(t *testing.T) { + before := blockTx(wireTx(100), 0) + after := blockTx(wireTx(101), 3) + txs := append(append([]*blockpb.Transaction{before}, bad...), after) + events, p := collect(t, func(p *BlockPipeline) { p.HandleBlock(blockMsg(10, txs...), time.Now()) }, nil, 10) + + require.Len(t, events, 3) + assert.Equal(t, uint64(1), events[0].(BundleEvent).Seq) + invalid, ok := events[1].(InvalidBundleEvent) + require.Truef(t, ok, "got %#v", events[1]) + assert.Equal(t, uint64(2), invalid.Seq, "an invalid bundle keeps its place") + assert.Error(t, invalid.Reason) + assert.Equal(t, uint64(3), events[2].(BundleEvent).Seq, "bundles around it are unaffected") + assert.Equal(t, uint64(1), p.Stats().InvalidBundles) + }) + } +} + +func TestBlockPipelineAcceptsWhatIngestMustNotReject(t *testing.T) { + v1 := blockTx(txfixture.MustSignedV1Wire(1, 0), 0) + events, _ := collect(t, func(p *BlockPipeline) { + p.HandleBlock(blockMsg(10, + v1, + blockTx(wireTx(1), 4), blockTx(wireTx(2), 4), blockTx(wireTx(3), 4), blockTx(wireTx(4), 4), blockTx(wireTx(5), 4), + // The same transaction in two bundles is for execution to refuse. + blockTx(wireTx(1), 0), + ), time.Now()) + }, func() bool { return true }, 10) + assert.Equal(t, []bundleShape{{1, false, 1}, {2, true, 5}, {3, false, 1}}, shapes(t, events), + "a v1 transaction once active, a five-member bundle, and a cross-bundle repeat all pass ingest") +} + +func TestBlockPipelineBlocksRatherThanDrops(t *testing.T) { + events := make(chan BlockEvent) + p := NewBlockPipeline(events, nil) + p.SetCurrentSlot(10) + done := make(chan struct{}) + go func() { + defer close(done) + var txs []*blockpb.Transaction + for i := range uint64(20) { + txs = append(txs, blockTx(wireTx(i), 0)) + } + p.HandleBlock(blockMsg(10, txs...), time.Now()) + }() + for want := uint64(1); want <= 20; want++ { + time.Sleep(time.Millisecond) + e := <-events + require.Equal(t, want, e.(BundleEvent).Seq) + } + <-done +} + +func TestBlockEngineSessionFeedsThePipeline(t *testing.T) { + h := newBlockEngineHarness(t, time.Hour) + events := make(chan BlockEvent, 16) + pipeline := NewBlockPipeline(events, nil) + h.session.cfg.Blocks = pipeline + h.start() + eventually(t, h.session.Healthy, "healthy") + + pipeline.SetCurrentSlot(50) + h.session.NotifyLeaderSlot(50, time.Now().Add(time.Hour)) + eventually(t, func() bool { return h.session.Stats().NoticesSent == 1 }, "notice sent") + h.be.blocks <- blockMsg(49, blockTx(wireTx(1), 0)) + h.be.blocks <- blockMsg(50, blockTx(wireTx(2), 8), blockTx(wireTx(3), 8)) + select { + case e := <-events: + bundle := e.(BundleEvent) + assert.Equal(t, uint64(50), bundle.Slot) + assert.True(t, bundle.RevertProtected) + assert.Len(t, bundle.Transactions, 2) + case <-time.After(2 * time.Second): + t.Fatal("no bundle") + } + + h.be.endBlocks <- status.Error(codes.Unavailable, "reset mid-slot") + select { + case e := <-events: + assert.Equal(t, StreamLostEvent{Slot: 50}, e) + case <-time.After(2 * time.Second): + t.Fatal("no stream-lost event") + } + eventually(t, h.session.Healthy, "reconnected") + h.be.blocks <- blockMsg(50, blockTx(wireTx(4), 0)) + eventually(t, func() bool { return pipeline.Stats().LostSlot == 1 }, "the lost slot stays discarded after reconnecting") + assert.Empty(t, events) +} From 7c19db7fdb239a450acc1e1370146bef4f85b2c8 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 21:03:15 +0000 Subject: [PATCH 17/25] harmonic: per-slot executor --- pkg/harmonic/executor.go | 406 ++++++++++++++++++++++++++++++++++ pkg/harmonic/executor_test.go | 352 +++++++++++++++++++++++++++++ 2 files changed, 758 insertions(+) create mode 100644 pkg/harmonic/executor.go create mode 100644 pkg/harmonic/executor_test.go diff --git a/pkg/harmonic/executor.go b/pkg/harmonic/executor.go new file mode 100644 index 00000000..0cbc2d4d --- /dev/null +++ b/pkg/harmonic/executor.go @@ -0,0 +1,406 @@ +package harmonic + +import ( + "context" + "fmt" + "strings" + "sync" + "time" + + "github.com/Overclock-Validator/mithril/pkg/blockprod" + "github.com/Overclock-Validator/mithril/pkg/mlog" + "github.com/gagliardetto/solana-go" +) + +const ( + // thresholdFraction of the slot's recording window passes before a slot + // with no block bundle falls back to local scheduling. + thresholdFraction = 0.5 + // voteTail is the end of the slot in which a reverting block bundle does + // not abort the block (spec §15.3). + voteTail = 20 * time.Millisecond +) + +// SlotMode is a leader slot's admission mode. +type SlotMode int + +const ( + ModeUndecided SlotMode = iota + ModeHarmonic + ModeFallback + ModeDone +) + +func (m SlotMode) String() string { + switch m { + case ModeUndecided: + return "undecided" + case ModeHarmonic: + return "harmonic" + case ModeFallback: + return "fallback" + default: + return "done" + } +} + +// EndFlags record why a slot's block stopped, for reporting. +type EndFlags uint8 + +const ( + FlagTimeout EndFlags = 1 << iota + FlagFailed + FlagReverted + // FlagUnavailable: Harmonic was not possible when the slot started. + FlagUnavailable +) + +func (f EndFlags) String() string { + var names []string + for flag, name := range map[EndFlags]string{FlagTimeout: "timeout", FlagFailed: "failed", FlagReverted: "reverted", FlagUnavailable: "unavailable"} { + if f&flag != 0 { + names = append(names, name) + } + } + if len(names) == 0 { + return "none" + } + return strings.Join(names, ",") +} + +// SlotReport summarizes one leader slot. +type SlotReport struct { + Slot uint64 + // Mode is the last mode before the slot ended: harmonic, fallback, or + // undecided if no block bundle arrived and the threshold never passed. + Mode SlotMode + Flags EndFlags + // Executed counts bundles that landed; Reverted, the ones that executed + // and landed nothing; Rejected, bundles refused without executing. + Executed, Reverted, Rejected uint64 + // Aborted is set when the leader abandoned the slot, with its reason. + Aborted bool + Reason string +} + +// BlockEngineControl is the part of the block engine session the executor +// uses. +type BlockEngineControl interface { + Healthy() bool + NotifyLeaderSlot(slot uint64, end time.Time) +} + +// SlotTracker is told the current block slot; the block pipeline admits only +// that slot's messages. +type SlotTracker interface { + SetCurrentSlot(slot uint64) +} + +// CrankBuilder returns the transactions that must land before a slot's first +// block bundle, or none when the tip state needs no change. +type CrankBuilder func(bank *blockprod.WorkingBank) ([]*solana.Transaction, error) + +// ExecutorConfig wires the executor. +type ExecutorConfig struct { + BlockEngine BlockEngineControl + Pipeline SlotTracker + Events <-chan BlockEvent + // Crank is optional; nil means no crank is ever needed. + Crank CrankBuilder + // OnReport, if set, receives each slot's report. + OnReport func(SlotReport) +} + +type hookMessage struct { + started *slotState + ended uint64 + abort bool + reason string +} + +type slotState struct { + slot uint64 + bank *blockprod.WorkingBank + start, end time.Time + threshold time.Time + mode SlotMode + stopped bool + nextSeq uint64 + crankDone bool + report SlotReport + thresholdTimer *time.Timer +} + +// Executor runs each leader slot's Harmonic block: it announces the slot, +// executes the block's bundles in stream order, and falls back to local +// scheduling when the block does not arrive or stops. It is the leader loop's +// LeaderHooks. +type Executor struct { + cfg ExecutorConfig + now func() time.Time + + hooksMu sync.Mutex + hooks []hookMessage + wake chan struct{} + // thresholds carries threshold expiries to the executor goroutine. + thresholds chan uint64 +} + +var _ blockprod.LeaderHooks = (*Executor)(nil) + +func NewExecutor(cfg ExecutorConfig) *Executor { + return &Executor{cfg: cfg, now: time.Now, wake: make(chan struct{}, 1), thresholds: make(chan uint64, 4)} +} + +// SlotStarted runs under the leader loop's lock, so it only does what cannot +// wait and queues the rest. The current slot is set before the notice is +// queued, because the block engine streams the slot only after the notice. +func (x *Executor) SlotStarted(info blockprod.LeaderSlotInfo) { + st := &slotState{ + slot: info.Slot, + bank: info.Bank, + start: info.Start, + end: info.End, + nextSeq: 1, + report: SlotReport{Slot: info.Slot}, + } + if info.End.IsZero() || !x.cfg.BlockEngine.Healthy() { + // Without a deadline or a healthy session no block can arrive; the + // TPU opens before the bank is published. + st.mode = ModeFallback + st.report.Flags |= FlagUnavailable + info.Bank.AllowTPU() + } else { + st.threshold = info.Start.Add(time.Duration(float64(info.End.Sub(info.Start)) * thresholdFraction)) + x.cfg.Pipeline.SetCurrentSlot(info.Slot) + x.cfg.BlockEngine.NotifyLeaderSlot(info.Slot, info.End) + } + x.post(hookMessage{started: st}) +} + +// SlotEnding runs under the leader loop's lock just before the bank freezes. +func (x *Executor) SlotEnding(slot uint64) { + x.cfg.Pipeline.SetCurrentSlot(0) + x.post(hookMessage{ended: slot}) +} + +// SlotAborted runs under the leader loop's lock when the slot is abandoned. +func (x *Executor) SlotAborted(slot uint64, reason string) { + x.cfg.Pipeline.SetCurrentSlot(0) + x.post(hookMessage{ended: slot, abort: true, reason: reason}) +} + +func (x *Executor) post(msg hookMessage) { + x.hooksMu.Lock() + x.hooks = append(x.hooks, msg) + x.hooksMu.Unlock() + select { + case x.wake <- struct{}{}: + default: + } +} + +// Run processes slots and block events until ctx ends. +func (x *Executor) Run(ctx context.Context) { + var st *slotState + for { + select { + case <-ctx.Done(): + return + case <-x.wake: + st = x.drainHooks(st) + case slot := <-x.thresholds: + st = x.drainHooks(st) + if st != nil && st.slot == slot && st.mode == ModeUndecided { + x.fallback(st, FlagTimeout) + } + case e := <-x.cfg.Events: + // A slot's start is always queued before its first event, since + // the block engine streams only after the notice. + st = x.drainHooks(st) + x.handleEvent(st, e) + } + } +} + +func (x *Executor) drainHooks(st *slotState) *slotState { + x.hooksMu.Lock() + hooks := x.hooks + x.hooks = nil + x.hooksMu.Unlock() + for _, msg := range hooks { + if msg.started != nil { + if st != nil && st.mode != ModeDone { + x.finish(st, "", false) + } + st = msg.started + x.armThreshold(st) + continue + } + if st != nil && st.slot == msg.ended && st.mode != ModeDone { + if msg.abort { + st.report.Flags |= FlagFailed + } + x.finish(st, msg.reason, msg.abort) + } + } + return st +} + +func (x *Executor) armThreshold(st *slotState) { + if st.mode != ModeUndecided { + return + } + slot := st.slot + st.thresholdTimer = time.AfterFunc(time.Until(st.threshold), func() { + select { + case x.thresholds <- slot: + default: + } + }) +} + +func (x *Executor) finish(st *slotState, reason string, aborted bool) { + if st.thresholdTimer != nil { + st.thresholdTimer.Stop() + } + st.report.Mode = st.mode + st.report.Aborted = aborted + st.report.Reason = reason + st.mode = ModeDone + r := st.report + mlog.Log.Infof("Harmonic slot %d: mode=%s flags=%s executed=%d reverted=%d rejected=%d aborted=%t%s", + r.Slot, r.Mode, r.Flags, r.Executed, r.Reverted, r.Rejected, r.Aborted, reasonSuffix(r.Reason)) + if x.cfg.OnReport != nil { + x.cfg.OnReport(r) + } +} + +func reasonSuffix(reason string) string { + if reason == "" { + return "" + } + return " reason=" + reason +} + +func (x *Executor) handleEvent(st *slotState, e BlockEvent) { + switch e := e.(type) { + case StreamLostEvent: + if st != nil && st.slot == e.Slot { + x.stop(st, FlagFailed) + } + case InvalidBundleEvent: + if !x.admits(st, e.Slot, e.Seq) { + return + } + x.stop(st, FlagFailed) + case BundleEvent: + if !x.admits(st, e.Slot, e.Seq) { + return + } + x.executeBundle(st, e) + } +} + +// admits checks a bundle event against the slot state and consumes its +// sequence number. A gap in the sequence means block data was lost. +func (x *Executor) admits(st *slotState, slot, seq uint64) bool { + if st == nil || st.slot != slot { + return false + } + if (st.mode != ModeUndecided && st.mode != ModeHarmonic) || st.stopped { + st.report.Rejected++ + return false + } + if seq != st.nextSeq { + st.report.Rejected++ + x.stop(st, FlagFailed) + return false + } + st.nextSeq++ + return true +} + +func (x *Executor) executeBundle(st *slotState, e BundleEvent) { + now := x.now() + if st.mode == ModeHarmonic && now.After(st.end) { + st.report.Rejected++ + x.stop(st, FlagTimeout) + return + } + if st.mode == ModeUndecided && !now.Before(st.threshold) { + st.report.Rejected++ + x.fallback(st, FlagTimeout) + return + } + if st.mode == ModeUndecided { + if !x.runCrank(st) { + st.report.Rejected++ + return + } + st.mode = ModeHarmonic + } + + var result blockprod.ForgeResult + if e.RevertProtected { + result = st.bank.ForgeBundle(blockprod.BundleInput{Transactions: e.Transactions, AdmitOnActual: true}).Result + } else { + result, _ = st.bank.ForgeStandaloneTransaction(e.Transactions[0], 0) + } + switch result { + case blockprod.ForgeAccepted: + st.report.Executed++ + case blockprod.ForgeDroppedNoLeader: + // The bank froze: the slot is over. + st.report.Rejected++ + st.stopped = true + case blockprod.ForgeDroppedCost: + // Execution is serial, so nothing in flight can free room. + st.report.Rejected++ + x.stop(st, FlagFailed) + default: + st.report.Reverted++ + if now.Before(st.end.Add(-voteTail)) { + x.stop(st, FlagReverted) + } + } +} + +// runCrank lands the slot's tip crank before its first block bundle. A crank +// that fails aborts the block. +func (x *Executor) runCrank(st *slotState) bool { + if st.crankDone || x.cfg.Crank == nil { + st.crankDone = true + return true + } + txs, err := x.cfg.Crank(st.bank) + if err == nil && len(txs) > 0 { + if r := st.bank.ForgeBundle(blockprod.BundleInput{Transactions: txs}); r.Result != blockprod.ForgeAccepted { + err = fmt.Errorf("crank bundle: %s", r.Result) + } + } + if err != nil { + mlog.Log.Infof("Harmonic slot %d: tip crank failed: %v", st.slot, err) + x.stop(st, FlagFailed) + return false + } + st.crankDone = true + return true +} + +// stop ends block admission for the slot. Before the cutoff the slot falls +// back to local scheduling; after it, the block is simply over. +func (x *Executor) stop(st *slotState, flag EndFlags) { + st.stopped = true + st.report.Flags |= flag + if st.mode == ModeUndecided || (st.mode == ModeHarmonic && x.now().Before(st.end)) { + x.fallback(st, 0) + } +} + +func (x *Executor) fallback(st *slotState, flag EndFlags) { + st.mode = ModeFallback + st.stopped = true + st.report.Flags |= flag + st.bank.AllowTPU() +} diff --git a/pkg/harmonic/executor_test.go b/pkg/harmonic/executor_test.go new file mode 100644 index 00000000..d94139c6 --- /dev/null +++ b/pkg/harmonic/executor_test.go @@ -0,0 +1,352 @@ +package harmonic + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/blockprod" + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/programs/system" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fakeBlockEngine records what the executor asks of the block engine. +type fakeBlockEngine struct { + mu sync.Mutex + healthy bool + calls []string + notices []uint64 +} + +func (f *fakeBlockEngine) Healthy() bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.healthy +} + +func (f *fakeBlockEngine) NotifyLeaderSlot(slot uint64, _ time.Time) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, "notice") + f.notices = append(f.notices, slot) +} + +// fakePipeline records the current slot and shares the call log. +type fakePipeline struct { + be *fakeBlockEngine + current []uint64 +} + +func (f *fakePipeline) SetCurrentSlot(slot uint64) { + f.be.mu.Lock() + defer f.be.mu.Unlock() + f.be.calls = append(f.be.calls, "current") + f.current = append(f.current, slot) +} + +type executorHarness struct { + exec *Executor + be *fakeBlockEngine + pipeline *fakePipeline + events chan BlockEvent + reports chan SlotReport +} + +func startExecutor(t *testing.T, healthy bool, crank CrankBuilder) *executorHarness { + t.Helper() + be := &fakeBlockEngine{healthy: healthy} + h := &executorHarness{be: be, pipeline: &fakePipeline{be: be}, events: make(chan BlockEvent), reports: make(chan SlotReport, 4)} + h.exec = NewExecutor(ExecutorConfig{ + BlockEngine: be, + Pipeline: h.pipeline, + Events: h.events, + Crank: crank, + OnReport: func(r SlotReport) { h.reports <- r }, + }) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + h.exec.Run(ctx) + close(done) + }() + t.Cleanup(func() { + cancel() + <-done + }) + return h +} + +func gatedBank(t *testing.T, limits costmodel.Limits) *blockprod.TestEnv { + t.Helper() + env := blockprod.NewTestEnv(blockprod.TestEnvConfig{TPUGated: true, Limits: limits}) + t.Cleanup(env.Close) + return env +} + +// startSlot opens a slot whose recording window is length long. +func (h *executorHarness) startSlot(slot uint64, env *blockprod.TestEnv, length time.Duration) { + now := time.Now() + h.exec.SlotStarted(blockprod.LeaderSlotInfo{Slot: slot, Bank: env.Bank, Start: now, End: now.Add(length)}) +} + +func (h *executorHarness) send(t *testing.T, e BlockEvent) { + t.Helper() + select { + case h.events <- e: + case <-time.After(2 * time.Second): + t.Fatal("executor not reading events") + } +} + +// sync waits until the executor has handled everything sent so far. +func (h *executorHarness) sync(t *testing.T) { + t.Helper() + h.send(t, StreamLostEvent{Slot: 1 << 63}) +} + +func (h *executorHarness) end(t *testing.T, slot uint64) SlotReport { + t.Helper() + h.exec.SlotEnding(slot) + select { + case r := <-h.reports: + return r + case <-time.After(2 * time.Second): + t.Fatal("no slot report") + return SlotReport{} + } +} + +func transfer(t *testing.T, lamports uint64) *solana.Transaction { + t.Helper() + tx, err := solana.NewTransaction( + []solana.Instruction{system.NewTransferInstruction(lamports, txfixture.PayerPubkey(), txfixture.DestPubkey()).Build()}, + txfixture.TestBlockhash(), solana.TransactionPayer(txfixture.PayerPubkey())) + require.NoError(t, err) + key := txfixture.PayerPrivateKey() + _, err = tx.Sign(func(solana.PublicKey) *solana.PrivateKey { return &key }) + require.NoError(t, err) + return tx +} + +func blockCost(units uint64) costmodel.Limits { + limits := costmodel.DefaultLimits() + limits.BlockCost = units + return limits +} + +func bundleEvent(slot, seq uint64, revertProtected bool, txs ...*solana.Transaction) BundleEvent { + return BundleEvent{Slot: slot, Seq: seq, Arrival: time.Now(), RevertProtected: revertProtected, Transactions: txs} +} + +func TestExecutorFallsBackAtOnceWhenHarmonicIsUnavailable(t *testing.T) { + h := startExecutor(t, false, nil) + env := gatedBank(t, costmodel.Limits{}) + h.startSlot(10, env, 400*time.Millisecond) + assert.True(t, env.Bank.TPUAllowed(), "the TPU opens before the bank is published") + assert.Empty(t, h.be.notices, "an unhealthy session is not notified") + + report := h.end(t, 10) + assert.Equal(t, ModeFallback, report.Mode) + assert.Equal(t, FlagUnavailable, report.Flags) + + noDeadline := startExecutor(t, true, nil) + env2 := gatedBank(t, costmodel.Limits{}) + noDeadline.exec.SlotStarted(blockprod.LeaderSlotInfo{Slot: 11, Bank: env2.Bank, Start: time.Now()}) + assert.True(t, env2.Bank.TPUAllowed(), "a slot with no deadline cannot run a block") +} + +func TestExecutorRunsTheBlockInOrderWithTheTPUClosed(t *testing.T) { + h := startExecutor(t, true, nil) + env := gatedBank(t, costmodel.Limits{}) + h.startSlot(10, env, 400*time.Millisecond) + assert.Equal(t, []string{"current", "notice"}, h.be.calls, "the slot is current before it is announced") + assert.Equal(t, []uint64{10}, h.pipeline.current) + assert.Equal(t, []uint64{10}, h.be.notices) + + a, b, c := transfer(t, 1), transfer(t, 2), transfer(t, 3) + h.send(t, bundleEvent(10, 1, true, a, b)) + h.send(t, bundleEvent(10, 2, false, c)) + h.sync(t) + assert.Equal(t, []*solana.Transaction{a, b, c}, env.Bank.ForgedTransactions()) + assert.False(t, env.Bank.TPUAllowed(), "the TPU stays closed during a Harmonic block") + + report := h.end(t, 10) + assert.Equal(t, ModeHarmonic, report.Mode) + assert.Equal(t, uint64(2), report.Executed) + assert.Zero(t, report.Flags) + assert.Equal(t, []uint64{10, 0}, h.pipeline.current, "the slot stops being current when it ends") +} + +func TestExecutorFallsBackWhenNoBlockArrivesByTheThreshold(t *testing.T) { + h := startExecutor(t, true, nil) + env := gatedBank(t, costmodel.Limits{}) + h.startSlot(10, env, 100*time.Millisecond) + require.Eventually(t, env.Bank.TPUAllowed, time.Second, time.Millisecond, "the threshold timer opens the TPU") + + h.send(t, bundleEvent(10, 1, false, transfer(t, 1))) + h.sync(t) + assert.Empty(t, env.Bank.ForgedTransactions(), "a bundle after the switch is never executed") + report := h.end(t, 10) + assert.Equal(t, ModeFallback, report.Mode) + assert.Equal(t, FlagTimeout, report.Flags) + assert.Equal(t, uint64(1), report.Rejected) +} + +func TestExecutorStopTriggers(t *testing.T) { + overdraw := func(t *testing.T) *solana.Transaction { return transfer(t, 20_000_000_000) } + cases := map[string]struct { + events func(t *testing.T) []BlockEvent + limits costmodel.Limits + flag EndFlags + }{ + "revert": { + events: func(t *testing.T) []BlockEvent { + return []BlockEvent{bundleEvent(10, 1, false, transfer(t, 1)), bundleEvent(10, 2, true, transfer(t, 2), overdraw(t))} + }, + flag: FlagReverted, + }, + "invalid bundle": { + events: func(t *testing.T) []BlockEvent { + return []BlockEvent{bundleEvent(10, 1, false, transfer(t, 1)), InvalidBundleEvent{Slot: 10, Seq: 2, Reason: errors.New("bad")}} + }, + flag: FlagFailed, + }, + "sequence gap": { + events: func(t *testing.T) []BlockEvent { + return []BlockEvent{bundleEvent(10, 1, false, transfer(t, 1)), bundleEvent(10, 3, false, transfer(t, 2))} + }, + flag: FlagFailed, + }, + "stream lost": { + events: func(t *testing.T) []BlockEvent { + return []BlockEvent{bundleEvent(10, 1, false, transfer(t, 1)), StreamLostEvent{Slot: 10}} + }, + flag: FlagFailed, + }, + "does not fit": { + events: func(t *testing.T) []BlockEvent { + return []BlockEvent{bundleEvent(10, 1, false, transfer(t, 1)), bundleEvent(10, 2, true, transfer(t, 2), transfer(t, 3))} + }, + limits: blockCost(4_000), + flag: FlagFailed, + }, + } + for name, tc := range cases { + t.Run(name, func(t *testing.T) { + h := startExecutor(t, true, nil) + env := gatedBank(t, tc.limits) + h.startSlot(10, env, time.Hour) + for _, e := range tc.events(t) { + h.send(t, e) + } + later := transfer(t, 4) + h.send(t, bundleEvent(10, 99, false, later)) + h.sync(t) + assert.True(t, env.Bank.TPUAllowed(), "a stop before the cutoff falls back") + assert.Len(t, env.Bank.ForgedTransactions(), 1, "only the first bundle landed") + report := h.end(t, 10) + assert.Equal(t, ModeFallback, report.Mode) + assert.Equal(t, tc.flag, report.Flags) + assert.Equal(t, uint64(1), report.Executed) + }) + } +} + +func TestExecutorCutoffAndVoteTail(t *testing.T) { + t.Run("a bundle after the cutoff stops the block without falling back", func(t *testing.T) { + h := startExecutor(t, true, nil) + env := gatedBank(t, costmodel.Limits{}) + h.startSlot(10, env, 80*time.Millisecond) + h.send(t, bundleEvent(10, 1, false, transfer(t, 1))) + time.Sleep(100 * time.Millisecond) + h.send(t, bundleEvent(10, 2, false, transfer(t, 2))) + h.sync(t) + assert.Len(t, env.Bank.ForgedTransactions(), 1) + assert.False(t, env.Bank.TPUAllowed(), "past the cutoff the block is simply over") + report := h.end(t, 10) + assert.Equal(t, ModeHarmonic, report.Mode) + assert.Equal(t, FlagTimeout, report.Flags) + }) + t.Run("a revert in the vote tail lands nothing and the block continues", func(t *testing.T) { + h := startExecutor(t, true, nil) + env := gatedBank(t, costmodel.Limits{}) + now := time.Now() + h.exec.SlotStarted(blockprod.LeaderSlotInfo{Slot: 10, Bank: env.Bank, Start: now, End: now.Add(time.Second)}) + h.send(t, bundleEvent(10, 1, false, transfer(t, 1))) + h.sync(t) + h.exec.now = func() time.Time { return now.Add(time.Second - 10*time.Millisecond) } + h.send(t, bundleEvent(10, 2, true, transfer(t, 2), transfer(t, 20_000_000_000))) + h.send(t, bundleEvent(10, 3, false, transfer(t, 3))) + h.sync(t) + assert.Len(t, env.Bank.ForgedTransactions(), 2) + report := h.end(t, 10) + assert.Equal(t, ModeHarmonic, report.Mode) + assert.Zero(t, report.Flags) + assert.Equal(t, uint64(1), report.Reverted) + }) +} + +func TestExecutorCrankRunsFirstAndItsFailureFallsBack(t *testing.T) { + crank := transfer(t, 9) + h := startExecutor(t, true, func(*blockprod.WorkingBank) ([]*solana.Transaction, error) { + return []*solana.Transaction{crank}, nil + }) + env := gatedBank(t, costmodel.Limits{}) + h.startSlot(10, env, time.Hour) + first := transfer(t, 1) + h.send(t, bundleEvent(10, 1, false, first)) + h.send(t, bundleEvent(10, 2, false, transfer(t, 2))) + h.sync(t) + forged := env.Bank.ForgedTransactions() + require.Len(t, forged, 3) + assert.Equal(t, []*solana.Transaction{crank, first}, forged[:2], "the crank lands before the first block bundle, once") + + failing := startExecutor(t, true, func(*blockprod.WorkingBank) ([]*solana.Transaction, error) { + return nil, errors.New("tip accounts unreadable") + }) + env2 := gatedBank(t, costmodel.Limits{}) + failing.startSlot(10, env2, time.Hour) + failing.send(t, bundleEvent(10, 1, false, transfer(t, 1))) + failing.sync(t) + assert.Empty(t, env2.Bank.ForgedTransactions()) + assert.True(t, env2.Bank.TPUAllowed()) + report := failing.end(t, 10) + assert.Equal(t, ModeFallback, report.Mode) + assert.Equal(t, FlagFailed, report.Flags) +} + +func TestExecutorSlotsAreIndependent(t *testing.T) { + h := startExecutor(t, true, nil) + env10 := gatedBank(t, costmodel.Limits{}) + h.startSlot(10, env10, time.Hour) + h.send(t, bundleEvent(10, 1, false, transfer(t, 1))) + h.send(t, InvalidBundleEvent{Slot: 10, Seq: 2, Reason: errors.New("bad")}) + h.sync(t) + r10 := h.end(t, 10) + assert.Equal(t, FlagFailed, r10.Flags) + + env11 := gatedBank(t, costmodel.Limits{}) + h.startSlot(11, env11, time.Hour) + h.send(t, bundleEvent(10, 3, false, transfer(t, 2))) + h.send(t, bundleEvent(11, 1, false, transfer(t, 3))) + h.sync(t) + assert.Len(t, env11.Bank.ForgedTransactions(), 1, "the new slot starts clean at sequence 1") + assert.False(t, env11.Bank.TPUAllowed()) + + h.exec.SlotAborted(11, "parent_changed_before_freeze") + select { + case r := <-h.reports: + assert.True(t, r.Aborted) + assert.Equal(t, "parent_changed_before_freeze", r.Reason) + assert.Equal(t, FlagFailed, r.Flags) + case <-time.After(2 * time.Second): + t.Fatal("no report for the aborted slot") + } + assert.Equal(t, []uint64{10, 0, 11, 0}, h.pipeline.current) +} From 2d883aee8ade26f590165d620d4adaf52389d541 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 21:15:46 +0000 Subject: [PATCH 18/25] harmonic: tip crank --- pkg/blockprod/bank.go | 30 +++ pkg/blockprod/bank_bundle_test.go | 33 +++ pkg/harmonic/blockengine.go | 5 + pkg/harmonic/crank.go | 260 ++++++++++++++++++++ pkg/harmonic/crank_test.go | 378 ++++++++++++++++++++++++++++++ 5 files changed, 706 insertions(+) create mode 100644 pkg/harmonic/crank.go create mode 100644 pkg/harmonic/crank_test.go diff --git a/pkg/blockprod/bank.go b/pkg/blockprod/bank.go index ff97dec0..27cdb84b 100644 --- a/pkg/blockprod/bank.go +++ b/pkg/blockprod/bank.go @@ -4,6 +4,7 @@ import ( "sync" "sync/atomic" + "github.com/Overclock-Validator/mithril/pkg/accounts" "github.com/Overclock-Validator/mithril/pkg/arena" "github.com/Overclock-Validator/mithril/pkg/costmodel" "github.com/Overclock-Validator/mithril/pkg/features" @@ -159,6 +160,35 @@ func (b *WorkingBank) TxFeeAccumulator() fees.TxFeeInfoAccumulator { return b.txFees } +// ReadAccount returns pubkey's account as this bank sees it: as written +// earlier in the slot, otherwise from parent state. ok is false for an +// account that does not exist. +func (b *WorkingBank) ReadAccount(pubkey solana.PublicKey) (*accounts.Account, bool) { + b.mu.Lock() + defer b.mu.Unlock() + if b.slotCtx == nil { + return nil, false + } + acct, err := b.slotCtx.GetAccount(pubkey) + if err != nil && (b.slotCtx.UnrootedRead != nil || b.slotCtx.AccountsDb != nil) { + acct, err = b.slotCtx.GetAccountFromAccountsDb(pubkey) + } + if err != nil || acct == nil || acct.Lamports == 0 { + return nil, false + } + return acct, true +} + +// Epoch is the bank's epoch. +func (b *WorkingBank) Epoch() uint64 { + return b.slotCtx.Epoch +} + +// LastBlockhash is the most recent blockhash the bank accepts: its parent's. +func (b *WorkingBank) LastBlockhash() solana.Hash { + return b.slotCtx.LastBlockhash +} + // ForgeResult reports whether a transaction was committed into the working bank. type ForgeResult int diff --git a/pkg/blockprod/bank_bundle_test.go b/pkg/blockprod/bank_bundle_test.go index aff6457d..873bbf2b 100644 --- a/pkg/blockprod/bank_bundle_test.go +++ b/pkg/blockprod/bank_bundle_test.go @@ -1,12 +1,14 @@ package blockprod import ( + "errors" "math" "math/rand/v2" "sync" "testing" "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/accountsdb" "github.com/Overclock-Validator/mithril/pkg/addresses" "github.com/Overclock-Validator/mithril/pkg/costmodel" "github.com/Overclock-Validator/mithril/pkg/replay" @@ -401,3 +403,34 @@ func TestForgeBundleFailureNeitherChargesFeesNorAdvancesNonce(t *testing.T) { }) } } + +func TestWorkingBankReadAccountSeesSlotWritesThenParentState(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + parentOnly := solana.PublicKey{0x51} + env.SlotCtx.AccountsDb = &accountsdb.AccountsDb{} + env.SlotCtx.UnrootedRead = parentStateAccounts{parentOnly: {Key: parentOnly, Lamports: 7}} + + acct, ok := env.Bank.ReadAccount(txfixture.PayerPubkey()) + require.True(t, ok, "written in the slot") + assert.Equal(t, uint64(10_000_000_000), acct.Lamports) + acct, ok = env.Bank.ReadAccount(parentOnly) + require.True(t, ok, "falls back to parent state") + assert.Equal(t, uint64(7), acct.Lamports) + _, ok = env.Bank.ReadAccount(solana.PublicKey{0x52}) + assert.False(t, ok, "missing everywhere") + require.NoError(t, env.SlotCtx.SetAccount(parentOnly, &accounts.Account{Key: parentOnly})) + _, ok = env.Bank.ReadAccount(parentOnly) + assert.False(t, ok, "an account the slot emptied does not exist") + assert.Equal(t, env.SlotCtx.Epoch, env.Bank.Epoch()) + assert.Equal(t, txfixture.TestBlockhash(), env.Bank.LastBlockhash()) +} + +type parentStateAccounts map[solana.PublicKey]*accounts.Account + +func (p parentStateAccounts) GetAccount(_ uint64, pubkey solana.PublicKey) (*accounts.Account, error) { + if acct, ok := p[pubkey]; ok { + return acct.Clone(), nil + } + return nil, errors.New("not found") +} diff --git a/pkg/harmonic/blockengine.go b/pkg/harmonic/blockengine.go index 078cab5a..ba01cd5f 100644 --- a/pkg/harmonic/blockengine.go +++ b/pkg/harmonic/blockengine.go @@ -189,6 +189,11 @@ func (b *BlockEngineSession) Healthy() bool { return b.healthy } +// Strategy is the scheduling strategy the session applies. +func (b *BlockEngineSession) Strategy() blockenginepb.SchedulingStrategy { + return blockenginepb.SchedulingStrategy(b.strategy.Load()) +} + // SetStrategy changes the scheduling strategy. The session reconnects so the // block engine applies it. func (b *BlockEngineSession) SetStrategy(strategy blockenginepb.SchedulingStrategy) { diff --git a/pkg/harmonic/crank.go b/pkg/harmonic/crank.go new file mode 100644 index 00000000..2a534bc6 --- /dev/null +++ b/pkg/harmonic/crank.go @@ -0,0 +1,260 @@ +package harmonic + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/blockprod" + blockenginepb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/blockengine" + "github.com/gagliardetto/solana-go" +) + +var ( + memoProgram = solana.MustPublicKeyFromBase58("MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr") + + // Anchor discriminators: the first 8 bytes of sha256("account:Config") + // and of sha256("global:"). + tipPaymentConfigDiscriminator = []byte{0x9b, 0x0c, 0xaa, 0xe0, 0x1e, 0xfa, 0xcc, 0x82} + changeTipReceiverDiscriminator = []byte{0x45, 0x63, 0x16, 0x47, 0x0b, 0xe7, 0x56, 0x8f} + changeBlockBuilderDiscriminator = []byte{0x86, 0x50, 0x26, 0x89, 0xa5, 0x15, 0x72, 0x7b} + initTipDistributionDiscriminator = []byte{0x78, 0xbf, 0x19, 0xb6, 0x6f, 0x31, 0xb3, 0x37} + errNoFeeInfo = errors.New("no valid block builder fee info") + errTipPaymentProgramUninitialized = errors.New("tip payment program is not initialized") +) + +const ( + tipAccounts = 8 + // tipPaymentConfigSize is the discriminator, tip receiver, block builder, + // commission and nine bumps. + tipPaymentConfigSize = 8 + 32 + 32 + 8 + 9 + maxCommissionBps = 10_000 +) + +// TipConfig configures the tip crank. +type TipConfig struct { + TipPaymentProgram solana.PublicKey + TipDistributionProgram solana.PublicKey + MerkleRootUploadAuthority solana.PublicKey + VoteAccount solana.PublicKey + // CommissionBps is the validator's commission on tips, 0 to 10000. + CommissionBps uint16 + // Identity pays for and signs the crank; it must be an authorized voter + // of VoteAccount. + Identity solana.PrivateKey +} + +// TipPaymentConfig is the tip payment program's config account. +type TipPaymentConfig struct { + TipReceiver solana.PublicKey + BlockBuilder solana.PublicKey + CommissionPct uint64 +} + +// ParseTipPaymentConfig decodes the tip payment config account's data. +func ParseTipPaymentConfig(data []byte) (TipPaymentConfig, error) { + if len(data) < tipPaymentConfigSize { + return TipPaymentConfig{}, fmt.Errorf("tip payment config has %d bytes, want %d", len(data), tipPaymentConfigSize) + } + if !bytes.Equal(data[:8], tipPaymentConfigDiscriminator) { + return TipPaymentConfig{}, fmt.Errorf("tip payment config discriminator %x", data[:8]) + } + cfg := TipPaymentConfig{ + TipReceiver: solana.PublicKeyFromBytes(data[8:40]), + BlockBuilder: solana.PublicKeyFromBytes(data[40:72]), + CommissionPct: binary.LittleEndian.Uint64(data[72:80]), + } + if cfg.TipReceiver.IsZero() || cfg.BlockBuilder.IsZero() { + return TipPaymentConfig{}, errors.New("tip payment config holds a zero pubkey") + } + return cfg, nil +} + +// TipState is the bank state the crank reads. +type TipState interface { + ReadAccount(pubkey solana.PublicKey) (*accounts.Account, bool) + Epoch() uint64 + LastBlockhash() solana.Hash +} + +// TipCrank builds the transactions that route a slot's tips to this +// validator's tip distribution account with the block engine's block builder +// and commission, before any block bundle executes. +type TipCrank struct { + cfg TipConfig + config solana.PublicKey + distributionConfig solana.PublicKey + tipAccounts [tipAccounts]solana.PublicKey +} + +func NewTipCrank(cfg TipConfig) (*TipCrank, error) { + switch { + case cfg.TipPaymentProgram.IsZero(), cfg.TipDistributionProgram.IsZero(), cfg.VoteAccount.IsZero(): + return nil, errors.New("tip crank needs both tip programs and the vote account") + case cfg.CommissionBps > maxCommissionBps: + return nil, fmt.Errorf("tip commission %d bps is over %d", cfg.CommissionBps, maxCommissionBps) + case len(cfg.Identity) != 64: + return nil, errors.New("tip crank needs the identity key") + } + c := &TipCrank{cfg: cfg} + var err error + if c.config, _, err = solana.FindProgramAddress([][]byte{[]byte("CONFIG_ACCOUNT")}, cfg.TipPaymentProgram); err != nil { + return nil, err + } + if c.distributionConfig, _, err = solana.FindProgramAddress([][]byte{[]byte("CONFIG_ACCOUNT")}, cfg.TipDistributionProgram); err != nil { + return nil, err + } + for i := range c.tipAccounts { + seed := []byte(fmt.Sprintf("TIP_ACCOUNT_%d", i)) + if c.tipAccounts[i], _, err = solana.FindProgramAddress([][]byte{seed}, cfg.TipPaymentProgram); err != nil { + return nil, err + } + } + return c, nil +} + +// TipPaymentConfigAddress is the tip payment program's config account. +func (c *TipCrank) TipPaymentConfigAddress() solana.PublicKey { + return c.config +} + +// TipAccounts are the tip payment program's eight tip accounts. +func (c *TipCrank) TipAccounts() [tipAccounts]solana.PublicKey { + return c.tipAccounts +} + +// TipDistributionAccount is this validator's tip distribution account for +// epoch, and its bump. +func (c *TipCrank) TipDistributionAccount(epoch uint64) (solana.PublicKey, uint8, error) { + var epochLE [8]byte + binary.LittleEndian.PutUint64(epochLE[:], epoch) + return solana.FindProgramAddress([][]byte{[]byte("TIP_DISTRIBUTION_ACCOUNT"), c.cfg.VoteAccount[:], epochLE[:]}, c.cfg.TipDistributionProgram) +} + +// Build returns the crank for the slot whose bank is state, or no +// transactions when the tip state is already right. memo is the 3-byte +// strategy code recorded when the tip receiver changes. +func (c *TipCrank) Build(state TipState, fee FeeInfo, memo string) ([]*solana.Transaction, error) { + if fee.Pubkey.IsZero() { + return nil, errNoFeeInfo + } + configAcct, ok := state.ReadAccount(c.config) + if !ok || solana.PublicKeyFromBytes(configAcct.Owner[:]) != c.cfg.TipPaymentProgram { + return nil, errTipPaymentProgramUninitialized + } + config, err := ParseTipPaymentConfig(configAcct.Data) + if err != nil { + return nil, err + } + tda, bump, err := c.TipDistributionAccount(state.Epoch()) + if err != nil { + return nil, err + } + // Anyone can send lamports to the PDA, so it only exists once the tip + // distribution program owns it. + tdaAcct, ok := state.ReadAccount(tda) + needInit := !ok || solana.PublicKeyFromBytes(tdaAcct.Owner[:]) != c.cfg.TipDistributionProgram + needChange := config.TipReceiver != tda || config.BlockBuilder != fee.Pubkey || config.CommissionPct != fee.Commission + + var instructions [][]solana.Instruction + if needInit { + instructions = append(instructions, []solana.Instruction{c.initTipDistributionAccount(tda, bump)}) + } + if needChange { + instructions = append(instructions, []solana.Instruction{ + c.changeTipReceiver(config, tda), + c.changeBlockBuilder(config, tda, fee), + solana.NewInstruction(memoProgram, solana.AccountMetaSlice{}, []byte(memo)), + }) + } + txs := make([]*solana.Transaction, 0, len(instructions)) + for _, ixs := range instructions { + tx, err := solana.NewTransaction(ixs, state.LastBlockhash(), solana.TransactionPayer(c.cfg.Identity.PublicKey())) + if err != nil { + return nil, err + } + if _, err := tx.Sign(func(solana.PublicKey) *solana.PrivateKey { return &c.cfg.Identity }); err != nil { + return nil, err + } + txs = append(txs, tx) + } + return txs, nil +} + +func (c *TipCrank) initTipDistributionAccount(tda solana.PublicKey, bump uint8) solana.Instruction { + data := append([]byte(nil), initTipDistributionDiscriminator...) + data = append(data, c.cfg.MerkleRootUploadAuthority[:]...) + data = binary.LittleEndian.AppendUint16(data, c.cfg.CommissionBps) + data = append(data, bump) + return solana.NewInstruction(c.cfg.TipDistributionProgram, solana.AccountMetaSlice{ + solana.NewAccountMeta(c.distributionConfig, false, false), + solana.NewAccountMeta(tda, true, false), + solana.NewAccountMeta(c.cfg.VoteAccount, false, false), + solana.NewAccountMeta(c.cfg.Identity.PublicKey(), true, true), + solana.NewAccountMeta(solana.SystemProgramID, false, false), + }, data) +} + +// tipPaymentAccounts is the layout both tip payment instructions use: the +// config, three instruction-specific accounts, the tip accounts, and the +// signer, all writable. +func (c *TipCrank) tipPaymentAccounts(first, second, third solana.PublicKey) solana.AccountMetaSlice { + metas := solana.AccountMetaSlice{ + solana.NewAccountMeta(c.config, true, false), + solana.NewAccountMeta(first, true, false), + solana.NewAccountMeta(second, true, false), + solana.NewAccountMeta(third, true, false), + } + for _, tip := range c.tipAccounts { + metas = append(metas, solana.NewAccountMeta(tip, true, false)) + } + return append(metas, solana.NewAccountMeta(c.cfg.Identity.PublicKey(), true, true)) +} + +func (c *TipCrank) changeTipReceiver(config TipPaymentConfig, tda solana.PublicKey) solana.Instruction { + return solana.NewInstruction(c.cfg.TipPaymentProgram, + c.tipPaymentAccounts(config.TipReceiver, tda, config.BlockBuilder), + append([]byte(nil), changeTipReceiverDiscriminator...)) +} + +func (c *TipCrank) changeBlockBuilder(config TipPaymentConfig, tda solana.PublicKey, fee FeeInfo) solana.Instruction { + data := append([]byte(nil), changeBlockBuilderDiscriminator...) + data = binary.LittleEndian.AppendUint64(data, fee.Commission) + // The tip receiver changed in the instruction before. + return solana.NewInstruction(c.cfg.TipPaymentProgram, + c.tipPaymentAccounts(tda, config.BlockBuilder, fee.Pubkey), data) +} + +// MemoForStrategy is the 3-byte code a Harmonic crank records for strategy. +func MemoForStrategy(strategy blockenginepb.SchedulingStrategy) string { + switch strategy { + case blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_FBA: + return "FBA" + case blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_MREV: + return "MRV" + case blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_FIFO: + return "FIF" + default: + return "???" + } +} + +// FeeSource supplies the block builder fee info and scheduling strategy. +type FeeSource interface { + FeeInfo() (FeeInfo, bool) + Strategy() blockenginepb.SchedulingStrategy +} + +// Builder adapts the crank to the executor, with fee info and strategy from +// fees. Without valid fee info the crank fails, so the slot falls back. +func (c *TipCrank) Builder(fees FeeSource) CrankBuilder { + return func(bank *blockprod.WorkingBank) ([]*solana.Transaction, error) { + fee, ok := fees.FeeInfo() + if !ok { + return nil, errNoFeeInfo + } + return c.Build(bank, fee, MemoForStrategy(fees.Strategy())) + } +} diff --git a/pkg/harmonic/crank_test.go b/pkg/harmonic/crank_test.go new file mode 100644 index 00000000..a30b607d --- /dev/null +++ b/pkg/harmonic/crank_test.go @@ -0,0 +1,378 @@ +package harmonic + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/blockprod" + "github.com/Overclock-Validator/mithril/pkg/costmodel" + blockenginepb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/blockengine" + "github.com/Overclock-Validator/mithril/pkg/txverify" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The mainnet tip programs, and the inputs of Harmonic Major's crank in +// transaction tVoDkAas...PxttW (slot 447985272, epoch 1037). +var ( + mainnetTipPayment = solana.MustPublicKeyFromBase58("T1pyyaTNZsKv2WcRAB8oVnk93mLJw2XzjtVYqCsaHqt") + mainnetTipDistribution = solana.MustPublicKeyFromBase58("4R3gSG8BpU4t19KYj8CfnbtRpnT8gtk4dvTHxVRwc2r7") + majorVote = solana.MustPublicKeyFromBase58("MajorF3gAYEmUhqkoRXoL546Zim8nMa82tuUTz9LkmE") + majorMerkleAuthority = solana.MustPublicKeyFromBase58("8F4jGUmxF36vQ6yabnsxX6AQVXdKBhs8kGSUuRKSg8Xt") + // majorTDA is Harmonic Major's tip distribution account for epoch 1037. + majorTDA = solana.MustPublicKeyFromBase58("BufWFj7nbLcgEq7GN67ntPGJTRpepDDP3rA4c3WnRLGt") + // The tip receiver and block builder the crank replaced. + majorOldReceiver = solana.MustPublicKeyFromBase58("FeHQ4R45LncZ2jXdYmPuZpVzM8YFHot6sANqbKUmjkPX") + majorOldBuilder = solana.MustPublicKeyFromBase58("shftkxnsXmqAkmLgz9Mn7bNB5Fr6mKgFc58kFHfVikj") + // otherBuilder is fee info naming the block builder already in place. + otherBuilder = FeeInfo{Pubkey: majorOldBuilder, Commission: 3} +) + +// harmonicMajorCrank is transaction tVoDkAas...PxttW: a compute budget, then +// Harmonic Major initializing its tip distribution account, routing tips to +// it with itself as block builder at 3%, and the FBA memo. +const harmonicMajorCrank = "ASxom6nw/Z3iXqzjx8uTxv1HcP2Ks8RjZnmqZAUhD20Du29vpFRAJF+SERsg+DgI0Uoo3qS+Xi83ohQWvg0l9wgBAAUVgLeLcjD8+FglNNlJnga0pOs4bkzc6zTVeve4QlrBTqV4Uhyxec67hYm1VqLV7JTSSYaC/fm7KvWtZOSRzEFT2vGH7IfR90XLOgM4SiamntoMotGqD0HkJBY3fpH/W10xsU4N5V6fuoY5br/VSM/4ySAR6se3W6qbLZxqhvWhcUGI8f+jot/mF73E41cyUaMi4/yugeWkVzkOZHUcAKRl4rwrVwZe8d1mVDC+YGumWWwClTAbre+LWvxBAUFQ9BJ0iQd9VaW7EzB2Prdn9V7Ad7QaDQdffeHXP7rKPGPVVHG/lxtZEItbhaBPsJPx4htOP9TEyPSH3Qm5V1J2nw3YwyAmEB7CAyiWSjKrqxNsVAW5HzrjjuT2TLa96Hm4aDjSBoU4E1+QmgRpJJcD4xsLIzzYvKtaM6BEUVsC8WtuSbX3+ZoJQvrHiAVJQ6u8+Uax+bkWy+EK7c+plnZlNxyigNmQpc8umUDWUzX0Mtn+XQ3Sq6myPWCcJxYkGkx0QXbWDP0G3GcM+JGk0PvEOwY6JyH6YvyUBuw1pM48s3zaYWyiEpu+ZRlk81P2fGTpAMNrXt6MDmxYuHl7z6bTHV7M9YC4i3Iw/PhYJTTZSZ4GtKTrOG5M3Os01Xr3uEJawU6lBUXDkRcvkc62zsNrOYLVMPhvlR+AWi+pPPN0WAbGFSkDBkZv5SEXMv/srbpyw5vnvIzlu8X3EmssQ5s6QAAAAAaqCVSLUEdq1GL5H4mjAVAzJk/Jq9UnACCp0UIzR0L7BUpTWpkpIQZNJOhxYNo4fHw1td28kruB5B+oQEEFRI0AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADK8B8f95T8sn0WK6FHyWCqexPsACofWZ8R3DxbR0fycRk/Uk/arvshwYWKLA7MkSIQx+fLazpisW2BQT9+jPJoFEAAFAtD7AQAUBQkNDwATK3i/GbZvMbM3a5wn6b8s4ocbhqDkf2GsZbAAwy/v3JwieWFXTYHOS+MAAP8RDQoLDQwBAgMEBQYHCAAIRWMWRwvnVo8RDQoNDAABAgMEBQYHCAAQhlAmiaUVcnsDAAAAAAAAABIAA0ZCQQ==" + +type fakeTipState struct { + accounts map[solana.PublicKey]*accounts.Account + epoch uint64 + blockhash solana.Hash +} + +func (f *fakeTipState) ReadAccount(pubkey solana.PublicKey) (*accounts.Account, bool) { + acct, ok := f.accounts[pubkey] + return acct, ok +} + +func (f *fakeTipState) Epoch() uint64 { return f.epoch } +func (f *fakeTipState) LastBlockhash() solana.Hash { return f.blockhash } + +func tipConfigData(receiver, builder solana.PublicKey, commission uint64) []byte { + data := append([]byte(nil), tipPaymentConfigDiscriminator...) + data = append(data, receiver[:]...) + data = append(data, builder[:]...) + data = binary.LittleEndian.AppendUint64(data, commission) + return append(data, 254, 255, 254, 255, 255, 252, 255, 252, 255) +} + +func owned(owner solana.PublicKey, data []byte) *accounts.Account { + return &accounts.Account{Lamports: 1_000_000, Owner: owner, Data: data} +} + +// majorCrank returns a crank with the mainnet programs and Harmonic Major's +// vote account, signed by identity, and a bank state at epoch 1037 in which +// tips go to another tip receiver and block builder. +func majorCrank(t *testing.T, identity solana.PrivateKey) (*TipCrank, *fakeTipState) { + t.Helper() + crank, err := NewTipCrank(TipConfig{ + TipPaymentProgram: mainnetTipPayment, + TipDistributionProgram: mainnetTipDistribution, + MerkleRootUploadAuthority: majorMerkleAuthority, + VoteAccount: majorVote, + Identity: identity, + }) + require.NoError(t, err) + state := &fakeTipState{epoch: 1037, blockhash: solana.Hash{7}, accounts: map[solana.PublicKey]*accounts.Account{ + crank.TipPaymentConfigAddress(): owned(mainnetTipPayment, tipConfigData(majorOldReceiver, majorOldBuilder, 3)), + majorTDA: owned(mainnetTipDistribution, nil), + }} + return crank, state +} + +func newIdentity(t *testing.T) solana.PrivateKey { + t.Helper() + key, err := solana.NewRandomPrivateKey() + require.NoError(t, err) + return key +} + +// instructionKeys resolves an instruction's account indexes to pubkeys. +func instructionKeys(tx *solana.Transaction, ix solana.CompiledInstruction) []solana.PublicKey { + keys := make([]solana.PublicKey, len(ix.Accounts)) + for i, idx := range ix.Accounts { + keys[i] = tx.Message.AccountKeys[idx] + } + return keys +} + +func requireWellFormed(t *testing.T, tx *solana.Transaction, identity solana.PublicKey) { + t.Helper() + seen := map[solana.PublicKey]bool{} + for _, key := range tx.Message.AccountKeys { + require.Falsef(t, seen[key], "duplicate account key %s", key) + seen[key] = true + } + require.Equal(t, identity, tx.Message.AccountKeys[0], "the identity pays") + require.NoError(t, txverify.SanitizeTransaction(tx)) + require.NoError(t, txverify.VerifyTransaction(tx), "signed by the identity") +} + +func TestTipAddressesMatchMainnet(t *testing.T) { + crank, _ := majorCrank(t, newIdentity(t)) + assert.Equal(t, solana.MustPublicKeyFromBase58("HgzT81VF1xZ3FT9Eq1pHhea7Wcfq2bv4tWTP3VvJ8Y9D"), crank.TipPaymentConfigAddress()) + want := [tipAccounts]solana.PublicKey{} + for i, s := range []string{ + "96gYZGLnJYVFmbjzopPSU6QiEV5fGqZNyN9nmNhvrZU5", "HFqU5x63VTqvQss8hp11i4wVV8bD44PvwucfZ2bU7gRe", + "Cw8CFyM9FkoMi7K7Crf6HNQqf4uEMzpKw6QNghXLvLkY", "ADaUMid9yfUytqMBgopwjb2DTLSokTSzL1zt6iGPaS49", + "DfXygSm4jCyNCybVYYK6DwvWqjKee8pbDmJGcLWNDXjh", "ADuUkR4vqLUMWXxW9gh6D6L8pMSawimctcNZ5pGwDcEt", + "DttWaMuVvTiduZRnguLF7jNxTgiMBZ1hyAumKUiL2KRL", "3AVi9Tg9Uo68tJfuvoKvqKNWKkC5wPdSSdeBnizKZ6jT", + } { + want[i] = solana.MustPublicKeyFromBase58(s) + } + assert.Equal(t, want, crank.TipAccounts(), "tip accounts 0..7, in order") + tda, _, err := crank.TipDistributionAccount(1037) + require.NoError(t, err) + assert.Equal(t, majorTDA, tda) +} + +func TestTipDiscriminatorsAreAnchorSighashes(t *testing.T) { + sighash := func(preimage string) []byte { + sum := sha256.Sum256([]byte(preimage)) + return sum[:8] + } + assert.Equal(t, sighash("account:Config"), tipPaymentConfigDiscriminator) + assert.Equal(t, sighash("global:change_tip_receiver"), changeTipReceiverDiscriminator) + assert.Equal(t, sighash("global:change_block_builder"), changeBlockBuilderDiscriminator) + assert.Equal(t, sighash("global:initialize_tip_distribution_account"), initTipDistributionDiscriminator) +} + +// TestTipCrankMatchesHarmonicMajorsCrank builds the crank from the inputs of +// Harmonic Major's and compares the instructions: same programs, accounts and +// data. Harmonic Major sent them in one transaction after a compute budget; +// ours are an initialization and a change, signed by a test key in its place. +func TestTipCrankMatchesHarmonicMajorsCrank(t *testing.T) { + raw, err := base64.StdEncoding.DecodeString(harmonicMajorCrank) + require.NoError(t, err) + mainnet, err := solana.TransactionFromBytes(raw) + require.NoError(t, err) + require.Len(t, mainnet.Message.Instructions, 5) + majorIdentity := mainnet.Message.AccountKeys[0] + + key := newIdentity(t) + identity := key.PublicKey() + crank, state := majorCrank(t, key) + delete(state.accounts, majorTDA) + txs, err := crank.Build(state, FeeInfo{Pubkey: identity, Commission: 3}, "FBA") + require.NoError(t, err) + require.Len(t, txs, 2, "initialize, then change") + for _, tx := range txs { + requireWellFormed(t, tx, identity) + assert.Equal(t, solana.Hash{7}, tx.Message.RecentBlockhash) + } + init, change := txs[0], txs[1] + require.Len(t, init.Message.Instructions, 1) + require.Len(t, change.Message.Instructions, 3) + + type instruction struct { + tx *solana.Transaction + ix solana.CompiledInstruction + } + ours := []instruction{{init, init.Message.Instructions[0]}} + for _, ix := range change.Message.Instructions { + ours = append(ours, instruction{change, ix}) + } + for i, want := range mainnet.Message.Instructions[1:] { + got := ours[i] + assert.Equal(t, mainnet.Message.AccountKeys[want.ProgramIDIndex], got.tx.Message.AccountKeys[got.ix.ProgramIDIndex]) + wantKeys := instructionKeys(mainnet, want) + for j := range wantKeys { + if wantKeys[j] == majorIdentity { + wantKeys[j] = identity + } + } + assert.Equalf(t, wantKeys, instructionKeys(got.tx, got.ix), "instruction %d accounts", i+1) + assert.Equalf(t, []byte(want.Data), []byte(got.ix.Data), "instruction %d data", i+1) + } + + // The tip payment program needs every account of the change writable. + for _, key := range change.Message.AccountKeys { + if key == mainnetTipPayment || key == memoProgram { + continue + } + writable, err := change.Message.IsWritable(key) + require.NoError(t, err) + assert.Truef(t, writable, "%s is writable", key) + } +} + +func TestTipCrankIsEmptyWhenTipsAlreadyRouteHere(t *testing.T) { + crank, state := majorCrank(t, newIdentity(t)) + state.accounts[crank.TipPaymentConfigAddress()] = owned(mainnetTipPayment, tipConfigData(majorTDA, majorOldBuilder, 3)) + txs, err := crank.Build(state, otherBuilder, "FBA") + require.NoError(t, err) + assert.Empty(t, txs) + + for name, fee := range map[string]FeeInfo{ + "new block builder": {Pubkey: majorOldReceiver, Commission: 3}, + "new commission": {Pubkey: majorOldBuilder, Commission: 4}, + } { + txs, err := crank.Build(state, fee, "FBA") + require.NoError(t, err) + assert.Lenf(t, txs, 1, "%s needs a change", name) + } +} + +func TestTipCrankInitializesThisEpochsTipDistributionAccount(t *testing.T) { + crank, state := majorCrank(t, newIdentity(t)) + delete(state.accounts, majorTDA) + txs, err := crank.Build(state, otherBuilder, "MRV") + require.NoError(t, err) + require.Len(t, txs, 2, "initialize, then change") + tdaWritable, _ := txs[0].Message.IsWritable(majorTDA) + voteWritable, _ := txs[0].Message.IsWritable(majorVote) + assert.True(t, tdaWritable) + assert.False(t, voteWritable) + assert.Equal(t, "MRV", string(txs[1].Message.Instructions[2].Data)) + + // Lamports alone do not make the account exist. + state.accounts[majorTDA] = owned(solana.SystemProgramID, nil) + txs, err = crank.Build(state, otherBuilder, "MRV") + require.NoError(t, err) + assert.Len(t, txs, 2) + + // Tips already point at the account, which is missing: only initialize. + delete(state.accounts, majorTDA) + state.accounts[crank.TipPaymentConfigAddress()] = owned(mainnetTipPayment, tipConfigData(majorTDA, majorOldBuilder, 3)) + txs, err = crank.Build(state, otherBuilder, "MRV") + require.NoError(t, err) + require.Len(t, txs, 1) + assert.Equal(t, mainnetTipDistribution, txs[0].Message.AccountKeys[txs[0].Message.Instructions[0].ProgramIDIndex]) +} + +// TestTipCrankValidatorAsItsOwnBlockBuilder covers the identity being the +// block builder: the instructions reference the identity's existing key +// instead of listing it twice. +func TestTipCrankValidatorAsItsOwnBlockBuilder(t *testing.T) { + key := newIdentity(t) + identity := key.PublicKey() + crank, state := majorCrank(t, key) + + txs, err := crank.Build(state, FeeInfo{Pubkey: identity, Commission: 0}, "FBA") + require.NoError(t, err) + require.Len(t, txs, 1) + tx := txs[0] + requireWellFormed(t, tx, identity) + changeBuilder := tx.Message.Instructions[1] + assert.Equal(t, uint16(0), changeBuilder.Accounts[3], "the new block builder is the identity at index 0") + assert.Equal(t, uint16(0), changeBuilder.Accounts[12], "as is the signer") + assert.True(t, tx.Message.IsSigner(identity)) + + // The previous slot's leader made itself the builder; now it is us. + state.accounts[crank.TipPaymentConfigAddress()] = owned(mainnetTipPayment, tipConfigData(majorOldReceiver, identity, 0)) + txs, err = crank.Build(state, otherBuilder, "FBA") + require.NoError(t, err) + require.Len(t, txs, 1) + requireWellFormed(t, txs[0], identity) + assert.Equal(t, uint16(0), txs[0].Message.Instructions[0].Accounts[3], "the old block builder is the identity at index 0") +} + +func TestTipCrankRefusesToBuildFromBadState(t *testing.T) { + crank, _ := majorCrank(t, newIdentity(t)) + fee := otherBuilder + configKey := crank.TipPaymentConfigAddress() + cases := map[string]func(*fakeTipState) FeeInfo{ + "no fee info": func(*fakeTipState) FeeInfo { return FeeInfo{} }, + "tip payment config missing": func(s *fakeTipState) FeeInfo { delete(s.accounts, configKey); return fee }, + "config wrong owner": func(s *fakeTipState) FeeInfo { + s.accounts[configKey] = owned(solana.SystemProgramID, tipConfigData(majorOldReceiver, majorOldBuilder, 3)) + return fee + }, + "config wrong discriminator": func(s *fakeTipState) FeeInfo { + data := tipConfigData(majorOldReceiver, majorOldBuilder, 3) + data[0] ^= 1 + s.accounts[configKey] = owned(mainnetTipPayment, data) + return fee + }, + "config truncated": func(s *fakeTipState) FeeInfo { + s.accounts[configKey] = owned(mainnetTipPayment, tipConfigData(majorOldReceiver, majorOldBuilder, 3)[:80]) + return fee + }, + "config zero receiver": func(s *fakeTipState) FeeInfo { + s.accounts[configKey] = owned(mainnetTipPayment, tipConfigData(solana.PublicKey{}, majorOldBuilder, 3)) + return fee + }, + } + for name, setup := range cases { + _, state := majorCrank(t, crank.cfg.Identity) + txs, err := crank.Build(state, setup(state), "FBA") + assert.Errorf(t, err, "%s", name) + assert.Emptyf(t, txs, "%s", name) + } + + _, err := NewTipCrank(TipConfig{TipPaymentProgram: mainnetTipPayment, TipDistributionProgram: mainnetTipDistribution, + VoteAccount: majorVote, CommissionBps: 10_001, Identity: newIdentity(t)}) + assert.Error(t, err, "commission over 100%") +} + +func TestMemoForStrategy(t *testing.T) { + assert.Equal(t, "FBA", MemoForStrategy(blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_FBA)) + assert.Equal(t, "MRV", MemoForStrategy(blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_MREV)) + assert.Equal(t, "FIF", MemoForStrategy(blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_FIFO)) + assert.Equal(t, "???", MemoForStrategy(blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_UNSPECIFIED)) +} + +type fixedFees struct { + fee FeeInfo + ok bool +} + +func (f fixedFees) FeeInfo() (FeeInfo, bool) { return f.fee, f.ok } +func (f fixedFees) Strategy() blockenginepb.SchedulingStrategy { + return blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_FBA +} + +var _ FeeSource = (*BlockEngineSession)(nil) + +func TestExecutorRunsTheTipCrankFromTheSlotsBank(t *testing.T) { + identity := newIdentity(t) + crank, _ := majorCrank(t, identity) + installTips := func(env *blockprod.TestEnv, receiver solana.PublicKey) { + tda, _, err := crank.TipDistributionAccount(env.Bank.Epoch()) + require.NoError(t, err) + if receiver.IsZero() { + receiver = tda + } + require.NoError(t, env.SlotCtx.SetAccount(crank.TipPaymentConfigAddress(), owned(mainnetTipPayment, tipConfigData(receiver, majorOldBuilder, 3)))) + require.NoError(t, env.SlotCtx.SetAccount(tda, owned(mainnetTipDistribution, nil))) + } + + t.Run("tips already route here: the block runs without a crank", func(t *testing.T) { + h := startExecutor(t, true, crank.Builder(fixedFees{otherBuilder, true})) + env := gatedBank(t, costmodel.Limits{}) + installTips(env, solana.PublicKey{}) + h.startSlot(10, env, time.Hour) + h.send(t, bundleEvent(10, 1, false, transfer(t, 1))) + h.sync(t) + assert.Len(t, env.Bank.ForgedTransactions(), 1) + assert.Equal(t, ModeHarmonic, h.end(t, 10).Mode) + }) + t.Run("a crank that cannot land falls back", func(t *testing.T) { + // The test bank has no tip programs deployed, so the change cannot + // execute; the block must not start. + h := startExecutor(t, true, crank.Builder(fixedFees{otherBuilder, true})) + env := gatedBank(t, costmodel.Limits{}) + installTips(env, majorOldReceiver) + h.startSlot(10, env, time.Hour) + h.send(t, bundleEvent(10, 1, false, transfer(t, 1))) + h.sync(t) + assert.Empty(t, env.Bank.ForgedTransactions()) + report := h.end(t, 10) + assert.Equal(t, ModeFallback, report.Mode) + assert.Equal(t, FlagFailed, report.Flags) + }) + t.Run("no fee info falls back", func(t *testing.T) { + h := startExecutor(t, true, crank.Builder(fixedFees{})) + env := gatedBank(t, costmodel.Limits{}) + installTips(env, solana.PublicKey{}) + h.startSlot(10, env, time.Hour) + h.send(t, bundleEvent(10, 1, false, transfer(t, 1))) + h.sync(t) + assert.Empty(t, env.Bank.ForgedTransactions()) + assert.True(t, env.Bank.TPUAllowed()) + }) +} From 0e2b9f074abd9f538d6bdfd7a6653cf5e0e4fbdf Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 21:39:31 +0000 Subject: [PATCH 19/25] harmonic: wire into the validator --- cmd/mithril/node/harmonic.go | 111 +++++++++++++++ cmd/mithril/node/harmonic_test.go | 94 +++++++++++++ cmd/mithril/node/node.go | 25 ++++ config.example.toml | 37 +++++ pkg/config/config.go | 4 + pkg/harmonic/blocks.go | 29 +++- pkg/harmonic/executor.go | 40 ++++-- pkg/harmonic/executor_test.go | 29 ++++ pkg/harmonic/service.go | 215 ++++++++++++++++++++++++++++++ pkg/harmonic/service_test.go | 194 +++++++++++++++++++++++++++ 10 files changed, 759 insertions(+), 19 deletions(-) create mode 100644 cmd/mithril/node/harmonic.go create mode 100644 cmd/mithril/node/harmonic_test.go create mode 100644 pkg/harmonic/service.go create mode 100644 pkg/harmonic/service_test.go diff --git a/cmd/mithril/node/harmonic.go b/cmd/mithril/node/harmonic.go new file mode 100644 index 00000000..fe7b605f --- /dev/null +++ b/cmd/mithril/node/harmonic.go @@ -0,0 +1,111 @@ +package node + +import ( + "context" + "crypto/ed25519" + "fmt" + + "github.com/Overclock-Validator/mithril/pkg/blockprod" + "github.com/Overclock-Validator/mithril/pkg/config" + "github.com/Overclock-Validator/mithril/pkg/gossip" + "github.com/Overclock-Validator/mithril/pkg/harmonic" + blockenginepb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/blockengine" + "github.com/Overclock-Validator/mithril/pkg/mlog" + "github.com/Overclock-Validator/mithril/pkg/tpu" + solana "github.com/gagliardetto/solana-go" +) + +// harmonicSettings is the validated [harmonic] section. +type harmonicSettings struct { + enabled bool + blockEngine harmonic.Endpoint + relayer harmonic.Endpoint + insecureSkipVerify bool + strategy blockenginepb.SchedulingStrategy + // tips lacks the vote account and identity, which come from [validator]. + tips harmonic.TipConfig +} + +// loadHarmonicSettings reads [harmonic]. Harmonic is off unless enabled, and +// then every key it needs must be valid. +func loadHarmonicSettings() (harmonicSettings, error) { + s := harmonicSettings{enabled: config.GetBool("harmonic.enabled")} + if !s.enabled { + return s, nil + } + var err error + if s.blockEngine, err = harmonic.ParseEndpoint(config.GetString("harmonic.block_engine_url"), config.GetString("harmonic.block_engine_tls_domain")); err != nil { + return s, fmt.Errorf("harmonic.block_engine_url: %w", err) + } + if url := config.GetString("harmonic.relayer_url"); url != "" { + if s.relayer, err = harmonic.ParseEndpoint(url, config.GetString("harmonic.relayer_tls_domain")); err != nil { + return s, fmt.Errorf("harmonic.relayer_url: %w", err) + } + } + s.insecureSkipVerify = !config.GetBool("harmonic.tls_cert_verify") + if s.strategy, err = harmonic.ParseStrategy(config.GetString("harmonic.strategy")); err != nil { + return s, fmt.Errorf("harmonic.strategy: %w", err) + } + for _, key := range []struct { + name string + dst *solana.PublicKey + }{ + {"harmonic.tip_payment_program", &s.tips.TipPaymentProgram}, + {"harmonic.tip_distribution_program", &s.tips.TipDistributionProgram}, + {"harmonic.merkle_root_upload_authority", &s.tips.MerkleRootUploadAuthority}, + } { + value := config.GetString(key.name) + if value == "" { + return s, fmt.Errorf("%s is required when harmonic.enabled=true", key.name) + } + if *key.dst, err = solana.PublicKeyFromBase58(value); err != nil { + return s, fmt.Errorf("%s: %w", key.name, err) + } + } + bps := config.GetInt64("harmonic.commission_bps") + if bps < 0 || bps > 10_000 { + return s, fmt.Errorf("harmonic.commission_bps must be between 0 and 10000, got %d", bps) + } + s.tips.CommissionBps = uint16(bps) + return s, nil +} + +// startHarmonic runs the Harmonic service until the returned stop function, +// which waits for it to finish. The service's hooks must be given to the +// leader loop with GateTPU set. +func startHarmonic(ctx context.Context, s harmonicSettings, identity ed25519.PrivateKey, voteAccount solana.PublicKey, + tpuService *tpu.TPU, gossipClient *gossip.Client, localTPU gossip.TPUSockets, txV1Enabled func() bool, +) (blockprod.LeaderHooks, func(), error) { + tips := s.tips + tips.VoteAccount = voteAccount + tips.Identity = solana.PrivateKey(identity) + service, err := harmonic.NewService(harmonic.ServiceConfig{ + BlockEngine: s.blockEngine, + Relayer: s.relayer, + InsecureSkipVerify: s.insecureSkipVerify, + Strategy: s.strategy, + Tips: tips, + TPU: tpuService, + Gossip: gossipClient, + LocalTPU: localTPU, + TxV1Enabled: txV1Enabled, + }) + if err != nil { + return nil, nil, err + } + ctx, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + go func() { + defer close(done) + service.Run(ctx) + }() + relayer := "none" + if s.relayer.Target != "" { + relayer = s.relayer.Target + } + mlog.Log.Infof("Harmonic enabled: block engine %s, relayer %s, strategy %s", s.blockEngine.Target, relayer, s.strategy) + return service.Hooks(), func() { + cancel() + <-done + }, nil +} diff --git a/cmd/mithril/node/harmonic_test.go b/cmd/mithril/node/harmonic_test.go new file mode 100644 index 00000000..7244dbed --- /dev/null +++ b/cmd/mithril/node/harmonic_test.go @@ -0,0 +1,94 @@ +package node + +import ( + "fmt" + "maps" + "strings" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/config" + blockenginepb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/blockengine" + "github.com/spf13/viper" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// loadHarmonicTOML loads a [harmonic] section of the given keys; values are +// TOML literals. +func loadHarmonicTOML(t *testing.T, values map[string]string) (harmonicSettings, error) { + t.Helper() + var toml strings.Builder + toml.WriteString("[harmonic]\n") + for key, value := range values { + fmt.Fprintf(&toml, "%s = %s\n", key, value) + } + viper.Reset() + t.Cleanup(viper.Reset) + config.ApplyDefaults(viper.GetViper()) + viper.SetConfigType("toml") + require.NoError(t, viper.ReadConfig(strings.NewReader(toml.String()))) + return loadHarmonicSettings() +} + +// enabledHarmonic is a minimal enabled [harmonic] section with overrides. +func enabledHarmonic(overrides map[string]string) map[string]string { + values := map[string]string{ + "enabled": "true", + "block_engine_url": `"https://be.example.com"`, + "tip_payment_program": `"T1pyyaTNZsKv2WcRAB8oVnk93mLJw2XzjtVYqCsaHqt"`, + "tip_distribution_program": `"4R3gSG8BpU4t19KYj8CfnbtRpnT8gtk4dvTHxVRwc2r7"`, + "merkle_root_upload_authority": `"8F4jGUmxF36vQ6yabnsxX6AQVXdKBhs8kGSUuRKSg8Xt"`, + } + maps.Copy(values, overrides) + return values +} + +func TestHarmonicIsOffByDefault(t *testing.T) { + s, err := loadHarmonicTOML(t, nil) + require.NoError(t, err) + assert.False(t, s.enabled) + + // Nothing else is read while disabled. + s, err = loadHarmonicTOML(t, map[string]string{"block_engine_url": `"not a url"`}) + require.NoError(t, err) + assert.False(t, s.enabled) +} + +func TestHarmonicSettings(t *testing.T) { + s, err := loadHarmonicTOML(t, enabledHarmonic(nil)) + require.NoError(t, err) + assert.True(t, s.enabled) + assert.Equal(t, "be.example.com:443", s.blockEngine.Target) + assert.True(t, s.blockEngine.TLS) + assert.Empty(t, s.relayer.Target, "the relayer is optional") + assert.False(t, s.insecureSkipVerify, "certificates are verified by default") + assert.Equal(t, blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_FBA, s.strategy) + assert.Equal(t, "8F4jGUmxF36vQ6yabnsxX6AQVXdKBhs8kGSUuRKSg8Xt", s.tips.MerkleRootUploadAuthority.String()) + assert.Zero(t, s.tips.CommissionBps) + + s, err = loadHarmonicTOML(t, enabledHarmonic(map[string]string{ + "relayer_url": `"http://relayer.example.com:11226"`, + "tls_cert_verify": "false", + "strategy": `"MREV"`, + "commission_bps": "800", + })) + require.NoError(t, err) + assert.Equal(t, "relayer.example.com:11226", s.relayer.Target) + assert.True(t, s.insecureSkipVerify) + assert.Equal(t, blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_MREV, s.strategy) + assert.Equal(t, uint16(800), s.tips.CommissionBps) + + for name, bad := range map[string]map[string]string{ + "no block engine url": {"block_engine_url": `""`}, + "bad block engine url": {"block_engine_url": `"be.example.com"`}, + "bad relayer url": {"relayer_url": `"ftp://relayer.example.com"`}, + "bad strategy": {"strategy": `"fastest"`}, + "no tip program": {"tip_payment_program": `""`}, + "bad authority": {"merkle_root_upload_authority": `"not base58"`}, + "commission over 100%": {"commission_bps": "10001"}, + "negative commission": {"commission_bps": "-1"}, + } { + _, err := loadHarmonicTOML(t, enabledHarmonic(bad)) + assert.Errorf(t, err, "%s", name) + } +} diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index f68eb2d0..e86055e5 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -109,6 +109,7 @@ var ( validatorInitializeReservation bool validatorCompletionReserveMs int validatorMaxBufferedTransactions int + harmonicConfig harmonicSettings // Mode thresholds blockNearTipThreshold int // Enter near-tip when gap <= this @@ -1058,6 +1059,14 @@ func initConfigAndBindFlags(cmd *cobra.Command) error { mlog.Log.Infof("consensus.mode=validator: Alpenglow TPU ingress, block production, and fail-closed Votor voting enabled") } + // [harmonic] section — optional block engine integration for leader slots. + if harmonicConfig, err = loadHarmonicSettings(); err != nil { + return err + } + if harmonicConfig.enabled && consensusMode != "validator" { + return fmt.Errorf("harmonic.enabled=true requires consensus.mode=validator") + } + blockMaxRPS = getInt("block-max-rps", "block.max_rps") blockMaxInflight = getInt("block-max-inflight", "block.max_inflight") blockTipPollIntervalMs = getInt("block-tip-poll-ms", "block.tip_poll_interval_ms") @@ -2785,6 +2794,20 @@ postBootstrap: if err := sharedGossip.SetTPUQUIC(tpuAdvertise); err != nil { mlog.Log.Warnf("validator gossip TPU advertisement: %v", err) } + // Harmonic is optional. Without it the leader loop has no hooks and + // its banks admit TPU transactions from the start. + var leaderHooks blockprod.LeaderHooks + if harmonicConfig.enabled { + hooks, stopHarmonic, err := startHarmonic(ctx, harmonicConfig, validatorIdentity, validatorVoteAccount, tpuService, sharedGossip, + gossip.TPUSockets{QUIC: tpuAdvertise, ForwardsQUIC: tpuAdvertise}, tpuCfg.Pipeline.TxV1Enabled) + if err != nil { + klog.Fatalf("start Harmonic: %v", err) + } + // Defers run in reverse: the leader loop, which calls the hooks, + // stops first, then Harmonic, then the TPU it submits to. + defer stopHarmonic() + leaderHooks = hooks + } // Bind and validate local transports before consuming the durable clean // voting marker. Startup configuration failures must not force recovery. @@ -2892,6 +2915,8 @@ postBootstrap: }, LeaderForSlot: global.LeaderForSlot, RewardCerts: rewardBuilder, + Hooks: leaderHooks, + GateTPU: leaderHooks != nil, OnBlock: func(produced *block.Block) { select { case localBlocks <- produced: diff --git a/config.example.toml b/config.example.toml index c6212a86..8374f371 100644 --- a/config.example.toml +++ b/config.example.toml @@ -361,6 +361,43 @@ name = "mithril" # completion margins; this does not change the protocol slot deadline. block_completion_reserve_ms = 0 +# ============================================================================ +# [harmonic] - Harmonic Block Engine (optional, consensus.mode = "validator") +# ============================================================================ +# +# When enabled, each leader slot asks the Harmonic block engine for its block +# and executes the block's bundles in order, with local TPU scheduling closed. +# A slot falls back to local scheduling when the block engine is unavailable +# at slot start (immediately), when no block arrives by the middle of the +# slot, or when the block stops early. Harmonic failures never stop the +# validator. When disabled, leader slots schedule locally from the start. + +[harmonic] + enabled = false + # Block engine gRPC endpoint (http:// or https://), required when enabled. + block_engine_url = "" + # Optional TLS server name override for the block engine. + block_engine_tls_domain = "" + # Optional relayer (remote TPU) endpoint. While it and the block engine + # are healthy, gossip advertises the relayer's TPU instead of this one. + relayer_url = "" + relayer_tls_domain = "" + # Development only: false disables TLS certificate verification. + tls_cert_verify = true + # Block engine scheduling strategy: "fba" (default), "fifo" or "mrev". + strategy = "fba" + + # Tip programs, required when enabled. Consult the block engine provider + # for the cluster's values. Jito's mainnet programs, for reference: + # tip_payment_program = "T1pyyaTNZsKv2WcRAB8oVnk93mLJw2XzjtVYqCsaHqt" + # tip_distribution_program = "4R3gSG8BpU4t19KYj8CfnbtRpnT8gtk4dvTHxVRwc2r7" + # merkle_root_upload_authority = "8F4jGUmxF36vQ6yabnsxX6AQVXdKBhs8kGSUuRKSg8Xt" + tip_payment_program = "" + tip_distribution_program = "" + merkle_root_upload_authority = "" + # Validator commission on tips, in basis points (0-10000). + commission_bps = 0 + # ============================================================================ # [consensus] - Alpenglow Consensus # ============================================================================ diff --git a/pkg/config/config.go b/pkg/config/config.go index 501dfa15..2d0820a1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -26,6 +26,10 @@ func ApplyDefaults(v *viper.Viper) { v.SetDefault("validator.advertised_ip", "") v.SetDefault("validator.tpu_sigverify_workers", 0) v.SetDefault("validator.wait_to_vote_slot", uint64(0)) + v.SetDefault("harmonic.enabled", false) + v.SetDefault("harmonic.tls_cert_verify", true) + v.SetDefault("harmonic.strategy", "fba") + v.SetDefault("harmonic.commission_bps", 0) } // LedgerConfig holds ledger-related configuration (matches Firedancer [ledger] section) diff --git a/pkg/harmonic/blocks.go b/pkg/harmonic/blocks.go index 634bd376..b5ef9f16 100644 --- a/pkg/harmonic/blocks.go +++ b/pkg/harmonic/blocks.go @@ -64,6 +64,8 @@ type BlockPipelineStats struct { // them, and emits them in order. It is the session's BlockHandler. type BlockPipeline struct { events chan<- BlockEvent + done chan struct{} + closeOnce sync.Once txV1Enabled func() bool current atomic.Uint64 verifier txverify.BatchVerifier @@ -78,10 +80,23 @@ type BlockPipeline struct { } // NewBlockPipeline sends events on events, blocking rather than dropping -// when the executor falls behind. txV1Enabled reports whether the current bank -// accepts v1 transactions; nil rejects them. +// when the executor falls behind, until Close. txV1Enabled reports whether the +// current bank accepts v1 transactions; nil rejects them. func NewBlockPipeline(events chan<- BlockEvent, txV1Enabled func() bool) *BlockPipeline { - return &BlockPipeline{events: events, txV1Enabled: txV1Enabled, lost: make(map[uint64]bool)} + return &BlockPipeline{events: events, done: make(chan struct{}), txV1Enabled: txV1Enabled, lost: make(map[uint64]bool)} +} + +// Close drops every later event, so a stream goroutine blocked on an +// executor that has stopped returns. +func (p *BlockPipeline) Close() { + p.closeOnce.Do(func() { close(p.done) }) +} + +func (p *BlockPipeline) emit(e BlockEvent) { + select { + case p.events <- e: + case <-p.done: + } } func (p *BlockPipeline) Stats() BlockPipelineStats { @@ -114,7 +129,7 @@ func (p *BlockPipeline) StreamLost(slot uint64) { p.mu.Lock() p.lost[slot] = true p.mu.Unlock() - p.events <- StreamLostEvent{Slot: slot} + p.emit(StreamLostEvent{Slot: slot}) } // HandleBlock admits, segments and validates one block message. @@ -142,17 +157,17 @@ func (p *BlockPipeline) HandleBlock(block *blockpb.Block, arrival time.Time) { seq := p.takeSeq() if err := validateBundle(txs, parsed, parseErrs, bundle); err != nil { p.stats.invalid.Add(1) - p.events <- InvalidBundleEvent{Slot: slot, Seq: seq, Arrival: arrival, Reason: err} + p.emit(InvalidBundleEvent{Slot: slot, Seq: seq, Arrival: arrival, Reason: err}) continue } p.stats.bundles.Add(1) - p.events <- BundleEvent{ + p.emit(BundleEvent{ Slot: slot, Seq: seq, Arrival: arrival, RevertProtected: txs[bundle.start].GetBundleId() != 0, Transactions: append([]*solana.Transaction(nil), parsed[bundle.start:bundle.end]...), - } + }) } } diff --git a/pkg/harmonic/executor.go b/pkg/harmonic/executor.go index 0cbc2d4d..9f5f3718 100644 --- a/pkg/harmonic/executor.go +++ b/pkg/harmonic/executor.go @@ -116,6 +116,10 @@ type hookMessage struct { ended uint64 abort bool reason string + // threshold reports that slot's threshold passed. + threshold uint64 + // unhealthy reports the block engine session went down. + unhealthy bool } type slotState struct { @@ -142,14 +146,12 @@ type Executor struct { hooksMu sync.Mutex hooks []hookMessage wake chan struct{} - // thresholds carries threshold expiries to the executor goroutine. - thresholds chan uint64 } var _ blockprod.LeaderHooks = (*Executor)(nil) func NewExecutor(cfg ExecutorConfig) *Executor { - return &Executor{cfg: cfg, now: time.Now, wake: make(chan struct{}, 1), thresholds: make(chan uint64, 4)} + return &Executor{cfg: cfg, now: time.Now, wake: make(chan struct{}, 1)} } // SlotStarted runs under the leader loop's lock, so it only does what cannot @@ -190,6 +192,16 @@ func (x *Executor) SlotAborted(slot uint64, reason string) { x.post(hookMessage{ended: slot, abort: true, reason: reason}) } +// BlockEngineHealthChanged reports the block engine session's health. A +// session that goes down before the slot's block starts takes the block with +// it, since it reconnects only after a backoff, so the slot falls back at once +// rather than at the threshold. A block already running ends with its stream. +func (x *Executor) BlockEngineHealthChanged(healthy bool) { + if !healthy { + x.post(hookMessage{unhealthy: true}) + } +} + func (x *Executor) post(msg hookMessage) { x.hooksMu.Lock() x.hooks = append(x.hooks, msg) @@ -209,11 +221,6 @@ func (x *Executor) Run(ctx context.Context) { return case <-x.wake: st = x.drainHooks(st) - case slot := <-x.thresholds: - st = x.drainHooks(st) - if st != nil && st.slot == slot && st.mode == ModeUndecided { - x.fallback(st, FlagTimeout) - } case e := <-x.cfg.Events: // A slot's start is always queued before its first event, since // the block engine streams only after the notice. @@ -229,6 +236,18 @@ func (x *Executor) drainHooks(st *slotState) *slotState { x.hooks = nil x.hooksMu.Unlock() for _, msg := range hooks { + if msg.threshold != 0 { + if st != nil && st.slot == msg.threshold && st.mode == ModeUndecided { + x.fallback(st, FlagTimeout) + } + continue + } + if msg.unhealthy { + if st != nil && st.mode == ModeUndecided { + x.stop(st, FlagFailed) + } + continue + } if msg.started != nil { if st != nil && st.mode != ModeDone { x.finish(st, "", false) @@ -253,10 +272,7 @@ func (x *Executor) armThreshold(st *slotState) { } slot := st.slot st.thresholdTimer = time.AfterFunc(time.Until(st.threshold), func() { - select { - case x.thresholds <- slot: - default: - } + x.post(hookMessage{threshold: slot}) }) } diff --git a/pkg/harmonic/executor_test.go b/pkg/harmonic/executor_test.go index d94139c6..54a65cf8 100644 --- a/pkg/harmonic/executor_test.go +++ b/pkg/harmonic/executor_test.go @@ -321,6 +321,35 @@ func TestExecutorCrankRunsFirstAndItsFailureFallsBack(t *testing.T) { assert.Equal(t, FlagFailed, report.Flags) } +func TestExecutorFallsBackAtOnceWhenTheBlockEngineGoesDown(t *testing.T) { + h := startExecutor(t, true, nil) + env := gatedBank(t, costmodel.Limits{}) + h.startSlot(10, env, time.Hour) + h.exec.BlockEngineHealthChanged(true) + h.sync(t) + assert.False(t, env.Bank.TPUAllowed()) + h.exec.BlockEngineHealthChanged(false) + require.Eventually(t, env.Bank.TPUAllowed, time.Second, time.Millisecond, "no block can arrive, so the slot falls back before the threshold") + report := h.end(t, 10) + assert.Equal(t, ModeFallback, report.Mode) + assert.Equal(t, FlagFailed, report.Flags) + + // A running block ends with its stream, in order with its bundles. + env = gatedBank(t, costmodel.Limits{}) + h.startSlot(11, env, time.Hour) + h.send(t, bundleEvent(11, 1, false, transfer(t, 1))) + h.sync(t) + h.exec.BlockEngineHealthChanged(false) + h.send(t, bundleEvent(11, 2, false, transfer(t, 2))) + h.sync(t) + assert.Len(t, env.Bank.ForgedTransactions(), 2) + assert.False(t, env.Bank.TPUAllowed()) + h.send(t, StreamLostEvent{Slot: 11}) + h.sync(t) + assert.True(t, env.Bank.TPUAllowed()) + assert.Equal(t, ModeFallback, h.end(t, 11).Mode) +} + func TestExecutorSlotsAreIndependent(t *testing.T) { h := startExecutor(t, true, nil) env10 := gatedBank(t, costmodel.Limits{}) diff --git a/pkg/harmonic/service.go b/pkg/harmonic/service.go new file mode 100644 index 00000000..7f418a89 --- /dev/null +++ b/pkg/harmonic/service.go @@ -0,0 +1,215 @@ +package harmonic + +import ( + "context" + "crypto/ed25519" + "errors" + "fmt" + "net" + "strings" + "sync" + "time" + + "github.com/Overclock-Validator/mithril/pkg/blockprod" + "github.com/Overclock-Validator/mithril/pkg/gossip" + blockenginepb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/blockengine" + "github.com/Overclock-Validator/mithril/pkg/mlog" +) + +const ( + // blockEventBuffer lets the block stream run ahead of execution. + blockEventBuffer = 256 + clientIDPollInterval = 50 * time.Millisecond + // The Harmonic client ID is advertised once the block engine session has + // stayed healthy this long, and withdrawn once it has stayed unhealthy + // for baseClientIDAfter, so a flapping session does not churn gossip. + harmonicClientIDAfter = 5 * time.Second + baseClientIDAfter = 2 * time.Second +) + +// ParseStrategy parses a scheduling strategy name: fifo, fba or mrev. +func ParseStrategy(name string) (blockenginepb.SchedulingStrategy, error) { + switch strings.ToLower(name) { + case "fifo": + return blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_FIFO, nil + case "fba": + return blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_FBA, nil + case "mrev": + return blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_MREV, nil + default: + return 0, fmt.Errorf("scheduling strategy %q: want fifo, fba or mrev", name) + } +} + +// Gossip is the part of the gossip client the Harmonic service changes. +type Gossip interface { + TPUAdvertiser + SetClientID(id uint16) error +} + +// ServiceConfig configures the Harmonic service. +type ServiceConfig struct { + BlockEngine Endpoint + // Relayer is optional; with no Target, no relayer session runs. + Relayer Endpoint + InsecureSkipVerify bool + // Strategy defaults to FBA. + Strategy blockenginepb.SchedulingStrategy + // Tips configures the tip crank. Its Identity also authenticates the + // sessions. + Tips TipConfig + // TPU takes the block engine's and the relayer's packets. + TPU PacketSink + // Gossip advertises the TPU sockets and the client ID. LocalTPU is this + // validator's own TPU, advertised whenever the relayer's is not. + Gossip Gossip + LocalTPU gossip.TPUSockets + // TxV1Enabled reports whether the current bank accepts v1 transactions. + TxV1Enabled func() bool + // Dialer replaces the TCP dialer, for tests. + Dialer func(ctx context.Context, addr string) (net.Conn, error) +} + +// Service runs a validator's Harmonic integration: the block engine and +// relayer sessions, the block pipeline, the executor, and the gossip +// advertisement of the relayer's TPU and the Harmonic client ID. Harmonic +// failures never stop the validator; its leader slots fall back to local +// scheduling. +type Service struct { + cfg ServiceConfig + pipeline *BlockPipeline + blockEngine *BlockEngineSession + relayer *RelayerSession + executor *Executor + tpu *TPUAdvertisement + + harmonicClientIDAfter, baseClientIDAfter time.Duration +} + +func NewService(cfg ServiceConfig) (*Service, error) { + if cfg.TPU == nil || cfg.Gossip == nil { + return nil, errors.New("harmonic service needs the TPU and gossip") + } + crank, err := NewTipCrank(cfg.Tips) + if err != nil { + return nil, err + } + events := make(chan BlockEvent, blockEventBuffer) + s := &Service{ + cfg: cfg, + pipeline: NewBlockPipeline(events, cfg.TxV1Enabled), + tpu: NewTPUAdvertisement(cfg.Gossip, cfg.LocalTPU), + harmonicClientIDAfter: harmonicClientIDAfter, + baseClientIDAfter: baseClientIDAfter, + } + identity := ed25519.PrivateKey(cfg.Tips.Identity) + s.blockEngine = NewBlockEngineSession(BlockEngineConfig{ + Endpoint: cfg.BlockEngine, + InsecureSkipVerify: cfg.InsecureSkipVerify, + Identity: identity, + Strategy: cfg.Strategy, + Packets: cfg.TPU, + Blocks: s.pipeline, + OnHealth: s.blockEngineHealthChanged, + Dialer: cfg.Dialer, + }) + s.executor = NewExecutor(ExecutorConfig{ + BlockEngine: s.blockEngine, + Pipeline: s.pipeline, + Events: events, + Crank: crank.Builder(s.blockEngine), + }) + if cfg.Relayer.Target != "" { + s.relayer = NewRelayerSession(RelayerConfig{ + Endpoint: cfg.Relayer, + InsecureSkipVerify: cfg.InsecureSkipVerify, + Identity: identity, + Packets: cfg.TPU, + OnStatus: s.relayerStatusChanged, + Dialer: cfg.Dialer, + }) + } + return s, nil +} + +// Hooks is the leader loop's LeaderHooks. The leader loop must gate the TPU, +// which the executor opens whenever a slot falls back. +func (s *Service) Hooks() blockprod.LeaderHooks { + return s.executor +} + +func (s *Service) blockEngineHealthChanged(healthy bool) { + mlog.Log.Infof("Harmonic block engine %s healthy=%t", s.cfg.BlockEngine.Target, healthy) + s.executor.BlockEngineHealthChanged(healthy) + if err := s.tpu.SetBlockEngineHealthy(healthy); err != nil { + mlog.Log.Warnf("Harmonic gossip TPU advertisement: %v", err) + } +} + +func (s *Service) relayerStatusChanged(cfg *TPUConfig) { + if err := s.tpu.SetRelayer(cfg); err != nil { + mlog.Log.Warnf("Harmonic gossip TPU advertisement: %v", err) + } +} + +// Run runs the service until ctx ends. The sessions restore the local TPU in +// gossip as they stop; Run then restores mithril's own client ID. +func (s *Service) Run(ctx context.Context) { + var wg sync.WaitGroup + run := func(f func(context.Context)) { + wg.Add(1) + go func() { + defer wg.Done() + f(ctx) + }() + } + run(s.executor.Run) + run(s.blockEngine.Run) + if s.relayer != nil { + run(s.relayer.Run) + } + run(s.advertiseClientID) + <-ctx.Done() + // The block stream may be blocked on the stopped executor. + s.pipeline.Close() + wg.Wait() + if err := s.cfg.Gossip.SetClientID(gossip.ClientIDMithril); err != nil { + mlog.Log.Warnf("Harmonic gossip client ID: %v", err) + } +} + +// advertiseClientID advertises the Harmonic client ID while the block engine +// session is healthy, with hysteresis. +func (s *Service) advertiseClientID(ctx context.Context) { + tick := time.NewTicker(clientIDPollInterval) + defer tick.Stop() + advertised := gossip.ClientIDMithril + healthy, since := false, time.Now() + for { + var now time.Time + select { + case <-ctx.Done(): + return + case now = <-tick.C: + } + if h := s.blockEngine.Healthy(); h != healthy { + healthy, since = h, now + } + want := advertised + switch { + case healthy && now.Sub(since) >= s.harmonicClientIDAfter: + want = gossip.ClientIDMithrilHarmonic + case !healthy && now.Sub(since) >= s.baseClientIDAfter: + want = gossip.ClientIDMithril + } + if want == advertised { + continue + } + if err := s.cfg.Gossip.SetClientID(want); err != nil { + mlog.Log.Warnf("Harmonic gossip client ID: %v", err) + continue + } + advertised = want + mlog.Log.Infof("Harmonic: advertising gossip client ID %d", want) + } +} diff --git a/pkg/harmonic/service_test.go b/pkg/harmonic/service_test.go new file mode 100644 index 00000000..0dc6a02b --- /dev/null +++ b/pkg/harmonic/service_test.go @@ -0,0 +1,194 @@ +package harmonic + +import ( + "context" + "errors" + "fmt" + "slices" + "sync" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/blockprod" + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/gossip" + authpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/auth" + blockenginepb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/blockengine" + relayerpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/relayer" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" +) + +// gossipRecorder records what the service advertises. +type gossipRecorder struct { + mu sync.Mutex + sockets []string + clientIDs []uint16 +} + +func socketsString(s gossip.TPUSockets) string { + return fmt.Sprintf("udp=%v quic=%v fwd_udp=%v fwd_quic=%v", s.UDP, s.QUIC, s.ForwardsUDP, s.ForwardsQUIC) +} + +func (g *gossipRecorder) SetTPUSockets(s gossip.TPUSockets) error { + g.mu.Lock() + defer g.mu.Unlock() + g.sockets = append(g.sockets, socketsString(s)) + return nil +} + +func (g *gossipRecorder) SetClientID(id uint16) error { + g.mu.Lock() + defer g.mu.Unlock() + g.clientIDs = append(g.clientIDs, id) + return nil +} + +func (g *gossipRecorder) advertisedSockets() string { + g.mu.Lock() + defer g.mu.Unlock() + if len(g.sockets) == 0 { + return "" + } + return g.sockets[len(g.sockets)-1] +} + +func (g *gossipRecorder) clientID() uint16 { + g.mu.Lock() + defer g.mu.Unlock() + if len(g.clientIDs) == 0 { + return gossip.ClientIDMithril + } + return g.clientIDs[len(g.clientIDs)-1] +} + +var serviceLocalTPU = gossip.TPUSockets{QUIC: netipUDP("203.0.113.10:8009"), ForwardsQUIC: netipUDP("203.0.113.10:8009")} + +type serviceHarness struct { + be *mockBlockEngine + gossip *gossipRecorder + service *Service + crank *TipCrank + stop func() +} + +// startService runs a service against a mock block engine and relayer. +func startService(t *testing.T) *serviceHarness { + t.Helper() + h := &serviceHarness{be: newMockBlockEngine(), gossip: &gossipRecorder{}} + dial := serveBufconn(t, func(s *grpc.Server) { + authpb.RegisterAuthServiceServer(s, newMockAuth(time.Now)) + blockenginepb.RegisterBlockEngineValidatorServer(s, h.be) + relayerpb.RegisterRelayerServer(s, newMockRelayer(tpuConfigResponse("198.51.100.7", 11222, 11223))) + }) + tips := TipConfig{ + TipPaymentProgram: mainnetTipPayment, + TipDistributionProgram: mainnetTipDistribution, + MerkleRootUploadAuthority: majorMerkleAuthority, + VoteAccount: majorVote, + Identity: solana.PrivateKey(rfc8032Key(t)), + } + var err error + h.crank, err = NewTipCrank(tips) + require.NoError(t, err) + h.service, err = NewService(ServiceConfig{ + BlockEngine: dial.Endpoint, + Relayer: dial.Endpoint, + Tips: tips, + TPU: &packetRecorder{}, + Gossip: h.gossip, + LocalTPU: serviceLocalTPU, + Dialer: dial.Dialer, + }) + require.NoError(t, err) + h.service.harmonicClientIDAfter, h.service.baseClientIDAfter = 100*time.Millisecond, 50*time.Millisecond + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + h.service.Run(ctx) + close(done) + }() + var once sync.Once + h.stop = func() { + once.Do(func() { + cancel() + <-done + }) + } + t.Cleanup(h.stop) + return h +} + +// routeTipsHere sets the bank's tip state to route tips here already, with +// the mock block engine's block builder, so the slot needs no crank. +func (h *serviceHarness) routeTipsHere(t *testing.T, env *blockprod.TestEnv) { + t.Helper() + tda, _, err := h.crank.TipDistributionAccount(env.Bank.Epoch()) + require.NoError(t, err) + builder := solana.MustPublicKeyFromBase58(h.be.feeInfo.GetPubkey()) + require.NoError(t, env.SlotCtx.SetAccount(h.crank.TipPaymentConfigAddress(), owned(mainnetTipPayment, tipConfigData(tda, builder, h.be.feeInfo.GetCommission())))) + require.NoError(t, env.SlotCtx.SetAccount(tda, owned(mainnetTipDistribution, nil))) +} + +func TestServiceAdvertisesHarmonicWhileTheBlockEngineIsHealthy(t *testing.T) { + h := startService(t) + relayer, err := ParseTPUConfig(tpuConfigResponse("198.51.100.7", 11222, 11223)) + require.NoError(t, err) + local, relayed := socketsString(serviceLocalTPU), socketsString(relayer.Sockets()) + + eventually(t, func() bool { return h.gossip.advertisedSockets() == relayed }, "the relayer's TPU is advertised once both sessions are up") + eventually(t, func() bool { return h.gossip.clientID() == gossip.ClientIDMithrilHarmonic }, "then the Harmonic client ID") + + // The session reconnects only after a backoff of seconds. + h.be.endBlocks <- errors.New("block engine restarting") + eventually(t, func() bool { return h.gossip.advertisedSockets() == local }, "a lost block engine restores the local TPU") + eventually(t, func() bool { return h.gossip.clientID() == gossip.ClientIDMithril }, "and mithril's own client ID") + + h.stop() + assert.Equal(t, local, h.gossip.advertisedSockets()) + assert.Equal(t, gossip.ClientIDMithril, h.gossip.clientID()) +} + +func TestServiceRestoresGossipOnShutdown(t *testing.T) { + h := startService(t) + eventually(t, func() bool { return h.gossip.clientID() == gossip.ClientIDMithrilHarmonic }, "Harmonic advertised") + h.stop() + assert.Equal(t, socketsString(serviceLocalTPU), h.gossip.advertisedSockets()) + assert.Equal(t, gossip.ClientIDMithril, h.gossip.clientID()) +} + +func TestServiceRunsTheBlockAndFallsBackWhenTheSessionDrops(t *testing.T) { + h := startService(t) + eventually(t, h.service.blockEngine.Healthy, "block engine session up") + hooks := h.service.Hooks() + startSlot := func(slot uint64) *blockprod.TestEnv { + env := gatedBank(t, costmodel.Limits{}) + h.routeTipsHere(t, env) + now := time.Now() + hooks.SlotStarted(blockprod.LeaderSlotInfo{Slot: slot, Bank: env.Bank, Start: now, End: now.Add(time.Hour)}) + eventually(t, func() bool { return slices.Contains(h.be.noticeSlots(), slot) }, "the block engine is told the slot") + return env + } + + env := startSlot(10) + wire, err := transfer(t, 1).MarshalBinary() + require.NoError(t, err) + h.be.blocks <- blockMsg(10, blockTx(wire, 0)) + eventually(t, func() bool { return len(env.Bank.ForgedTransactions()) == 1 }, "the block's transaction executes") + assert.False(t, env.Bank.TPUAllowed(), "the TPU stays closed while the block runs") + hooks.SlotEnding(10) + + env = startSlot(11) + h.be.endBlocks <- errors.New("block engine restarting") + eventually(t, env.Bank.TPUAllowed, "a lost session falls back at once, not at the threshold") + assert.Empty(t, env.Bank.ForgedTransactions()) + hooks.SlotEnding(11) + + // The session is down, so the next slot opens the TPU before it starts. + env = gatedBank(t, costmodel.Limits{}) + now := time.Now() + hooks.SlotStarted(blockprod.LeaderSlotInfo{Slot: 12, Bank: env.Bank, Start: now, End: now.Add(time.Hour)}) + assert.True(t, env.Bank.TPUAllowed()) +} From e0723a3227acac342cf08450b374d6728662e380 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 21:45:52 +0000 Subject: [PATCH 20/25] harmonic: metrics, 1ms scheduler idle poll --- pkg/blockprod/scheduler/scheduler.go | 3 +- pkg/harmonic/blockengine.go | 5 + pkg/harmonic/blocks.go | 6 +- pkg/harmonic/blocks_test.go | 1 + pkg/harmonic/executor.go | 49 +++++++++- pkg/harmonic/executor_test.go | 10 +- pkg/harmonic/metrics.go | 110 ++++++++++++++++++++++ pkg/harmonic/metrics_test.go | 134 +++++++++++++++++++++++++++ pkg/harmonic/relayer.go | 7 ++ pkg/harmonic/service.go | 8 ++ pkg/harmonic/service_test.go | 1 + pkg/statsd/statsd.go | 78 ++++++++++++++++ 12 files changed, 407 insertions(+), 5 deletions(-) create mode 100644 pkg/harmonic/metrics.go create mode 100644 pkg/harmonic/metrics_test.go diff --git a/pkg/blockprod/scheduler/scheduler.go b/pkg/blockprod/scheduler/scheduler.go index 59febede..55e88731 100644 --- a/pkg/blockprod/scheduler/scheduler.go +++ b/pkg/blockprod/scheduler/scheduler.go @@ -265,7 +265,8 @@ func (s *Scheduler) drainLoop(ctx context.Context) { bank := s.tpuBank() s.noteBank(bank) if bank == nil { - if !s.wait(ctx, 5*time.Millisecond) { + // Polling is how a new bank, or a gated one opening, is noticed. + if !s.wait(ctx, time.Millisecond) { return } continue diff --git a/pkg/harmonic/blockengine.go b/pkg/harmonic/blockengine.go index ba01cd5f..5806b409 100644 --- a/pkg/harmonic/blockengine.go +++ b/pkg/harmonic/blockengine.go @@ -14,6 +14,7 @@ import ( blockpb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/block" blockenginepb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/blockengine" "github.com/Overclock-Validator/mithril/pkg/mlog" + "github.com/Overclock-Validator/mithril/pkg/statsd" tpupacket "github.com/Overclock-Validator/mithril/pkg/tpu/packet" "github.com/Overclock-Validator/mithril/pkg/version" "github.com/gagliardetto/solana-go" @@ -498,11 +499,15 @@ func (b *BlockEngineSession) takeNotice() *leaderNotice { func (b *BlockEngineSession) sendNotice(ctx context.Context, client blockenginepb.BlockEngineValidatorClient, notice *leaderNotice) error { ctx, cancel := context.WithTimeout(ctx, leaderNoticeTimeout) defer cancel() + sent := time.Now() _, err := client.SubmitLeaderWindowInfo(ctx, &blockenginepb.SubmitLeaderWindowInfoRequest{ Slot: notice.slot, StartTimestamp: timestamppb.New(b.now()), EndTimestamp: timestamppb.New(notice.end), }) + if err == nil { + _ = statsd.Duration(statsd.HarmonicLeaderNoticeRTT, time.Since(sent), nil) + } return err } diff --git a/pkg/harmonic/blocks.go b/pkg/harmonic/blocks.go index b5ef9f16..bed6b9f3 100644 --- a/pkg/harmonic/blocks.go +++ b/pkg/harmonic/blocks.go @@ -57,6 +57,8 @@ type BlockPipelineStats struct { LostSlot uint64 Bundles uint64 InvalidBundles uint64 + // Transactions counts the transactions of valid bundles. + Transactions uint64 } // BlockPipeline turns the block stream into executor events. It admits only @@ -70,7 +72,7 @@ type BlockPipeline struct { current atomic.Uint64 verifier txverify.BatchVerifier stats struct { - messages, wrongSlot, lostSlot, bundles, invalid atomic.Uint64 + messages, wrongSlot, lostSlot, bundles, invalid, transactions atomic.Uint64 } mu sync.Mutex @@ -106,6 +108,7 @@ func (p *BlockPipeline) Stats() BlockPipelineStats { LostSlot: p.stats.lostSlot.Load(), Bundles: p.stats.bundles.Load(), InvalidBundles: p.stats.invalid.Load(), + Transactions: p.stats.transactions.Load(), } } @@ -161,6 +164,7 @@ func (p *BlockPipeline) HandleBlock(block *blockpb.Block, arrival time.Time) { continue } p.stats.bundles.Add(1) + p.stats.transactions.Add(uint64(bundle.end - bundle.start)) p.emit(BundleEvent{ Slot: slot, Seq: seq, diff --git a/pkg/harmonic/blocks_test.go b/pkg/harmonic/blocks_test.go index 772e21f0..45724d4e 100644 --- a/pkg/harmonic/blocks_test.go +++ b/pkg/harmonic/blocks_test.go @@ -82,6 +82,7 @@ func TestBlockPipelineSegmentsPerMessage(t *testing.T) { fifth := events[4].(BundleEvent) assert.Equal(t, solana.SignatureFromBytes(wireTx(4)[1:65]), fifth.Transactions[0].Signatures[0], "members keep stream order") assert.Equal(t, uint64(7), p.Stats().Bundles) + assert.Equal(t, uint64(9), p.Stats().Transactions) } func TestBlockPipelineAdmitsOnlyTheCurrentSlot(t *testing.T) { diff --git a/pkg/harmonic/executor.go b/pkg/harmonic/executor.go index 9f5f3718..26374c5e 100644 --- a/pkg/harmonic/executor.go +++ b/pkg/harmonic/executor.go @@ -68,6 +68,30 @@ func (f EndFlags) String() string { return strings.Join(names, ",") } +// CrankResult is what a slot's tip crank did. +type CrankResult int + +const ( + // CrankNotRun: no block bundle reached execution, or there is no crank. + CrankNotRun CrankResult = iota + CrankNotNeeded + CrankLanded + CrankFailed +) + +func (c CrankResult) String() string { + switch c { + case CrankNotNeeded: + return "not_needed" + case CrankLanded: + return "landed" + case CrankFailed: + return "failed" + default: + return "not_run" + } +} + // SlotReport summarizes one leader slot. type SlotReport struct { Slot uint64 @@ -78,6 +102,11 @@ type SlotReport struct { // Executed counts bundles that landed; Reverted, the ones that executed // and landed nothing; Rejected, bundles refused without executing. Executed, Reverted, Rejected uint64 + Crank CrankResult + // FirstArrival is how long after the bank opened the slot's first block + // bundle arrived, admitted or not, if Arrived. + FirstArrival time.Duration + Arrived bool // Aborted is set when the leader abandoned the slot, with its reason. Aborted bool Reason string @@ -285,8 +314,8 @@ func (x *Executor) finish(st *slotState, reason string, aborted bool) { st.report.Reason = reason st.mode = ModeDone r := st.report - mlog.Log.Infof("Harmonic slot %d: mode=%s flags=%s executed=%d reverted=%d rejected=%d aborted=%t%s", - r.Slot, r.Mode, r.Flags, r.Executed, r.Reverted, r.Rejected, r.Aborted, reasonSuffix(r.Reason)) + mlog.Log.Infof("Harmonic slot %d: mode=%s flags=%s executed=%d reverted=%d rejected=%d crank=%s aborted=%t%s", + r.Slot, r.Mode, r.Flags, r.Executed, r.Reverted, r.Rejected, r.Crank, r.Aborted, reasonSuffix(r.Reason)) if x.cfg.OnReport != nil { x.cfg.OnReport(r) } @@ -306,11 +335,13 @@ func (x *Executor) handleEvent(st *slotState, e BlockEvent) { x.stop(st, FlagFailed) } case InvalidBundleEvent: + x.noteArrival(st, e.Slot, e.Arrival) if !x.admits(st, e.Slot, e.Seq) { return } x.stop(st, FlagFailed) case BundleEvent: + x.noteArrival(st, e.Slot, e.Arrival) if !x.admits(st, e.Slot, e.Seq) { return } @@ -318,6 +349,15 @@ func (x *Executor) handleEvent(st *slotState, e BlockEvent) { } } +// noteArrival records when the slot's first block bundle arrived. +func (x *Executor) noteArrival(st *slotState, slot uint64, arrival time.Time) { + if st == nil || st.slot != slot || st.report.Arrived || arrival.IsZero() { + return + } + st.report.Arrived = true + st.report.FirstArrival = max(arrival.Sub(st.start), 0) +} + // admits checks a bundle event against the slot state and consumes its // sequence number. A gap in the sequence means block data was lost. func (x *Executor) admits(st *slotState, slot, seq uint64) bool { @@ -397,9 +437,14 @@ func (x *Executor) runCrank(st *slotState) bool { } if err != nil { mlog.Log.Infof("Harmonic slot %d: tip crank failed: %v", st.slot, err) + st.report.Crank = CrankFailed x.stop(st, FlagFailed) return false } + st.report.Crank = CrankNotNeeded + if len(txs) > 0 { + st.report.Crank = CrankLanded + } st.crankDone = true return true } diff --git a/pkg/harmonic/executor_test.go b/pkg/harmonic/executor_test.go index 54a65cf8..760e1074 100644 --- a/pkg/harmonic/executor_test.go +++ b/pkg/harmonic/executor_test.go @@ -170,7 +170,8 @@ func TestExecutorRunsTheBlockInOrderWithTheTPUClosed(t *testing.T) { assert.Equal(t, []uint64{10}, h.be.notices) a, b, c := transfer(t, 1), transfer(t, 2), transfer(t, 3) - h.send(t, bundleEvent(10, 1, true, a, b)) + first := bundleEvent(10, 1, true, a, b) + h.send(t, first) h.send(t, bundleEvent(10, 2, false, c)) h.sync(t) assert.Equal(t, []*solana.Transaction{a, b, c}, env.Bank.ForgedTransactions()) @@ -180,6 +181,9 @@ func TestExecutorRunsTheBlockInOrderWithTheTPUClosed(t *testing.T) { assert.Equal(t, ModeHarmonic, report.Mode) assert.Equal(t, uint64(2), report.Executed) assert.Zero(t, report.Flags) + assert.Equal(t, CrankNotRun, report.Crank, "no crank is configured") + assert.True(t, report.Arrived) + assert.Less(t, report.FirstArrival, 400*time.Millisecond) assert.Equal(t, []uint64{10, 0}, h.pipeline.current, "the slot stops being current when it ends") } @@ -196,6 +200,8 @@ func TestExecutorFallsBackWhenNoBlockArrivesByTheThreshold(t *testing.T) { assert.Equal(t, ModeFallback, report.Mode) assert.Equal(t, FlagTimeout, report.Flags) assert.Equal(t, uint64(1), report.Rejected) + assert.True(t, report.Arrived, "a late bundle still records its arrival") + assert.GreaterOrEqual(t, report.FirstArrival, 50*time.Millisecond) } func TestExecutorStopTriggers(t *testing.T) { @@ -306,6 +312,7 @@ func TestExecutorCrankRunsFirstAndItsFailureFallsBack(t *testing.T) { forged := env.Bank.ForgedTransactions() require.Len(t, forged, 3) assert.Equal(t, []*solana.Transaction{crank, first}, forged[:2], "the crank lands before the first block bundle, once") + assert.Equal(t, CrankLanded, h.end(t, 10).Crank) failing := startExecutor(t, true, func(*blockprod.WorkingBank) ([]*solana.Transaction, error) { return nil, errors.New("tip accounts unreadable") @@ -319,6 +326,7 @@ func TestExecutorCrankRunsFirstAndItsFailureFallsBack(t *testing.T) { report := failing.end(t, 10) assert.Equal(t, ModeFallback, report.Mode) assert.Equal(t, FlagFailed, report.Flags) + assert.Equal(t, CrankFailed, report.Crank) } func TestExecutorFallsBackAtOnceWhenTheBlockEngineGoesDown(t *testing.T) { diff --git a/pkg/harmonic/metrics.go b/pkg/harmonic/metrics.go new file mode 100644 index 00000000..0e12bd30 --- /dev/null +++ b/pkg/harmonic/metrics.go @@ -0,0 +1,110 @@ +package harmonic + +import ( + "context" + "time" + + "github.com/Overclock-Validator/mithril/pkg/statsd" +) + +const metricsInterval = time.Second + +// statsSnapshot is the cumulative counters last exported. +type statsSnapshot struct { + blockEngine BlockEngineStats + relayer RelayerStats + pipeline BlockPipelineStats +} + +// exportMetrics publishes the sessions' and the pipeline's counters, and the +// state gauges, every interval until ctx ends. +func (s *Service) exportMetrics(ctx context.Context) { + tick := time.NewTicker(s.metricsInterval) + defer tick.Stop() + var last statsSnapshot + for { + select { + case <-ctx.Done(): + return + case <-tick.C: + last = s.publishStats(last) + } + } +} + +// publishStats exports what changed since last and returns the new snapshot. +func (s *Service) publishStats(last statsSnapshot) statsSnapshot { + now := statsSnapshot{blockEngine: s.blockEngine.Stats(), pipeline: s.pipeline.Stats()} + if s.relayer != nil { + now.relayer = s.relayer.Stats() + } + + be, prevBE := now.blockEngine, last.blockEngine + countDelta(statsd.HarmonicSessions, be.Sessions, prevBE.Sessions, "block_engine") + countDelta(statsd.HarmonicSessionResets, be.Resets, prevBE.Resets, "block_engine") + countDelta(statsd.HarmonicPackets, be.Packets, prevBE.Packets, "block_engine") + countDelta(statsd.HarmonicPacketsDropped, be.DroppedSize, prevBE.DroppedSize, "block_engine", "size") + countDelta(statsd.HarmonicPacketsDropped, be.DroppedFull, prevBE.DroppedFull, "block_engine", "full") + countDelta(statsd.HarmonicBlockEngineErrors, be.StrategyErrors, prevBE.StrategyErrors, "set_strategy") + countDelta(statsd.HarmonicBlockEngineErrors, be.InvalidFeeInfo, prevBE.InvalidFeeInfo, "fee_info") + countDelta(statsd.HarmonicLeaderNotices, be.NoticesSent, prevBE.NoticesSent, "sent") + countDelta(statsd.HarmonicLeaderNotices, be.NoticeErrors, prevBE.NoticeErrors, "failed") + countDelta(statsd.HarmonicLeaderNotices, be.NoticesExpired, prevBE.NoticesExpired, "expired") + countDelta(statsd.HarmonicBlockStreamsLost, be.StreamsLost, prevBE.StreamsLost) + + r, prevR := now.relayer, last.relayer + countDelta(statsd.HarmonicSessions, r.Sessions, prevR.Sessions, "relayer") + countDelta(statsd.HarmonicSessionResets, r.Resets, prevR.Resets, "relayer") + countDelta(statsd.HarmonicPackets, r.Packets, prevR.Packets, "relayer") + countDelta(statsd.HarmonicPacketsDropped, r.DroppedSize, prevR.DroppedSize, "relayer", "size") + countDelta(statsd.HarmonicPacketsDropped, r.DroppedFull, prevR.DroppedFull, "relayer", "full") + countDelta(statsd.HarmonicRelayerHeartbeats, r.Heartbeats, prevR.Heartbeats) + countDelta(statsd.HarmonicRelayerInvalidTPUConfigs, r.InvalidTPUConfigs, prevR.InvalidTPUConfigs) + + p, prevP := now.pipeline, last.pipeline + countDelta(statsd.HarmonicBlockMessages, p.Messages, prevP.Messages) + countDelta(statsd.HarmonicBlockMessagesDropped, p.WrongSlot, prevP.WrongSlot, "wrong_slot") + countDelta(statsd.HarmonicBlockMessagesDropped, p.LostSlot, prevP.LostSlot, "lost_slot") + countDelta(statsd.HarmonicBlockBundles, p.Bundles, prevP.Bundles, "valid") + countDelta(statsd.HarmonicBlockBundles, p.InvalidBundles, prevP.InvalidBundles, "invalid") + countDelta(statsd.HarmonicBlockTransactions, p.Transactions, prevP.Transactions) + + setGauge(statsd.HarmonicBlockEngineHealthy, s.blockEngine.Healthy()) + setGauge(statsd.HarmonicRelayerConnected, s.relayer != nil && s.relayer.Status() != nil) + setGauge(statsd.HarmonicRelayerTPUAdvertised, s.tpu.RelayerAdvertised()) + setGauge(statsd.HarmonicClientIDAdvertised, s.clientIDAdvertised.Load()) + return now +} + +func countDelta(m statsd.Metric, now, last uint64, labels ...string) { + if now > last { + _ = statsd.Count(m, int64(now-last), labels) + } +} + +func setGauge(m statsd.Metric, on bool) { + value := 0.0 + if on { + value = 1 + } + _ = statsd.Gauge(m, value, nil) +} + +// reportSlot exports a leader slot's report. +func reportSlot(r SlotReport) { + _ = statsd.Count(statsd.HarmonicSlots, 1, []string{r.Mode.String()}) + for _, flag := range []EndFlags{FlagTimeout, FlagFailed, FlagReverted, FlagUnavailable} { + if r.Flags&flag != 0 { + _ = statsd.Count(statsd.HarmonicSlotEndFlags, 1, []string{flag.String()}) + } + } + countDelta(statsd.HarmonicSlotBundles, r.Executed, 0, "executed") + countDelta(statsd.HarmonicSlotBundles, r.Reverted, 0, "reverted") + countDelta(statsd.HarmonicSlotBundles, r.Rejected, 0, "rejected") + if r.Crank != CrankNotRun { + _ = statsd.Count(statsd.HarmonicTipCranks, 1, []string{r.Crank.String()}) + } + if r.Arrived { + _ = statsd.Duration(statsd.HarmonicFirstBundleDelay, r.FirstArrival, nil) + } +} diff --git a/pkg/harmonic/metrics_test.go b/pkg/harmonic/metrics_test.go new file mode 100644 index 00000000..9b2904cd --- /dev/null +++ b/pkg/harmonic/metrics_test.go @@ -0,0 +1,134 @@ +package harmonic + +import ( + "errors" + "slices" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/blockprod" + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// metricValue reads a counter or gauge, or a histogram's sample count, from +// the default registry. labels are name, value pairs. A series not yet +// created reads as zero. +func metricValue(t *testing.T, name string, labels ...string) float64 { + t.Helper() + families, err := prometheus.DefaultGatherer.Gather() + require.NoError(t, err) + for _, family := range families { + if family.GetName() != name { + continue + } + for _, m := range family.GetMetric() { + if !labelsMatch(m.GetLabel(), labels) { + continue + } + switch { + case m.Counter != nil: + return m.Counter.GetValue() + case m.Gauge != nil: + return m.Gauge.GetValue() + case m.Histogram != nil: + return float64(m.Histogram.GetSampleCount()) + } + } + } + return 0 +} + +func labelsMatch(have []*dto.LabelPair, want []string) bool { + if len(have)*2 != len(want) { + return false + } + for i := 0; i < len(want); i += 2 { + if !slices.ContainsFunc(have, func(l *dto.LabelPair) bool { return l.GetName() == want[i] && l.GetValue() == want[i+1] }) { + return false + } + } + return true +} + +// metricDelta returns how much a metric has grown since the call. +func metricDelta(t *testing.T, name string, labels ...string) func() float64 { + t.Helper() + before := metricValue(t, name, labels...) + return func() float64 { return metricValue(t, name, labels...) - before } +} + +func TestReportSlotExportsTheSlot(t *testing.T) { + slots := metricDelta(t, "harmonic_slots_total", "mode", "harmonic") + reverted := metricDelta(t, "harmonic_slot_end_flags_total", "flag", "reverted") + timeout := metricDelta(t, "harmonic_slot_end_flags_total", "flag", "timeout") + failed := metricDelta(t, "harmonic_slot_end_flags_total", "flag", "failed") + executed := metricDelta(t, "harmonic_slot_bundles_total", "result", "executed") + revertedBundles := metricDelta(t, "harmonic_slot_bundles_total", "result", "reverted") + cranks := metricDelta(t, "harmonic_tip_cranks_total", "result", "landed") + arrivals := metricDelta(t, "harmonic_first_bundle_delay_seconds") + + reportSlot(SlotReport{Mode: ModeHarmonic, Flags: FlagReverted | FlagTimeout, Executed: 3, Reverted: 1, + Crank: CrankLanded, Arrived: true, FirstArrival: 30 * time.Millisecond}) + assert.Equal(t, 1.0, slots()) + assert.Equal(t, 1.0, reverted()) + assert.Equal(t, 1.0, timeout()) + assert.Zero(t, failed()) + assert.Equal(t, 3.0, executed()) + assert.Equal(t, 1.0, revertedBundles()) + assert.Equal(t, 1.0, cranks()) + assert.Equal(t, 1.0, arrivals()) + + // A slot that fell back before any bundle has no crank or arrival. + reportSlot(SlotReport{Mode: ModeFallback, Flags: FlagUnavailable}) + assert.Equal(t, 1.0, cranks()) + assert.Equal(t, 1.0, arrivals()) +} + +func TestServiceExportsMetrics(t *testing.T) { + sessions := metricDelta(t, "harmonic_sessions_total", "session", "block_engine") + relayerSessions := metricDelta(t, "harmonic_sessions_total", "session", "relayer") + notices := metricDelta(t, "harmonic_leader_notices_total", "result", "sent") + rtts := metricDelta(t, "harmonic_leader_notice_rtt_seconds") + messages := metricDelta(t, "harmonic_block_messages_received_total") + bundles := metricDelta(t, "harmonic_block_bundles_received_total", "result", "valid") + txs := metricDelta(t, "harmonic_block_transactions_received_total") + slots := metricDelta(t, "harmonic_slots_total", "mode", "harmonic") + cranks := metricDelta(t, "harmonic_tip_cranks_total", "result", "not_needed") + arrivals := metricDelta(t, "harmonic_first_bundle_delay_seconds") + + h := startService(t) + gaugeOn := func(name string) func() bool { + return func() bool { return metricValue(t, name) == 1 } + } + eventually(t, gaugeOn("harmonic_block_engine_healthy"), "block engine healthy") + eventually(t, gaugeOn("harmonic_relayer_connected"), "relayer connected") + eventually(t, gaugeOn("harmonic_relayer_tpu_advertised"), "relayer TPU advertised") + eventually(t, gaugeOn("harmonic_client_id_advertised"), "Harmonic client ID advertised") + // Gauges are global, so they may hold an earlier service's values. + eventually(t, func() bool { return sessions() == 1 && relayerSessions() == 1 }, "sessions exported") + + env := gatedBank(t, costmodel.Limits{}) + h.routeTipsHere(t, env) + now := time.Now() + h.service.Hooks().SlotStarted(blockprod.LeaderSlotInfo{Slot: 10, Bank: env.Bank, Start: now, End: now.Add(time.Hour)}) + eventually(t, func() bool { return slices.Contains(h.be.noticeSlots(), 10) }, "notice sent") + wire, err := transfer(t, 1).MarshalBinary() + require.NoError(t, err) + h.be.blocks <- blockMsg(10, blockTx(wire, 0)) + eventually(t, func() bool { return len(env.Bank.ForgedTransactions()) == 1 }, "block executed") + h.service.Hooks().SlotEnding(10) + + eventually(t, func() bool { return slots() == 1 }, "slot reported") + assert.Equal(t, 1.0, cranks()) + assert.Equal(t, 1.0, arrivals()) + eventually(t, func() bool { return notices() == 1 && rtts() == 1 }, "notice exported") + eventually(t, func() bool { return messages() == 1 && bundles() == 1 && txs() == 1 }, "block stream exported") + + h.be.endBlocks <- errors.New("block engine restarting") + eventually(t, func() bool { return metricValue(t, "harmonic_block_engine_healthy") == 0 }, "block engine down") + eventually(t, func() bool { return metricValue(t, "harmonic_relayer_tpu_advertised") == 0 }, "local TPU advertised") +} diff --git a/pkg/harmonic/relayer.go b/pkg/harmonic/relayer.go index b1db47cc..6d12c8dc 100644 --- a/pkg/harmonic/relayer.go +++ b/pkg/harmonic/relayer.go @@ -376,6 +376,13 @@ func (a *TPUAdvertisement) SetBlockEngineHealthy(healthy bool) error { return a.applyLocked() } +// RelayerAdvertised reports whether gossip advertises the relayer's TPU. +func (a *TPUAdvertisement) RelayerAdvertised() bool { + a.mu.Lock() + defer a.mu.Unlock() + return a.advertised != nil +} + func (a *TPUAdvertisement) applyLocked() error { var want *TPUConfig if a.relayer != nil && a.blockEngineHealth { diff --git a/pkg/harmonic/service.go b/pkg/harmonic/service.go index 7f418a89..40a9ad39 100644 --- a/pkg/harmonic/service.go +++ b/pkg/harmonic/service.go @@ -8,6 +8,7 @@ import ( "net" "strings" "sync" + "sync/atomic" "time" "github.com/Overclock-Validator/mithril/pkg/blockprod" @@ -83,7 +84,9 @@ type Service struct { executor *Executor tpu *TPUAdvertisement + clientIDAdvertised atomic.Bool harmonicClientIDAfter, baseClientIDAfter time.Duration + metricsInterval time.Duration } func NewService(cfg ServiceConfig) (*Service, error) { @@ -101,6 +104,7 @@ func NewService(cfg ServiceConfig) (*Service, error) { tpu: NewTPUAdvertisement(cfg.Gossip, cfg.LocalTPU), harmonicClientIDAfter: harmonicClientIDAfter, baseClientIDAfter: baseClientIDAfter, + metricsInterval: metricsInterval, } identity := ed25519.PrivateKey(cfg.Tips.Identity) s.blockEngine = NewBlockEngineSession(BlockEngineConfig{ @@ -118,6 +122,7 @@ func NewService(cfg ServiceConfig) (*Service, error) { Pipeline: s.pipeline, Events: events, Crank: crank.Builder(s.blockEngine), + OnReport: reportSlot, }) if cfg.Relayer.Target != "" { s.relayer = NewRelayerSession(RelayerConfig{ @@ -169,6 +174,7 @@ func (s *Service) Run(ctx context.Context) { run(s.relayer.Run) } run(s.advertiseClientID) + run(s.exportMetrics) <-ctx.Done() // The block stream may be blocked on the stopped executor. s.pipeline.Close() @@ -176,6 +182,7 @@ func (s *Service) Run(ctx context.Context) { if err := s.cfg.Gossip.SetClientID(gossip.ClientIDMithril); err != nil { mlog.Log.Warnf("Harmonic gossip client ID: %v", err) } + s.clientIDAdvertised.Store(false) } // advertiseClientID advertises the Harmonic client ID while the block engine @@ -210,6 +217,7 @@ func (s *Service) advertiseClientID(ctx context.Context) { continue } advertised = want + s.clientIDAdvertised.Store(want == gossip.ClientIDMithrilHarmonic) mlog.Log.Infof("Harmonic: advertising gossip client ID %d", want) } } diff --git a/pkg/harmonic/service_test.go b/pkg/harmonic/service_test.go index 0dc6a02b..fd02f18f 100644 --- a/pkg/harmonic/service_test.go +++ b/pkg/harmonic/service_test.go @@ -104,6 +104,7 @@ func startService(t *testing.T) *serviceHarness { }) require.NoError(t, err) h.service.harmonicClientIDAfter, h.service.baseClientIDAfter = 100*time.Millisecond, 50*time.Millisecond + h.service.metricsInterval = 10 * time.Millisecond ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { diff --git a/pkg/statsd/statsd.go b/pkg/statsd/statsd.go index 5a9eda42..39ce1716 100644 --- a/pkg/statsd/statsd.go +++ b/pkg/statsd/statsd.go @@ -143,6 +143,34 @@ var ( ReplaySigverifyGroupSignatures = Metric{"replay_sigverify_group_signatures_total"} TurbineReplayAdmission = Metric{"turbine_replay_admission_duration_seconds"} + // Harmonic block engine integration. Session and pipeline counters are + // exported once a second; slot metrics when each leader slot ends. + HarmonicBlockEngineHealthy = Metric{"harmonic_block_engine_healthy"} + HarmonicRelayerConnected = Metric{"harmonic_relayer_connected"} + HarmonicRelayerTPUAdvertised = Metric{"harmonic_relayer_tpu_advertised"} + HarmonicClientIDAdvertised = Metric{"harmonic_client_id_advertised"} + HarmonicSessions = Metric{"harmonic_sessions_total"} + HarmonicSessionResets = Metric{"harmonic_session_resets_total"} + HarmonicPackets = Metric{"harmonic_packets_received_total"} + HarmonicPacketsDropped = Metric{"harmonic_packets_dropped_total"} + HarmonicRelayerHeartbeats = Metric{"harmonic_relayer_heartbeats_total"} + HarmonicRelayerInvalidTPUConfigs = Metric{"harmonic_relayer_invalid_tpu_configs_total"} + HarmonicBlockEngineErrors = Metric{"harmonic_block_engine_errors_total"} + HarmonicLeaderNotices = Metric{"harmonic_leader_notices_total"} + HarmonicLeaderNoticeRTT = Metric{"harmonic_leader_notice_rtt_seconds"} + HarmonicBlockStreamsLost = Metric{"harmonic_block_streams_lost_total"} + HarmonicBlockMessages = Metric{"harmonic_block_messages_received_total"} + HarmonicBlockMessagesDropped = Metric{"harmonic_block_messages_dropped_total"} + HarmonicBlockBundles = Metric{"harmonic_block_bundles_received_total"} + HarmonicBlockTransactions = Metric{"harmonic_block_transactions_received_total"} + HarmonicSlots = Metric{"harmonic_slots_total"} + HarmonicSlotEndFlags = Metric{"harmonic_slot_end_flags_total"} + HarmonicSlotBundles = Metric{"harmonic_slot_bundles_total"} + HarmonicTipCranks = Metric{"harmonic_tip_cranks_total"} + // HarmonicFirstBundleDelay is when a leader slot's first block bundle + // arrived, after the bank opened, against the threshold at mid-slot. + HarmonicFirstBundleDelay = Metric{"harmonic_first_bundle_delay_seconds"} + SnapshotWorkerPoolUtilization = Metric{"snapshot_worker_pool_utilization"} TasksSetIfSlotHigherQueueSize = Metric{"tasks_set_if_slot_higher_queue_size"} Epoch = Metric{"epoch"} @@ -266,6 +294,30 @@ var MetricToType = map[Metric]metricType{ ReplaySigverifyGroupSignatures: CountT, TurbineReplayAdmission: TimingT, + HarmonicBlockEngineHealthy: GaugeT, + HarmonicRelayerConnected: GaugeT, + HarmonicRelayerTPUAdvertised: GaugeT, + HarmonicClientIDAdvertised: GaugeT, + HarmonicSessions: CountT, + HarmonicSessionResets: CountT, + HarmonicPackets: CountT, + HarmonicPacketsDropped: CountT, + HarmonicRelayerHeartbeats: CountT, + HarmonicRelayerInvalidTPUConfigs: CountT, + HarmonicBlockEngineErrors: CountT, + HarmonicLeaderNotices: CountT, + HarmonicLeaderNoticeRTT: TimingT, + HarmonicBlockStreamsLost: CountT, + HarmonicBlockMessages: CountT, + HarmonicBlockMessagesDropped: CountT, + HarmonicBlockBundles: CountT, + HarmonicBlockTransactions: CountT, + HarmonicSlots: CountT, + HarmonicSlotEndFlags: CountT, + HarmonicSlotBundles: CountT, + HarmonicTipCranks: CountT, + HarmonicFirstBundleDelay: TimingT, + TestCount: CountT, SnapshotWorkerPoolUtilization: GaugeT, @@ -381,6 +433,30 @@ var MetricToLabels = map[Metric][]string{ ReplaySigverifyGroupSignatures: {}, TurbineReplayAdmission: {}, + HarmonicBlockEngineHealthy: {}, + HarmonicRelayerConnected: {}, + HarmonicRelayerTPUAdvertised: {}, + HarmonicClientIDAdvertised: {}, + HarmonicSessions: {"session"}, + HarmonicSessionResets: {"session"}, + HarmonicPackets: {"source"}, + HarmonicPacketsDropped: {"source", "reason"}, + HarmonicRelayerHeartbeats: {}, + HarmonicRelayerInvalidTPUConfigs: {}, + HarmonicBlockEngineErrors: {"kind"}, + HarmonicLeaderNotices: {"result"}, + HarmonicLeaderNoticeRTT: {}, + HarmonicBlockStreamsLost: {}, + HarmonicBlockMessages: {}, + HarmonicBlockMessagesDropped: {"reason"}, + HarmonicBlockBundles: {"result"}, + HarmonicBlockTransactions: {}, + HarmonicSlots: {"mode"}, + HarmonicSlotEndFlags: {"flag"}, + HarmonicSlotBundles: {"result"}, + HarmonicTipCranks: {"result"}, + HarmonicFirstBundleDelay: {}, + SnapshotWorkerPoolUtilization: {"task"}, TasksSetIfSlotHigherQueueSize: {}, Epoch: {}, @@ -437,6 +513,8 @@ var MetricToBuckets = map[Metric][]float64{ TxFailedPublicationPreparation: turbinePipelineDurationBuckets, TxFailedPayerPublication: turbinePipelineDurationBuckets, TxFailedNoncePublication: turbinePipelineDurationBuckets, + HarmonicLeaderNoticeRTT: blockProductionDurationBuckets, + HarmonicFirstBundleDelay: blockProductionDurationBuckets, } type Prometheusmetrics struct { From abddc14f2cd2dd152e7d6610343332d27fc707f8 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 21:48:34 +0000 Subject: [PATCH 21/25] harmonic: CLI flags --- cmd/mithril/node/harmonic.go | 54 +++++++++++++------ cmd/mithril/node/harmonic_test.go | 88 +++++++++++++++++++++++++++++-- cmd/mithril/node/node.go | 5 +- config.example.toml | 1 + pkg/harmonic/executor.go | 2 +- 5 files changed, 129 insertions(+), 21 deletions(-) diff --git a/cmd/mithril/node/harmonic.go b/cmd/mithril/node/harmonic.go index fe7b605f..892b5ec7 100644 --- a/cmd/mithril/node/harmonic.go +++ b/cmd/mithril/node/harmonic.go @@ -6,13 +6,13 @@ import ( "fmt" "github.com/Overclock-Validator/mithril/pkg/blockprod" - "github.com/Overclock-Validator/mithril/pkg/config" "github.com/Overclock-Validator/mithril/pkg/gossip" "github.com/Overclock-Validator/mithril/pkg/harmonic" blockenginepb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/blockengine" "github.com/Overclock-Validator/mithril/pkg/mlog" "github.com/Overclock-Validator/mithril/pkg/tpu" solana "github.com/gagliardetto/solana-go" + "github.com/spf13/pflag" ) // harmonicSettings is the validated [harmonic] section. @@ -26,35 +26,59 @@ type harmonicSettings struct { tips harmonic.TipConfig } +// registerHarmonicFlags adds a flag for each [harmonic] key. +func registerHarmonicFlags(flags *pflag.FlagSet) { + flags.Bool("harmonic-enabled", false, "Run leader slots from a Harmonic block engine (validator mode)") + flags.String("harmonic-block-engine-url", "", "Harmonic block engine URL (http:// or https://)") + flags.String("harmonic-block-engine-tls-domain", "", "TLS server name override for the block engine") + flags.String("harmonic-relayer-url", "", "Optional Harmonic relayer URL") + flags.String("harmonic-relayer-tls-domain", "", "TLS server name override for the relayer") + flags.Bool("harmonic-tls-cert-verify", true, "Verify Harmonic TLS certificates (false for development only)") + flags.String("harmonic-strategy", "fba", "Block engine scheduling strategy: fba, fifo or mrev") + flags.String("harmonic-tip-payment-program", "", "Tip payment program") + flags.String("harmonic-tip-distribution-program", "", "Tip distribution program") + flags.String("harmonic-merkle-root-upload-authority", "", "Merkle root upload authority for tip distribution accounts") + flags.Int64("harmonic-commission-bps", 0, "Validator commission on tips, in basis points (0-10000)") +} + +// optionGetters resolve an option from its CLI flag when set, else from the +// config file. +type optionGetters struct { + getString func(cliKey, tomlKey string) string + getInt64 func(cliKey, tomlKey string) int64 + getBool func(cliKey, tomlKey string) bool +} + // loadHarmonicSettings reads [harmonic]. Harmonic is off unless enabled, and // then every key it needs must be valid. -func loadHarmonicSettings() (harmonicSettings, error) { - s := harmonicSettings{enabled: config.GetBool("harmonic.enabled")} +func loadHarmonicSettings(get optionGetters) (harmonicSettings, error) { + s := harmonicSettings{enabled: get.getBool("harmonic-enabled", "harmonic.enabled")} if !s.enabled { return s, nil } var err error - if s.blockEngine, err = harmonic.ParseEndpoint(config.GetString("harmonic.block_engine_url"), config.GetString("harmonic.block_engine_tls_domain")); err != nil { + if s.blockEngine, err = harmonic.ParseEndpoint(get.getString("harmonic-block-engine-url", "harmonic.block_engine_url"), + get.getString("harmonic-block-engine-tls-domain", "harmonic.block_engine_tls_domain")); err != nil { return s, fmt.Errorf("harmonic.block_engine_url: %w", err) } - if url := config.GetString("harmonic.relayer_url"); url != "" { - if s.relayer, err = harmonic.ParseEndpoint(url, config.GetString("harmonic.relayer_tls_domain")); err != nil { + if url := get.getString("harmonic-relayer-url", "harmonic.relayer_url"); url != "" { + if s.relayer, err = harmonic.ParseEndpoint(url, get.getString("harmonic-relayer-tls-domain", "harmonic.relayer_tls_domain")); err != nil { return s, fmt.Errorf("harmonic.relayer_url: %w", err) } } - s.insecureSkipVerify = !config.GetBool("harmonic.tls_cert_verify") - if s.strategy, err = harmonic.ParseStrategy(config.GetString("harmonic.strategy")); err != nil { + s.insecureSkipVerify = !get.getBool("harmonic-tls-cert-verify", "harmonic.tls_cert_verify") + if s.strategy, err = harmonic.ParseStrategy(get.getString("harmonic-strategy", "harmonic.strategy")); err != nil { return s, fmt.Errorf("harmonic.strategy: %w", err) } for _, key := range []struct { - name string - dst *solana.PublicKey + flag, name string + dst *solana.PublicKey }{ - {"harmonic.tip_payment_program", &s.tips.TipPaymentProgram}, - {"harmonic.tip_distribution_program", &s.tips.TipDistributionProgram}, - {"harmonic.merkle_root_upload_authority", &s.tips.MerkleRootUploadAuthority}, + {"harmonic-tip-payment-program", "harmonic.tip_payment_program", &s.tips.TipPaymentProgram}, + {"harmonic-tip-distribution-program", "harmonic.tip_distribution_program", &s.tips.TipDistributionProgram}, + {"harmonic-merkle-root-upload-authority", "harmonic.merkle_root_upload_authority", &s.tips.MerkleRootUploadAuthority}, } { - value := config.GetString(key.name) + value := get.getString(key.flag, key.name) if value == "" { return s, fmt.Errorf("%s is required when harmonic.enabled=true", key.name) } @@ -62,7 +86,7 @@ func loadHarmonicSettings() (harmonicSettings, error) { return s, fmt.Errorf("%s: %w", key.name, err) } } - bps := config.GetInt64("harmonic.commission_bps") + bps := get.getInt64("harmonic-commission-bps", "harmonic.commission_bps") if bps < 0 || bps > 10_000 { return s, fmt.Errorf("harmonic.commission_bps must be between 0 and 10000, got %d", bps) } diff --git a/cmd/mithril/node/harmonic_test.go b/cmd/mithril/node/harmonic_test.go index 7244dbed..fbcc2ee0 100644 --- a/cmd/mithril/node/harmonic_test.go +++ b/cmd/mithril/node/harmonic_test.go @@ -3,19 +3,50 @@ package node import ( "fmt" "maps" + "strconv" "strings" "testing" "github.com/Overclock-Validator/mithril/pkg/config" blockenginepb "github.com/Overclock-Validator/mithril/pkg/harmonic/proto/blockengine" + "github.com/spf13/pflag" "github.com/spf13/viper" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -// loadHarmonicTOML loads a [harmonic] section of the given keys; values are -// TOML literals. -func loadHarmonicTOML(t *testing.T, values map[string]string) (harmonicSettings, error) { +// flagsThenConfig resolves options as runLive does: a set flag wins over the +// config file. +func flagsThenConfig(flags *pflag.FlagSet) optionGetters { + changed := func(cliKey string) *pflag.Flag { + if f := flags.Lookup(cliKey); f != nil && f.Changed { + return f + } + return nil + } + return optionGetters{ + getString: func(cliKey, tomlKey string) string { + if f := changed(cliKey); f != nil { + return f.Value.String() + } + return config.GetString(tomlKey) + }, + getInt64: func(cliKey, tomlKey string) int64 { + if f := changed(cliKey); f != nil { + v, _ := strconv.ParseInt(f.Value.String(), 10, 64) + return v + } + return config.GetInt64(tomlKey) + }, + getBool: func(cliKey, tomlKey string) bool { + return resolveBoolOption(flags.Lookup(cliKey), config.IsSet(tomlKey), config.GetBool(tomlKey)) + }, + } +} + +// loadHarmonic loads a [harmonic] section of the given keys, whose values are +// TOML literals, under the given flag values. +func loadHarmonic(t *testing.T, values map[string]string, flagValues map[string]string) (harmonicSettings, error) { t.Helper() var toml strings.Builder toml.WriteString("[harmonic]\n") @@ -27,7 +58,17 @@ func loadHarmonicTOML(t *testing.T, values map[string]string) (harmonicSettings, config.ApplyDefaults(viper.GetViper()) viper.SetConfigType("toml") require.NoError(t, viper.ReadConfig(strings.NewReader(toml.String()))) - return loadHarmonicSettings() + flags := pflag.NewFlagSet("run", pflag.ContinueOnError) + registerHarmonicFlags(flags) + for name, value := range flagValues { + require.NoError(t, flags.Set(name, value)) + } + return loadHarmonicSettings(flagsThenConfig(flags)) +} + +func loadHarmonicTOML(t *testing.T, values map[string]string) (harmonicSettings, error) { + t.Helper() + return loadHarmonic(t, values, nil) } // enabledHarmonic is a minimal enabled [harmonic] section with overrides. @@ -92,3 +133,42 @@ func TestHarmonicSettings(t *testing.T) { assert.Errorf(t, err, "%s", name) } } + +func TestHarmonicFlagsOverrideTheConfigFile(t *testing.T) { + require.NotNil(t, Run.Flags().Lookup("harmonic-enabled")) + + // Every key from flags alone. + s, err := loadHarmonic(t, nil, map[string]string{ + "harmonic-enabled": "true", + "harmonic-block-engine-url": "https://be.example.com", + "harmonic-block-engine-tls-domain": "be.internal", + "harmonic-relayer-url": "http://relayer.example.com:11226", + "harmonic-relayer-tls-domain": "relayer.internal", + "harmonic-tls-cert-verify": "false", + "harmonic-strategy": "fifo", + "harmonic-tip-payment-program": "T1pyyaTNZsKv2WcRAB8oVnk93mLJw2XzjtVYqCsaHqt", + "harmonic-tip-distribution-program": "4R3gSG8BpU4t19KYj8CfnbtRpnT8gtk4dvTHxVRwc2r7", + "harmonic-merkle-root-upload-authority": "8F4jGUmxF36vQ6yabnsxX6AQVXdKBhs8kGSUuRKSg8Xt", + "harmonic-commission-bps": "500", + }) + require.NoError(t, err) + assert.True(t, s.enabled) + assert.Equal(t, "be.example.com:443", s.blockEngine.Target) + assert.Equal(t, "be.internal", s.blockEngine.ServerName) + assert.Equal(t, "relayer.example.com:11226", s.relayer.Target) + assert.Equal(t, "relayer.internal", s.relayer.ServerName) + assert.True(t, s.insecureSkipVerify) + assert.Equal(t, blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_FIFO, s.strategy) + assert.Equal(t, "T1pyyaTNZsKv2WcRAB8oVnk93mLJw2XzjtVYqCsaHqt", s.tips.TipPaymentProgram.String()) + assert.Equal(t, "4R3gSG8BpU4t19KYj8CfnbtRpnT8gtk4dvTHxVRwc2r7", s.tips.TipDistributionProgram.String()) + assert.Equal(t, "8F4jGUmxF36vQ6yabnsxX6AQVXdKBhs8kGSUuRKSg8Xt", s.tips.MerkleRootUploadAuthority.String()) + assert.Equal(t, uint16(500), s.tips.CommissionBps) + + // A flag beats the config file, including an explicit false. + s, err = loadHarmonic(t, enabledHarmonic(map[string]string{"strategy": `"mrev"`}), map[string]string{"harmonic-strategy": "fba"}) + require.NoError(t, err) + assert.Equal(t, blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_FBA, s.strategy) + s, err = loadHarmonic(t, enabledHarmonic(nil), map[string]string{"harmonic-enabled": "false"}) + require.NoError(t, err) + assert.False(t, s.enabled) +} diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index e86055e5..f0e41b14 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -579,6 +579,9 @@ func init() { Run.Flags().IntVar(&validatorCompletionReserveMs, "leader-completion-reserve-ms", 0, "Time reserved for leader finalization and broadcast (0 = 75ms default; tune from measured completion times)") Run.Flags().IntVar(&validatorMaxBufferedTransactions, "tpu-max-buffered-transactions", 0, "Maximum queued TPU transactions (0 = 131072 default)") + // [harmonic] section flags + registerHarmonicFlags(Run.Flags()) + // [tuning] section flags Run.Flags().Uint64Var(¶mArenaSizeMB, "param-arena-size-mb", 512, "Size in MB for serialized parameter arena (0 to disable)") Run.Flags().Uint64Var(&borrowedAccountArenaSize, "borrowed-account-arena-size", 1024, "Number of borrowed accounts to preallocate in arena (0 to disable)") @@ -1060,7 +1063,7 @@ func initConfigAndBindFlags(cmd *cobra.Command) error { } // [harmonic] section — optional block engine integration for leader slots. - if harmonicConfig, err = loadHarmonicSettings(); err != nil { + if harmonicConfig, err = loadHarmonicSettings(optionGetters{getString: getString, getInt64: getInt64, getBool: getBool}); err != nil { return err } if harmonicConfig.enabled && consensusMode != "validator" { diff --git a/config.example.toml b/config.example.toml index 8374f371..2a8d4853 100644 --- a/config.example.toml +++ b/config.example.toml @@ -371,6 +371,7 @@ name = "mithril" # at slot start (immediately), when no block arrives by the middle of the # slot, or when the block stops early. Harmonic failures never stop the # validator. When disabled, leader slots schedule locally from the start. +# Every key has a flag: --harmonic-, e.g. --harmonic-block-engine-url. [harmonic] enabled = false diff --git a/pkg/harmonic/executor.go b/pkg/harmonic/executor.go index 26374c5e..13de7487 100644 --- a/pkg/harmonic/executor.go +++ b/pkg/harmonic/executor.go @@ -17,7 +17,7 @@ const ( // with no block bundle falls back to local scheduling. thresholdFraction = 0.5 // voteTail is the end of the slot in which a reverting block bundle does - // not abort the block (spec §15.3). + // not abort the block. voteTail = 20 * time.Millisecond ) From de9fe4f506544be017ce825b19456fe322b47673 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Sun, 18 Oct 2026 23:54:23 +0000 Subject: [PATCH 22/25] harmonic: run the block until past the slot end and drained --- pkg/blockprod/leader.go | 19 +++++- pkg/blockprod/leader_hooks.go | 12 ++++ pkg/blockprod/leader_hooks_test.go | 58 +++++++++++++++++++ pkg/harmonic/blocks.go | 10 ++++ pkg/harmonic/executor.go | 81 ++++++++++++++++++++------ pkg/harmonic/executor_test.go | 93 ++++++++++++++++++++++++------ 6 files changed, 236 insertions(+), 37 deletions(-) diff --git a/pkg/blockprod/leader.go b/pkg/blockprod/leader.go index 84a5964e..f9a8576c 100644 --- a/pkg/blockprod/leader.go +++ b/pkg/blockprod/leader.go @@ -101,6 +101,7 @@ type LeaderLoop struct { onBlock func(*b.Block) commitLeaderSlot func(replay.CommitLeaderInput) (*sealevel.SlotCtx, error) hooks LeaderHooks + holder SlotHolder gateTPU bool currentSlot func() uint64 @@ -185,6 +186,7 @@ func NewLeaderLoop(cfg LeaderLoopConfig) *LeaderLoop { if cfg.CompletionReserve <= 0 { cfg.CompletionReserve = leaderBlockCompletionReserve } + holder, _ := cfg.Hooks.(SlotHolder) return &LeaderLoop{ controller: cfg.Controller, identity: cfg.Identity, @@ -206,6 +208,7 @@ func NewLeaderLoop(cfg LeaderLoopConfig) *LeaderLoop { completionReserve: cfg.CompletionReserve, now: cfg.Now, hooks: cfg.Hooks, + holder: holder, gateTPU: cfg.GateTPU && cfg.Hooks != nil, finishedLeaderSlots: make(map[uint64]struct{}), pendingFailures: make(map[uint64]leaderSlotFailure), @@ -215,6 +218,10 @@ func NewLeaderLoop(cfg LeaderLoopConfig) *LeaderLoop { func (l *LeaderLoop) Run(stop <-chan struct{}) { ticker := time.NewTicker(l.pollInterval) defer ticker.Stop() + var released <-chan struct{} + if l.holder != nil { + released = l.holder.Released() + } for { select { @@ -223,6 +230,8 @@ func (l *LeaderLoop) Run(stop <-chan struct{}) { return case scheduledAt := <-ticker.C: l.tickScheduled(scheduledAt) + case <-released: + l.tick() } } } @@ -263,7 +272,7 @@ func (l *LeaderLoop) tickScheduled(scheduledAt time.Time) { } now = l.now() if l.productionWindow.active { - if now.Before(l.productionWindowDeadlineLocked(l.activeSlot)) { + if now.Before(l.productionWindowDeadlineLocked(l.activeSlot)) || l.holdActiveSlotLocked(now) { return } } else if wallSlot <= l.activeSlot { @@ -617,6 +626,14 @@ func (l *LeaderLoop) productionWindowDeadlineLocked(slot uint64) time.Time { return deadline.Add(-reserve) } +// holdActiveSlotLocked reports whether the hooks keep the active slot open past +// its production deadline. The protocol deadline always ends the hold. +func (l *LeaderLoop) holdActiveSlotLocked(now time.Time) bool { + return l.holder != nil && + now.Before(l.productionWindowProtocolDeadlineLocked(l.activeSlot)) && + l.holder.HoldSlot(l.activeSlot) +} + func (l *LeaderLoop) productionWindowProtocolDeadlineLocked(slot uint64) time.Time { if !l.productionWindow.active || slot < l.productionWindow.startSlot { return time.Time{} diff --git a/pkg/blockprod/leader_hooks.go b/pkg/blockprod/leader_hooks.go index 609b3f48..a66df61e 100644 --- a/pkg/blockprod/leader_hooks.go +++ b/pkg/blockprod/leader_hooks.go @@ -22,6 +22,18 @@ type LeaderHooks interface { SlotAborted(slot uint64, reason string) } +// SlotHolder is implemented by LeaderHooks that can keep a slot open past its +// production deadline while they still have work for it. The protocol +// deadline ends a hold regardless. +type SlotHolder interface { + // HoldSlot reports whether slot must stay open. It runs under the leader + // loop's lock and must return promptly. + HoldSlot(slot uint64) bool + // Released is signaled when a held slot may end, so the loop need not wait + // for its next poll. + Released() <-chan struct{} +} + // LeaderSlotInfo describes a leader slot whose working bank is about to be // published. type LeaderSlotInfo struct { diff --git a/pkg/blockprod/leader_hooks_test.go b/pkg/blockprod/leader_hooks_test.go index a2dc546f..9cf9b796 100644 --- a/pkg/blockprod/leader_hooks_test.go +++ b/pkg/blockprod/leader_hooks_test.go @@ -322,3 +322,61 @@ func TestLeaderHooksSlotAbortedWithoutEndingWhenParentChanges(t *testing.T) { assert.Nil(t, controller.WorkingBank()) assert.Nil(t, loop.activeBank) } + +// holdingHooks keeps the active slot open while hold is set. +type holdingHooks struct { + *recordingHooks + hold bool + released chan struct{} +} + +func (h *holdingHooks) HoldSlot(uint64) bool { return h.hold } +func (h *holdingHooks) Released() <-chan struct{} { return h.released } + +func TestLeaderHoldsTheSlotPastItsDeadlineUntilReleased(t *testing.T) { + holding := &holdingHooks{recordingHooks: &recordingHooks{}} + require.Same(t, holding, NewLeaderLoop(LeaderLoopConfig{Hooks: holding}).holder) + + ready := time.Unix(1_700_000_000, 0) + for _, tc := range []struct { + name string + slot uint64 + // at is past the production deadline (125ms after ParentReady for the + // window's first slot); the protocol deadline is at 200ms. + at time.Duration + release bool + finishes bool + }{ + {"held past the production deadline", 120, 150 * time.Millisecond, false, false}, + {"released", 121, 150 * time.Millisecond, true, true}, + {"the protocol deadline ends a hold", 122, 200 * time.Millisecond, false, true}, + } { + t.Run(tc.name, func(t *testing.T) { + hooks := &recordingHooks{} + loop, _, _, delivered := newFinishingLeaderLoop(t, tc.slot, hooks, + func(*ParentContext, replay.CommitLeaderInput) error { return nil }) + holder := &holdingHooks{recordingHooks: hooks, hold: true} + loop.holder = holder + now := ready.Add(tc.at) + loop.now = func() time.Time { return now } + loop.currentSlot = func() uint64 { return tc.slot } + loop.leaderForSlot = func(uint64) (solana.PublicKey, bool) { return solana.PublicKey{}, false } + loop.productionWindow = leaderProductionWindow{ + active: true, startSlot: tc.slot, endSlot: tc.slot + 3, nextSlot: tc.slot, readyAt: ready, + } + if tc.release { + holder.hold = false + } + + loop.tick() + if !tc.finishes { + assert.NotNil(t, loop.activeBank) + assert.Empty(t, hooks.kinds()) + return + } + require.True(t, *delivered) + assert.Equal(t, []string{"ending"}, hooks.kinds()) + assert.Nil(t, loop.activeBank) + }) + } +} diff --git a/pkg/harmonic/blocks.go b/pkg/harmonic/blocks.go index bed6b9f3..b5dbf51d 100644 --- a/pkg/harmonic/blocks.go +++ b/pkg/harmonic/blocks.go @@ -70,6 +70,7 @@ type BlockPipeline struct { closeOnce sync.Once txV1Enabled func() bool current atomic.Uint64 + inFlight atomic.Int32 verifier txverify.BatchVerifier stats struct { messages, wrongSlot, lostSlot, bundles, invalid, transactions atomic.Uint64 @@ -126,6 +127,12 @@ func (p *BlockPipeline) SetCurrentSlot(slot uint64) { } } +// InFlight reports whether a current-slot message is being split into +// bundles whose events are not all sent yet. +func (p *BlockPipeline) InFlight() bool { + return p.inFlight.Load() > 0 +} + // StreamLost marks slot lost: the rest of its block is discarded even after // the session reconnects, since nothing says where a new stream resumes. func (p *BlockPipeline) StreamLost(slot uint64) { @@ -137,6 +144,9 @@ func (p *BlockPipeline) StreamLost(slot uint64) { // HandleBlock admits, segments and validates one block message. func (p *BlockPipeline) HandleBlock(block *blockpb.Block, arrival time.Time) { + // Until its last event is sent, a message counts as buffered. + p.inFlight.Add(1) + defer p.inFlight.Add(-1) p.stats.messages.Add(1) slot := block.GetSlot() if current := p.current.Load(); current == 0 || slot != current { diff --git a/pkg/harmonic/executor.go b/pkg/harmonic/executor.go index 13de7487..a2be6ed5 100644 --- a/pkg/harmonic/executor.go +++ b/pkg/harmonic/executor.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" "sync" + "sync/atomic" "time" "github.com/Overclock-Validator/mithril/pkg/blockprod" @@ -16,9 +17,6 @@ const ( // thresholdFraction of the slot's recording window passes before a slot // with no block bundle falls back to local scheduling. thresholdFraction = 0.5 - // voteTail is the end of the slot in which a reverting block bundle does - // not abort the block. - voteTail = 20 * time.Millisecond ) // SlotMode is a leader slot's admission mode. @@ -119,10 +117,13 @@ type BlockEngineControl interface { NotifyLeaderSlot(slot uint64, end time.Time) } -// SlotTracker is told the current block slot; the block pipeline admits only -// that slot's messages. +// SlotTracker is the block pipeline's side of the executor. It is told the +// current block slot, and admits only that slot's messages. type SlotTracker interface { SetCurrentSlot(slot uint64) + // InFlight reports whether a message is being split into bundles whose + // events are not all sent yet. + InFlight() bool } // CrankBuilder returns the transactions that must land before a slot's first @@ -175,12 +176,56 @@ type Executor struct { hooksMu sync.Mutex hooks []hookMessage wake chan struct{} + + // running is the slot whose block is executing, or 0. The leader loop + // holds that slot open while block bundles remain. + running atomic.Uint64 + released chan struct{} } -var _ blockprod.LeaderHooks = (*Executor)(nil) +var ( + _ blockprod.LeaderHooks = (*Executor)(nil) + _ blockprod.SlotHolder = (*Executor)(nil) +) func NewExecutor(cfg ExecutorConfig) *Executor { - return &Executor{cfg: cfg, now: time.Now, wake: make(chan struct{}, 1)} + return &Executor{cfg: cfg, now: time.Now, wake: make(chan struct{}, 1), released: make(chan struct{}, 1)} +} + +// HoldSlot keeps a running block's slot open while any block bundle received +// for it has yet to execute: the slot ends once it is past its end and the +// buffer is empty. It runs on the leader loop's goroutine. A bundle already +// executing when it returns false still lands, since freezing the bank waits +// for it. +func (x *Executor) HoldSlot(slot uint64) bool { + return x.running.Load() == slot && (len(x.cfg.Events) > 0 || x.cfg.Pipeline.InFlight()) +} + +// Released is signaled when the running block's slot may end. +func (x *Executor) Released() <-chan struct{} { + return x.released +} + +func (x *Executor) signalRelease() { + select { + case x.released <- struct{}{}: + default: + } +} + +// endHold stops holding st's slot open. +func (x *Executor) endHold(st *slotState) { + if x.running.CompareAndSwap(st.slot, 0) { + x.signalRelease() + } +} + +// releaseIfDrained lets the leader loop end a running block's slot as soon as +// it is past its end and every block bundle received has executed. +func (x *Executor) releaseIfDrained(st *slotState) { + if st != nil && x.running.Load() == st.slot && !x.now().Before(st.end) && !x.HoldSlot(st.slot) { + x.signalRelease() + } } // SlotStarted runs under the leader loop's lock, so it only does what cannot @@ -255,6 +300,7 @@ func (x *Executor) Run(ctx context.Context) { // the block engine streams only after the notice. st = x.drainHooks(st) x.handleEvent(st, e) + x.releaseIfDrained(st) } } } @@ -309,6 +355,7 @@ func (x *Executor) finish(st *slotState, reason string, aborted bool) { if st.thresholdTimer != nil { st.thresholdTimer.Stop() } + x.endHold(st) st.report.Mode = st.mode st.report.Aborted = aborted st.report.Reason = reason @@ -378,13 +425,7 @@ func (x *Executor) admits(st *slotState, slot, seq uint64) bool { } func (x *Executor) executeBundle(st *slotState, e BundleEvent) { - now := x.now() - if st.mode == ModeHarmonic && now.After(st.end) { - st.report.Rejected++ - x.stop(st, FlagTimeout) - return - } - if st.mode == ModeUndecided && !now.Before(st.threshold) { + if st.mode == ModeUndecided && !x.now().Before(st.threshold) { st.report.Rejected++ x.fallback(st, FlagTimeout) return @@ -395,6 +436,7 @@ func (x *Executor) executeBundle(st *slotState, e BundleEvent) { return } st.mode = ModeHarmonic + x.running.Store(st.slot) } var result blockprod.ForgeResult @@ -410,15 +452,14 @@ func (x *Executor) executeBundle(st *slotState, e BundleEvent) { // The bank froze: the slot is over. st.report.Rejected++ st.stopped = true + x.endHold(st) case blockprod.ForgeDroppedCost: // Execution is serial, so nothing in flight can free room. st.report.Rejected++ x.stop(st, FlagFailed) default: st.report.Reverted++ - if now.Before(st.end.Add(-voteTail)) { - x.stop(st, FlagReverted) - } + x.stop(st, FlagReverted) } } @@ -449,11 +490,12 @@ func (x *Executor) runCrank(st *slotState) bool { return true } -// stop ends block admission for the slot. Before the cutoff the slot falls -// back to local scheduling; after it, the block is simply over. +// stop ends block admission for the slot. Before the slot's end it falls back +// to local scheduling; after it, the block is simply over. func (x *Executor) stop(st *slotState, flag EndFlags) { st.stopped = true st.report.Flags |= flag + x.endHold(st) if st.mode == ModeUndecided || (st.mode == ModeHarmonic && x.now().Before(st.end)) { x.fallback(st, 0) } @@ -463,5 +505,6 @@ func (x *Executor) fallback(st *slotState, flag EndFlags) { st.mode = ModeFallback st.stopped = true st.report.Flags |= flag + x.endHold(st) st.bank.AllowTPU() } diff --git a/pkg/harmonic/executor_test.go b/pkg/harmonic/executor_test.go index 760e1074..f13d5cf9 100644 --- a/pkg/harmonic/executor_test.go +++ b/pkg/harmonic/executor_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "sync" + "sync/atomic" "testing" "time" @@ -39,8 +40,29 @@ func (f *fakeBlockEngine) NotifyLeaderSlot(slot uint64, _ time.Time) { // fakePipeline records the current slot and shares the call log. type fakePipeline struct { - be *fakeBlockEngine - current []uint64 + be *fakeBlockEngine + current []uint64 + inFlight atomic.Bool +} + +func (f *fakePipeline) InFlight() bool { + return f.inFlight.Load() +} + +// testClock is a clock tests can move while the executor runs. +type testClock struct{ nanos atomic.Int64 } + +func (c *testClock) now() time.Time { return time.Unix(0, c.nanos.Load()) } +func (c *testClock) set(t time.Time) { c.nanos.Store(t.UnixNano()) } + +// useClock gives the executor a settable clock, starting at the current time. +// Call it before the executor sees any slot or event. +func (h *executorHarness) useClock() (*testClock, time.Time) { + clock := &testClock{} + now := time.Now() + clock.set(now) + h.exec.now = clock.now + return clock, now } func (f *fakePipeline) SetCurrentSlot(slot uint64) { @@ -264,8 +286,8 @@ func TestExecutorStopTriggers(t *testing.T) { } } -func TestExecutorCutoffAndVoteTail(t *testing.T) { - t.Run("a bundle after the cutoff stops the block without falling back", func(t *testing.T) { +func TestExecutorRunsTheBlockUntilPastTheEndAndDrained(t *testing.T) { + t.Run("past its end, the block keeps executing what it receives", func(t *testing.T) { h := startExecutor(t, true, nil) env := gatedBank(t, costmodel.Limits{}) h.startSlot(10, env, 80*time.Millisecond) @@ -273,28 +295,65 @@ func TestExecutorCutoffAndVoteTail(t *testing.T) { time.Sleep(100 * time.Millisecond) h.send(t, bundleEvent(10, 2, false, transfer(t, 2))) h.sync(t) - assert.Len(t, env.Bank.ForgedTransactions(), 1) - assert.False(t, env.Bank.TPUAllowed(), "past the cutoff the block is simply over") + assert.Len(t, env.Bank.ForgedTransactions(), 2) + assert.False(t, env.Bank.TPUAllowed()) report := h.end(t, 10) assert.Equal(t, ModeHarmonic, report.Mode) - assert.Equal(t, FlagTimeout, report.Flags) + assert.Zero(t, report.Flags) }) - t.Run("a revert in the vote tail lands nothing and the block continues", func(t *testing.T) { + t.Run("the slot is held while a message is in flight, and released once drained", func(t *testing.T) { h := startExecutor(t, true, nil) + clock, now := h.useClock() env := gatedBank(t, costmodel.Limits{}) - now := time.Now() h.exec.SlotStarted(blockprod.LeaderSlotInfo{Slot: 10, Bank: env.Bank, Start: now, End: now.Add(time.Second)}) + assert.False(t, h.exec.HoldSlot(10), "no block is running yet") h.send(t, bundleEvent(10, 1, false, transfer(t, 1))) h.sync(t) - h.exec.now = func() time.Time { return now.Add(time.Second - 10*time.Millisecond) } - h.send(t, bundleEvent(10, 2, true, transfer(t, 2), transfer(t, 20_000_000_000))) - h.send(t, bundleEvent(10, 3, false, transfer(t, 3))) + h.pipeline.inFlight.Store(true) + assert.True(t, h.exec.HoldSlot(10)) + assert.False(t, h.exec.HoldSlot(11)) + h.pipeline.inFlight.Store(false) + assert.False(t, h.exec.HoldSlot(10), "the buffer is empty") + + // Draining before the end does not release; draining past it does. + h.send(t, bundleEvent(10, 2, false, transfer(t, 2))) h.sync(t) - assert.Len(t, env.Bank.ForgedTransactions(), 2) - report := h.end(t, 10) - assert.Equal(t, ModeHarmonic, report.Mode) - assert.Zero(t, report.Flags) - assert.Equal(t, uint64(1), report.Reverted) + select { + case <-h.exec.Released(): + t.Fatal("released before the slot's end") + default: + } + clock.set(now.Add(2 * time.Second)) + h.send(t, bundleEvent(10, 3, false, transfer(t, 3))) + select { + case <-h.exec.Released(): + case <-time.After(2 * time.Second): + t.Fatal("not released once past the end and drained") + } + assert.Len(t, env.Bank.ForgedTransactions(), 3) + }) + t.Run("a revert stops the block, falling back only before the end", func(t *testing.T) { + for _, pastEnd := range []bool{false, true} { + h := startExecutor(t, true, nil) + clock, now := h.useClock() + env := gatedBank(t, costmodel.Limits{}) + h.exec.SlotStarted(blockprod.LeaderSlotInfo{Slot: 10, Bank: env.Bank, Start: now, End: now.Add(time.Second)}) + h.send(t, bundleEvent(10, 1, false, transfer(t, 1))) + h.sync(t) + if pastEnd { + clock.set(now.Add(2 * time.Second)) + } + h.pipeline.inFlight.Store(true) + h.send(t, bundleEvent(10, 2, true, transfer(t, 2), transfer(t, 20_000_000_000))) + h.send(t, bundleEvent(10, 3, false, transfer(t, 3))) + h.sync(t) + assert.Len(t, env.Bank.ForgedTransactions(), 1) + assert.False(t, h.exec.HoldSlot(10), "a stopped block holds nothing open") + assert.Equal(t, !pastEnd, env.Bank.TPUAllowed()) + report := h.end(t, 10) + assert.Equal(t, FlagReverted, report.Flags) + assert.Equal(t, uint64(1), report.Reverted) + } }) } From d5bb0f36b32d642710d01df5580a8dd34e6e9117 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Mon, 19 Oct 2026 01:10:35 +0000 Subject: [PATCH 23/25] harmonic: execute block bundles in parallel --- cmd/mithril/node/harmonic.go | 10 +- cmd/mithril/node/harmonic_test.go | 7 + config.example.toml | 3 + pkg/blockprod/bank_bundle.go | 90 ++------ pkg/blockprod/bank_parallel.go | 278 +++++++++++++++++++++++++ pkg/blockprod/bank_parallel_test.go | 203 ++++++++++++++++++ pkg/blockprod/testenv.go | 10 + pkg/config/config.go | 1 + pkg/harmonic/executor.go | 191 ++++++++++++++--- pkg/harmonic/executor_parallel_test.go | 240 +++++++++++++++++++++ pkg/harmonic/executor_test.go | 23 +- pkg/harmonic/metrics.go | 3 + pkg/harmonic/metrics_test.go | 4 +- pkg/harmonic/schedule.go | 157 ++++++++++++++ pkg/harmonic/schedule_test.go | 112 ++++++++++ pkg/harmonic/service.go | 4 + pkg/statsd/statsd.go | 5 + 17 files changed, 1233 insertions(+), 108 deletions(-) create mode 100644 pkg/blockprod/bank_parallel.go create mode 100644 pkg/blockprod/bank_parallel_test.go create mode 100644 pkg/harmonic/executor_parallel_test.go create mode 100644 pkg/harmonic/schedule.go create mode 100644 pkg/harmonic/schedule_test.go diff --git a/cmd/mithril/node/harmonic.go b/cmd/mithril/node/harmonic.go index 892b5ec7..d117aca1 100644 --- a/cmd/mithril/node/harmonic.go +++ b/cmd/mithril/node/harmonic.go @@ -22,6 +22,7 @@ type harmonicSettings struct { relayer harmonic.Endpoint insecureSkipVerify bool strategy blockenginepb.SchedulingStrategy + workers int // tips lacks the vote account and identity, which come from [validator]. tips harmonic.TipConfig } @@ -39,6 +40,7 @@ func registerHarmonicFlags(flags *pflag.FlagSet) { flags.String("harmonic-tip-distribution-program", "", "Tip distribution program") flags.String("harmonic-merkle-root-upload-authority", "", "Merkle root upload authority for tip distribution accounts") flags.Int64("harmonic-commission-bps", 0, "Validator commission on tips, in basis points (0-10000)") + flags.Int64("harmonic-num-workers", 8, "Threads executing block bundles that do not conflict in parallel") } // optionGetters resolve an option from its CLI flag when set, else from the @@ -91,6 +93,11 @@ func loadHarmonicSettings(get optionGetters) (harmonicSettings, error) { return s, fmt.Errorf("harmonic.commission_bps must be between 0 and 10000, got %d", bps) } s.tips.CommissionBps = uint16(bps) + workers := get.getInt64("harmonic-num-workers", "harmonic.num_workers") + if workers < 1 || workers > 64 { + return s, fmt.Errorf("harmonic.num_workers must be between 1 and 64, got %d", workers) + } + s.workers = int(workers) return s, nil } @@ -113,6 +120,7 @@ func startHarmonic(ctx context.Context, s harmonicSettings, identity ed25519.Pri Gossip: gossipClient, LocalTPU: localTPU, TxV1Enabled: txV1Enabled, + Workers: s.workers, }) if err != nil { return nil, nil, err @@ -127,7 +135,7 @@ func startHarmonic(ctx context.Context, s harmonicSettings, identity ed25519.Pri if s.relayer.Target != "" { relayer = s.relayer.Target } - mlog.Log.Infof("Harmonic enabled: block engine %s, relayer %s, strategy %s", s.blockEngine.Target, relayer, s.strategy) + mlog.Log.Infof("Harmonic enabled: block engine %s, relayer %s, strategy %s, workers %d", s.blockEngine.Target, relayer, s.strategy, s.workers) return service.Hooks(), func() { cancel() <-done diff --git a/cmd/mithril/node/harmonic_test.go b/cmd/mithril/node/harmonic_test.go index fbcc2ee0..89bbdb64 100644 --- a/cmd/mithril/node/harmonic_test.go +++ b/cmd/mithril/node/harmonic_test.go @@ -106,18 +106,21 @@ func TestHarmonicSettings(t *testing.T) { assert.Equal(t, blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_FBA, s.strategy) assert.Equal(t, "8F4jGUmxF36vQ6yabnsxX6AQVXdKBhs8kGSUuRKSg8Xt", s.tips.MerkleRootUploadAuthority.String()) assert.Zero(t, s.tips.CommissionBps) + assert.Equal(t, 8, s.workers, "eight workers by default") s, err = loadHarmonicTOML(t, enabledHarmonic(map[string]string{ "relayer_url": `"http://relayer.example.com:11226"`, "tls_cert_verify": "false", "strategy": `"MREV"`, "commission_bps": "800", + "num_workers": "4", })) require.NoError(t, err) assert.Equal(t, "relayer.example.com:11226", s.relayer.Target) assert.True(t, s.insecureSkipVerify) assert.Equal(t, blockenginepb.SchedulingStrategy_SCHEDULING_STRATEGY_MREV, s.strategy) assert.Equal(t, uint16(800), s.tips.CommissionBps) + assert.Equal(t, 4, s.workers) for name, bad := range map[string]map[string]string{ "no block engine url": {"block_engine_url": `""`}, @@ -128,6 +131,8 @@ func TestHarmonicSettings(t *testing.T) { "bad authority": {"merkle_root_upload_authority": `"not base58"`}, "commission over 100%": {"commission_bps": "10001"}, "negative commission": {"commission_bps": "-1"}, + "no workers": {"num_workers": "0"}, + "too many workers": {"num_workers": "65"}, } { _, err := loadHarmonicTOML(t, enabledHarmonic(bad)) assert.Errorf(t, err, "%s", name) @@ -150,6 +155,7 @@ func TestHarmonicFlagsOverrideTheConfigFile(t *testing.T) { "harmonic-tip-distribution-program": "4R3gSG8BpU4t19KYj8CfnbtRpnT8gtk4dvTHxVRwc2r7", "harmonic-merkle-root-upload-authority": "8F4jGUmxF36vQ6yabnsxX6AQVXdKBhs8kGSUuRKSg8Xt", "harmonic-commission-bps": "500", + "harmonic-num-workers": "2", }) require.NoError(t, err) assert.True(t, s.enabled) @@ -163,6 +169,7 @@ func TestHarmonicFlagsOverrideTheConfigFile(t *testing.T) { assert.Equal(t, "4R3gSG8BpU4t19KYj8CfnbtRpnT8gtk4dvTHxVRwc2r7", s.tips.TipDistributionProgram.String()) assert.Equal(t, "8F4jGUmxF36vQ6yabnsxX6AQVXdKBhs8kGSUuRKSg8Xt", s.tips.MerkleRootUploadAuthority.String()) assert.Equal(t, uint16(500), s.tips.CommissionBps) + assert.Equal(t, 2, s.workers) // A flag beats the config file, including an explicit false. s, err = loadHarmonic(t, enabledHarmonic(map[string]string{"strategy": `"mrev"`}), map[string]string{"harmonic-strategy": "fba"}) diff --git a/config.example.toml b/config.example.toml index 2a8d4853..0994eebc 100644 --- a/config.example.toml +++ b/config.example.toml @@ -398,6 +398,9 @@ name = "mithril" merkle_root_upload_authority = "" # Validator commission on tips, in basis points (0-10000). commission_bps = 0 + # Threads executing block bundles that do not conflict on accounts in + # parallel (1-64). Conflicting bundles always execute in stream order. + num_workers = 8 # ============================================================================ # [consensus] - Alpenglow Consensus diff --git a/pkg/blockprod/bank_bundle.go b/pkg/blockprod/bank_bundle.go index 062e3289..c3ed7ac5 100644 --- a/pkg/blockprod/bank_bundle.go +++ b/pkg/blockprod/bank_bundle.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" - "github.com/Overclock-Validator/mithril/pkg/accounts" "github.com/Overclock-Validator/mithril/pkg/costmodel" "github.com/Overclock-Validator/mithril/pkg/replay" "github.com/Overclock-Validator/mithril/pkg/sealevel" @@ -36,30 +35,16 @@ type BundleResult struct { Err error } -// ForgeBundle forges a bundle all or nothing. Transactions execute in order -// against a scratch view of the bank, each seeing the writes of the ones -// before it, and are only applied once all have succeeded. Any failure, -// including a fees-only failure, discards the whole bundle. +// ForgeBundle forges a bundle all or nothing, under the bank lock throughout. +// Transactions execute in order against a scratch view of the bank, each +// seeing the writes of the ones before it, and are only applied once all have +// succeeded. Any failure, including a fees-only failure, discards the whole +// bundle. func (b *WorkingBank) ForgeBundle(in BundleInput) BundleResult { - txs := in.Transactions - if len(txs) == 0 { + p, prepared := b.PrepareBundle(in.Transactions, true) + if prepared.Result != ForgeAccepted { b.RebateSchedule(in.Reservation) - return BundleResult{Result: ForgeDroppedParse, Index: -1} - } - candidates := make([]forgeCandidate, len(txs)) - seen := make(map[[32]byte]struct{}, len(txs)) - for idx, tx := range txs { - c, result := b.prepareForge(tx, 0, nil) - if result != ForgeAccepted { - b.RebateSchedule(in.Reservation) - return BundleResult{Result: result, Index: idx} - } - if _, dup := seen[c.messageHash]; dup { - b.RebateSchedule(in.Reservation) - return BundleResult{Result: ForgeDroppedAlreadyProcessed, Index: idx} - } - seen[c.messageHash] = struct{}{} - candidates[idx] = c + return prepared } b.mu.Lock() @@ -69,9 +54,9 @@ func (b *WorkingBank) ForgeBundle(in BundleInput) BundleResult { if !b.accepting { return BundleResult{Result: ForgeDroppedNoLeader, Index: -1} } - requested := make([]costmodel.TransactionCost, len(candidates)) - sizes := make([]int, len(candidates)) - for idx, c := range candidates { + requested := make([]costmodel.TransactionCost, len(p.candidates)) + sizes := make([]int, len(p.candidates)) + for idx, c := range p.candidates { if result := b.statusLocked(c); result != ForgeAccepted { return BundleResult{Result: result, Index: idx} } @@ -94,36 +79,12 @@ func (b *WorkingBank) ForgeBundle(in BundleInput) BundleResult { committed := false defer func() { if !committed { - b.evictProgramCacheAdds(scratch) + evictProgramCacheAdds(scratch) } }() - outputs := make([]replay.LoadAndExecuteTransactionOutput, len(txs)) - actual := make([]costmodel.TransactionCost, len(txs)) - for idx, c := range candidates { - if err := b.preparer.PayerCanFund(scratch, c.execTx, nil); err != nil { - return BundleResult{Result: ForgeDroppedExecution, Index: idx, Err: err} - } - // Every output must outlive the next execution, so none may use the - // bank's per-transaction arena. - output := b.preparer.LoadAndExecute(replay.LoadAndExecuteTransactionInput{ - SlotCtx: scratch, - Transaction: c.execTx, - LeanResult: true, - SkipTimingMetrics: true, - }, nil) - if txErr := output.ProcessingResult.TransactionError; txErr != nil { - return BundleResult{Result: ForgeDroppedExecution, Index: idx, Err: transactionErr(txErr)} - } - if output.ExecCtx == nil { - return BundleResult{Result: ForgeDroppedExecution, Index: idx, Err: errors.New("missing execution context")} - } - txAccounts := &output.ExecCtx.TransactionContext.Accounts - if err := accounts.SetTransactionAccounts(scratch.Accounts, txAccounts.Accounts, txAccounts.Touched); err != nil { - return BundleResult{Result: ForgeDroppedExecution, Index: idx, Err: err} - } - execCU, loadedCost := actualExecutionUsage(output) - actual[idx] = candidates[idx].cost.WithActualUsage(execCU, loadedCost) - outputs[idx] = output + outputs, actual, result := b.executeBundleOn(scratch, p.candidates) + if result.Result != ForgeAccepted { + return result } if in.AdmitOnActual { if reason := b.costs.WouldExceedAll(actual); reason != costmodel.ExceedNone { @@ -134,31 +95,14 @@ func (b *WorkingBank) ForgeBundle(in BundleInput) BundleResult { return BundleResult{Result: ForgeDroppedExecution, Index: -1, Err: err} } committed = true - - if b.seenMessages == nil { - b.seenMessages = make(map[[32]byte]struct{}) - } - for idx, tx := range txs { - b.seenMessages[candidates[idx].messageHash] = struct{}{} - if outputs[idx].FeeInfo != nil { - b.txFees.Add(outputs[idx].FeeInfo) - } - b.forgedTxs = append(b.forgedTxs, tx) - b.numSigs += uint64(tx.Message.Header.NumRequiredSignatures) - // Equivalent to recording the requested cost and rebating it. - b.costs.Record(actual[idx]) - if flushed, batchBytes, didFlush := b.entries.appendSerialized(*tx, sizes[idx], sizes[idx]); didFlush { - b.entryHash = b.entries.CurrentEntryHash() - b.sink.OnEntryBatch(flushed, batchBytes) - } - } + b.recordForgedLocked(in.Transactions, p.candidates, outputs, actual) return BundleResult{Result: ForgeAccepted, Index: -1} } // evictProgramCacheAdds drops programs a discarded scratch execution loaded // into the shared program cache. Eviction only forces a reload from account // data, so it is always safe. -func (b *WorkingBank) evictProgramCacheAdds(scratch *sealevel.SlotCtx) { +func evictProgramCacheAdds(scratch *sealevel.SlotCtx) { for _, key := range scratch.TakeProgramCacheAdds() { if scratch.AccountsDb != nil { scratch.AccountsDb.RemoveProgramFromCache(key) diff --git a/pkg/blockprod/bank_parallel.go b/pkg/blockprod/bank_parallel.go new file mode 100644 index 00000000..eab5671a --- /dev/null +++ b/pkg/blockprod/bank_parallel.go @@ -0,0 +1,278 @@ +package blockprod + +import ( + "errors" + + "github.com/Overclock-Validator/mithril/pkg/accounts" + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/replay" + "github.com/Overclock-Validator/mithril/pkg/sealevel" + "github.com/gagliardetto/solana-go" +) + +// PreparedBundle is a block bundle checked for forging, with its address-table +// lookups resolved, so its account set is final. A bundle that is not revert +// protected is a single transaction that lands even if it fails, as long as +// the failure is one the protocol records. +type PreparedBundle struct { + txs []*solana.Transaction + candidates []forgeCandidate + revertProtected bool +} + +// PrepareBundle does the lock-free checks of a block bundle's transactions. +// Unless the result is ForgeAccepted, the bundle cannot be forged. +func (b *WorkingBank) PrepareBundle(txs []*solana.Transaction, revertProtected bool) (*PreparedBundle, BundleResult) { + if len(txs) == 0 || (!revertProtected && len(txs) != 1) { + return nil, BundleResult{Result: ForgeDroppedParse, Index: -1} + } + p := &PreparedBundle{txs: txs, candidates: make([]forgeCandidate, len(txs)), revertProtected: revertProtected} + seen := make(map[[32]byte]struct{}, len(txs)) + for idx, tx := range txs { + c, result := b.prepareForge(tx, 0, nil) + if result != ForgeAccepted { + return nil, BundleResult{Result: result, Index: idx} + } + if _, dup := seen[c.messageHash]; dup { + return nil, BundleResult{Result: ForgeDroppedAlreadyProcessed, Index: idx} + } + seen[c.messageHash] = struct{}{} + p.candidates[idx] = c + } + return p, BundleResult{Result: ForgeAccepted, Index: -1} +} + +// Accounts calls fn once for each account the bundle loads, with whether any +// of its transactions write-locks it. Bundles that share an account one of +// them writes conflict and must not execute concurrently. +func (p *PreparedBundle) Accounts(fn func(key solana.PublicKey, writable bool)) { + if len(p.candidates) == 1 { + lockedAccounts(&p.candidates[0].execTx.Message, fn) + return + } + writable := make(map[solana.PublicKey]bool) + var order []solana.PublicKey + for _, c := range p.candidates { + lockedAccounts(&c.execTx.Message, func(key solana.PublicKey, w bool) { + prev, ok := writable[key] + if !ok { + order = append(order, key) + } + writable[key] = prev || w + }) + } + for _, key := range order { + fn(key, writable[key]) + } +} + +// lockedAccounts calls fn for each account m loads, with whether it is write +// locked. Lookups in m must be resolved, if it has any. +func lockedAccounts(m *solana.Message, fn func(key solana.PublicKey, writable bool)) { + keys := m.AccountKeys + numStatic := len(keys) + if m.IsResolved() { + numStatic -= m.NumLookups() + } + h := m.Header + signers := int(h.NumRequiredSignatures) + writableSigners := max(signers-int(h.NumReadonlySignedAccounts), 0) + writableUnsigned := max(numStatic-signers-int(h.NumReadonlyUnsignedAccounts), 0) + writableLookups := m.AddressTableLookups.NumWritableLookups() + for i, key := range keys { + var writable bool + switch { + case i >= numStatic: + writable = i-numStatic < writableLookups + case i >= signers: + writable = i-signers < writableUnsigned + default: + writable = i < writableSigners + } + fn(key, writable) + } +} + +// ExecutedBundle is a bundle executed on its own view of the bank, not yet +// committed. Commit it with CommitBundle or drop it with Discard. +type ExecutedBundle struct { + prepared *PreparedBundle + scratch *sealevel.SlotCtx + outputs []replay.LoadAndExecuteTransactionOutput + actual []costmodel.TransactionCost + // result is set when the bundle cannot commit whatever the bank's state. + result BundleResult +} + +// Discard drops an executed bundle that will not be committed. +func (e *ExecutedBundle) Discard() { + if e.scratch != nil { + evictProgramCacheAdds(e.scratch) + e.scratch = nil + } +} + +// ExecuteBundle executes a prepared bundle on a scratch view of the bank, +// without the bank lock: nothing lands until CommitBundle. Bundles may execute +// concurrently only if their account sets do not conflict, and a bundle must +// not execute before a conflicting earlier one has committed. Anything that +// forges into the bank concurrently, such as the TPU, must respect the same +// account locks. +func (b *WorkingBank) ExecuteBundle(p *PreparedBundle) *ExecutedBundle { + e := &ExecutedBundle{prepared: p, result: BundleResult{Result: ForgeAccepted, Index: -1}} + b.mu.Lock() + accepting := b.accepting + for idx, c := range p.candidates { + if !accepting { + break + } + // Copies of one transaction always conflict on their fee payer, so a + // copy committed earlier is already recorded here. + if result := b.statusLocked(c); result != ForgeAccepted { + e.result = BundleResult{Result: result, Index: idx} + break + } + } + b.mu.Unlock() + if !accepting { + e.result = BundleResult{Result: ForgeDroppedNoLeader, Index: -1} + } + if e.result.Result != ForgeAccepted { + return e + } + + e.scratch = sealevel.NewScratchSlotCtx(b.slotCtx) + if p.revertProtected { + e.outputs, e.actual, e.result = b.executeBundleOn(e.scratch, p.candidates) + } else { + e.outputs, e.actual, e.result = b.executeStandaloneOn(e.scratch, p.candidates[0]) + } + if e.result.Result != ForgeAccepted { + e.Discard() + } + return e +} + +// executeBundleOn executes a revert-protected bundle's transactions in order +// on scratch, each seeing the writes of the ones before it. Any failure stops +// the bundle. +func (b *WorkingBank) executeBundleOn(scratch *sealevel.SlotCtx, candidates []forgeCandidate) ([]replay.LoadAndExecuteTransactionOutput, []costmodel.TransactionCost, BundleResult) { + outputs := make([]replay.LoadAndExecuteTransactionOutput, len(candidates)) + actual := make([]costmodel.TransactionCost, len(candidates)) + for idx, c := range candidates { + if err := b.preparer.PayerCanFund(scratch, c.execTx, nil); err != nil { + return nil, nil, BundleResult{Result: ForgeDroppedExecution, Index: idx, Err: err} + } + // Every output must outlive the next execution, so none may use an + // arena that is reset per transaction. + output := b.preparer.LoadAndExecute(replay.LoadAndExecuteTransactionInput{ + SlotCtx: scratch, + Transaction: c.execTx, + LeanResult: true, + SkipTimingMetrics: true, + }, nil) + if txErr := output.ProcessingResult.TransactionError; txErr != nil { + return nil, nil, BundleResult{Result: ForgeDroppedExecution, Index: idx, Err: transactionErr(txErr)} + } + if output.ExecCtx == nil { + return nil, nil, BundleResult{Result: ForgeDroppedExecution, Index: idx, Err: errors.New("missing execution context")} + } + txAccounts := &output.ExecCtx.TransactionContext.Accounts + if err := accounts.SetTransactionAccounts(scratch.Accounts, txAccounts.Accounts, txAccounts.Touched); err != nil { + return nil, nil, BundleResult{Result: ForgeDroppedExecution, Index: idx, Err: err} + } + execCU, loadedCost := actualExecutionUsage(output) + actual[idx] = c.cost.WithActualUsage(execCU, loadedCost) + outputs[idx] = output + } + return outputs, actual, BundleResult{Result: ForgeAccepted, Index: -1} +} + +// executeStandaloneOn executes a transaction that is not revert protected on +// scratch. A failed execution still commits if the protocol records the +// failure; CommitBundle decides. +func (b *WorkingBank) executeStandaloneOn(scratch *sealevel.SlotCtx, c forgeCandidate) ([]replay.LoadAndExecuteTransactionOutput, []costmodel.TransactionCost, BundleResult) { + if err := b.preparer.PayerCanFund(scratch, c.execTx, nil); err != nil { + return nil, nil, BundleResult{Result: ForgeDroppedExecution, Index: 0, Err: err} + } + output := b.preparer.LoadAndExecute(replay.LoadAndExecuteTransactionInput{ + SlotCtx: scratch, + Transaction: c.execTx, + LeanResult: true, + SkipTimingMetrics: true, + }, nil) + execCU, loadedCost := actualExecutionUsage(output) + return []replay.LoadAndExecuteTransactionOutput{output}, + []costmodel.TransactionCost{c.cost.WithActualUsage(execCU, loadedCost)}, + BundleResult{Result: ForgeAccepted, Index: -1} +} + +// CommitBundle lands an executed bundle under the bank lock, checking block +// limits against the units it actually used. Unless it returns ForgeAccepted, +// nothing from the bundle landed. +func (b *WorkingBank) CommitBundle(e *ExecutedBundle) BundleResult { + if e.result.Result != ForgeAccepted { + return e.result + } + committed := false + defer func() { + if !committed { + e.Discard() + } + }() + b.mu.Lock() + defer b.mu.Unlock() + if !b.accepting { + return BundleResult{Result: ForgeDroppedNoLeader, Index: -1} + } + p := e.prepared + if reason := b.costs.WouldExceedAll(e.actual); reason != costmodel.ExceedNone { + return BundleResult{Result: ForgeDroppedCost, Reason: reason, Index: -1} + } + sizes := make([]int, len(p.candidates)) + for idx, c := range p.candidates { + sizes[idx] = c.serializedSize + } + if b.entries.wouldOverflowBatch(sizes[0]) { + b.flushEntriesLocked() + } + if b.entries.wouldExceedSlotN(sizes) { + return BundleResult{Result: ForgeDroppedCost, Reason: costmodel.ExceedBatchBytes, Index: -1} + } + + if !p.revertProtected && e.outputs[0].ProcessingResult.TransactionError != nil { + feeInfo, err := replay.ApplyFailedTransaction(b.slotCtx, p.candidates[0].execTx, e.outputs[0]) + if err != nil { + return BundleResult{Result: ForgeDroppedExecution, Index: 0, Err: err} + } + e.outputs[0].FeeInfo = feeInfo + } else if err := replay.ApplySuccessfulTransactions(b.slotCtx, e.outputs); err != nil { + return BundleResult{Result: ForgeDroppedExecution, Index: -1, Err: err} + } + committed = true + b.recordForgedLocked(p.txs, p.candidates, e.outputs, e.actual) + return BundleResult{Result: ForgeAccepted, Index: -1} +} + +// recordForgedLocked records committed transactions: their statuses, fees, +// signatures and costs, and their entries, in order. +func (b *WorkingBank) recordForgedLocked(txs []*solana.Transaction, candidates []forgeCandidate, outputs []replay.LoadAndExecuteTransactionOutput, actual []costmodel.TransactionCost) { + if b.seenMessages == nil { + b.seenMessages = make(map[[32]byte]struct{}) + } + for idx, tx := range txs { + b.seenMessages[candidates[idx].messageHash] = struct{}{} + if outputs[idx].FeeInfo != nil { + b.txFees.Add(outputs[idx].FeeInfo) + } + b.forgedTxs = append(b.forgedTxs, tx) + b.numSigs += uint64(tx.Message.Header.NumRequiredSignatures) + // Equivalent to recording the requested cost and rebating it. + b.costs.Record(actual[idx]) + size := candidates[idx].serializedSize + if flushed, batchBytes, didFlush := b.entries.appendSerialized(*tx, size, size); didFlush { + b.entryHash = b.entries.CurrentEntryHash() + b.sink.OnEntryBatch(flushed, batchBytes) + } + } +} diff --git a/pkg/blockprod/bank_parallel_test.go b/pkg/blockprod/bank_parallel_test.go new file mode 100644 index 00000000..ec407b0e --- /dev/null +++ b/pkg/blockprod/bank_parallel_test.go @@ -0,0 +1,203 @@ +package blockprod + +import ( + "sync" + "testing" + + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// fundedPayer returns a new key funded with lamports in env. +func fundedPayer(t *testing.T, env *TestEnv, lamports uint64) solana.PrivateKey { + t.Helper() + key, err := solana.NewRandomPrivateKey() + require.NoError(t, err) + require.NoError(t, env.Fund(key.PublicKey(), lamports)) + return key +} + +// paidTransfer is a transfer that payer signs and pays for. +func paidTransfer(t *testing.T, payer solana.PrivateKey, to solana.PublicKey, lamports uint64) *solana.Transaction { + t.Helper() + tx, err := solana.NewTransaction([]solana.Instruction{transferIx(lamports, payer.PublicKey(), to)}, + txfixture.TestBlockhash(), solana.TransactionPayer(payer.PublicKey())) + require.NoError(t, err) + _, err = tx.Sign(func(solana.PublicKey) *solana.PrivateKey { return &payer }) + require.NoError(t, err) + return tx +} + +func forgeParallel(t *testing.T, bank *WorkingBank, revertProtected bool, txs ...*solana.Transaction) BundleResult { + t.Helper() + p, prepared := bank.PrepareBundle(txs, revertProtected) + if prepared.Result != ForgeAccepted { + return prepared + } + return bank.CommitBundle(bank.ExecuteBundle(p)) +} + +func TestParallelForgeMatchesForgeBundle(t *testing.T) { + funded, err := solana.NewRandomPrivateKey() + require.NoError(t, err) + fund := signedBundleTx(t, nil, transferIx(2_000_000, txfixture.PayerPubkey(), funded.PublicKey())) + spend := signedBundleTx(t, []solana.PrivateKey{funded}, transferIx(500_000, funded.PublicKey(), txfixture.DestPubkey())) + + serial := NewTestEnv(TestEnvConfig{}) + defer serial.Close() + require.Equal(t, ForgeAccepted, serial.Bank.ForgeBundle(BundleInput{Transactions: []*solana.Transaction{fund, spend}, AdmitOnActual: true}).Result) + parallel := NewTestEnv(TestEnvConfig{}) + defer parallel.Close() + require.Equal(t, ForgeAccepted, forgeParallel(t, parallel.Bank, true, fund, spend).Result) + + for _, key := range []solana.PublicKey{txfixture.PayerPubkey(), funded.PublicKey(), txfixture.DestPubkey()} { + assert.Equal(t, lamportsOf(t, serial, key), lamportsOf(t, parallel, key), "%s", key) + } + assert.Equal(t, serial.Bank.TxFeeAccumulator(), parallel.Bank.TxFeeAccumulator()) + assert.Equal(t, serial.Bank.NumSignatures(), parallel.Bank.NumSignatures()) + assert.Equal(t, serial.Bank.ForgedTransactions(), parallel.Bank.ForgedTransactions()) + assert.Equal(t, serial.Bank.CostTracker().BlockCost(), parallel.Bank.CostTracker().BlockCost()) + assert.Equal(t, serial.Bank.EntryBytes(), parallel.Bank.EntryBytes()) + + assert.Equal(t, ForgeDroppedAlreadyProcessed, forgeParallel(t, parallel.Bank, true, fund).Result, "a committed transaction is recorded") +} + +func TestParallelForgeRevertProtectedFailureLandsNothing(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + payerBefore := lamportsOf(t, env, txfixture.PayerPubkey()) + destBefore := lamportsOf(t, env, txfixture.DestPubkey()) + ok := signedBundleTx(t, nil, transferIx(1, txfixture.PayerPubkey(), txfixture.DestPubkey())) + fails := signedBundleTx(t, nil, transferIx(20_000_000_000, txfixture.PayerPubkey(), txfixture.DestPubkey())) + got := forgeParallel(t, env.Bank, true, ok, fails) + assert.Equal(t, ForgeDroppedExecution, got.Result) + assert.Equal(t, 1, got.Index) + requireNothingLanded(t, env, payerBefore, destBefore) +} + +func TestParallelForgeStandaloneFailureLandsAndPays(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + payerBefore := lamportsOf(t, env, txfixture.PayerPubkey()) + destBefore := lamportsOf(t, env, txfixture.DestPubkey()) + fails := signedBundleTx(t, nil, transferIx(20_000_000_000, txfixture.PayerPubkey(), txfixture.DestPubkey())) + require.Equal(t, ForgeAccepted, forgeParallel(t, env.Bank, false, fails).Result) + assert.Equal(t, destBefore, lamportsOf(t, env, txfixture.DestPubkey())) + assert.Equal(t, payerBefore-5_000, lamportsOf(t, env, txfixture.PayerPubkey()), "the failure is recorded and pays its fee") + assert.Equal(t, []*solana.Transaction{fails}, env.Bank.ForgedTransactions()) + + _, prepared := env.Bank.PrepareBundle([]*solana.Transaction{fails, fails}, false) + assert.Equal(t, ForgeDroppedParse, prepared.Result, "a standalone bundle is one transaction") +} + +func TestParallelForgeExecutesDisjointBundlesConcurrently(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + const n = 8 + payers := make([]solana.PrivateKey, n) + dests := make([]solana.PublicKey, n) + prepared := make([]*PreparedBundle, n) + for i := range payers { + payers[i] = fundedPayer(t, env, 1_000_000_000) + dests[i] = solana.NewWallet().PublicKey() + p, r := env.Bank.PrepareBundle([]*solana.Transaction{paidTransfer(t, payers[i], dests[i], uint64(1_000_000+i))}, true) + require.Equal(t, ForgeAccepted, r.Result) + prepared[i] = p + } + executed := make([]*ExecutedBundle, n) + var wg sync.WaitGroup + for i := range prepared { + wg.Add(1) + go func() { + defer wg.Done() + executed[i] = env.Bank.ExecuteBundle(prepared[i]) + }() + } + wg.Wait() + // Disjoint bundles commit in any order. + for i := n - 1; i >= 0; i-- { + require.Equal(t, ForgeAccepted, env.Bank.CommitBundle(executed[i]).Result) + } + for i := range payers { + assert.Equal(t, uint64(1_000_000+i), lamportsOf(t, env, dests[i])) + assert.Equal(t, uint64(1_000_000_000-1_000_000-i-5_000), lamportsOf(t, env, payers[i].PublicKey())) + } + assert.Len(t, env.Bank.ForgedTransactions(), n) +} + +func TestParallelForgeAccountSet(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + payer := fundedPayer(t, env, 1_000_000_000) + other := solana.NewWallet().PublicKey() + p, r := env.Bank.PrepareBundle([]*solana.Transaction{ + paidTransfer(t, payer, other, 1), + signedBundleTx(t, nil, transferIx(1, txfixture.PayerPubkey(), txfixture.DestPubkey())), + }, true) + require.Equal(t, ForgeAccepted, r.Result) + got := map[solana.PublicKey]bool{} + p.Accounts(func(key solana.PublicKey, writable bool) { + _, dup := got[key] + require.False(t, dup, "each account once") + got[key] = writable + }) + assert.Equal(t, map[solana.PublicKey]bool{ + payer.PublicKey(): true, + other: true, + txfixture.PayerPubkey(): true, + txfixture.DestPubkey(): true, + solana.SystemProgramID: false, + }, got) + + // Loaded addresses are part of the set, with their lookup's lock. + withLookupSlotHashes(t, env) + addLookupTable(t, env, ActiveLookupTable(lookupTestTableKey, txfixture.DestPubkey())) + v0, _ := lookupTransfer(t, 1) + p, r = env.Bank.PrepareBundle([]*solana.Transaction{v0}, false) + require.Equal(t, ForgeAccepted, r.Result) + got = map[solana.PublicKey]bool{} + p.Accounts(func(key solana.PublicKey, writable bool) { got[key] = writable }) + assert.Equal(t, map[solana.PublicKey]bool{ + txfixture.PayerPubkey(): true, + solana.SystemProgramID: false, + txfixture.DestPubkey(): true, + }, got) +} + +func TestParallelForgeCommitChecks(t *testing.T) { + t.Run("a frozen bank lands nothing", func(t *testing.T) { + env := NewTestEnv(TestEnvConfig{}) + defer env.Close() + payerBefore := lamportsOf(t, env, txfixture.PayerPubkey()) + destBefore := lamportsOf(t, env, txfixture.DestPubkey()) + p, _ := env.Bank.PrepareBundle([]*solana.Transaction{mustSignedTransfer(t, 1)}, true) + executed := env.Bank.ExecuteBundle(p) + env.Bank.Freeze() + assert.Equal(t, ForgeDroppedNoLeader, env.Bank.CommitBundle(executed).Result) + requireNothingLanded(t, env, payerBefore, destBefore) + + p, _ = env.Bank.PrepareBundle([]*solana.Transaction{mustSignedTransfer(t, 2)}, true) + assert.Equal(t, ForgeDroppedNoLeader, env.Bank.CommitBundle(env.Bank.ExecuteBundle(p)).Result) + }) + t.Run("block limits are checked against actual usage at commit", func(t *testing.T) { + probe := NewTestEnv(TestEnvConfig{}) + defer probe.Close() + require.Equal(t, ForgeAccepted, forgeParallel(t, probe.Bank, true, mustSignedTransfer(t, 1)).Result) + limits := costmodel.DefaultLimits() + limits.BlockCost = probe.Bank.CostTracker().BlockCost() + 1 + env := NewTestEnv(TestEnvConfig{Limits: limits}) + defer env.Close() + // Both execute before either commits; only the first fits. + first, _ := env.Bank.PrepareBundle([]*solana.Transaction{mustSignedTransfer(t, 1)}, true) + second, _ := env.Bank.PrepareBundle([]*solana.Transaction{mustSignedTransfer(t, 2)}, true) + firstExec, secondExec := env.Bank.ExecuteBundle(first), env.Bank.ExecuteBundle(second) + require.Equal(t, ForgeAccepted, env.Bank.CommitBundle(firstExec).Result) + got := env.Bank.CommitBundle(secondExec) + assert.Equal(t, ForgeDroppedCost, got.Result) + assert.Equal(t, costmodel.ExceedBlockCost, got.Reason) + assert.Len(t, env.Bank.ForgedTransactions(), 1) + }) +} diff --git a/pkg/blockprod/testenv.go b/pkg/blockprod/testenv.go index 43e34117..c6aea415 100644 --- a/pkg/blockprod/testenv.go +++ b/pkg/blockprod/testenv.go @@ -101,6 +101,16 @@ func (e *TestEnv) Close() { } } +// Fund creates a system account at key holding lamports. +func (e *TestEnv) Fund(key solana.PublicKey, lamports uint64) error { + return e.SlotCtx.SetAccount(key, &accounts.Account{ + Key: key, + Lamports: lamports, + Owner: addresses.SystemProgramAddr, + RentEpoch: math.MaxUint64, + }) +} + func seedTestAccounts(mem accounts.MemAccounts) { systemAcct := &accounts.Account{ Key: addresses.SystemProgramAddr, diff --git a/pkg/config/config.go b/pkg/config/config.go index 2d0820a1..949055fa 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -30,6 +30,7 @@ func ApplyDefaults(v *viper.Viper) { v.SetDefault("harmonic.tls_cert_verify", true) v.SetDefault("harmonic.strategy", "fba") v.SetDefault("harmonic.commission_bps", 0) + v.SetDefault("harmonic.num_workers", 8) } // LedgerConfig holds ledger-related configuration (matches Firedancer [ledger] section) diff --git a/pkg/harmonic/executor.go b/pkg/harmonic/executor.go index a2be6ed5..8df39f8d 100644 --- a/pkg/harmonic/executor.go +++ b/pkg/harmonic/executor.go @@ -100,7 +100,9 @@ type SlotReport struct { // Executed counts bundles that landed; Reverted, the ones that executed // and landed nothing; Rejected, bundles refused without executing. Executed, Reverted, Rejected uint64 - Crank CrankResult + // MaxParallel is the most bundles that executed at once. + MaxParallel int + Crank CrankResult // FirstArrival is how long after the bank opened the slot's first block // bundle arrived, admitted or not, if Arrived. FirstArrival time.Duration @@ -139,6 +141,9 @@ type ExecutorConfig struct { Crank CrankBuilder // OnReport, if set, receives each slot's report. OnReport func(SlotReport) + // Workers execute block bundles that do not conflict in parallel. Zero + // means one. + Workers int } type hookMessage struct { @@ -163,6 +168,15 @@ type slotState struct { crankDone bool report SlotReport thresholdTimer *time.Timer + + sched schedule + // running counts bundles executing. After a fallback the TPU opens only + // once none are, since they keep their account locks until they finish. + running int + tpuOnDrain bool + // outstanding counts admitted bundles that have not finished; the leader + // loop reads it. + outstanding atomic.Int64 } // Executor runs each leader slot's Harmonic block: it announces the slot, @@ -177,9 +191,20 @@ type Executor struct { hooks []hookMessage wake chan struct{} - // running is the slot whose block is executing, or 0. The leader loop + workers int + work chan *bundleNode + done chan *bundleNode + // busy counts bundles handed to workers and not yet back. + busy int + // beforeExecute, if set, runs on the worker before each bundle executes. + // Tests use it to hold workers. + beforeExecute func(seq uint64) + // pending counts admitted bundles of any slot that have not finished. + pending atomic.Int64 + + // holding is the slot whose block is running, or nil. The leader loop // holds that slot open while block bundles remain. - running atomic.Uint64 + holding atomic.Pointer[slotState] released chan struct{} } @@ -189,16 +214,25 @@ var ( ) func NewExecutor(cfg ExecutorConfig) *Executor { - return &Executor{cfg: cfg, now: time.Now, wake: make(chan struct{}, 1), released: make(chan struct{}, 1)} + workers := max(cfg.Workers, 1) + return &Executor{ + cfg: cfg, + now: time.Now, + wake: make(chan struct{}, 1), + workers: workers, + work: make(chan *bundleNode, workers), + done: make(chan *bundleNode, workers), + released: make(chan struct{}, 1), + } } // HoldSlot keeps a running block's slot open while any block bundle received -// for it has yet to execute: the slot ends once it is past its end and the -// buffer is empty. It runs on the leader loop's goroutine. A bundle already -// executing when it returns false still lands, since freezing the bank waits -// for it. +// for it has yet to finish: the slot ends once it is past its end and the +// buffer is empty. It runs on the leader loop's goroutine. func (x *Executor) HoldSlot(slot uint64) bool { - return x.running.Load() == slot && (len(x.cfg.Events) > 0 || x.cfg.Pipeline.InFlight()) + st := x.holding.Load() + return st != nil && st.slot == slot && + (len(x.cfg.Events) > 0 || x.cfg.Pipeline.InFlight() || st.outstanding.Load() > 0) } // Released is signaled when the running block's slot may end. @@ -215,15 +249,15 @@ func (x *Executor) signalRelease() { // endHold stops holding st's slot open. func (x *Executor) endHold(st *slotState) { - if x.running.CompareAndSwap(st.slot, 0) { + if x.holding.CompareAndSwap(st, nil) { x.signalRelease() } } // releaseIfDrained lets the leader loop end a running block's slot as soon as -// it is past its end and every block bundle received has executed. +// it is past its end and every block bundle received has finished. func (x *Executor) releaseIfDrained(st *slotState) { - if st != nil && x.running.Load() == st.slot && !x.now().Before(st.end) && !x.HoldSlot(st.slot) { + if st != nil && x.holding.Load() == st && !x.now().Before(st.end) && !x.HoldSlot(st.slot) { x.signalRelease() } } @@ -286,8 +320,12 @@ func (x *Executor) post(msg hookMessage) { } } -// Run processes slots and block events until ctx ends. +// Run processes slots and block events until ctx ends. Bundles execute on +// worker goroutines; everything else, including commits, happens here. func (x *Executor) Run(ctx context.Context) { + for range x.workers { + go x.worker(ctx) + } var st *slotState for { select { @@ -301,6 +339,29 @@ func (x *Executor) Run(ctx context.Context) { st = x.drainHooks(st) x.handleEvent(st, e) x.releaseIfDrained(st) + case n := <-x.done: + st = x.drainHooks(st) + x.complete(n) + x.releaseIfDrained(st) + } + } +} + +func (x *Executor) worker(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case n := <-x.work: + if x.beforeExecute != nil { + x.beforeExecute(n.seq) + } + n.executed = n.st.bank.ExecuteBundle(n.prepared) + select { + case x.done <- n: + case <-ctx.Done(): + return + } } } } @@ -356,13 +417,14 @@ func (x *Executor) finish(st *slotState, reason string, aborted bool) { st.thresholdTimer.Stop() } x.endHold(st) + x.dropPending(st) st.report.Mode = st.mode st.report.Aborted = aborted st.report.Reason = reason st.mode = ModeDone r := st.report - mlog.Log.Infof("Harmonic slot %d: mode=%s flags=%s executed=%d reverted=%d rejected=%d crank=%s aborted=%t%s", - r.Slot, r.Mode, r.Flags, r.Executed, r.Reverted, r.Rejected, r.Crank, r.Aborted, reasonSuffix(r.Reason)) + mlog.Log.Infof("Harmonic slot %d: mode=%s flags=%s executed=%d reverted=%d rejected=%d parallel=%d crank=%s aborted=%t%s", + r.Slot, r.Mode, r.Flags, r.Executed, r.Reverted, r.Rejected, r.MaxParallel, r.Crank, r.Aborted, reasonSuffix(r.Reason)) if x.cfg.OnReport != nil { x.cfg.OnReport(r) } @@ -392,7 +454,7 @@ func (x *Executor) handleEvent(st *slotState, e BlockEvent) { if !x.admits(st, e.Slot, e.Seq) { return } - x.executeBundle(st, e) + x.admitBundle(st, e) } } @@ -424,7 +486,9 @@ func (x *Executor) admits(st *slotState, slot, seq uint64) bool { return true } -func (x *Executor) executeBundle(st *slotState, e BundleEvent) { +// admitBundle schedules a block bundle. The slot's first one waits for the +// tip crank, which lands before any block bundle executes. +func (x *Executor) admitBundle(st *slotState, e BundleEvent) { if st.mode == ModeUndecided && !x.now().Before(st.threshold) { st.report.Rejected++ x.fallback(st, FlagTimeout) @@ -436,31 +500,99 @@ func (x *Executor) executeBundle(st *slotState, e BundleEvent) { return } st.mode = ModeHarmonic - x.running.Store(st.slot) + x.holding.Store(st) + } + prepared, result := st.bank.PrepareBundle(e.Transactions, e.RevertProtected) + if result.Result != blockprod.ForgeAccepted { + // It cannot execute: unparsable, already processed, breaking the + // account lock rules, or with lookups that do not resolve. + st.report.Rejected++ + x.stop(st, FlagFailed) + return } + st.outstanding.Add(1) + x.pending.Add(1) + st.sched.add(&bundleNode{st: st, seq: e.Seq, prepared: prepared}, prepared.Accounts) + x.dispatch(st) +} - var result blockprod.ForgeResult - if e.RevertProtected { - result = st.bank.ForgeBundle(blockprod.BundleInput{Transactions: e.Transactions, AdmitOnActual: true}).Result - } else { - result, _ = st.bank.ForgeStandaloneTransaction(e.Transactions[0], 0) +// dispatch hands ready bundles to idle workers. +func (x *Executor) dispatch(st *slotState) { + for !st.stopped && x.busy < x.workers { + n := st.sched.next() + if n == nil { + return + } + x.busy++ + st.running++ + st.report.MaxParallel = max(st.report.MaxParallel, st.running) + x.work <- n } - switch result { +} + +// complete commits a bundle a worker executed. Bundles commit in the order +// they finish, which keeps stream order for any two that conflict. +func (x *Executor) complete(n *bundleNode) { + x.busy-- + st := n.st + st.running-- + if st.mode == ModeDone { + // The slot is over; its bank no longer takes bundles. + n.executed.Discard() + x.finishBundle(n) + return + } + result := st.bank.CommitBundle(n.executed) + n.executed = nil + switch result.Result { case blockprod.ForgeAccepted: st.report.Executed++ + x.finishBundle(n) case blockprod.ForgeDroppedNoLeader: // The bank froze: the slot is over. st.report.Rejected++ + x.finishBundle(n) st.stopped = true x.endHold(st) + x.dropPending(st) case blockprod.ForgeDroppedCost: - // Execution is serial, so nothing in flight can free room. + if st.running > 0 && !st.stopped { + // What is executing may still free room. + st.sched.retryLater(n) + break + } st.report.Rejected++ + x.finishBundle(n) x.stop(st, FlagFailed) default: st.report.Reverted++ + x.finishBundle(n) x.stop(st, FlagReverted) } + if !st.stopped { + st.sched.requeueRetries() + } + if st.running == 0 && st.tpuOnDrain { + st.tpuOnDrain = false + st.bank.AllowTPU() + } + x.dispatch(st) +} + +// finishBundle releases a bundle that will not execute again. +func (x *Executor) finishBundle(n *bundleNode) { + n.st.sched.finish(n) + n.st.outstanding.Add(-1) + x.pending.Add(-1) +} + +// dropPending drops the slot's bundles that are not executing. +func (x *Executor) dropPending(st *slotState) { + if dropped := int64(st.sched.close()); dropped > 0 { + st.report.Rejected += uint64(dropped) + st.outstanding.Add(-dropped) + x.pending.Add(-dropped) + } } // runCrank lands the slot's tip crank before its first block bundle. A crank @@ -496,15 +628,24 @@ func (x *Executor) stop(st *slotState, flag EndFlags) { st.stopped = true st.report.Flags |= flag x.endHold(st) + x.dropPending(st) if st.mode == ModeUndecided || (st.mode == ModeHarmonic && x.now().Before(st.end)) { x.fallback(st, 0) } } +// fallback switches the slot to local scheduling. Bundles still executing +// keep their account locks until they finish, and the TPU ignores them, so it +// opens only then. func (x *Executor) fallback(st *slotState, flag EndFlags) { st.mode = ModeFallback st.stopped = true st.report.Flags |= flag x.endHold(st) + x.dropPending(st) + if st.running > 0 { + st.tpuOnDrain = true + return + } st.bank.AllowTPU() } diff --git a/pkg/harmonic/executor_parallel_test.go b/pkg/harmonic/executor_parallel_test.go new file mode 100644 index 00000000..72b6fbe1 --- /dev/null +++ b/pkg/harmonic/executor_parallel_test.go @@ -0,0 +1,240 @@ +package harmonic + +import ( + "math/rand/v2" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/blockprod" + "github.com/Overclock-Validator/mithril/pkg/costmodel" + "github.com/Overclock-Validator/mithril/pkg/tpu/txfixture" + "github.com/gagliardetto/solana-go" + "github.com/gagliardetto/solana-go/programs/system" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// newKeys returns n fresh keys, funded with lamports in each env given. +func newKeys(t *testing.T, n int, lamports uint64, envs ...*blockprod.TestEnv) []solana.PrivateKey { + t.Helper() + keys := make([]solana.PrivateKey, n) + for i := range keys { + key, err := solana.NewRandomPrivateKey() + require.NoError(t, err) + keys[i] = key + for _, env := range envs { + require.NoError(t, env.Fund(key.PublicKey(), lamports)) + } + } + return keys +} + +// transfers is a transaction of transfers, each from[i] to to[i], paid by +// from[0] and signed by every sender. +func transfers(t *testing.T, from []solana.PrivateKey, to []solana.PublicKey, lamports uint64) *solana.Transaction { + t.Helper() + ixs := make([]solana.Instruction, len(from)) + for i := range from { + ixs[i] = system.NewTransferInstruction(lamports, from[i].PublicKey(), to[i]).Build() + } + tx, err := solana.NewTransaction(ixs, txfixture.TestBlockhash(), solana.TransactionPayer(from[0].PublicKey())) + require.NoError(t, err) + _, err = tx.Sign(func(pub solana.PublicKey) *solana.PrivateKey { + for i := range from { + if from[i].PublicKey() == pub { + return &from[i] + } + } + return nil + }) + require.NoError(t, err) + return tx +} + +func pay(t *testing.T, from solana.PrivateKey, to solana.PublicKey, lamports uint64) *solana.Transaction { + return transfers(t, []solana.PrivateKey{from}, []solana.PublicKey{to}, lamports) +} + +func TestExecutorRunsDisjointBundlesInParallel(t *testing.T) { + entered := make(chan uint64, 8) + release := make(chan struct{}) + // A fifth worker is free for bundle 5, so only its dependency holds it. + h := startExecutorWith(t, ExecutorConfig{Workers: 5}, true, func(seq uint64) { + entered <- seq + <-release + }) + env := gatedBank(t, costmodel.Limits{}) + payers := newKeys(t, 4, 1_000_000_000, env) + h.startSlot(10, env, time.Hour) + for i, payer := range payers { + h.send(t, bundleEvent(10, uint64(i+1), false, pay(t, payer, solana.NewWallet().PublicKey(), 1_000_000))) + } + // Conflicts with bundle 1, so it waits for it. + h.send(t, bundleEvent(10, 5, false, pay(t, payers[0], solana.NewWallet().PublicKey(), 1_000_000))) + + got := map[uint64]bool{} + for range 4 { + select { + case seq := <-entered: + got[seq] = true + case <-time.After(2 * time.Second): + t.Fatal("disjoint bundles did not execute at once") + } + } + assert.Equal(t, map[uint64]bool{1: true, 2: true, 3: true, 4: true}, got) + select { + case seq := <-entered: + t.Fatalf("bundle %d started before the bundle it conflicts with finished", seq) + case <-time.After(20 * time.Millisecond): + } + close(release) + h.sync(t) + assert.Len(t, env.Bank.ForgedTransactions(), 5) + report := h.end(t, 10) + assert.Equal(t, uint64(5), report.Executed) + assert.Equal(t, 4, report.MaxParallel) + assert.Zero(t, report.Flags) +} + +func TestExecutorKeepsConflictingBundlesInStreamOrder(t *testing.T) { + // The first link is held until every bundle has arrived, so later links + // would run ahead of it if dependencies were not kept. + release := make(chan struct{}) + h := startExecutorWith(t, ExecutorConfig{Workers: 8}, true, func(seq uint64) { + if seq == 1 { + <-release + } + }) + env := gatedBank(t, costmodel.Limits{}) + funder := newKeys(t, 1, 1_000_000_000, env)[0] + // Each link starts unfunded and pays its fee from the transfer before it, + // so a link that ran early would revert and stop the block. + chain := newKeys(t, 4, 0) + others := newKeys(t, 4, 1_000_000_000, env) + h.startSlot(10, env, time.Hour) + seq := uint64(0) + next := func() uint64 { seq++; return seq } + var ordered []*solana.Transaction + link := pay(t, funder, chain[0].PublicKey(), 100_000_000) + ordered = append(ordered, link) + h.send(t, bundleEvent(10, next(), true, link)) + for i := 1; i < len(chain); i++ { + h.send(t, bundleEvent(10, next(), false, pay(t, others[i], solana.NewWallet().PublicKey(), 1_000))) + link := pay(t, chain[i-1], chain[i].PublicKey(), uint64(100_000_000-i*1_000_000)) + ordered = append(ordered, link) + h.send(t, bundleEvent(10, next(), true, link)) + } + close(release) + h.sync(t) + report := h.end(t, 10) + assert.Zero(t, report.Reverted) + assert.Equal(t, seq, report.Executed) + + position := map[*solana.Transaction]int{} + for i, tx := range env.Bank.ForgedTransactions() { + position[tx] = i + } + for i := 1; i < len(ordered); i++ { + assert.Less(t, position[ordered[i-1]], position[ordered[i]], "link %d lands after link %d", i, i-1) + } +} + +func TestExecutorRevertKeepsInFlightBundlesAndOpensTheTPUAfterThem(t *testing.T) { + release := make(chan struct{}) + h := startExecutorWith(t, ExecutorConfig{Workers: 2}, true, func(seq uint64) { + if seq == 1 { + <-release + } + }) + env := gatedBank(t, costmodel.Limits{}) + keys := newKeys(t, 2, 1_000_000_000, env) + h.startSlot(10, env, time.Hour) + inFlight := pay(t, keys[0], solana.NewWallet().PublicKey(), 1_000) + h.send(t, bundleEvent(10, 1, false, inFlight)) + h.send(t, bundleEvent(10, 2, true, pay(t, keys[1], solana.NewWallet().PublicKey(), 5_000_000_000))) + h.send(t, bundleEvent(10, 3, false, pay(t, keys[1], solana.NewWallet().PublicKey(), 1_000))) + require.Eventually(t, func() bool { return h.exec.pending.Load() == 1 }, 2*time.Second, time.Millisecond, + "the revert stops the block and drops what has not started") + assert.False(t, env.Bank.TPUAllowed(), "the TPU waits for the bundle still executing") + + close(release) + h.sync(t) + assert.True(t, env.Bank.TPUAllowed()) + assert.Equal(t, []*solana.Transaction{inFlight}, env.Bank.ForgedTransactions(), "a disjoint bundle already executing lands") + report := h.end(t, 10) + assert.Equal(t, ModeFallback, report.Mode) + assert.Equal(t, FlagReverted, report.Flags) + assert.Equal(t, uint64(1), report.Executed) + assert.Equal(t, uint64(1), report.Reverted) + assert.Equal(t, uint64(1), report.Rejected) +} + +// TestExecutorParallelMatchesSerial runs one random block of transfers among +// a few accounts with one worker and with eight. Committed state must match, +// and any two bundles that share a written account land in stream order. +func TestExecutorParallelMatchesSerial(t *testing.T) { + serialEnv, parallelEnv := gatedBank(t, costmodel.Limits{}), gatedBank(t, costmodel.Limits{}) + keys := newKeys(t, 6, 1_000_000_000, serialEnv, parallelEnv) + rng := rand.New(rand.NewPCG(1, 2)) + var bundles []BundleEvent + for seq := uint64(1); seq <= 120; seq++ { + perm := rng.Perm(len(keys)) + a, b, c := keys[perm[0]], keys[perm[1]], keys[perm[2]] + amount := uint64(1 + rng.IntN(1_000)) + if rng.IntN(3) == 0 { + tx := transfers(t, []solana.PrivateKey{a, b}, []solana.PublicKey{b.PublicKey(), c.PublicKey()}, amount) + bundles = append(bundles, bundleEvent(10, seq, true, tx)) + } else { + bundles = append(bundles, bundleEvent(10, seq, false, pay(t, a, b.PublicKey(), amount))) + } + } + run := func(env *blockprod.TestEnv, workers int) SlotReport { + h := startExecutorWith(t, ExecutorConfig{Workers: workers}, true, nil) + h.startSlot(10, env, time.Hour) + for _, e := range bundles { + h.send(t, e) + } + h.sync(t) + return h.end(t, 10) + } + serial, parallel := run(serialEnv, 1), run(parallelEnv, 8) + require.Equal(t, uint64(len(bundles)), serial.Executed) + require.Equal(t, uint64(len(bundles)), parallel.Executed) + for _, key := range keys { + assert.Equal(t, balance(t, serialEnv, key.PublicKey()), balance(t, parallelEnv, key.PublicKey())) + } + + position := map[solana.Signature]int{} + for i, tx := range parallelEnv.Bank.ForgedTransactions() { + position[tx.Signatures[0]] = i + } + writes := func(e BundleEvent) map[solana.PublicKey]bool { + w := map[solana.PublicKey]bool{} + for _, key := range e.Transactions[0].Message.AccountKeys { + if ok, _ := e.Transactions[0].Message.IsWritable(key); ok { + w[key] = true + } + } + return w + } + for i := range bundles { + wi := writes(bundles[i]) + for j := i + 1; j < len(bundles); j++ { + for key := range writes(bundles[j]) { + if wi[key] { + require.Less(t, position[bundles[i].Transactions[0].Signatures[0]], position[bundles[j].Transactions[0].Signatures[0]], + "bundles %d and %d share %s", i+1, j+1, key) + break + } + } + } + } + t.Logf("max parallel: %d", parallel.MaxParallel) +} + +func balance(t *testing.T, env *blockprod.TestEnv, key solana.PublicKey) uint64 { + t.Helper() + acct, err := env.SlotCtx.GetAccount(key) + require.NoError(t, err) + return acct.Lamports +} diff --git a/pkg/harmonic/executor_test.go b/pkg/harmonic/executor_test.go index f13d5cf9..221ff052 100644 --- a/pkg/harmonic/executor_test.go +++ b/pkg/harmonic/executor_test.go @@ -81,16 +81,20 @@ type executorHarness struct { } func startExecutor(t *testing.T, healthy bool, crank CrankBuilder) *executorHarness { + t.Helper() + return startExecutorWith(t, ExecutorConfig{Crank: crank}, healthy, nil) +} + +// startExecutorWith runs an executor configured by cfg, with the harness's +// fakes. beforeExecute, if set, runs on a worker before each bundle executes. +func startExecutorWith(t *testing.T, cfg ExecutorConfig, healthy bool, beforeExecute func(seq uint64)) *executorHarness { t.Helper() be := &fakeBlockEngine{healthy: healthy} h := &executorHarness{be: be, pipeline: &fakePipeline{be: be}, events: make(chan BlockEvent), reports: make(chan SlotReport, 4)} - h.exec = NewExecutor(ExecutorConfig{ - BlockEngine: be, - Pipeline: h.pipeline, - Events: h.events, - Crank: crank, - OnReport: func(r SlotReport) { h.reports <- r }, - }) + cfg.BlockEngine, cfg.Pipeline, cfg.Events = be, h.pipeline, h.events + cfg.OnReport = func(r SlotReport) { h.reports <- r } + h.exec = NewExecutor(cfg) + h.exec.beforeExecute = beforeExecute ctx, cancel := context.WithCancel(context.Background()) done := make(chan struct{}) go func() { @@ -126,10 +130,12 @@ func (h *executorHarness) send(t *testing.T, e BlockEvent) { } } -// sync waits until the executor has handled everything sent so far. +// sync waits until the executor has handled everything sent so far, +// including the bundles its workers execute. func (h *executorHarness) sync(t *testing.T) { t.Helper() h.send(t, StreamLostEvent{Slot: 1 << 63}) + require.Eventually(t, func() bool { return h.exec.pending.Load() == 0 }, 2*time.Second, time.Millisecond, "bundles still executing") } func (h *executorHarness) end(t *testing.T, slot uint64) SlotReport { @@ -273,6 +279,7 @@ func TestExecutorStopTriggers(t *testing.T) { for _, e := range tc.events(t) { h.send(t, e) } + h.sync(t) later := transfer(t, 4) h.send(t, bundleEvent(10, 99, false, later)) h.sync(t) diff --git a/pkg/harmonic/metrics.go b/pkg/harmonic/metrics.go index 0e12bd30..13b2a3f9 100644 --- a/pkg/harmonic/metrics.go +++ b/pkg/harmonic/metrics.go @@ -107,4 +107,7 @@ func reportSlot(r SlotReport) { if r.Arrived { _ = statsd.Duration(statsd.HarmonicFirstBundleDelay, r.FirstArrival, nil) } + if r.MaxParallel > 0 { + _ = statsd.Timing(statsd.HarmonicSlotMaxParallel, uint64(r.MaxParallel), nil) + } } diff --git a/pkg/harmonic/metrics_test.go b/pkg/harmonic/metrics_test.go index 9b2904cd..f3435491 100644 --- a/pkg/harmonic/metrics_test.go +++ b/pkg/harmonic/metrics_test.go @@ -70,9 +70,10 @@ func TestReportSlotExportsTheSlot(t *testing.T) { revertedBundles := metricDelta(t, "harmonic_slot_bundles_total", "result", "reverted") cranks := metricDelta(t, "harmonic_tip_cranks_total", "result", "landed") arrivals := metricDelta(t, "harmonic_first_bundle_delay_seconds") + parallel := metricDelta(t, "harmonic_slot_max_parallel") reportSlot(SlotReport{Mode: ModeHarmonic, Flags: FlagReverted | FlagTimeout, Executed: 3, Reverted: 1, - Crank: CrankLanded, Arrived: true, FirstArrival: 30 * time.Millisecond}) + Crank: CrankLanded, Arrived: true, FirstArrival: 30 * time.Millisecond, MaxParallel: 3}) assert.Equal(t, 1.0, slots()) assert.Equal(t, 1.0, reverted()) assert.Equal(t, 1.0, timeout()) @@ -81,6 +82,7 @@ func TestReportSlotExportsTheSlot(t *testing.T) { assert.Equal(t, 1.0, revertedBundles()) assert.Equal(t, 1.0, cranks()) assert.Equal(t, 1.0, arrivals()) + assert.Equal(t, 1.0, parallel()) // A slot that fell back before any bundle has no crank or arrival. reportSlot(SlotReport{Mode: ModeFallback, Flags: FlagUnavailable}) diff --git a/pkg/harmonic/schedule.go b/pkg/harmonic/schedule.go new file mode 100644 index 00000000..fa5c390b --- /dev/null +++ b/pkg/harmonic/schedule.go @@ -0,0 +1,157 @@ +package harmonic + +import ( + "container/heap" + + "github.com/Overclock-Validator/mithril/pkg/blockprod" + "github.com/gagliardetto/solana-go" +) + +// bundleNode is an admitted block bundle in its slot's schedule. +type bundleNode struct { + st *slotState + seq uint64 + prepared *blockprod.PreparedBundle + executed *blockprod.ExecutedBundle + // deps counts the earlier conflicting bundles that have not finished. + deps int + dependents []*bundleNode + // lastDependent is the most recent bundle made to depend on this one, so a + // bundle sharing several accounts with it is linked once. + lastDependent *bundleNode + done bool +} + +// accountLocks is an account's latest unfinished writer and the bundles that +// read it since. +type accountLocks struct { + writer *bundleNode + readers []*bundleNode +} + +// schedule orders a slot's bundles so that executing them concurrently gives +// the same result as executing them one after another in stream order. +// +// A bundle depends on the latest earlier writer of every account it uses and, +// for each account it writes, on the readers since that writer. It is linked +// only to these direct predecessors: each of them waits for its own, so +// conflicts are ordered transitively. A bundle is ready once every +// predecessor has finished, and ready bundles start lowest sequence first. +// Bundles that conflict with no unfinished earlier one start at once, ahead +// of earlier bundles still waiting. +type schedule struct { + locks map[solana.PublicKey]*accountLocks + ready readyQueue + // waiting counts bundles with unfinished predecessors. + waiting int + // retry holds bundles that did not fit the block's limits while others + // were executing; they keep their place and try again later. + retry []*bundleNode + closed bool +} + +// accountSet calls fn once for each account a bundle uses, with whether it +// writes it. +type accountSet func(fn func(key solana.PublicKey, writable bool)) + +// add schedules n, which uses accounts, after every earlier bundle it +// conflicts with. +func (s *schedule) add(n *bundleNode, accounts accountSet) { + if s.locks == nil { + s.locks = make(map[solana.PublicKey]*accountLocks) + } + accounts(func(key solana.PublicKey, writable bool) { + l := s.locks[key] + if l == nil { + l = &accountLocks{} + s.locks[key] = l + } + s.dependOn(n, l.writer) + if !writable { + l.readers = append(l.readers, n) + return + } + for _, r := range l.readers { + s.dependOn(n, r) + } + l.writer, l.readers = n, nil + }) + if n.deps == 0 { + heap.Push(&s.ready, n) + } else { + s.waiting++ + } +} + +func (s *schedule) dependOn(n, p *bundleNode) { + if p == nil || p.done || p.lastDependent == n { + return + } + p.lastDependent = n + p.dependents = append(p.dependents, n) + n.deps++ +} + +// next returns the ready bundle earliest in the stream, or nil. +func (s *schedule) next() *bundleNode { + if s.closed || s.ready.Len() == 0 { + return nil + } + return heap.Pop(&s.ready).(*bundleNode) +} + +// finish marks n finished, landed or not, and readies the bundles that waited +// only on it. +func (s *schedule) finish(n *bundleNode) { + n.done = true + for _, d := range n.dependents { + d.deps-- + if d.deps == 0 && !s.closed { + s.waiting-- + heap.Push(&s.ready, d) + } + } + n.dependents = nil +} + +// retryLater keeps n, which did not fit the block's limits, to try again once +// another bundle finishes. It keeps its account locks meanwhile. +func (s *schedule) retryLater(n *bundleNode) { + s.retry = append(s.retry, n) +} + +// requeueRetries readies the bundles waiting to retry. +func (s *schedule) requeueRetries() { + if s.closed { + return + } + for _, n := range s.retry { + heap.Push(&s.ready, n) + } + s.retry = nil +} + +// close drops every bundle not executing and returns how many there were. +// Nothing is scheduled afterwards. +func (s *schedule) close() int { + if s.closed { + return 0 + } + dropped := s.ready.Len() + s.waiting + len(s.retry) + s.ready, s.waiting, s.retry, s.closed = nil, 0, nil, true + return dropped +} + +// readyQueue is a min-heap of bundles by stream sequence. +type readyQueue []*bundleNode + +func (q readyQueue) Len() int { return len(q) } +func (q readyQueue) Less(i, j int) bool { return q[i].seq < q[j].seq } +func (q readyQueue) Swap(i, j int) { q[i], q[j] = q[j], q[i] } +func (q *readyQueue) Push(x any) { *q = append(*q, x.(*bundleNode)) } +func (q *readyQueue) Pop() any { + old := *q + n := old[len(old)-1] + *q = old[:len(old)-1] + return n +} diff --git a/pkg/harmonic/schedule_test.go b/pkg/harmonic/schedule_test.go new file mode 100644 index 00000000..ff1d11f2 --- /dev/null +++ b/pkg/harmonic/schedule_test.go @@ -0,0 +1,112 @@ +package harmonic + +import ( + "testing" + + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// uses describes a bundle's accounts: keys prefixed with w are written, the +// rest read. +func uses(keys ...string) accountSet { + return func(fn func(solana.PublicKey, bool)) { + for _, k := range keys { + var key solana.PublicKey + copy(key[:], k[1:]) + fn(key, k[0] == 'w') + } + } +} + +// scheduleOf adds bundles 1..n, the ith using accounts[i-1]. +func scheduleOf(accounts ...accountSet) (*schedule, []*bundleNode) { + s := &schedule{} + nodes := make([]*bundleNode, len(accounts)) + for i, a := range accounts { + nodes[i] = &bundleNode{seq: uint64(i + 1)} + s.add(nodes[i], a) + } + return s, nodes +} + +// drain returns the ready bundles' sequence numbers, in order. +func drain(s *schedule) []uint64 { + var seqs []uint64 + for n := s.next(); n != nil; n = s.next() { + seqs = append(seqs, n.seq) + } + return seqs +} + +func TestScheduleOrdersConflictsTransitively(t *testing.T) { + // 1 writes x; 2 writes x and y; 3 writes y; 4 shares nothing. 3 has no + // account in common with 1 but must still wait for it, through 2. + s, nodes := scheduleOf(uses("wx"), uses("wx", "wy"), uses("wy"), uses("wz")) + assert.Equal(t, []uint64{1, 4}, drain(s), "4 overtakes the waiting 2 and 3") + s.finish(nodes[3]) + assert.Empty(t, drain(s)) + s.finish(nodes[0]) + assert.Equal(t, []uint64{2}, drain(s)) + s.finish(nodes[1]) + assert.Equal(t, []uint64{3}, drain(s)) +} + +func TestScheduleLockRules(t *testing.T) { + t.Run("readers share", func(t *testing.T) { + s, _ := scheduleOf(uses("rx"), uses("rx"), uses("rx")) + assert.Equal(t, []uint64{1, 2, 3}, drain(s)) + }) + t.Run("a write waits for every earlier reader", func(t *testing.T) { + s, nodes := scheduleOf(uses("rx"), uses("rx"), uses("wx")) + assert.Equal(t, []uint64{1, 2}, drain(s)) + s.finish(nodes[1]) + assert.Empty(t, drain(s)) + s.finish(nodes[0]) + assert.Equal(t, []uint64{3}, drain(s)) + }) + t.Run("a read waits for the earlier writer, not for other readers", func(t *testing.T) { + s, nodes := scheduleOf(uses("wx"), uses("rx"), uses("rx")) + assert.Equal(t, []uint64{1}, drain(s)) + s.finish(nodes[0]) + assert.Equal(t, []uint64{2, 3}, drain(s)) + }) + t.Run("a write after a write", func(t *testing.T) { + s, nodes := scheduleOf(uses("wx"), uses("wx")) + assert.Equal(t, []uint64{1}, drain(s)) + s.finish(nodes[0]) + assert.Equal(t, []uint64{2}, drain(s)) + }) + t.Run("a bundle sharing several accounts depends once", func(t *testing.T) { + s, nodes := scheduleOf(uses("wx", "wy", "rz"), uses("wx", "wy", "wz")) + assert.Equal(t, []uint64{1}, drain(s)) + assert.Equal(t, 1, nodes[1].deps) + s.finish(nodes[0]) + assert.Equal(t, []uint64{2}, drain(s)) + }) + t.Run("only unfinished bundles are depended on", func(t *testing.T) { + s, nodes := scheduleOf(uses("wx")) + require.Equal(t, []uint64{1}, drain(s)) + s.finish(nodes[0]) + late := &bundleNode{seq: 2} + s.add(late, uses("wx")) + assert.Equal(t, []uint64{2}, drain(s)) + }) +} + +func TestScheduleRetryAndClose(t *testing.T) { + s, nodes := scheduleOf(uses("wx"), uses("wx"), uses("wy")) + require.Equal(t, []uint64{1, 3}, drain(s)) + s.retryLater(nodes[0]) + assert.Empty(t, drain(s), "a retried bundle keeps its locks") + s.finish(nodes[2]) + s.requeueRetries() + assert.Equal(t, []uint64{1}, drain(s)) + + s.retryLater(nodes[0]) + assert.Equal(t, 2, s.close(), "the retry and the waiting bundle are dropped") + s.finish(nodes[0]) + assert.Nil(t, s.next(), "nothing is scheduled after close") + assert.Zero(t, s.close()) +} diff --git a/pkg/harmonic/service.go b/pkg/harmonic/service.go index 40a9ad39..c993e273 100644 --- a/pkg/harmonic/service.go +++ b/pkg/harmonic/service.go @@ -67,6 +67,9 @@ type ServiceConfig struct { LocalTPU gossip.TPUSockets // TxV1Enabled reports whether the current bank accepts v1 transactions. TxV1Enabled func() bool + // Workers execute block bundles that do not conflict in parallel. Zero + // means one. + Workers int // Dialer replaces the TCP dialer, for tests. Dialer func(ctx context.Context, addr string) (net.Conn, error) } @@ -123,6 +126,7 @@ func NewService(cfg ServiceConfig) (*Service, error) { Events: events, Crank: crank.Builder(s.blockEngine), OnReport: reportSlot, + Workers: cfg.Workers, }) if cfg.Relayer.Target != "" { s.relayer = NewRelayerSession(RelayerConfig{ diff --git a/pkg/statsd/statsd.go b/pkg/statsd/statsd.go index 39ce1716..11bdca88 100644 --- a/pkg/statsd/statsd.go +++ b/pkg/statsd/statsd.go @@ -167,6 +167,8 @@ var ( HarmonicSlotEndFlags = Metric{"harmonic_slot_end_flags_total"} HarmonicSlotBundles = Metric{"harmonic_slot_bundles_total"} HarmonicTipCranks = Metric{"harmonic_tip_cranks_total"} + // HarmonicSlotMaxParallel is the most block bundles a slot executed at once. + HarmonicSlotMaxParallel = Metric{"harmonic_slot_max_parallel"} // HarmonicFirstBundleDelay is when a leader slot's first block bundle // arrived, after the bank opened, against the threshold at mid-slot. HarmonicFirstBundleDelay = Metric{"harmonic_first_bundle_delay_seconds"} @@ -316,6 +318,7 @@ var MetricToType = map[Metric]metricType{ HarmonicSlotEndFlags: CountT, HarmonicSlotBundles: CountT, HarmonicTipCranks: CountT, + HarmonicSlotMaxParallel: TimingT, HarmonicFirstBundleDelay: TimingT, TestCount: CountT, @@ -455,6 +458,7 @@ var MetricToLabels = map[Metric][]string{ HarmonicSlotEndFlags: {"flag"}, HarmonicSlotBundles: {"result"}, HarmonicTipCranks: {"result"}, + HarmonicSlotMaxParallel: {}, HarmonicFirstBundleDelay: {}, SnapshotWorkerPoolUtilization: {"task"}, @@ -515,6 +519,7 @@ var MetricToBuckets = map[Metric][]float64{ TxFailedNoncePublication: turbinePipelineDurationBuckets, HarmonicLeaderNoticeRTT: blockProductionDurationBuckets, HarmonicFirstBundleDelay: blockProductionDurationBuckets, + HarmonicSlotMaxParallel: {1, 2, 3, 4, 6, 8, 12, 16, 24, 32, 48, 64}, } type Prometheusmetrics struct { From cf6365d3889e580534591cbdc8780d510f9686c8 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Mon, 19 Oct 2026 01:10:41 +0000 Subject: [PATCH 24/25] harmonic: parse and verify block messages in parallel --- pkg/harmonic/blocks.go | 252 +++++++++++++++++++++++++----------- pkg/harmonic/blocks_test.go | 48 ++++++- pkg/harmonic/service.go | 2 +- 3 files changed, 223 insertions(+), 79 deletions(-) diff --git a/pkg/harmonic/blocks.go b/pkg/harmonic/blocks.go index b5dbf51d..599f9f05 100644 --- a/pkg/harmonic/blocks.go +++ b/pkg/harmonic/blocks.go @@ -61,9 +61,15 @@ type BlockPipelineStats struct { Transactions uint64 } +// pipelineBatch is the most messages a worker parses and verifies together, +// so short messages still fill signature verification batches. +const pipelineBatch = 16 + // BlockPipeline turns the block stream into executor events. It admits only // messages for the current block slot, splits each into bundles, validates -// them, and emits them in order. It is the session's BlockHandler. +// them, and emits them in stream order. Workers parse and verify messages in +// parallel; their results are put back in stream order before anything is +// emitted. It is the session's BlockHandler. type BlockPipeline struct { events chan<- BlockEvent done chan struct{} @@ -71,26 +77,54 @@ type BlockPipeline struct { txV1Enabled func() bool current atomic.Uint64 inFlight atomic.Int32 - verifier txverify.BatchVerifier + jobs chan *pipelineJob + results chan *pipelineJob stats struct { messages, wrongSlot, lostSlot, bundles, invalid, transactions atomic.Uint64 } - mu sync.Mutex - lost map[uint64]bool - seqSlot uint64 - nextSeq uint64 + mu sync.Mutex + lost map[uint64]bool + // nextOrder is the stream position of the next admitted message. + nextOrder uint64 +} + +// pipelineJob is an admitted message, or a lost-stream marker, at its stream +// position. +type pipelineJob struct { + order uint64 + slot uint64 + arrival time.Time + txs []*blockpb.Transaction + lost bool + parsed []*solana.Transaction + errs []error } // NewBlockPipeline sends events on events, blocking rather than dropping // when the executor falls behind, until Close. txV1Enabled reports whether the -// current bank accepts v1 transactions; nil rejects them. -func NewBlockPipeline(events chan<- BlockEvent, txV1Enabled func() bool) *BlockPipeline { - return &BlockPipeline{events: events, done: make(chan struct{}), txV1Enabled: txV1Enabled, lost: make(map[uint64]bool)} +// current bank accepts v1 transactions; nil rejects them. workers parse and +// verify messages in parallel. +func NewBlockPipeline(events chan<- BlockEvent, txV1Enabled func() bool, workers int) *BlockPipeline { + workers = max(workers, 1) + p := &BlockPipeline{ + events: events, + done: make(chan struct{}), + txV1Enabled: txV1Enabled, + jobs: make(chan *pipelineJob, workers*pipelineBatch), + results: make(chan *pipelineJob, workers*pipelineBatch), + lost: make(map[uint64]bool), + nextOrder: 1, + } + for range workers { + go p.work() + } + go p.mux() + return p } -// Close drops every later event, so a stream goroutine blocked on an -// executor that has stopped returns. +// Close stops the pipeline and drops every later event, so a stream +// goroutine blocked on an executor that has stopped returns. func (p *BlockPipeline) Close() { p.closeOnce.Do(func() { close(p.done) }) } @@ -127,26 +161,27 @@ func (p *BlockPipeline) SetCurrentSlot(slot uint64) { } } -// InFlight reports whether a current-slot message is being split into -// bundles whose events are not all sent yet. +// InFlight reports whether an admitted message has events not yet sent. func (p *BlockPipeline) InFlight() bool { return p.inFlight.Load() > 0 } // StreamLost marks slot lost: the rest of its block is discarded even after -// the session reconnects, since nothing says where a new stream resumes. +// the session reconnects, since nothing says where a new stream resumes. The +// loss is reported after every message admitted before it. func (p *BlockPipeline) StreamLost(slot uint64) { p.mu.Lock() p.lost[slot] = true + job := &pipelineJob{order: p.nextOrder, slot: slot, lost: true} + p.nextOrder++ p.mu.Unlock() - p.emit(StreamLostEvent{Slot: slot}) + p.inFlight.Add(1) + p.queue(job) } -// HandleBlock admits, segments and validates one block message. +// HandleBlock admits one block message and queues it for parsing. It blocks +// while the pipeline is full, holding back the stream. func (p *BlockPipeline) HandleBlock(block *blockpb.Block, arrival time.Time) { - // Until its last event is sent, a message counts as buffered. - p.inFlight.Add(1) - defer p.inFlight.Add(-1) p.stats.messages.Add(1) slot := block.GetSlot() if current := p.current.Load(); current == 0 || slot != current { @@ -154,79 +189,150 @@ func (p *BlockPipeline) HandleBlock(block *blockpb.Block, arrival time.Time) { return } p.mu.Lock() - lost := p.lost[slot] - if !lost && p.seqSlot != slot { - p.seqSlot, p.nextSeq = slot, 1 - } - p.mu.Unlock() - if lost { + if p.lost[slot] { + p.mu.Unlock() p.stats.lostSlot.Add(1) return } + job := &pipelineJob{order: p.nextOrder, slot: slot, arrival: arrival, txs: block.GetTransactions()} + p.nextOrder++ + // Until its last event is sent, a message counts as buffered. + p.inFlight.Add(1) + p.mu.Unlock() + p.queue(job) +} + +func (p *BlockPipeline) queue(job *pipelineJob) { + select { + case p.jobs <- job: + case <-p.done: + } +} - txs := block.GetTransactions() - parsed, parseErrs := p.parse(txs) - for _, bundle := range segment(txs) { - seq := p.takeSeq() - if err := validateBundle(txs, parsed, parseErrs, bundle); err != nil { - p.stats.invalid.Add(1) - p.emit(InvalidBundleEvent{Slot: slot, Seq: seq, Arrival: arrival, Reason: err}) - continue +// work parses and verifies queued messages, several at a time. +func (p *BlockPipeline) work() { + batch := make([]*pipelineJob, 0, pipelineBatch) + var verifier txverify.BatchVerifier + for { + select { + case <-p.done: + return + case job := <-p.jobs: + batch = append(batch[:0], job) + } + gather: + for len(batch) < pipelineBatch { + select { + case job := <-p.jobs: + batch = append(batch, job) + default: + break gather + } + } + p.parse(batch, &verifier) + for _, job := range batch { + select { + case p.results <- job: + case <-p.done: + return + } } - p.stats.bundles.Add(1) - p.stats.transactions.Add(uint64(bundle.end - bundle.start)) - p.emit(BundleEvent{ - Slot: slot, - Seq: seq, - Arrival: arrival, - RevertProtected: txs[bundle.start].GetBundleId() != 0, - Transactions: append([]*solana.Transaction(nil), parsed[bundle.start:bundle.end]...), - }) } } -func (p *BlockPipeline) takeSeq() uint64 { - p.mu.Lock() - defer p.mu.Unlock() - seq := p.nextSeq - p.nextSeq++ - return seq +// mux emits results in stream order, holding each until every earlier one is +// out. It alone assigns bundle sequence numbers. +func (p *BlockPipeline) mux() { + held := make(map[uint64]*pipelineJob) + next := uint64(1) + var seqSlot, nextSeq uint64 + for { + select { + case <-p.done: + return + case job := <-p.results: + held[job.order] = job + } + for job, ok := held[next]; ok; job, ok = held[next] { + delete(held, next) + next++ + if job.lost { + p.emit(StreamLostEvent{Slot: job.slot}) + } else { + if seqSlot != job.slot { + seqSlot, nextSeq = job.slot, 1 + } + nextSeq = p.emitMessage(job, nextSeq) + } + p.inFlight.Add(-1) + } + } } -// parse parses every transaction in a message and verifies all their -// signatures as one batch. errs[i] is set for a transaction that is unusable. -func (p *BlockPipeline) parse(txs []*blockpb.Transaction) ([]*solana.Transaction, []error) { - parsed := make([]*solana.Transaction, len(txs)) - errs := make([]error, len(txs)) - verify := make([]*solana.Transaction, 0, len(txs)) - verifyAt := make([]int, 0, len(txs)) - for i, t := range txs { - data := t.GetTransaction() - if len(data) == 0 || len(data) > packet.DataSize { - errs[i] = fmt.Errorf("transaction %d has %d bytes", i, len(data)) - continue +// emitMessage splits a parsed message into bundles and emits them, numbered +// from seq. It returns the next sequence number. +func (p *BlockPipeline) emitMessage(job *pipelineJob, seq uint64) uint64 { + for _, bundle := range segment(job.txs) { + if err := validateBundle(job.txs, job.parsed, job.errs, bundle); err != nil { + p.stats.invalid.Add(1) + p.emit(InvalidBundleEvent{Slot: job.slot, Seq: seq, Arrival: job.arrival, Reason: err}) + } else { + p.stats.bundles.Add(1) + p.stats.transactions.Add(uint64(bundle.end - bundle.start)) + p.emit(BundleEvent{ + Slot: job.slot, + Seq: seq, + Arrival: job.arrival, + RevertProtected: job.txs[bundle.start].GetBundleId() != 0, + Transactions: append([]*solana.Transaction(nil), job.parsed[bundle.start:bundle.end]...), + }) } - tx, err := sigverify.ParseTx(data) - if err != nil { - errs[i] = fmt.Errorf("transaction %d: %w", i, err) + seq++ + } + return seq +} + +// parse parses every transaction of the batch's messages and verifies all +// their signatures together. errs[i] is set for a transaction that is +// unusable. +func (p *BlockPipeline) parse(batch []*pipelineJob, verifier *txverify.BatchVerifier) { + var verify []*solana.Transaction + type at struct{ job, tx int } + var verifyAt []at + for j, job := range batch { + if job.lost { continue } - if tx.Message.GetVersion() == solana.MessageVersionV1 && (p.txV1Enabled == nil || !p.txV1Enabled()) { - errs[i] = fmt.Errorf("transaction %d: v1 transactions are not enabled", i) - continue + job.parsed = make([]*solana.Transaction, len(job.txs)) + job.errs = make([]error, len(job.txs)) + for i, t := range job.txs { + data := t.GetTransaction() + if len(data) == 0 || len(data) > packet.DataSize { + job.errs[i] = fmt.Errorf("transaction %d has %d bytes", i, len(data)) + continue + } + tx, err := sigverify.ParseTx(data) + if err != nil { + job.errs[i] = fmt.Errorf("transaction %d: %w", i, err) + continue + } + if tx.Message.GetVersion() == solana.MessageVersionV1 && (p.txV1Enabled == nil || !p.txV1Enabled()) { + job.errs[i] = fmt.Errorf("transaction %d: v1 transactions are not enabled", i) + continue + } + job.parsed[i] = tx + verify = append(verify, tx) + verifyAt = append(verifyAt, at{j, i}) } - parsed[i] = tx - verify = append(verify, tx) - verifyAt = append(verifyAt, i) } verifyErrs := make([]error, len(verify)) - p.verifier.Verify(verify, verifyErrs) - for j, err := range verifyErrs { + verifier.Verify(verify, verifyErrs) + for k, err := range verifyErrs { if err != nil { - errs[verifyAt[j]] = fmt.Errorf("transaction %d: %w", verifyAt[j], err) + a := verifyAt[k] + batch[a.job].errs[a.tx] = fmt.Errorf("transaction %d: %w", a.tx, err) } } - return parsed, errs } type bundleSpan struct { diff --git a/pkg/harmonic/blocks_test.go b/pkg/harmonic/blocks_test.go index 45724d4e..1d306e42 100644 --- a/pkg/harmonic/blocks_test.go +++ b/pkg/harmonic/blocks_test.go @@ -27,13 +27,16 @@ func blockTx(data []byte, bundleID uint64) *blockpb.Transaction { return &blockpb.Transaction{Transaction: data, BundleId: bundleID} } -// collect runs fn and returns every event it emitted. +// collect runs fn on a pipeline with several workers and returns every event +// it emitted. func collect(t *testing.T, fn func(*BlockPipeline), txV1Enabled func() bool, current uint64) ([]BlockEvent, *BlockPipeline) { t.Helper() - events := make(chan BlockEvent, 64) - p := NewBlockPipeline(events, txV1Enabled) + events := make(chan BlockEvent, 4096) + p := NewBlockPipeline(events, txV1Enabled, 4) p.SetCurrentSlot(current) fn(p) + require.Eventually(t, func() bool { return !p.InFlight() }, 2*time.Second, time.Millisecond, "messages still in the pipeline") + p.Close() close(events) var out []BlockEvent for e := range events { @@ -171,7 +174,8 @@ func TestBlockPipelineAcceptsWhatIngestMustNotReject(t *testing.T) { func TestBlockPipelineBlocksRatherThanDrops(t *testing.T) { events := make(chan BlockEvent) - p := NewBlockPipeline(events, nil) + p := NewBlockPipeline(events, nil, 4) + t.Cleanup(p.Close) p.SetCurrentSlot(10) done := make(chan struct{}) go func() { @@ -193,7 +197,8 @@ func TestBlockPipelineBlocksRatherThanDrops(t *testing.T) { func TestBlockEngineSessionFeedsThePipeline(t *testing.T) { h := newBlockEngineHarness(t, time.Hour) events := make(chan BlockEvent, 16) - pipeline := NewBlockPipeline(events, nil) + pipeline := NewBlockPipeline(events, nil, 4) + t.Cleanup(pipeline.Close) h.session.cfg.Blocks = pipeline h.start() eventually(t, h.session.Healthy, "healthy") @@ -225,3 +230,36 @@ func TestBlockEngineSessionFeedsThePipeline(t *testing.T) { eventually(t, func() bool { return pipeline.Stats().LostSlot == 1 }, "the lost slot stays discarded after reconnecting") assert.Empty(t, events) } + +// Messages are parsed and verified in parallel, but their bundles come out in +// exact stream order, numbered consecutively, with a loss reported after +// everything received before it. +func TestBlockPipelineKeepsStreamOrder(t *testing.T) { + const messages = 300 + var want []solana.Signature + events, _ := collect(t, func(p *BlockPipeline) { + n := uint64(0) + for m := range messages { + // Varying sizes make workers finish out of order. + var txs []*blockpb.Transaction + for range 1 + m%7 { + n++ + wire := wireTx(n) + txs = append(txs, blockTx(wire, 0)) + want = append(want, solana.SignatureFromBytes(wire[1:65])) + } + p.HandleBlock(blockMsg(10, txs...), time.Now()) + } + p.StreamLost(10) + p.HandleBlock(blockMsg(10, blockTx(wireTx(n+1), 0)), time.Now()) + }, nil, 10) + + require.Len(t, events, len(want)+1) + for i, sig := range want { + bundle, ok := events[i].(BundleEvent) + require.Truef(t, ok, "event %d is %#v", i, events[i]) + assert.Equal(t, uint64(i+1), bundle.Seq) + assert.Equal(t, sig, bundle.Transactions[0].Signatures[0], "bundle %d", i+1) + } + assert.Equal(t, StreamLostEvent{Slot: 10}, events[len(want)], "the loss follows everything received before it") +} diff --git a/pkg/harmonic/service.go b/pkg/harmonic/service.go index c993e273..49121a3a 100644 --- a/pkg/harmonic/service.go +++ b/pkg/harmonic/service.go @@ -103,7 +103,7 @@ func NewService(cfg ServiceConfig) (*Service, error) { events := make(chan BlockEvent, blockEventBuffer) s := &Service{ cfg: cfg, - pipeline: NewBlockPipeline(events, cfg.TxV1Enabled), + pipeline: NewBlockPipeline(events, cfg.TxV1Enabled, cfg.Workers), tpu: NewTPUAdvertisement(cfg.Gossip, cfg.LocalTPU), harmonicClientIDAfter: harmonicClientIDAfter, baseClientIDAfter: baseClientIDAfter, From 034521f1d1dba251f1d06fa5cc9716cf988c3015 Mon Sep 17 00:00:00 2001 From: cavemanloverboy <93507302+cavemanloverboy@users.noreply.github.com> Date: Mon, 19 Oct 2026 01:10:48 +0000 Subject: [PATCH 25/25] turbine: forward shreds to configured receivers --- cmd/mithril/node/node.go | 27 ++++- cmd/mithril/node/shred_receivers.go | 28 +++++ cmd/mithril/node/shred_receivers_test.go | 25 ++++ config.example.toml | 8 ++ pkg/blockstream/block_source.go | 21 ++-- pkg/blockstream/prewarm.go | 22 ++-- pkg/blockstream/turbine_stream.go | 15 +-- pkg/replay/block.go | 4 + pkg/turbine/retransmit.go | 32 +++++- pkg/turbine/shred_receivers_test.go | 140 +++++++++++++++++++++++ pkg/turbine/turbine_broadcaster.go | 21 ++-- 11 files changed, 304 insertions(+), 39 deletions(-) create mode 100644 cmd/mithril/node/shred_receivers.go create mode 100644 cmd/mithril/node/shred_receivers_test.go create mode 100644 pkg/turbine/shred_receivers_test.go diff --git a/cmd/mithril/node/node.go b/cmd/mithril/node/node.go index f0e41b14..43baba53 100644 --- a/cmd/mithril/node/node.go +++ b/cmd/mithril/node/node.go @@ -167,6 +167,10 @@ var ( turbineGossipBindAddr string turbineAdvertisedIP string turbineShredVersion int + // turbineShredReceivers get copies of the shreds this validator + // broadcasts, and turbineShredRetransmitReceivers of the shreds it relays. + turbineShredReceivers []*net.UDPAddr + turbineShredRetransmitReceivers []*net.UDPAddr ) func snapshotEpochForState(manifest *snapshot.SnapshotManifest) uint64 { @@ -628,6 +632,8 @@ func init() { Run.Flags().BoolVar(&blockRPCFallback, "rpc-fallback", false, "Allow RPC to fetch blocks when replay is more than repair-catchup-max-gap-slots behind (default false: shreds via turbine + repair are the only block path)") Run.Flags().StringVar(&turbineAdvertisedIP, "turbine-advertised-ip", "", "Public IP advertised by native turbine gossip (optional)") Run.Flags().IntVar(&turbineShredVersion, "turbine-shred-version", 0, "Shred version for native turbine gossip (0 = discover from entrypoint)") + Run.Flags().StringSlice("shred-receiver-address", nil, "Also send every shred this validator broadcasts to this host:port (repeatable), e.g. a block builder") + Run.Flags().StringSlice("shred-retransmit-receiver-address", nil, "Also send every shred this node relays to this host:port (repeatable), e.g. a block builder") Run.Flags().IntVar(&blockMaxRPS, "block-max-rps", 0, "Max RPC requests per second for block fetching (0 = use default)") Run.Flags().IntVar(&blockMaxInflight, "block-max-inflight", 0, "Max concurrent block fetch workers (0 = use default)") Run.Flags().IntVar(&blockTipPollIntervalMs, "block-tip-poll-ms", 0, "Tip poll interval in milliseconds (0 = use default)") @@ -938,6 +944,16 @@ func initConfigAndBindFlags(cmd *cobra.Command) error { turbineGossipBindAddr = getString("turbine-gossip-bind-addr", "turbine.gossip_bind_addr") turbineAdvertisedIP = getString("turbine-advertised-ip", "turbine.advertised_ip") turbineShredVersion = getInt("turbine-shred-version", "turbine.shred_version") + shredReceivers, err := parseShredReceivers("turbine.shred_receiver_addresses", + getStringSlice("shred-receiver-address", "turbine.shred_receiver_addresses")) + if err != nil { + return err + } + turbineShredReceivers = shredReceivers + if turbineShredRetransmitReceivers, err = parseShredReceivers("turbine.shred_retransmit_receiver_addresses", + getStringSlice("shred-retransmit-receiver-address", "turbine.shred_retransmit_receiver_addresses")); err != nil { + return err + } repairCatchupMaxGapSlots = getInt("repair-catchup-max-gap-slots", "block.repair_catchup_max_gap_slots") if repairCatchupMaxGapSlots < 0 { repairCatchupMaxGapSlots = 0 @@ -1566,6 +1582,7 @@ func runLive(c *cobra.Command, args []string) { RootSlot: func() uint64 { return rootSlot }, UseChaCha8: true, DedupAddrs: false, + ShredReceivers: turbineShredRetransmitReceivers, FloorSlot: m.Bank.Slot + 1, ShredSpoolDir: catchupShredSpoolDir(), RepairMaxRequestsPerSecond: repairMaxRequestsPerSecond, @@ -2749,7 +2766,8 @@ postBootstrap: UseChaCha8: true, // Agave treats custom/development clusters as non-deduplicating so // every validator derives the same deterministic Turbine node list. - DedupAddrs: false, + DedupAddrs: false, + ShredReceivers: turbineShredReceivers, }) if err != nil { klog.Fatalf("validator turbine broadcaster: %v", err) @@ -2963,9 +2981,10 @@ postBootstrap: } return alpenglowEpochSchedule.GetEpoch(slot) }, - TurbineRootSlot: global.ReplayFrontier, - TurbineUseChaCha8: true, - TurbineDedupAddrs: false, + TurbineRootSlot: global.ReplayFrontier, + TurbineUseChaCha8: true, + TurbineDedupAddrs: false, + TurbineShredReceivers: turbineShredRetransmitReceivers, } if consensusMode == "validator" { identity := solana.PrivateKey(validatorIdentity).PublicKey() diff --git a/cmd/mithril/node/shred_receivers.go b/cmd/mithril/node/shred_receivers.go new file mode 100644 index 00000000..81cc70b1 --- /dev/null +++ b/cmd/mithril/node/shred_receivers.go @@ -0,0 +1,28 @@ +package node + +import ( + "fmt" + "net" + "strings" +) + +// parseShredReceivers resolves host:port addresses that get copies of shreds, +// such as a block builder's shred listener. +func parseShredReceivers(key string, addrs []string) ([]*net.UDPAddr, error) { + var out []*net.UDPAddr + for _, raw := range addrs { + raw = strings.TrimSpace(raw) + if raw == "" { + continue + } + addr, err := net.ResolveUDPAddr("udp4", raw) + if err != nil { + return nil, fmt.Errorf("%s: %q: %w", key, raw, err) + } + if addr.Port == 0 || addr.IP == nil || addr.IP.IsUnspecified() { + return nil, fmt.Errorf("%s: %q needs a host and a port", key, raw) + } + out = append(out, addr) + } + return out, nil +} diff --git a/cmd/mithril/node/shred_receivers_test.go b/cmd/mithril/node/shred_receivers_test.go new file mode 100644 index 00000000..a77c6a2c --- /dev/null +++ b/cmd/mithril/node/shred_receivers_test.go @@ -0,0 +1,25 @@ +package node + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParseShredReceivers(t *testing.T) { + got, err := parseShredReceivers("turbine.shred_receiver_addresses", []string{"64.130.42.198:8003", " ", "127.0.0.1:9000"}) + require.NoError(t, err) + require.Len(t, got, 2) + assert.Equal(t, "64.130.42.198:8003", got[0].String()) + assert.Equal(t, "127.0.0.1:9000", got[1].String()) + + none, err := parseShredReceivers("turbine.shred_receiver_addresses", nil) + require.NoError(t, err) + assert.Empty(t, none) + + for _, bad := range []string{"64.130.42.198", "64.130.42.198:0", "0.0.0.0:8003", "not a host:8003"} { + _, err := parseShredReceivers("turbine.shred_receiver_addresses", []string{bad}) + assert.Errorf(t, err, "%q", bad) + } +} diff --git a/config.example.toml b/config.example.toml index 0994eebc..13a8a4b6 100644 --- a/config.example.toml +++ b/config.example.toml @@ -478,6 +478,14 @@ name = "mithril" # Optional override. Leave at zero/empty to discover from gossip_entrypoint. # shred_version = 0 + # Optional host:port receivers that get a copy of every shred, in addition + # to normal Turbine operation, such as a block builder's shred listener. + # shred_receiver_addresses: shreds of blocks this validator produces. + # shred_retransmit_receiver_addresses: shreds this node relays for others. + # Also --shred-receiver-address / --shred-retransmit-receiver-address. + # shred_receiver_addresses = ["127.0.0.1:8003"] + # shred_retransmit_receiver_addresses = ["127.0.0.1:8003"] + # ============================================================================ # [lightbringer] - Lightbringer Sidecar (block.source = "lightbringer") diff --git a/pkg/blockstream/block_source.go b/pkg/blockstream/block_source.go index 32a7fec1..7e64aaf5 100644 --- a/pkg/blockstream/block_source.go +++ b/pkg/blockstream/block_source.go @@ -7,6 +7,7 @@ import ( "errors" "fmt" "math" + "net" "os" "path/filepath" "sync" @@ -49,14 +50,16 @@ type BlockSourceOpts struct { TurbineAlpenglowBlockIDHints bool // TurbineStreamingExecution subscribes replay's streaming executor to the // assembler's decoded-batch feed (StreamEvents). Off by default. - TurbineStreamingExecution bool - TurbineIdentity ed25519.PrivateKey - LeaderForSlot func(slot uint64) (solana.PublicKey, bool) - TurbineStakesForSlot func(slot uint64) map[solana.PublicKey]uint64 - TurbineEpochForSlot func(slot uint64) uint64 - TurbineRootSlot func() uint64 - TurbineUseChaCha8 bool - TurbineDedupAddrs bool + TurbineStreamingExecution bool + TurbineIdentity ed25519.PrivateKey + LeaderForSlot func(slot uint64) (solana.PublicKey, bool) + TurbineStakesForSlot func(slot uint64) map[solana.PublicKey]uint64 + TurbineEpochForSlot func(slot uint64) uint64 + TurbineRootSlot func() uint64 + TurbineUseChaCha8 bool + TurbineDedupAddrs bool + // TurbineShredReceivers get a copy of every shred this node relays. + TurbineShredReceivers []*net.UDPAddr LocalLeaderForSlot func(slot uint64) bool GossipClient *gossip.Client AlpenglowDecisionSource func(anchorSlot uint64) (alpenglow.ChainDecision, bool) @@ -422,6 +425,7 @@ type BlockSource struct { turbineRootSlot func() uint64 turbineUseChaCha8 bool turbineDedupAddrs bool + turbineShredReceivers []*net.UDPAddr localLeaderForSlot func(slot uint64) bool localBlocks <-chan *b.Block gossipClient *gossip.Client @@ -785,6 +789,7 @@ func NewBlockSource(opts *BlockSourceOpts) *BlockSource { turbineRootSlot: opts.TurbineRootSlot, turbineUseChaCha8: opts.TurbineUseChaCha8, turbineDedupAddrs: opts.TurbineDedupAddrs, + turbineShredReceivers: opts.TurbineShredReceivers, localLeaderForSlot: opts.LocalLeaderForSlot, localBlocks: opts.LocalBlocks, gossipClient: opts.GossipClient, diff --git a/pkg/blockstream/prewarm.go b/pkg/blockstream/prewarm.go index 9dc8a957..008f893c 100644 --- a/pkg/blockstream/prewarm.go +++ b/pkg/blockstream/prewarm.go @@ -4,6 +4,7 @@ import ( "context" "crypto/ed25519" "fmt" + "net" "sort" "sync" "time" @@ -69,8 +70,10 @@ type TurbinePrewarmConfig struct { RootSlot func() uint64 UseChaCha8 bool DedupAddrs bool - FloorSlot uint64 // resume frontier: spool floor and assembler retention floor - MaxSpoolBlocks int + // ShredReceivers get a copy of every shred the prewarm relays. + ShredReceivers []*net.UDPAddr + FloorSlot uint64 // resume frontier: spool floor and assembler retention floor + MaxSpoolBlocks int // ShredSpoolDir: shared on-disk shred spool. Everything the prewarm // hears goes to disk too, so even blocks beyond the in-RAM spool cap // hydrate later instead of re-repairing. @@ -119,13 +122,14 @@ func StartTurbinePrewarm(cfg TurbinePrewarmConfig) (*TurbinePrewarm, error) { } if cfg.StakesForSlot != nil { if err := receiver.SetRetransmit(turbine.RetransmitConfig{ - Identity: client.Identity(), - Peers: client, - Stakes: cfg.StakesForSlot, - EpochForSlot: cfg.EpochForSlot, - RootSlot: cfg.RootSlot, - UseChaCha8: cfg.UseChaCha8, - DedupAddrs: cfg.DedupAddrs, + Identity: client.Identity(), + Peers: client, + Stakes: cfg.StakesForSlot, + EpochForSlot: cfg.EpochForSlot, + RootSlot: cfg.RootSlot, + UseChaCha8: cfg.UseChaCha8, + DedupAddrs: cfg.DedupAddrs, + ShredReceivers: cfg.ShredReceivers, }); err != nil { cancel() return nil, fmt.Errorf("prewarm retransmit setup: %w", err) diff --git a/pkg/blockstream/turbine_stream.go b/pkg/blockstream/turbine_stream.go index 5e53b32a..f5ab4c72 100644 --- a/pkg/blockstream/turbine_stream.go +++ b/pkg/blockstream/turbine_stream.go @@ -365,13 +365,14 @@ func (bs *BlockSource) runTurbineStream() { if gossipClient != nil { if bs.turbineStakesForSlot != nil { if err := receiver.SetRetransmit(turbine.RetransmitConfig{ - Identity: gossipClient.Identity(), - Peers: gossipClient, - Stakes: bs.turbineStakesForSlot, - EpochForSlot: bs.turbineEpochForSlot, - RootSlot: bs.turbineRootSlot, - UseChaCha8: bs.turbineUseChaCha8, - DedupAddrs: bs.turbineDedupAddrs, + Identity: gossipClient.Identity(), + Peers: gossipClient, + Stakes: bs.turbineStakesForSlot, + EpochForSlot: bs.turbineEpochForSlot, + RootSlot: bs.turbineRootSlot, + UseChaCha8: bs.turbineUseChaCha8, + DedupAddrs: bs.turbineDedupAddrs, + ShredReceivers: bs.turbineShredReceivers, }); err != nil { cancelStream() bs.handleLiveShredStreamClosed(fmt.Sprintf("native turbine retransmit setup failed: %v", err)) diff --git a/pkg/replay/block.go b/pkg/replay/block.go index 191f1b73..3f96f42a 100644 --- a/pkg/replay/block.go +++ b/pkg/replay/block.go @@ -11,6 +11,7 @@ import ( "fmt" "io" "math" + "net" "os" "path/filepath" "runtime" @@ -103,6 +104,8 @@ type BlockFetchOpts struct { TurbineRootSlot func() uint64 TurbineUseChaCha8 bool TurbineDedupAddrs bool + // TurbineShredReceivers get a copy of every shred this node relays. + TurbineShredReceivers []*net.UDPAddr } var SerializedParameterArena *arena.Arena[byte] @@ -2533,6 +2536,7 @@ func ReplayBlocks( opts.TurbineRootSlot = blockFetchOpts.TurbineRootSlot opts.TurbineUseChaCha8 = blockFetchOpts.TurbineUseChaCha8 opts.TurbineDedupAddrs = blockFetchOpts.TurbineDedupAddrs + opts.TurbineShredReceivers = blockFetchOpts.TurbineShredReceivers // Mode thresholds opts.NearTipThreshold = blockFetchOpts.NearTipThreshold diff --git a/pkg/turbine/retransmit.go b/pkg/turbine/retransmit.go index dfb0a04a..04b4dd14 100644 --- a/pkg/turbine/retransmit.go +++ b/pkg/turbine/retransmit.go @@ -56,6 +56,9 @@ type RetransmitConfig struct { DedupAddrs bool Workers int QueueDepth int + // ShredReceivers get a copy of every shred this node relays, in addition + // to its Turbine children, such as a block builder on the same host. + ShredReceivers []*net.UDPAddr } type retransmitWork struct { @@ -179,6 +182,8 @@ type Retransmitter struct { parentSourceDirectLeader atomic.Uint64 parentSourceRepairSocket atomic.Uint64 parentSourceUnexplained atomic.Uint64 + shredReceiverSent atomic.Uint64 + shredReceiverFailed atomic.Uint64 rootDistance [maxTurbineHops]atomic.Uint64 } @@ -216,9 +221,13 @@ type RetransmitStats struct { ParentSourceRepairSocket uint64 ParentSourceUnexplained uint64 SendBufferBytes int - LastSendDiagnostic string - LastParentDiagnostic string - RootDistance [maxTurbineHops]uint64 + // ShredReceiverSent and ShredReceiverFailed count copies to the + // configured shred receivers. + ShredReceiverSent uint64 + ShredReceiverFailed uint64 + LastSendDiagnostic string + LastParentDiagnostic string + RootDistance [maxTurbineHops]uint64 } func NewRetransmitter(cfg RetransmitConfig) (*Retransmitter, error) { @@ -497,9 +506,22 @@ func (r *Retransmitter) send(work retransmitWork, sender packetBatchSender, scra } if len(peers) == 0 { r.noPeers.Add(1) + } else { + r.sendToPeers(work.packet, peers, sender) + } + // After the fanout, so receivers never delay it. A leaf forwards too. + r.sendToShredReceivers(work.packet, sender) +} + +func (r *Retransmitter) sendToShredReceivers(packet []byte, sender packetBatchSender) { + receivers := r.cfg.ShredReceivers + if len(receivers) == 0 { return } - r.sendToPeers(work.packet, peers, sender) + sent, _ := sender.Send(packet, receivers) + sent = min(max(sent, 0), len(receivers)) + r.shredReceiverSent.Add(uint64(sent)) + r.shredReceiverFailed.Add(uint64(len(receivers) - sent)) } func (r *Retransmitter) sendToPeers(packet []byte, peers []*net.UDPAddr, sender packetBatchSender) { @@ -789,6 +811,8 @@ func (r *Retransmitter) Stats() RetransmitStats { ParentSourceRepairSocket: r.parentSourceRepairSocket.Load(), ParentSourceUnexplained: r.parentSourceUnexplained.Load(), SendBufferBytes: r.sendBufferBytes, + ShredReceiverSent: r.shredReceiverSent.Load(), + ShredReceiverFailed: r.shredReceiverFailed.Load(), } r.diagnosticMu.Lock() stats.LastSendDiagnostic = r.lastSendDiag diff --git a/pkg/turbine/shred_receivers_test.go b/pkg/turbine/shred_receivers_test.go new file mode 100644 index 00000000..51079fab --- /dev/null +++ b/pkg/turbine/shred_receivers_test.go @@ -0,0 +1,140 @@ +package turbine + +import ( + "context" + "crypto/ed25519" + "net" + "testing" + "time" + + "github.com/Overclock-Validator/mithril/pkg/gossip" + "github.com/gagliardetto/solana-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func listenUDP(t *testing.T) *net.UDPConn { + t.Helper() + conn, err := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + require.NoError(t, err) + t.Cleanup(func() { _ = conn.Close() }) + return conn +} + +func readPacket(t *testing.T, conn *net.UDPConn) []byte { + t.Helper() + buf := make([]byte, packetDataSize) + require.NoError(t, conn.SetReadDeadline(time.Now().Add(2*time.Second))) + n, _, err := conn.ReadFromUDP(buf) + require.NoError(t, err) + return buf[:n] +} + +func TestTurbineBroadcasterSendsToShredReceivers(t *testing.T) { + root, receiver := listenUDP(t), listenUDP(t) + leaderPub, leaderKey := testIdentity(t) + rootPub, _ := testIdentity(t) + b, err := NewTurbineBroadcaster(TurbineBroadcasterConfig{ + Self: leaderPub, + Peers: &mutableTVUPeers{peers: []gossip.TVUPeer{{Pubkey: gossip.Pubkey(rootPub), TVUAddr: root.LocalAddr().(*net.UDPAddr)}}}, + Stakes: func(uint64) map[solana.PublicKey]uint64 { + return map[solana.PublicKey]uint64{leaderPub: 1, rootPub: 1} + }, + ShredReceivers: []*net.UDPAddr{receiver.LocalAddr().(*net.UDPAddr)}, + }) + require.NoError(t, err) + t.Cleanup(func() { _ = b.Close() }) + + gen := ShredGenerator{Slot: 90, ParentSlot: 89, Version: 44, ReferenceTick: 63} + packets, _, _, _, err := gen.MakeShredsFromData(solana.PrivateKey(leaderKey), []byte("leader block"), true, solana.Hash{9}, 0, 0) + require.NoError(t, err) + require.NoError(t, b.Broadcast(packets[:1])) + assert.Equal(t, packets[0], readPacket(t, root), "the Turbine root") + assert.Equal(t, packets[0], readPacket(t, receiver), "and the shred receiver") +} + +// A relay sends each shred it retransmits to the shred receivers too, after +// its Turbine children, and a leaf with no children still does. +func TestRetransmitterSendsToShredReceivers(t *testing.T) { + for name, position := range map[string]int{"relay": 1, "leaf": dataPlaneFanout + 1} { + t.Run(name, func(t *testing.T) { + leaderPub, leaderKey := testIdentity(t) + const identityCount = dataPlaneFanout + 5 + identities := make(map[solana.PublicKey]ed25519.PrivateKey) + allPeers := make([]gossip.TVUPeer, identityCount) + stakes := map[solana.PublicKey]uint64{leaderPub: 1_000} + for i := range allPeers { + pubkey, private := testIdentity(t) + identities[pubkey] = private + allPeers[i] = gossip.TVUPeer{ + Pubkey: gossip.Pubkey(pubkey), + TVUAddr: &net.UDPAddr{IP: net.IPv4(127, 2, byte(i/250), byte(i%250+1)), Port: 9000 + i}, + } + stakes[pubkey] = uint64(identityCount - i) + } + gen := ShredGenerator{Slot: 90, ParentSlot: 89, Version: 44, ReferenceTick: 63} + packets, _, _, _, err := gen.MakeShredsFromData(solana.PrivateKey(leaderKey), []byte("relayed block"), true, solana.Hash{9}, 0, 0) + require.NoError(t, err) + packet := packets[0] + shred, err := ParseShred(packet) + require.NoError(t, err) + id := ShredID{Slot: shred.Slot, Index: shred.Index, Type: shred.Type} + + nodes := NewRetransmitClusterNodes(ClusterNodesConfig{Self: solana.PublicKey(allPeers[0].Pubkey), TVUPeers: allPeers, Stakes: stakes, UseChaCha8: true}) + shuffle := nodes.retransmitShuffle(leaderPub, id) + self := nodes.nodes[shuffle[position]].pubkey + nodes.selfPubkey, nodes.selfIndex = self, shuffle[position] + parent, ok, err := nodes.RetransmitParent(leaderPub, id, dataPlaneFanout) + require.NoError(t, err) + require.True(t, ok) + if parent != leaderPub { + merkleRoot, err := shred.MerkleRoot() + require.NoError(t, err) + offset, err := shred.retransmitterSignatureOffset() + require.NoError(t, err) + copy(packet[offset:offset+ed25519.SignatureSize], ed25519.Sign(identities[parent], merkleRoot[:])) + shred, err = ParseShred(packet) + require.NoError(t, err) + } + + receiver := &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 8003} + sender := newCaptureBatchSender() + retransmitter, err := newRetransmitterWithSenders(RetransmitConfig{ + Identity: identities[self], + Peers: &mutableTVUPeers{peers: allPeers}, + Stakes: func(uint64) map[solana.PublicKey]uint64 { return stakes }, + UseChaCha8: true, + ShredReceivers: []*net.UDPAddr{receiver}, + }, []packetBatchSender{sender}) + require.NoError(t, err) + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan struct{}) + go func() { + retransmitter.Run(ctx) + close(done) + }() + t.Cleanup(func() { + cancel() + <-done + }) + + require.NoError(t, retransmitter.Submit(packet, shred, leaderPub, false)) + require.Eventually(t, func() bool { return retransmitter.Stats().ShredReceiverSent == 1 }, time.Second, time.Millisecond) + sent := sender.packets() + last := sent[len(sent)-1] + assert.Equal(t, []*net.UDPAddr{receiver}, last.peers, "the shred receiver comes after the children") + if name == "relay" { + require.Len(t, sent, 2) + assert.NotEmpty(t, sent[0].peers) + assert.Equal(t, sent[0].packet, last.packet, "the receiver gets what the children get") + } else { + assert.Len(t, sent, 1, "a leaf has no children") + } + + // A duplicate is not forwarded again. + require.NoError(t, retransmitter.Submit(packet, shred, leaderPub, false)) + time.Sleep(20 * time.Millisecond) + assert.Equal(t, uint64(1), retransmitter.Stats().ShredReceiverSent) + }) + } +} diff --git a/pkg/turbine/turbine_broadcaster.go b/pkg/turbine/turbine_broadcaster.go index fe1196c9..668691c5 100644 --- a/pkg/turbine/turbine_broadcaster.go +++ b/pkg/turbine/turbine_broadcaster.go @@ -33,6 +33,7 @@ type TurbineBroadcaster struct { useChaCha8 bool dedupAddrs bool + receivers []*net.UDPAddr cacheSlot uint64 cacheValid bool @@ -49,6 +50,9 @@ type TurbineBroadcasterConfig struct { LeaderSlots uint64 UseChaCha8 bool DedupAddrs bool + // ShredReceivers get a copy of every shred this node broadcasts, in + // addition to the Turbine root, such as a block builder on the same host. + ShredReceivers []*net.UDPAddr } func NewTurbineBroadcaster(cfg TurbineBroadcasterConfig) (*TurbineBroadcaster, error) { @@ -83,6 +87,7 @@ func NewTurbineBroadcaster(cfg TurbineBroadcasterConfig) (*TurbineBroadcaster, e leaderSlots: leaderSlots, useChaCha8: cfg.UseChaCha8, dedupAddrs: cfg.DedupAddrs, + receivers: cfg.ShredReceivers, }, nil } @@ -109,13 +114,10 @@ func (b *TurbineBroadcaster) broadcastPacket(packet []byte) error { } nodes := b.clusterNodesForSlot(shredID.Slot) dests := b.broadcastDestinations(shredID, nodes) - if len(dests) == 0 { - // The deterministic root can be a staked identity whose contact info is - // absent. Match Agave by dropping that route and relying on the FEC set. - // With no external route whatsoever, however, fail the local slot closed. - if nodes.HasRoutablePeer() { - return nil - } + // The deterministic root can be a staked identity whose contact info is + // absent. Match Agave by dropping that route and relying on the FEC set. + // With no external route whatsoever, however, fail the local slot closed. + if len(dests) == 0 && !nodes.HasRoutablePeer() { return fmt.Errorf("turbine broadcast: no routable TVU destination for slot %d shred %d", shredID.Slot, shredID.Index) } for _, peer := range dests { @@ -123,6 +125,11 @@ func (b *TurbineBroadcaster) broadcastPacket(packet []byte) error { return err } } + // After Turbine, so receivers never delay it. A receiver that cannot be + // reached does not fail the slot. + for _, receiver := range b.receivers { + _ = b.sendTo(packet, receiver) + } return nil }