Skip to content

refactor: type benchmark workload choices #39

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
0xcadams merged 1 commit into from
Oct 1, 2026
Merged

refactor: type benchmark workload choices #39

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
109 changes: 66 additions & 43 deletions cmd/tnlbench/benchmark_plan.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,29 +8,45 @@ import (
"os"
"time"

"github.com/tnldotdev/tnl/internal/benchworkload"
"github.com/tnldotdev/tnl/internal/clientstate"
)

type benchmarkSuite string

const (
benchmarkSmoke benchmarkSuite = "smoke"
benchmarkTarget benchmarkSuite = "target"
)

type visitorNetwork string

const (
visitorTCP visitorNetwork = "tcp"
visitorTCP4 visitorNetwork = "tcp4"
visitorTCP6 visitorNetwork = "tcp6"
)

type workloadOptions struct {
Suite string `name:"suite" env:"BENCH_SUITE" default:"smoke" enum:"smoke,target" help:"Small smoke or an explicit target workload."`
Server string `name:"server" env:"BENCH_SERVER" required:"" help:"HTTPS control URL of the approved tnl server."`
Transport string `name:"transport" env:"BENCH_TRANSPORT" default:"mixed" enum:"mixed,quic,tcp,auto" help:"Forced cohorts or normal QUIC/TLS-TCP selection."`
QUICDisablePathMTUDiscovery bool `name:"quic-disable-path-mtu-discovery" env:"BENCH_QUIC_DISABLE_PATH_MTU_DISCOVERY" help:"Disable publisher QUIC path-MTU discovery for a diagnostic run."`
QUICQlog bool `name:"quic-qlog" env:"BENCH_QUIC_QLOG" help:"Record publisher QUIC qlogs in the benchmark result directory."`
QUICKeepAlive time.Duration `name:"quic-keepalive" env:"BENCH_QUIC_KEEPALIVE" help:"Publisher QUIC keepalive period override for a diagnostic run."`
VisitorNetwork string `name:"visitor-network" env:"BENCH_VISITOR_NETWORK" default:"tcp" enum:"tcp,tcp4,tcp6" help:"Network used by public visitor TCP sockets."`
VisitorInterface string `name:"visitor-interface" env:"BENCH_VISITOR_INTERFACE" help:"Local interface whose IPv4 address is used by public visitors only."`
PublicURLs int `name:"public-urls" env:"BENCH_PUBLIC_URLS" default:"4" help:"Public URLs to publish."`
FreshRate int `name:"fresh-connections-per-second" env:"BENCH_FRESH_CONNECTIONS_PER_SECOND" default:"16" help:"Offered visitor requests per second."`
HeldStreams int `name:"held-streams" env:"BENCH_HELD_STREAMS" default:"4" help:"Held visitor streams."`
Concurrency int `name:"concurrency" env:"BENCH_CONCURRENCY" default:"128" help:"Concurrent visitor request workers."`
QueueSlots int `name:"queue-slots" env:"BENCH_QUEUE_SLOTS" default:"8" help:"Waiting visitor request slots."`
PayloadBytes int `name:"payload-bytes" env:"BENCH_PAYLOAD_BYTES" default:"32768" help:"Verified response bytes."`
Repetitions int `name:"repetitions" env:"BENCH_REPETITIONS" default:"1" help:"Measurement windows."`
Warmup time.Duration `name:"warmup" env:"BENCH_WARMUP" default:"5s" help:"Time before measuring."`
Duration time.Duration `name:"duration" env:"BENCH_DURATION" default:"10s" help:"Length of each measurement window."`
StateDir string `name:"state-dir" env:"BENCH_STATE_DIR" type:"path" help:"Client state with an existing login for the selected server."`
ResultsRoot string `name:"results-root" env:"BENCH_RESULTS_ROOT" default:"bench-results" type:"path" help:"Result directory."`
Suite benchmarkSuite `name:"suite" env:"BENCH_SUITE" default:"smoke" enum:"smoke,target" help:"Small smoke or an explicit target workload."`
Server string `name:"server" env:"BENCH_SERVER" required:"" help:"HTTPS control URL of the approved tnl server."`
Transport benchworkload.TransportChoice `name:"transport" env:"BENCH_TRANSPORT" default:"mixed" enum:"mixed,quic,tcp,auto" help:"Forced cohorts or normal QUIC/TLS-TCP selection."`
QUICDisablePathMTUDiscovery bool `name:"quic-disable-path-mtu-discovery" env:"BENCH_QUIC_DISABLE_PATH_MTU_DISCOVERY" help:"Disable publisher QUIC path-MTU discovery for a diagnostic run."`
QUICQlog bool `name:"quic-qlog" env:"BENCH_QUIC_QLOG" help:"Record publisher QUIC qlogs in the benchmark result directory."`
QUICKeepAlive time.Duration `name:"quic-keepalive" env:"BENCH_QUIC_KEEPALIVE" help:"Publisher QUIC keepalive period override for a diagnostic run."`
VisitorNetwork visitorNetwork `name:"visitor-network" env:"BENCH_VISITOR_NETWORK" default:"tcp" enum:"tcp,tcp4,tcp6" help:"Network used by public visitor TCP sockets."`
VisitorInterface string `name:"visitor-interface" env:"BENCH_VISITOR_INTERFACE" help:"Local interface whose IPv4 address is used by public visitors only."`
PublicURLs int `name:"public-urls" env:"BENCH_PUBLIC_URLS" default:"4" help:"Public URLs to publish."`
FreshRate int `name:"fresh-connections-per-second" env:"BENCH_FRESH_CONNECTIONS_PER_SECOND" default:"16" help:"Offered visitor requests per second."`
HeldStreams int `name:"held-streams" env:"BENCH_HELD_STREAMS" default:"4" help:"Held visitor streams."`
Concurrency int `name:"concurrency" env:"BENCH_CONCURRENCY" default:"128" help:"Concurrent visitor request workers."`
QueueSlots int `name:"queue-slots" env:"BENCH_QUEUE_SLOTS" default:"8" help:"Waiting visitor request slots."`
PayloadBytes int `name:"payload-bytes" env:"BENCH_PAYLOAD_BYTES" default:"32768" help:"Verified response bytes."`
Repetitions int `name:"repetitions" env:"BENCH_REPETITIONS" default:"1" help:"Measurement windows."`
Warmup time.Duration `name:"warmup" env:"BENCH_WARMUP" default:"5s" help:"Time before measuring."`
Duration time.Duration `name:"duration" env:"BENCH_DURATION" default:"10s" help:"Length of each measurement window."`
StateDir string `name:"state-dir" env:"BENCH_STATE_DIR" type:"path" help:"Client state with an existing login for the selected server."`
ResultsRoot string `name:"results-root" env:"BENCH_RESULTS_ROOT" default:"bench-results" type:"path" help:"Result directory."`
}

type benchmarkPlan struct {
Expand All @@ -41,22 +57,22 @@ type benchmarkPlan struct {
}

type workloadSummary struct {
Suite string `json:"suite"`
Transport string `json:"transport"`
QUICDisablePathMTUDiscovery bool `json:"quic_disable_path_mtu_discovery,omitempty"`
QUICQlog bool `json:"quic_qlog,omitempty"`
QUICKeepAlive time.Duration `json:"quic_keepalive,omitempty"`
VisitorNetwork string `json:"visitor_network"`
VisitorInterface string `json:"visitor_interface,omitempty"`
PublicURLs int `json:"public_urls"`
FreshRate int `json:"fresh_connections_per_second"`
HeldStreams int `json:"held_streams"`
Concurrency int `json:"concurrency"`
QueueSlots int `json:"queue_slots"`
PayloadBytes int `json:"payload_bytes"`
Repetitions int `json:"repetitions"`
Warmup time.Duration `json:"warmup"`
Duration time.Duration `json:"duration"`
Suite benchmarkSuite `json:"suite"`
Transport benchworkload.TransportChoice `json:"transport"`
QUICDisablePathMTUDiscovery bool `json:"quic_disable_path_mtu_discovery,omitempty"`
QUICQlog bool `json:"quic_qlog,omitempty"`
QUICKeepAlive time.Duration `json:"quic_keepalive,omitempty"`
VisitorNetwork visitorNetwork `json:"visitor_network"`
VisitorInterface string `json:"visitor_interface,omitempty"`
PublicURLs int `json:"public_urls"`
FreshRate int `json:"fresh_connections_per_second"`
HeldStreams int `json:"held_streams"`
Concurrency int `json:"concurrency"`
QueueSlots int `json:"queue_slots"`
PayloadBytes int `json:"payload_bytes"`
Repetitions int `json:"repetitions"`
Warmup time.Duration `json:"warmup"`
Duration time.Duration `json:"duration"`
}

func (c workloadOptions) plan() (benchmarkPlan, error) {
Expand All @@ -67,20 +83,20 @@ func (c workloadOptions) plan() (benchmarkPlan, error) {
if err != nil {
return benchmarkPlan{}, err
}
if c.Suite != "smoke" && c.Suite != "target" {
if c.Suite != benchmarkSmoke && c.Suite != benchmarkTarget {
return benchmarkPlan{}, errors.New("suite must be smoke or target")
}
if c.Transport != "mixed" && c.Transport != "quic" && c.Transport != "tcp" && c.Transport != "auto" {
if !c.Transport.Valid() {
return benchmarkPlan{}, errors.New("transport must be mixed, quic, tcp, or auto")
}
if c.VisitorNetwork != "tcp" && c.VisitorNetwork != "tcp4" && c.VisitorNetwork != "tcp6" {
if c.VisitorNetwork != visitorTCP && c.VisitorNetwork != visitorTCP4 && c.VisitorNetwork != visitorTCP6 {
return benchmarkPlan{}, errors.New("visitor network must be tcp, tcp4, or tcp6")
}
if c.QUICKeepAlive < 0 || c.QUICKeepAlive > 0 && (c.QUICKeepAlive < time.Second || c.QUICKeepAlive > time.Minute) {
return benchmarkPlan{}, errors.New("QUIC keepalive must be zero or between 1s and 1m")
}
if c.VisitorInterface != "" {
if c.VisitorNetwork == "tcp6" {
if c.VisitorNetwork == visitorTCP6 {
return benchmarkPlan{}, errors.New("a selected visitor interface requires IPv4 visitor sockets")
}
if _, err := visitorSourceAddress(c.VisitorInterface); err != nil {
Expand All @@ -90,7 +106,7 @@ func (c workloadOptions) plan() (benchmarkPlan, error) {
if c.PublicURLs < 1 || c.PublicURLs > 10_000 || c.FreshRate < 1 || c.FreshRate > 10_000 || c.HeldStreams < 0 || c.HeldStreams > 100_000 || c.Concurrency < 1 || c.Concurrency > 100_000 || c.QueueSlots < 0 || c.QueueSlots > 10_000 || c.PayloadBytes < 1 || c.PayloadBytes > 16<<20 || c.Repetitions < 1 || c.Repetitions > 10 || c.Warmup < 0 || c.Warmup > 5*time.Minute || c.Duration < time.Second || c.Duration > time.Hour {
return benchmarkPlan{}, errors.New("invalid workload shape or duration")
}
if c.Suite == "smoke" && (c.Transport != "mixed" || c.PublicURLs > 4 || c.FreshRate > 16 ||
if c.Suite == benchmarkSmoke && (c.Transport != benchworkload.TransportMixed || c.PublicURLs > 4 || c.FreshRate > 16 ||
c.HeldStreams > 4 || c.Concurrency > 128 || c.QueueSlots > 8 || c.PayloadBytes > 32<<10 ||
c.Repetitions != 1 || c.Warmup > 5*time.Second || c.Duration > 30*time.Second) {
return benchmarkPlan{}, errors.New("larger workloads require suite target")
Expand All @@ -101,17 +117,24 @@ func (c workloadOptions) plan() (benchmarkPlan, error) {
PayloadBytes: c.PayloadBytes, Repetitions: c.Repetitions, Warmup: c.Warmup, Duration: c.Duration}}, nil
}

type benchmarkPlanFormat string

const (
benchmarkPlanHuman benchmarkPlanFormat = "human"
benchmarkPlanJSON benchmarkPlanFormat = "json"
)

type planCommand struct {
workloadOptions
Format string `name:"format" default:"human" enum:"human,json" help:"Plan output format."`
Format benchmarkPlanFormat `name:"format" default:"human" enum:"human,json" help:"Plan output format."`
}

func (c planCommand) run(stdout io.Writer) error {
plan, err := c.plan()
if err != nil {
return err
}
if c.Format == "json" {
if c.Format == benchmarkPlanJSON {
return json.NewEncoder(stdout).Encode(plan)
}
fmt.Fprintf(stdout, "Benchmark plan (READ ONLY)\nServer: %s\nGenerators: local publisher and visitor\n", plan.Server)
Expand Down Expand Up @@ -149,7 +172,7 @@ func (c runCommand) validate() (benchmarkPlan, error) {
if c.Approved != "1" {
return plan, errors.New("BENCH_APPROVED must be exactly 1; planning does not grant execution approval")
}
if value, found := os.LookupEnv("BENCH_SUITE"); !found || value == "" || value != c.Suite {
if value, found := os.LookupEnv("BENCH_SUITE"); !found || value == "" || value != string(c.Suite) {
return plan, errors.New("execution requires an explicit BENCH_SUITE environment variable")
}
return plan, nil
Expand Down
8 changes: 4 additions & 4 deletions cmd/tnlbench/benchmark_result.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -133,14 +133,14 @@ type generatorResult struct {
Elapsed time.Duration `json:"elapsed"`
}

func recordReadyPublicURLs(result *benchmarkResult, ready []benchworkload.PublishedPublicURL, transport string, count int) []string {
func recordReadyPublicURLs(result *benchmarkResult, ready []benchworkload.PublishedPublicURL, transport benchworkload.TransportChoice, count int) []string {
urls := make([]string, count)
for _, publicURL := range ready {
urls[publicURL.Index] = publicURL.Ready.PublicURL
selected := transport
if selected == "tcp" || selected == "mixed" && publicURL.Index%2 != 0 {
selected := string(transport)
if transport == benchworkload.TransportTCP || transport == benchworkload.TransportMixed && publicURL.Index%2 != 0 {
selected = string(tunnel.TransportTLSTCP)
} else if selected == "quic" || selected == "mixed" {
} else if transport == benchworkload.TransportQUIC || transport == benchworkload.TransportMixed {
selected = string(tunnel.TransportQUIC)
}
result.PublicURLInfo = append(result.PublicURLInfo, benchmarkPublicURL{
Expand Down
2 changes: 1 addition & 1 deletion cmd/tnlbench/benchmark_run.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ func (c runCommand) measure(parent context.Context, plan benchmarkPlan, stateDir
}); err != nil {
return result, err
}
visitor := benchworkload.Visitor{PayloadBytes: c.PayloadBytes, Network: c.VisitorNetwork, SourceAddress: source}
visitor := benchworkload.Visitor{PayloadBytes: c.PayloadBytes, Network: string(c.VisitorNetwork), SourceAddress: source}
fmt.Fprintln(progress, "tnlbench: verifying each public URL")
for _, url := range urls {
if check := visitor.Request(ctx, url, time.Now()); check.Error != "" {
Expand Down
28 changes: 21 additions & 7 deletions internal/benchworkload/publishers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,20 @@ import (
"github.com/tnldotdev/tnl/pkg/api/controlv1"
)

// TransportChoice selects a benchmark publisher cohort or production fallback.
type TransportChoice string

const (
TransportAuto TransportChoice = "auto"
TransportMixed TransportChoice = "mixed"
TransportQUIC TransportChoice = "quic"
TransportTCP TransportChoice = "tcp"
)

func (transport TransportChoice) Valid() bool {
return transport == TransportAuto || transport == TransportMixed || transport == TransportQUIC || transport == TransportTCP
}

type PublisherConfig struct {
Server, LoginToken, Domain, StateRoot, Target string
HostnamePrefix string
Expand All @@ -31,7 +45,7 @@ type PublisherConfig struct {
RelayTLS *tls.Config
AllowedIPPrefixes []string
// mixed forces alternating QUIC/TLS-TCP cohorts; auto uses production fallback.
Transport string
Transport TransportChoice
QUICDisablePathMTUDiscovery bool
QUICQlog bool
QUICKeepAlive time.Duration
Expand Down Expand Up @@ -85,7 +99,7 @@ func OpenPublishers(ctx context.Context, config PublisherConfig) (_ *Publishers,
if config.Parallel < 1 || config.StartParallel < 0 || config.ReadyTimeout <= 0 || config.StopTimeout <= 0 || config.DrainTime < 0 {
return nil, errors.New("publisher concurrency and deadlines must be positive")
}
if config.Transport != "auto" && config.Transport != "mixed" && config.Transport != "quic" && config.Transport != "tcp" {
if !config.Transport.Valid() {
return nil, errors.New("publisher transport must be auto, mixed, quic, or tcp")
}
database, err := clientstate.Open(ctx, config.StateRoot)
Expand Down Expand Up @@ -233,17 +247,17 @@ func (g *Publishers) Start(ctx context.Context, indexes []int) ([]PublishedPubli
}
}
transport := g.config.Transport
if transport == "mixed" {
if transport == TransportMixed {
if index%2 == 0 {
transport = "quic"
transport = TransportQUIC
} else {
transport = "tcp"
transport = TransportTCP
}
}
if transport == "quic" {
if transport == TransportQUIC {
cfg.TCPConnector = disabledConnector{}
}
if transport == "tcp" {
if transport == TransportTCP {
cfg.QUICConnector = disabledConnector{}
}
go func() {
Expand Down
4 changes: 2 additions & 2 deletions internal/tnldruntime/separated_components_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -306,9 +306,9 @@ func runSeparatedPublishers(t *testing.T, ctx context.Context, count, shard int)
}
}
}
transport := "mixed"
transport := benchworkload.TransportMixed
if *runtimeLoadScenario == "udp-fallback" {
transport = "auto"
transport = benchworkload.TransportAuto
}
var fallbacks atomic.Int64
client := separatedHTTP(t)
Expand Down
Loading