Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 19 additions & 9 deletions cli/client/client_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -586,16 +586,17 @@ func TestBackends(t *testing.T) {
}
}

// The report is served as the backend writes it, not as the Go struct
// encodes it, so a phase key the CLI names wrongly fails here.
func TestReport(t *testing.T) {
c, _ := testServer(t, jsonHandler(t, "GET", "/api/tests/", Report{
Summary: &ReportSummary{
TotalRecords: 100000, AvgLatencyMs: 5.2, P99LatencyMs: 12.3,
AvgThroughputRecPerSec: 50000, ErrorRate: 0.001,
},
OverallSlaVerdict: &SlaVerdict{
Passed: true,
},
}))
c, _ := testServer(t, func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{"summary":{"totalRecords":100000,"avgThroughputRecPerSec":50000,` +
`"avgLatencyMs":5.2,"p99LatencyMs":12.3,"errorRate":0.001},` +
`"overallSlaVerdict":{"passed":true},"phases":[` +
`{"phaseName":"produce","metrics":{"totalRecords":60000,"avgThroughputRecPerSec":30000,"p99LatencyMs":9.5},"slaVerdict":{"passed":true}},` +
`{"phaseName":"consume","metrics":{"totalRecords":40000,"avgThroughputRecPerSec":20000},"slaVerdict":{"passed":false}}]}`))
})
resp, err := c.Report(context.Background(), "run-1")
if err != nil {
t.Fatal(err)
Expand All @@ -606,6 +607,15 @@ func TestReport(t *testing.T) {
if !resp.OverallSlaVerdict.Passed {
t.Error("SLA should have passed")
}
if len(resp.Phases) != 2 {
t.Fatalf("Phases = %+v", resp.Phases)
}
if p := resp.Phases[0]; p.Metrics == nil || p.Metrics.AvgThroughputRecPerSec != 30000 || p.Metrics.P99LatencyMs != 9.5 {
t.Errorf("produce = %+v", p.Metrics)
}
if p := resp.Phases[1]; p.Name != "consume" || p.SlaVerdict == nil || p.SlaVerdict.Passed {
t.Errorf("consume = %+v", p)
}
}

func TestReportSummary(t *testing.T) {
Expand Down
11 changes: 5 additions & 6 deletions cli/client/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -286,13 +286,12 @@ type Report struct {
Phases []ReportPhase `json:"phases,omitempty"`
}

// ReportPhase is the backend's PhaseReport: a phase's name, its own summary
// and its own SLA verdict.
type ReportPhase struct {
Name string `json:"name"`
Status string `json:"status"`
RecordsSent float64 `json:"recordsSent"`
ThroughputRecPerSec float64 `json:"throughputRecordsPerSec"`
P99LatencyMs float64 `json:"p99LatencyMs"`
ErrorRate float64 `json:"errorRate"`
Name string `json:"phaseName"`
Metrics *ReportSummary `json:"metrics,omitempty"`
SlaVerdict *SlaVerdict `json:"slaVerdict,omitempty"`
}

type SlaVerdict struct {
Expand Down
66 changes: 22 additions & 44 deletions cli/cmd/report.go
Original file line number Diff line number Diff line change
Expand Up @@ -35,61 +35,39 @@ var reportShowCmd = &cobra.Command{
summary := result.Summary

if summary == nil || summary.AvgThroughputRecPerSec == 0 {
if len(result.Phases) > 0 {
run, fetchErr := apiClient.GetTest(context.Background(), args[0])
if fetchErr == nil && len(run.Results) > 0 && !isStaleResult(run.Results) {
s := &client.ReportSummary{}
count := 0
for _, p := range result.Phases {
s.TotalRecords += p.RecordsSent
if p.ThroughputRecPerSec > s.PeakThroughputRecPerSec {
s.PeakThroughputRecPerSec = p.ThroughputRecPerSec
for _, r := range run.Results {
s.TotalRecords += r.RecordsSent
if r.ThroughputRecordsPerSec > s.PeakThroughputRecPerSec {
s.PeakThroughputRecPerSec = r.ThroughputRecordsPerSec
}
if p.ThroughputRecPerSec > 0 {
s.AvgThroughputRecPerSec += p.ThroughputRecPerSec
if r.ThroughputRecordsPerSec > 0 {
s.AvgThroughputRecPerSec += r.ThroughputRecordsPerSec
count++
}
if p.P99LatencyMs > s.P99LatencyMs {
s.P99LatencyMs = p.P99LatencyMs
if r.P99LatencyMs > s.P99LatencyMs {
s.P99LatencyMs = r.P99LatencyMs
}
if r.AvgLatencyMs > s.AvgLatencyMs {
s.AvgLatencyMs = r.AvgLatencyMs
}
if r.P50LatencyMs > s.P50LatencyMs {
s.P50LatencyMs = r.P50LatencyMs
}
if r.P95LatencyMs > s.P95LatencyMs {
s.P95LatencyMs = r.P95LatencyMs
}
if r.MaxLatencyMs > s.MaxLatencyMs {
s.MaxLatencyMs = r.MaxLatencyMs
}
}
if count > 0 {
s.AvgThroughputRecPerSec /= float64(count)
}
summary = s
} else {
run, fetchErr := apiClient.GetTest(context.Background(), args[0])
if fetchErr == nil && len(run.Results) > 0 && !isStaleResult(run.Results) {
s := &client.ReportSummary{}
count := 0
for _, r := range run.Results {
s.TotalRecords += r.RecordsSent
if r.ThroughputRecordsPerSec > s.PeakThroughputRecPerSec {
s.PeakThroughputRecPerSec = r.ThroughputRecordsPerSec
}
if r.ThroughputRecordsPerSec > 0 {
s.AvgThroughputRecPerSec += r.ThroughputRecordsPerSec
count++
}
if r.P99LatencyMs > s.P99LatencyMs {
s.P99LatencyMs = r.P99LatencyMs
}
if r.AvgLatencyMs > s.AvgLatencyMs {
s.AvgLatencyMs = r.AvgLatencyMs
}
if r.P50LatencyMs > s.P50LatencyMs {
s.P50LatencyMs = r.P50LatencyMs
}
if r.P95LatencyMs > s.P95LatencyMs {
s.P95LatencyMs = r.P95LatencyMs
}
if r.MaxLatencyMs > s.MaxLatencyMs {
s.MaxLatencyMs = r.MaxLatencyMs
}
}
if count > 0 {
s.AvgThroughputRecPerSec /= float64(count)
}
summary = s
}
}
}

Expand Down
19 changes: 11 additions & 8 deletions cli/cmd/report_html.go
Original file line number Diff line number Diff line change
Expand Up @@ -213,17 +213,20 @@ func renderHTMLReport(id string, r *client.Report) string {
b.WriteString(`<div class="card">
<h2>Phase Breakdown</h2>
<table>
<thead><tr><th>Phase</th><th>Status</th><th>Records</th><th>Throughput</th><th>P99 (ms)</th><th>Errors</th></tr></thead>
<thead><tr><th>Phase</th><th>SLA</th><th>Records</th><th>Throughput</th><th>P99 (ms)</th><th>Errors</th></tr></thead>
<tbody>
`)
for _, p := range r.Phases {
phaseName := p.Name
if phaseName == "" {
phaseName = "main"
m := client.ReportSummary{}
if p.Metrics != nil {
m = *p.Metrics
}
statusBadge := `<span class="badge badge-pass">` + p.Status + `</span>`
if strings.EqualFold(p.Status, "FAILED") || strings.EqualFold(p.Status, "ERROR") {
statusBadge = `<span class="badge badge-fail">` + p.Status + `</span>`
slaBadge := "—"
if v := p.SlaVerdict; v != nil {
slaBadge = `<span class="badge badge-pass">✓ PASSED</span>`
if !v.Passed {
slaBadge = `<span class="badge badge-fail">✖ FAILED</span>`
}
}
b.WriteString(fmt.Sprintf(` <tr>
<td>%s</td>
Expand All @@ -233,7 +236,7 @@ func renderHTMLReport(id string, r *client.Report) string {
<td class="metric-value">%s</td>
<td class="metric-value">%.4f%%</td>
</tr>
`, phaseName, statusBadge, fmtNum(p.RecordsSent), fmtFloat(p.ThroughputRecPerSec, 1), fmtFloat(p.P99LatencyMs, 2), p.ErrorRate*100))
`, html.EscapeString(p.Name), slaBadge, fmtNum(m.TotalRecords), fmtFloat(m.AvgThroughputRecPerSec, 1), fmtFloat(m.P99LatencyMs, 2), m.ErrorRate*100))
}
b.WriteString(` </tbody>
</table>
Expand Down
13 changes: 7 additions & 6 deletions cli/cmd/report_html_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -118,8 +118,8 @@ func TestRenderHTMLReport_WithPhases(t *testing.T) {
report := &client.Report{
Summary: &client.ReportSummary{TotalRecords: 1},
Phases: []client.ReportPhase{
{Name: "warmup", Status: "DONE", RecordsSent: 500},
{Name: "burst", Status: "FAILED", RecordsSent: 200},
{Name: "warmup", Metrics: &client.ReportSummary{TotalRecords: 500}, SlaVerdict: &client.SlaVerdict{Passed: true}},
{Name: "<script>", Metrics: &client.ReportSummary{TotalRecords: 200}, SlaVerdict: &client.SlaVerdict{Passed: false}},
},
}

Expand All @@ -128,11 +128,12 @@ func TestRenderHTMLReport_WithPhases(t *testing.T) {
if !strings.Contains(html, "Phase Breakdown") {
t.Error("expected phase breakdown section")
}
if !strings.Contains(html, "warmup") {
t.Error("expected warmup phase")
if !strings.Contains(html, "<td>warmup</td>\n <td><span class=\"badge badge-pass\">") {
t.Errorf("a phase whose SLA passed has no pass badge:\n%s", html)
}
if !strings.Contains(html, "burst") {
t.Error("expected burst phase")
// A phase name comes from the user's scenario file, not markup.
if strings.Contains(html, "<script>") || !strings.Contains(html, "<td>&lt;script&gt;</td>\n <td><span class=\"badge badge-fail\">") {
t.Errorf("the phase name is not escaped, or a failed SLA has no fail badge:\n%s", html)
}
}

Expand Down
19 changes: 11 additions & 8 deletions cli/cmd/report_markdown.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,17 +62,20 @@ func renderMarkdownPhases(b *strings.Builder, r *client.Report) {
return
}
b.WriteString("\n## Phase Breakdown\n\n")
b.WriteString("| Phase | Status | Records | Throughput | P99 (ms) | Errors |\n|---|---|---|---|---|---|\n")
b.WriteString("| Phase | SLA | Records | Throughput | P99 (ms) | Errors |\n|---|---|---|---|---|---|\n")
for _, p := range r.Phases {
name := p.Name
if name == "" {
name = "main"
m := client.ReportSummary{}
if p.Metrics != nil {
m = *p.Metrics
}
status := "✓ " + p.Status
if strings.EqualFold(p.Status, "FAILED") || strings.EqualFold(p.Status, "ERROR") {
status = "✖ " + p.Status
sla := "—"
if v := p.SlaVerdict; v != nil {
sla = "✓"
if !v.Passed {
sla = "✖"
}
}
b.WriteString(fmt.Sprintf("| %s | %s | %s | %s rec/s | %.2f | %.4f%% |\n",
name, status, fmtNum(p.RecordsSent), fmtFloat(p.ThroughputRecPerSec, 1), p.P99LatencyMs, p.ErrorRate*100))
strings.ReplaceAll(p.Name, "|", `\|`), sla, fmtNum(m.TotalRecords), fmtFloat(m.AvgThroughputRecPerSec, 1), m.P99LatencyMs, m.ErrorRate*100))
}
}
22 changes: 16 additions & 6 deletions cli/cmd/report_markdown_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -110,8 +110,11 @@ func TestRenderMarkdownReport_WithPhases(t *testing.T) {
report := &client.Report{
Summary: &client.ReportSummary{TotalRecords: 1000},
Phases: []client.ReportPhase{
{Name: "warmup", Status: "DONE", RecordsSent: 500, ThroughputRecPerSec: 10000, P99LatencyMs: 15.5, ErrorRate: 0},
{Name: "main", Status: "DONE", RecordsSent: 500, ThroughputRecPerSec: 25000, P99LatencyMs: 22.3, ErrorRate: 0.001},
{Name: "produce", Metrics: &client.ReportSummary{TotalRecords: 600, AvgThroughputRecPerSec: 10000, P99LatencyMs: 15.5},
SlaVerdict: &client.SlaVerdict{Passed: true}},
{Name: "consume", Metrics: &client.ReportSummary{TotalRecords: 400, AvgThroughputRecPerSec: 25000, P99LatencyMs: 22.3, ErrorRate: 0.001},
SlaVerdict: &client.SlaVerdict{Passed: false}},
{Name: "a|b"},
},
}

Expand All @@ -120,11 +123,18 @@ func TestRenderMarkdownReport_WithPhases(t *testing.T) {
if !strings.Contains(md, "## Phase Breakdown") {
t.Error("expected Phase Breakdown section")
}
if !strings.Contains(md, "warmup") {
t.Error("expected warmup phase")
if !strings.Contains(md, "| produce | ✓ | 600 | 10000.0 rec/s | 15.50 | 0.0000% |") {
t.Errorf("the produce row does not carry its own numbers:\n%s", md)
}
if !strings.Contains(md, "main") {
t.Error("expected main phase")
if !strings.Contains(md, "| consume | ✖ |") {
t.Errorf("a phase whose SLA failed is not marked failed:\n%s", md)
}
// A phase name is the user's text; a | in it would split the row.
if !strings.Contains(md, `| a\|b | — |`) {
t.Errorf("the phase name is not escaped, or a phase with no verdict is not —:\n%s", md)
}
if strings.Contains(md, "| main |") {
t.Errorf("a phase is named main:\n%s", md)
}
}

Expand Down
Loading