Skip to content

Commit ebfd69c

Browse files
haoliuuclaude
andcommitted
perf: Increase benchmark iterations to 100 for statistical significance
- FirstCatalogQuery: 100 fresh processes (was 10) - SubsequentCatalogQuery: 100 iterations (was 20) Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent 4421d5a commit ebfd69c

2 files changed

Lines changed: 7 additions & 7 deletions

File tree

.github/workflows/foundry-local-benchmark.yml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,30 +63,30 @@ jobs:
6363
- name: Build Test Project
6464
run: dotnet build AIDevGallery.Tests -r win-x64 -f net9.0-windows10.0.26100.0 /p:Configuration=Release /p:Platform=x64
6565

66-
- name: Run FirstCatalogQuery (10 fresh processes)
66+
- name: Run FirstCatalogQuery (100 fresh processes)
6767
shell: pwsh
6868
run: |
6969
$env:PERFORMANCE_OUTPUT_PATH = "${{ github.workspace }}\PerfResults"
7070
New-Item -ItemType Directory -Force -Path $env:PERFORMANCE_OUTPUT_PATH | Out-Null
7171
7272
$testDll = "AIDevGallery.Tests\bin\x64\Release\net9.0-windows10.0.26100.0\win-x64\AIDevGallery.Tests.dll"
7373
74-
for ($i = 1; $i -le 10; $i++) {
75-
Write-Host "`n=== FirstCatalogQuery Run $i/10 ==="
74+
for ($i = 1; $i -le 100; $i++) {
75+
Write-Host "`n=== FirstCatalogQuery Run $i/100 ==="
7676
dotnet test $testDll `
7777
--filter "TestCategory=FirstCatalogQuery" `
7878
--logger "console;verbosity=detailed" `
7979
--verbosity quiet
8080
Write-Host "Exit code: $LASTEXITCODE"
8181
}
8282
83-
- name: Run SubsequentCatalogQuery (20 iterations in one process)
83+
- name: Run SubsequentCatalogQuery (100 iterations in one process)
8484
shell: pwsh
8585
run: |
8686
$env:PERFORMANCE_OUTPUT_PATH = "${{ github.workspace }}\PerfResults"
8787
$testDll = "AIDevGallery.Tests\bin\x64\Release\net9.0-windows10.0.26100.0\win-x64\AIDevGallery.Tests.dll"
8888
89-
Write-Host "`n=== SubsequentCatalogQuery (20 iterations) ==="
89+
Write-Host "`n=== SubsequentCatalogQuery (100 iterations) ==="
9090
New-Item -ItemType Directory -Force -Path "${{ github.workspace }}\TestResults" | Out-Null
9191
dotnet test $testDll `
9292
--filter "TestCategory=SubsequentCatalogQuery" `

AIDevGallery.Tests/IntegrationTests/FoundryLocalPerformanceBenchmark.cs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ public async Task BenchmarkFirstCatalogQuery()
106106

107107
/// <summary>
108108
/// Measures subsequent catalog queries after initialization (cached/warm path).
109-
/// Runs 20 iterations in a single process.
109+
/// Runs 100 iterations in a single process.
110110
/// </summary>
111111
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
112112
[TestMethod]
@@ -142,7 +142,7 @@ public async Task BenchmarkSubsequentCatalogQuery()
142142
// Warm-up call (not measured)
143143
await catalog.ListModelsAsync();
144144

145-
for (int i = 0; i < 20; i++)
145+
for (int i = 0; i < 100; i++)
146146
{
147147
var sw = Stopwatch.StartNew();
148148
var models = await catalog.ListModelsAsync();

0 commit comments

Comments
 (0)