Skip to content

Commit eca7ec3

Browse files
haoliuuclaude
andcommitted
perf: Increase benchmark iterations to 100 for statistical significance
- FirstCatalogQuery: 100 fresh processes (was 10) - SubsequentCatalogQuery: 100 iterations (was 20) - Increase SubsequentCatalogQuery timeout to 5min for 100 iterations Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
1 parent bc74c40 commit eca7ec3

2 files changed

Lines changed: 8 additions & 8 deletions

File tree

.github/workflows/foundry-local-benchmark.yml

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -63,30 +63,30 @@ jobs:
6363
- name: Build Test Project
6464
run: dotnet build AIDevGallery.Tests -r win-x64 -f net9.0-windows10.0.26100.0 /p:Configuration=Release /p:Platform=x64
6565

66-
- name: Run FirstCatalogQuery (10 fresh processes)
66+
- name: Run FirstCatalogQuery (100 fresh processes)
6767
shell: pwsh
6868
run: |
6969
$env:PERFORMANCE_OUTPUT_PATH = "${{ github.workspace }}\PerfResults"
7070
New-Item -ItemType Directory -Force -Path $env:PERFORMANCE_OUTPUT_PATH | Out-Null
7171
7272
$testDll = "AIDevGallery.Tests\bin\x64\Release\net9.0-windows10.0.26100.0\win-x64\AIDevGallery.Tests.dll"
7373
74-
for ($i = 1; $i -le 10; $i++) {
75-
Write-Host "`n=== FirstCatalogQuery Run $i/10 ==="
74+
for ($i = 1; $i -le 100; $i++) {
75+
Write-Host "`n=== FirstCatalogQuery Run $i/100 ==="
7676
dotnet test $testDll `
7777
--filter "TestCategory=FirstCatalogQuery" `
7878
--logger "console;verbosity=detailed" `
7979
--verbosity quiet
8080
Write-Host "Exit code: $LASTEXITCODE"
8181
}
8282
83-
- name: Run SubsequentCatalogQuery (20 iterations in one process)
83+
- name: Run SubsequentCatalogQuery (100 iterations in one process)
8484
shell: pwsh
8585
run: |
8686
$env:PERFORMANCE_OUTPUT_PATH = "${{ github.workspace }}\PerfResults"
8787
$testDll = "AIDevGallery.Tests\bin\x64\Release\net9.0-windows10.0.26100.0\win-x64\AIDevGallery.Tests.dll"
8888
89-
Write-Host "`n=== SubsequentCatalogQuery (20 iterations) ==="
89+
Write-Host "`n=== SubsequentCatalogQuery (100 iterations) ==="
9090
New-Item -ItemType Directory -Force -Path "${{ github.workspace }}\TestResults" | Out-Null
9191
dotnet test $testDll `
9292
--filter "TestCategory=SubsequentCatalogQuery" `

AIDevGallery.Tests/IntegrationTests/FoundryLocalPerformanceBenchmark.cs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -95,12 +95,12 @@ public async Task BenchmarkFirstCatalogQuery()
9595
/// Measures subsequent catalog queries via HTTP API (warm path).
9696
/// Each iteration creates a new client to force a fresh HTTP request,
9797
/// but the Foundry service is already running.
98-
/// Runs 20 iterations in a single process.
98+
/// Runs 100 iterations in a single process.
9999
/// </summary>
100100
/// <returns>A <see cref="Task"/> representing the asynchronous operation.</returns>
101101
[TestMethod]
102102
[TestCategory("SubsequentCatalogQuery")]
103-
[Timeout(120000)]
103+
[Timeout(300000)]
104104
public async Task BenchmarkSubsequentCatalogQuery()
105105
{
106106
EnsureCliAvailable();
@@ -117,7 +117,7 @@ public async Task BenchmarkSubsequentCatalogQuery()
117117
await _client.ListCatalogModels();
118118
_client.Dispose();
119119

120-
for (int i = 0; i < 20; i++)
120+
for (int i = 0; i < 100; i++)
121121
{
122122
// Create fresh client each time to force new HTTP connection + query
123123
var client = await FoundryCliClient.CreateAsync();

0 commit comments

Comments
 (0)