Skip to content

Commit d47bad7

Browse files
author
Mark Saroufim
committed
Fix test_submit_full mock to return mode-specific results
Use side_effect to return different FullResult for each call: - First call (PUBLIC mode) returns {"public": eval_result} - Second call (SECRET mode) returns {"secret": eval_result} This prevents the backend from storing all keys from both calls.
1 parent 2e022ca commit d47bad7

1 file changed

Lines changed: 20 additions & 3 deletions

File tree

tests/test_backend.py

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -232,7 +232,18 @@ async def test_submit_full(bot: backend.KernelBackend, task_directory):
232232
task = db.get_leaderboard("submit-leaderboard")["task"]
233233

234234
eval_result = create_eval_result("benchmark")
235-
mock_launcher = _mock_launcher(bot, {"public": eval_result, "secret": eval_result})
235+
# Use side_effect to return different results for each call
236+
# First call (PUBLIC mode) returns {"public": ...}, second call (SECRET mode) returns {"secret": ...}
237+
mock_launcher = MagicMock(spec=backend.Launcher)
238+
mock_launcher.name = "launcher"
239+
mock_launcher.gpus = [consts.ModalGPU.A100]
240+
mock_launcher.run_submission = AsyncMock(
241+
side_effect=[
242+
FullResult(success=True, error="", system=sample_system_info(), runs={"public": eval_result}),
243+
FullResult(success=True, error="", system=sample_system_info(), runs={"secret": eval_result}),
244+
]
245+
)
246+
bot.register_launcher(mock_launcher)
236247

237248
from libkernelbot.submission import ProcessedSubmissionRequest
238249

@@ -252,9 +263,15 @@ async def test_submit_full(bot: backend.KernelBackend, task_directory):
252263
req, mode=consts.SubmissionMode.PUBLIC, reporter=reporter
253264
)
254265

255-
expected_result = mock_launcher.run_submission.return_value
266+
expected_result_public = FullResult(
267+
success=True, error="", system=sample_system_info(), runs={"public": eval_result}
268+
)
269+
expected_result_secret = FullResult(
270+
success=True, error="", system=sample_system_info(), runs={"secret": eval_result}
271+
)
256272
assert len(results) == 2
257-
assert results == [expected_result, expected_result]
273+
assert results[0].success == expected_result_public.success
274+
assert results[1].success == expected_result_secret.success
258275

259276
r1, r2 = reporter.reporter_list
260277
assert r1.lines == [

0 commit comments

Comments
 (0)