diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a150cbdd..2ca670b1 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -3,7 +3,7 @@ on: push: branches: [ main, master ] pull_request: - branches: [ main, master ] + branches: [ main, master, polyworld-mailboxes ] workflow_dispatch: jobs: build: @@ -15,6 +15,12 @@ jobs: steps: - uses: actions/checkout@v5 - uses: treeform/setup-nim-action@v6 + - name: Install native HTTP library on Windows + if: runner.os == 'Windows' + shell: pwsh + run: | + vcpkg install curl:x64-windows + "$env:VCPKG_INSTALLATION_ROOT/installed/x64-windows/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - name: Install dependencies shell: bash run: | @@ -24,6 +30,8 @@ jobs: nimby sync "${{ github.event.repository.name }}/nimby.lock" - run: nim r tests/tests.nim - run: nim r -d:nimAllocStats tests/test_mailbox_allocations.nim + - run: nim r -d:headless -d:llmCta tests/test_llm_hosts.nim + - run: nim r -d:headless -d:llmLvd tests/test_llm_hosts.nim - run: nim r tests/test_gota_events.nim - run: nim r -d:replayEvents tests/test_gota_controls.nim - run: nim r -d:replayEvents tests/test_gota_camps.nim diff --git a/coworld/Dockerfile b/coworld/Dockerfile index 72adb3cb..4af20507 100644 --- a/coworld/Dockerfile +++ b/coworld/Dockerfile @@ -18,6 +18,9 @@ ARG GAME ARG ENTRY RUN nim c -d:coworld --nimcache:/tmp/nimcache -o:/out/game examples/${GAME}/${ENTRY}.nim FROM ubuntu:24.04 +RUN apt-get update && apt-get install -y --no-install-recommends \ + libcurl4t64 ca-certificates \ + && rm -rf /var/lib/apt/lists/* COPY --from=compiler /out/game /usr/local/bin/polyworld # The runner stages writable outputs as root in its shared volume. EXPOSE 8080 diff --git a/coworld/dependencies.lock b/coworld/dependencies.lock index 95e9cb58..91649a3f 100644 --- a/coworld/dependencies.lock +++ b/coworld/dependencies.lock @@ -1,4 +1,6 @@ bassy 0.1.0 https://github.com/treeform/bassy 669a7c4b94e3d5b0a38dc9557608a7e58c2a764d +curly 1.1.1 https://github.com/guzba/curly 4f36f01ac6ca881fff4bbf0f8daa362d4b9f0668 +libcurl 1.0.0 https://github.com/Araq/libcurl 7a420498f60a31d99fc8513886ce36c4e8c3a4ae fixxy 0.1.0 https://github.com/treeform/fixxy 05e5446dffb70093056cebb0c57721a60deaf52a silky 0.2.0 https://github.com/treeform/silky fb9b13910edd66cf1751056784c2f7d2932a59fc pixie 6.1.0 https://github.com/treeform/pixie 87cecced5c4c6f311c658a5f3ca0c9b43edb6aa7 diff --git a/coworld/tools/test_runtime.nim b/coworld/tools/test_runtime.nim index eb32bbd5..b5758b0a 100644 --- a/coworld/tools/test_runtime.nim +++ b/coworld/tools/test_runtime.nim @@ -102,6 +102,7 @@ proc episode( scripts: seq[string], failure = false, ticks = 240, + waitForLlm = false, expectedOutput = "" ) = ## Runs one local roster and inspects outputs at the completion marker. @@ -115,7 +116,8 @@ proc episode( listener.close() let config = %*{ - "tokens": [], "players": [], "seed": 2026, "max_ticks": ticks + "tokens": [], "players": [], "seed": 2026, "max_ticks": ticks, + "wait_for_llm": waitForLlm } seats = newJArray() env = newStringTable(modeCaseSensitive) @@ -267,7 +269,7 @@ for (game, count) in Games: sendChat(-2, "CHAT") print pullMailbox$(), mailboxId() """ - episode(game, count, scripts, ticks = 3, + episode(game, count, scripts, ticks = 3, waitForLlm = true, expectedOutput = "CHAT") echo game, ": hosted mailbox integration passed" diff --git a/docs/inference.md b/docs/inference.md new file mode 100644 index 00000000..c1abb7ce --- /dev/null +++ b/docs/inference.md @@ -0,0 +1,230 @@ +# OpenRouter, JEV, and headless time + +Gods of the Arena, Light vs Dark, and Call to Adventure share the OpenRouter +host in `src/polyworld/llms.nim`. The game runtime is Nim. +No Python program or subprocess is used for LLM calls. + +Each player has one `LlmClient` that owns HTTP requests, BASIC bindings, and +structured oracle state. `oracles.nim` builds JEV payloads and interprets +answers; it has no HTTP client or BASIC runtime dependency. Normal text and +JEV requests use the same client and pending-request limit. + +## Time controls + +| Control | Default | Behavior | +| --- | --- | --- | +| `--headless-tick-rate N` | `0` | Zero runs freely. Positive values limit wall-clock simulation ticks per second. | +| `--llm-mode async` | `async` | Continue simulation while LLM requests are pending. | +| `--llm-mode barrier` | | Run all due BASIC decisions, then wait for every submitted request to finish or fail before the next tick. | + +The rate and barrier controls compose. Barrier mode starts each seat's request +without waiting for earlier seats, then polls all seats together. It waits for +the slowest outstanding request, not the sum of their latencies. A seat that +does not submit a request contributes no wait. Timeouts and HTTP errors settle +the barrier too. The default request timeout is 30 seconds. + +On tick T, scripts consume previous replies and submit new requests. The tick +finishes, and the barrier waits while the world remains fixed. Replies are +available at the next scheduled BASIC decision after tick T. BASIC restarts +at its first line each decision with persistent variables. There is no suspended +statement or immediate rerun inside tick T. + +These controls affect headless wall-clock scheduling. Physics still uses the +game's fixed 24 Hz simulation clock. A tick rate of 24 approximates real time; +1 allows one simulation tick per wall-clock second. A slow LLM can make a +barrier run slower than that rate. The pacer never runs catch-up bursts. +Graphical playback keeps its existing controls. Replay playback uses recorded +actions and never calls an LLM. + +The hosted JSON config uses `headless_tick_rate` and `wait_for_llm`. +Local GotA `--config` accepts the same fields. They are stored in replay +metadata, defaulting to zero and false for older configs. + +```json +{ + "headless_tick_rate": 24, + "wait_for_llm": true +} +``` + +For a local run, set `COGAME_LLM_MODEL` to an available OpenRouter model and +configure either the sidecar or a local API key. Then, from the repository: + +```sh +nim r -d:headless examples/light_vs_dark/lvd.nim \ + --bot:examples/inference/chat.bas:2 --ticks=100 \ + --llm-mode:barrier --headless-tick-rate:24 +``` + +## HTTP transport and normal LLMs + +`llms.nim` calls Curly's `startRequest` and `pollForResponse` directly. +Each active player gets a Curly worker on their first request. Submission and +polling do not wait for HTTP responses. Polling happens at decision boundaries +or inside the headless barrier. Curly owns the HTTP handles and buffers. +The host prioritizes `AWS_ENDPOINT_URL_BEDROCK_RUNTIME`, the existing container +sidecar root, and sends `X-Coworld-Player-Slot` with the zero-based seat index. +It does not send an API key to the sidecar. The sidecar remains responsible for +provider authentication, model availability, spend limits, and accounting. + +Without a sidecar, local runs can use `COGAME_LLM_KEY` or +`OPENROUTER_API_KEY`. The default direct root is `https://openrouter.ai/api`; +`COGAME_LLM_BASE_URL` overrides it and must use HTTPS. Credentials and endpoints +come from the host environment, never from BASIC. + +| BASIC call | Result | +| --- | --- | +| `llmAvailable()` | 1 when a native endpoint is configured, otherwise 0. | +| `llmReady()` | 0 when ready, positive remaining spacing ticks, -1 when disabled or pending. | +| `llmAsk(model$, prompt$)` | Request ID for a simple chat message. Empty model uses `COGAME_LLM_MODEL`. | +| `llmRequest(method$, path$, body$)` | Request ID for raw API access under `/v1/`. | +| `llmPoll(id)` | 0 pending, 1 successful, -1 failed or expired. | +| `llmText$(id)` | Ordinary text from completed Chat Completions, Responses, or SSE replies. | +| `llmResponse$(id)` | Complete raw body, or empty while pending. | +| `llmRead$(id, offset, count)` | A zero-based byte slice of the raw body. | +| `llmStatus(id)` | Completed HTTP status, or 0 without one. | +| `llmError$(id)` | Failure message, or empty string for a successful retained reply. | +| `jsonQuote$(value$)` | JSON-escaped string including quotes. | +| `jsonGet$(json$, pointer$)` | Value at an RFC 6901 pointer, text for strings and JSON for other values. | + +The raw request body is forwarded unchanged. This lets scripts use model IDs, +message history, tool calls, structured output, provider routing, reasoning, +multimodal JSON content, generation parameters, and other JSON fields without +waiting for a new Nim wrapper. GET, POST, PUT, PATCH, DELETE, and HEAD are +supported. The sidecar must expose the requested route. Tool calls are returned +as JSON for the script to interpret, not executed automatically. Setting +`stream: true` preserves SSE events. Curly's polling API returns the complete +response, so `llmText$` and `llmResponse$` become available after the stream +finishes. Barrier mode also waits until it finishes. + +This is bounded JSON/SSE access, not an unbounded file upload API. Multipart +uploads and arbitrary custom headers are not exposed. Requests are limited to +64 KiB. Completed replies exceeding 256 KiB of body or 16 KiB of parsed headers +are rejected before being retained for BASIC. Curly buffers the download before +these checks; these are not transport memory limits. BASIC strings can +hold 64 KiB each, with 1024 string slots and a 256 KiB total string budget per +VM. Temporary strings are reclaimed between decisions. Use `llmRead$` for +larger responses. JSON extraction allows up to 64 nesting levels. Oversized +responses fail with an error instead of silently truncating successful results. + +Each seat has one pending request shared by normal LLM and JEV calls, and +retains only its latest completed response. Older request IDs expire when a +new response replaces it. JEV likewise retains only its latest answer set. +Request functions return 0 when busy, +rate-limited, or the body exceeds the request limit. A host deadline settles +the BASIC request and barrier; the seat stays busy until Curly's outstanding +transfer finishes, and its late reply is discarded. Curly's transport timeout +is rounded up to whole seconds. Closing a client waits for that bounded +outstanding transfer before freeing its worker. Host environment controls: + +| Variable | Default | Purpose | +| --- | --- | --- | +| `COGAME_LLM` | enabled when configured | `off` disables remote calls. | +| `COGAME_LLM_MODEL` | empty | Default model for `llmAsk`. | +| `COGAME_LLM_INTERVAL` | `1` | Minimum simulation ticks between submissions, 1 through 100000. | +| `COGAME_LLM_TIMEOUT_MS` | `30000` | Whole-request deadline, 1 through 120000 milliseconds. | +| `COGAME_ORACLE` | enabled | `off` disables JEV helpers independently. | +| `COGAME_ORACLE_MODEL` | `typesafe/jev-1.13` | Model for `/v1/systemone`. | + +`examples/inference/chat.bas` and `request.bas` demonstrate text and raw JSON. +Native installations need libcurl with asynchronous DNS support. The hosted +Docker image installs libcurl and CA certificates. Browser builds expose the +same function names but remote inference reports unavailable. + +## JEV structured judgments + +`oracles.nim` constructs SystemOne requests directly in Nim. The BASIC draft +API is `oracleState(key$, integer)`, `oracleStateText(key$, text$)`, +`oracleNote(text$)`, `oracleQuestion(key$, kind, instructions$)`, +`oracleCriterion(question$, label$, text$)`, and +`oracleCriterionField(question$, label$, field$, text$)`. State paths support +dotted objects and bounded indices, for example `candidates[0].hp`. + +`oracleAsk()` submits the draft to `/v1/systemone` and returns a request ID. +Drafts are cleared after submission and at the next decision. `oracleReady()` +and `oracleAvailable()` mirror the normal LLM helpers. `oraclePoll(id)` returns +zero while pending, the number of usable answers on success, or -1 on failure. + +| Question kind | Meaning of `oracleAnswer(id, key$)` | +| --- | --- | +| `0`, noul | Probability of true multiplied by 1000. | +| `1`, score | Score multiplied by 1000. | +| `2`, choice | Zero-based criterion index. | + +`oracleConfidence(id, key$)` and +`oracleProbability(id, key$, label$)` return thousandths. Missing values are +-1. Use `llmResponse$` to inspect the original response. There are at most +64 questions, 16 criteria per question, 256 state writes, 4096 state nodes, +16 notes, and 32 KiB of serialized draft JSON. `examples/inference/jev.bas` +demonstrates noul and choice questions. + +The existing sidecar must provide `/v1/systemone` for JEV. Local mock tests +verify this wire format; they do not certify a particular hosted deployment. +The inspected local Metta sidecar registers Chat Completions and Anthropic +routes, but no SystemOne route. JEV and other API paths need corresponding +sidecar routing before they work in that deployment. + +## Manual live JEV test + +With `OPENROUTER_API_KEY` set in your shell, run: + +```sh +nim r tests/manual_jev.nim +``` + +This submits exactly one paid request from BASIC directly to OpenRouter's +`/v1/systemone` endpoint, using `typesafe/jev-1.13`. It bypasses sidecar +configuration, asks for a GotA strategy and lane, and checks BASIC's readback +of both choices and the returned model. It prints the original response, +including the request ID, provider, token usage, and reported cost. +There are no retries or fallback answers. This test is never run by CI. + +The ordinary test suite uses a local mock, including a captured live JEV +response to verify the same BASIC script without additional paid requests. + +## GotA strategy bot + +`examples/gods_of_the_arena/players/jev.bas` is a playable strategy example. +Every 15 simulation seconds, each hero sends JEV a text situation report: +the estimated early/mid/late game stage, elapsed time, class and role, health, +mana, level, gold, deaths, position, current plan, god health, home threats, +and observed heroes, creeps, and towers in each lane. The summary uses only +the script's permitted observations. Lane sectors are approximate, and the +creep scan is bounded; missing enemies are not treated as known absences. + +Two choice questions ask for a strategy (`farm`, `gank`, `push`, `defend`, +or `regroup`) and a lane (`top`, `mid`, or `bottom`). The bot stores both +answers and follows them between calls. Farming favors lane creeps and last +hits; ganking prioritizes visible heroes; pushing advances toward structures +and the enemy god; defending returns home; regrouping joins allied heroes. +Top and bottom refer to the low-coordinate and high-coordinate outer routes +respectively, for both teams. + +Local BASIC handles drafting, ability upgrades, basic attacks, and movement. +It retreats to its spawn room when badly hurt, regardless of JEV's advice. +It starts with a farming plan, continues playing while requests are pending, +keeps its last plan for incomplete replies, and retries failures after five +simulation seconds. This is a macro-strategy example, not the full reference +policy's item, spell, or combat tactics. + +With a sidecar exposing `/v1/systemone`, or configured direct access: + +```sh +nim r -d:headless examples/gods_of_the_arena/gota.nim \ + --bot:examples/gods_of_the_arena/players/jev.bas:1 \ + --bot:examples/gods_of_the_arena/players/base.bas:9 \ + --ticks=1440 --llm-mode=barrier --headless-tick-rate=24 +``` + +`COGAME_ORACLE_MODEL` selects the JEV model. Each JEV-controlled hero has its +own request and refresh interval. Successful updates print the selected +strategy and lane to that player's log. + +## Using LLM responses in chat + +The [mailbox API](mailboxes.md) provides communication with routing defined +by each game. GotA supports global, team, and DM chat; LvD supports global +and DM chat; CTA supports global chat within 16 tiles on the same level. +For GotA and LvD, `examples/inference/mailbox_llm.bas` reads a DM, asks an +LLM for a text reply, and sends that reply to the original sender. The +player's unread messages stay queued while its LLM request is pending. diff --git a/examples/call_to_adventure/bots.nim b/examples/call_to_adventure/bots.nim index aab456a5..cd4668d4 100644 --- a/examples/call_to_adventure/bots.nim +++ b/examples/call_to_adventure/bots.nim @@ -6,7 +6,7 @@ import bassy, - polyworld/[mailboxes, bodies, metrics, cli, controllers, + polyworld/[llms, mailboxes, bodies, metrics, cli, controllers, pathing, profiles], content, sim, @@ -94,6 +94,8 @@ proc issueHeroAction(action: ReplayAction): int32 = proc heroLimits(): Limits = ## Defines one isolated hero VM's source, memory, and decision budgets. result = defaultLimits() + result.maxStrings = 1024 + result.maxStringLength = 64 * 1024 result.maxStringBytes = 256 * 1024 result.maxSourceBytes = 128 * 1024 result.maxCodeInstructions = 50_000 @@ -125,9 +127,11 @@ proc sendChat*( if distance in 0 .. 16 and game.inboxes[recipient].push(-2, text): inc result -proc buildHeroHost(heroId: int32): Host = +proc buildHeroHost(heroId: int32, llm: LlmClient = nil): Host = ## Builds the world-query and high-level action API for one hero. result = initHost() + let services = if llm == nil: newLlmClient(0, LlmConfig()) else: llm + services.addFunctions(result) let sendChatProc: NumericHostProc = proc(args: openArray[Value]): Value = ## Sends script text through the game's routing rules. let player = int(heroId - 100) @@ -279,6 +283,7 @@ proc loadBots*( for slot in 0 ..< PartySize: if kinds[slot] == PlayerController: continue + let llm = newLlmClient(slot) let source = sources[slot] let program = when defined(coworld): @@ -291,11 +296,14 @@ proc loadBots*( game.heroVms[slot] = HeroVm( runtime: initRuntime( program, - buildHeroHost(int32(100 + slot)), + buildHeroHost(int32(100 + slot), llm), limits ), ready: true, + prepareDecision: llm.decisionCallback(), + pollRequests: llm.requestPoller() ) + llm.bindRuntime(game.heroVms[slot].runtime) when defined(coworld): game.heroVms[slot].output = playerPrinter(int(slot)) @@ -313,6 +321,8 @@ proc runBotDecisions*(game: Game, slot: int32) {.measure.} = let objective = game.objectiveTile(slot) try: game.heroVms[slot].runtime.restart() + if game.heroVms[slot].prepareDecision != nil: + game.heroVms[slot].prepareDecision(game.world.tick) game.heroVms[slot].runtime.setData(heroDataIds[DataSelfId], actor.id) game.heroVms[slot].runtime.setData( heroDataIds[DataSelfClass], diff --git a/examples/call_to_adventure/game.nim b/examples/call_to_adventure/game.nim index 71b39067..b78c44b9 100644 --- a/examples/call_to_adventure/game.nim +++ b/examples/call_to_adventure/game.nim @@ -6,7 +6,7 @@ import std/[os, strformat, strutils, times], - polyworld/[cli, controllers, metrics, profiles, tapes], + polyworld/[cli, controllers, metrics, profiles, tapes, timings], content, maps, sim, @@ -31,6 +31,8 @@ proc usage() = echo " --seconds NUMBER Duration in seconds (default 1200)." echo " --minutes NUMBER Duration in minutes (default 20)." echo " --ticks NUMBER Duration in ticks (default 28800)." + echo " --headless-tick-rate N Wall-clock ticks per second, 0 is unlimited." + echo " --llm-mode:async|barrier Wait for all requests between headless ticks." echo " --speed NUMBER Graphical start speed: 1, 2, 4, or 16." echo " --play=false Start the graphical transport paused." echo " --windowSize WxH Graphical window, such as 800x400." @@ -158,7 +160,7 @@ proc summarize*(game: Game) = echo " party alive ", alive proc runHeadless*() = - ## Runs one live expedition or replay with no frame-rate waiting. + ## Runs fixed simulation ticks with optional pacing and request barriers. echo "Call to Adventure, seed ", run.world.setup.seed echo "dungeon: ", LevelCount, " levels, ", run.dungeon.ramps.len, " ramps" var monsters = 0 @@ -171,10 +173,19 @@ proc runHeadless*() = defer: finishGameProfile() let started = epochTime() - var reported = 0 + var + reported = 0 + pacer = initTickPacer(options.headlessTickRate) + pollers: seq[RequestPoll] + if options.waitForLlm and not run.replayMode: + for vm in run.heroVms: + if vm != nil: + pollers.add vm.pollRequests while run.world.tick < options.maximumTicks and run.world.phase notin {EscapedPhase, WipedPhase}: advanceGame() + waitForRequests(pollers) + pacer.pace() if profileShouldDump(run.world.tick): finishGameProfile() if options.verbose: diff --git a/examples/call_to_adventure/sim.nim b/examples/call_to_adventure/sim.nim index 9e7b55c5..900bd3a0 100644 --- a/examples/call_to_adventure/sim.nim +++ b/examples/call_to_adventure/sim.nim @@ -19,6 +19,8 @@ import type HeroVm* = ref object output*: PrintProc + prepareDecision*: proc(tick: int32) {.closure.} + pollRequests*: proc(): bool {.closure.} ## One compiled BASIC program for a party slot. Not simulation state. runtime*: Runtime ready*: bool diff --git a/examples/gods_of_the_arena/bots.nim b/examples/gods_of_the_arena/bots.nim index 286851ba..1db98085 100644 --- a/examples/gods_of_the_arena/bots.nim +++ b/examples/gods_of_the_arena/bots.nim @@ -3,7 +3,7 @@ import bassy, fixxy, - polyworld/[mailboxes, metrics, bodies, cli, controllers, + polyworld/[llms, mailboxes, metrics, bodies, cli, controllers, pathing, profiles, tapes], content, maps, @@ -103,6 +103,8 @@ proc bindHeroData(program: Program) = proc heroVmLimits(): Limits = ## Returns independent structural and per-decision limits for a hero VM. result = defaultLimits() + result.maxStrings = 1024 + result.maxStringLength = 64 * 1024 result.maxStringBytes = 256 * 1024 result.maxSourceBytes = 64 * 1024 result.maxCodeInstructions = 20_000 @@ -282,9 +284,11 @@ proc sendChat*( if game.inboxes[recipient].push(id, text): inc result -proc initHeroHost(heroId: int32): Host = +proc initHeroHost(heroId: int32, llm: LlmClient = nil): Host = ## Builds the bounded world-query and action interface for one hero. result = initHost() + let services = if llm == nil: newLlmClient(0, LlmConfig()) else: llm + services.addFunctions(result) let sendChatProc: NumericHostProc = proc(args: openArray[Value]): Value = ## Sends script text through the game's routing rules. let player = activeGame.world.heroIndex(heroId) @@ -862,6 +866,7 @@ proc loadBots*( for i in 0 ..< game.world.heroes.len: if kinds[i] == PlayerController: continue + let llm = newLlmClient(i) let source = sources[i] let program = when defined(coworld): @@ -874,12 +879,15 @@ proc loadBots*( game.heroVms[i] = HeroVm( runtime: initRuntime( program, - initHeroHost(game.world.heroes[i].id), + initHeroHost(game.world.heroes[i].id, llm), limits ), limits: limits, + prepareDecision: llm.decisionCallback(), + pollRequests: llm.requestPoller(), ready: true ) + llm.bindRuntime(game.heroVms[i].runtime) when defined(coworld): game.heroVms[i].output = playerPrinter(int(i)) @@ -894,6 +902,8 @@ proc runHeroScript(game: Game, index: int) = return try: vm.runtime.restart() + if vm.prepareDecision != nil: + vm.prepareDecision(game.world.tick) discard game.world.worldObjectCount(hero.id) vm.runtime.setData(heroDataIds[DataSelfId], hero.id) vm.runtime.setData(heroDataIds[DataSelfTeam], int32(hero.team.ord)) diff --git a/examples/gods_of_the_arena/game.nim b/examples/gods_of_the_arena/game.nim index cdebf0be..0dd06b64 100644 --- a/examples/gods_of_the_arena/game.nim +++ b/examples/gods_of_the_arena/game.nim @@ -6,7 +6,7 @@ import std/[math, os, strformat, strutils, times], - polyworld/[cli, controllers, metrics, profiles, tapes], + polyworld/[cli, controllers, metrics, profiles, tapes, timings], content, maps, sim, @@ -38,6 +38,8 @@ proc usage() = echo " --map-seed NUMBER Regenerate the arena from another seed (the" echo " league always plays the preset's own seed)." echo " --config PATH JSON match settings, including mapPreset." + echo " --headless-tick-rate N Wall-clock ticks per second, 0 is unlimited." + echo " --llm-mode:async|barrier Wait for all requests between headless ticks." echo " --spawn-interval NUMBER Seconds between waves." echo " --play=false Start the graphical transport paused." echo " --speed NUMBER Graphical start speed: 1, 2, 4, or 16." @@ -60,6 +62,8 @@ proc parseGameOptions(): GameOptions = maximumTicks: matchConfig.maxTicks, spawnIntervalTicks: matchConfig.spawnIntervalTicks, playerSlot: matchConfig.playerSlot, + headlessTickRate: matchConfig.headlessTickRate, + waitForLlm: matchConfig.waitForLlm, speed: 1, windowWidth: 1920, windowHeight: 1080 @@ -298,18 +302,27 @@ when defined(headless): &"{vmStatus.decisions} decisions" proc runHeadless*() = - ## Runs a live game or replay immediately with fixed simulation ticks. + ## Runs fixed simulation ticks with optional pacing and request barriers. let started = epochTime() if not run.replayMode: startReplayRecording(uint32(options.maximumTicks)) startGameProfile() defer: finishGameProfile() - var steps = 0 + var + steps = 0 + pacer = initTickPacer(options.headlessTickRate) + pollers: seq[RequestPoll] + if options.waitForLlm and not run.replayMode: + for vm in run.heroVms: + if vm != nil: + pollers.add vm.pollRequests while (if run.replayMode: steps < run.replayData.hashes.len else: not run.finished()) and run.recordingError.len == 0: advanceGame() + waitForRequests(pollers) + pacer.pace() inc steps if profileShouldDump(steps): finishGameProfile() diff --git a/examples/gods_of_the_arena/players/jev.bas b/examples/gods_of_the_arena/players/jev.bas new file mode 100644 index 00000000..be23827a --- /dev/null +++ b/examples/gods_of_the_arena/players/jev.bas @@ -0,0 +1,336 @@ +' JEV chooses the macro strategy and lane every 15 simulation seconds. +' Local BASIC handles movement, attacks, drafting, and emergency retreats. +' Decisions and pending request IDs persist across ticks. + +dim laneX(2) +dim laneY(2) +dim friends(2) +dim foes(2) +dim creeps(2) +dim towers(2) +dim friendlyTowers(2) +dim strategies$(4) +dim lanes$(2) + +sub chooseHero() + if draftTurnId <> selfId then + exit sub + end if + for candidate = 0 to 9 + if heroAvailable(candidate) then + draftHero(candidate) + exit sub + end if + next candidate +end sub + +sub moveToGoal(marching) + if selfRootTicks > 0 then + exit sub + end if + if marching then + accepted = attackMove(goalX, goalY) + else + accepted = walkTo(goalX, goalY) + end if + if accepted then + exit sub + end if + ' Generated terrain can block an approximate lane waypoint. + for offsetY = -2 to 2 + for offsetX = -2 to 2 + tileX = (goalX \ 1) + offsetX + tileY = (goalY \ 1) + offsetY + if tileX >= 0 and tileX < mapWidth and tileY >= 0 and tileY < mapHeight then + if terrainWalkable(tileX, tileY) then + if marching then + accepted = attackMove(tileX, tileY) + else + accepted = walkTo(tileX, tileY) + end if + if accepted then + exit sub + end if + end if + end if + next offsetX + next offsetY +end sub + +sub observe() + for laneIndex = 0 to 2 + friends(laneIndex) = 0 + foes(laneIndex) = 0 + creeps(laneIndex) = 0 + towers(laneIndex) = 0 + friendlyTowers(laneIndex) = 0 + next laneIndex + targetId = 0 + targetScore = -1000000 + allies = 0 + allyX = 0 + allyY = 0 + homeThreats = 0 + homeHp = 0 + enemyHp = 0 + objects = objectCount() + scanned = 0 + ' Structures and heroes precede creeps. Rotate the crowded creep tail. + for scan = 0 to 95 + index = scan + if scan >= 48 then + index = scan + scanOffset + end if + if index < objects and objectAlive(index) then + scanned = scanned + 1 + kind = objectKind(index) + team = objectTeam(index) + x = objectX(index) + y = objectY(index) + dx = x - selfX + dy = y - selfY + distance = dx * dx + dy * dy + ' Approximate lane sectors around the reference policy's waypoints. + sector = 1 + if x + y < (mapWidth + mapHeight) * 3 / 8 then + sector = 0 + elseif x + y > (mapWidth + mapHeight) * 5 / 8 then + sector = 2 + end if + if team = selfTeam then + if kind = 1 then + homeX = x + homeY = y + homeHp = objectHp(index) + elseif kind = 2 then + friends(sector) = friends(sector) + 1 + if objectId(index) <> selfId then + allies = allies + 1 + allyX = allyX + x + allyY = allyY + y + end if + elseif kind = 4 then + friendlyTowers(sector) = friendlyTowers(sector) + 1 + end if + elseif team >= 0 and kind <> 6 then + if kind = 1 then + enemyX = x + enemyY = y + enemyHp = objectHp(index) + elseif kind = 2 then + foes(sector) = foes(sector) + 1 + elseif kind = 3 then + creeps(sector) = creeps(sector) + 1 + elseif kind = 4 then + towers(sector) = towers(sector) + 1 + end if + homeDx = x - homeX + homeDy = y - homeY + if kind = 2 and homeDx * homeDx + homeDy * homeDy < 225 then + homeThreats = homeThreats + 1 + end if + if distance < 144 then + score = 100 - distance + eligible = sector = chosenLane + if strategy = 0 then + eligible = eligible and kind = 3 + if objectHp(index) <= selfAttackDamage then + score = score + 200 + end if + elseif strategy = 1 then + eligible = eligible and kind = 2 + score = score + 100 - objectHp(index) / 10 + elseif strategy = 2 then + if kind = 4 or kind = 1 or kind = 5 then + score = score + 150 + end if + elseif strategy = 3 then + eligible = homeDx * homeDx + homeDy * homeDy < 400 + else + eligible = distance < 25 + end if + if objectTarget(index) = selfId and distance < 16 then + eligible = 1 + end if + if eligible and score > targetScore then + targetScore = score + targetId = objectId(index) + end if + end if + end if + end if + next scan + scanOffset = scanOffset + 48 + if scanOffset >= objects - 48 then + scanOffset = 0 + end if +end sub + +sub askAdvice() + if request <> 0 or worldTick < nextAdvice then + exit sub + end if + if oracleReady() <> 0 then + exit sub + end if + stage$ = "early laning and farming" + if selfLevel >= 6 or worldTick >= tickRate * 180 then + stage$ = "mid game rotations and objectives" + end if + if selfLevel >= 12 or worldTick >= tickRate * 600 then + stage$ = "late game and finishing the enemy god" + end if + summary$ = "Stage: " + stage$ + ". Elapsed seconds:" + str$(worldTick / tickRate) + summary$ = summary$ + ". Player:" + str$(mailboxSelf()) + ", team:" + str$(selfTeam) + summary$ = summary$ + ", class:" + str$(selfClass) + ", role:" + str$(heroRole(selfClass)) + summary$ = summary$ + ", level:" + str$(selfLevel) + ", gold:" + str$(selfGold) + summary$ = summary$ + ", HP:" + str$(selfHp) + "/" + str$(selfMaxHp) + summary$ = summary$ + ", mana:" + str$(selfMana) + "/" + str$(selfMaxMana) + summary$ = summary$ + ", deaths:" + str$(selfDeaths) + summary$ = summary$ + ", position:(" + str$(selfX) + "," + str$(selfY) + ")." + summary$ = summary$ + " Current strategy: " + strategies$(strategy) + ", lane: " + lanes$(chosenLane) + "." + summary$ = summary$ + " Home god HP:" + str$(homeHp) + ", enemy god HP:" + str$(enemyHp) + summary$ = summary$ + ", visible enemy heroes near home:" + str$(homeThreats) + "." + oracleStateText("stage", stage$) + oracleStateText("situation", summary$) + for laneIndex = 0 to 2 + report$ = lanes$(laneIndex) + ": allied heroes=" + str$(friends(laneIndex)) + report$ = report$ + ", enemy heroes=" + str$(foes(laneIndex)) + report$ = report$ + ", enemy creeps=" + str$(creeps(laneIndex)) + report$ = report$ + ", allied towers=" + str$(friendlyTowers(laneIndex)) + report$ = report$ + ", enemy towers=" + str$(towers(laneIndex)) + "." + oracleNote(report$) + next laneIndex + oracleNote("Win by destroying the enemy god. Roles: 0 frontline, 1 carry, 2 mage, 3 support, 4 fighter. This example uses basic attacks and movement.") + oracleNote("All facts come from the permitted observation, not hidden enemy state. Lane counts are approximate sectors and a bounded sample; zero does not prove absence. Zero god HP can mean unavailable or protected, not destroyed.") + oracleNote("Top is the low-X/low-Y outer route, mid crosses the center, bottom is the high-X/high-Y outer route. Choose one coherent strategy and lane for the next 15 seconds. Emergency healing overrides advice.") + oracleQuestion("strategy", 2, "What should this hero do now, considering the stage, health, team, threats, and objectives?") + oracleCriterion("strategy", "farm", "Farm enemy lane creeps for gold and experience; favor safe last hits.") + oracleCriterion("strategy", "gank", "Rotate to the chosen lane and attack a visible enemy hero.") + oracleCriterion("strategy", "push", "Advance down the chosen lane, attack structures, and pressure the enemy god.") + oracleCriterion("strategy", "defend", "Return to our god and intercept nearby enemy attackers.") + oracleCriterion("strategy", "regroup", "Move toward the allied heroes before taking another fight.") + oracleQuestion("lane", 2, "Which lane best supports your selected strategy? Consider the lane reports and avoid unnecessary switching.") + oracleCriterion("lane", "top", "Use the top outer lane.") + oracleCriterion("lane", "mid", "Use the central lane.") + oracleCriterion("lane", "bottom", "Use the bottom outer lane.") + request = oracleAsk() + if request > 0 then + nextAdvice = worldTick + tickRate * 15 + else + nextAdvice = worldTick + tickRate * 5 + end if +end sub + +if drafting then + chooseHero() + end +end if + +if initialized = 0 then + initialized = 1 + strategy = 0 + chosenLane = mailboxSelf() mod 3 + strategies$(0) = "farm" + strategies$(1) = "gank" + strategies$(2) = "push" + strategies$(3) = "defend" + strategies$(4) = "regroup" + lanes$(0) = "top" + lanes$(1) = "mid" + lanes$(2) = "bottom" + for laneIndex = 0 to 2 + laneX(laneIndex) = mapWidth * (1 + laneIndex * 4) / 10 + laneY(laneIndex) = mapHeight * (1 + laneIndex * 4) / 10 + next laneIndex + homeX = selfX + homeY = selfY + spawnX = selfX + spawnY = selfY + enemyX = mapWidth - 1 - selfX + enemyY = mapHeight - 1 - selfY +end if + +if request > 0 then + status = oraclePoll(request) + if status > 0 then + advisedStrategy = oracleAnswer(request, "strategy") + advisedLane = oracleAnswer(request, "lane") + if advisedStrategy >= 0 and advisedStrategy <= 4 and advisedLane >= 0 and advisedLane <= 2 then + strategy = advisedStrategy + chosenLane = advisedLane + crossedLane = 0 + nextAction = 0 + print "JEV strategy: ", strategies$(strategy), ", lane: ", lanes$(chosenLane) + end if + request = 0 + elseif status = -1 then + print "JEV unavailable: ", llmError$(request) + request = 0 + nextAdvice = worldTick + tickRate * 5 + end if +end if + +if selfHp <= 0 then + crossedLane = 0 + end +end if +if selfChannelTicks > 0 or selfStunTicks > 0 then + end +end if +if worldTick < nextAction then + end +end if +nextAction = worldTick + 6 +observe() +askAdvice() + +for upgrade = 1 to 4 + if canLevelAbility(3) then + levelAbility(3) + elseif canLevelAbility(1) then + levelAbility(1) + elseif canLevelAbility(2) then + levelAbility(2) + elseif canLevelAbility(0) then + levelAbility(0) + end if +next upgrade + +if selfHp * 4 < selfMaxHp then + retreating = 1 +end if +if inOwnSpawn() and selfHp * 10 >= selfMaxHp * 9 then + retreating = 0 +end if +if retreating then + goalX = spawnX + goalY = spawnY + moveToGoal(0) + end +end if +if targetId <> 0 then + if selfTarget <> targetId then + attackTarget(targetId) + end if + end +end if + +goalX = laneX(chosenLane) +goalY = laneY(chosenLane) +dx = selfX - goalX +dy = selfY - goalY +if dx * dx + dy * dy < 36 then + crossedLane = 1 +end if +if strategy = 2 and crossedLane then + goalX = enemyX + goalY = enemyY +elseif strategy = 3 then + goalX = homeX + goalY = homeY +elseif strategy = 4 and allies > 0 then + goalX = allyX / allies + goalY = allyY / allies +end if +moveToGoal(1) diff --git a/examples/gods_of_the_arena/sim.nim b/examples/gods_of_the_arena/sim.nim index 0e23a3ad..97079e70 100644 --- a/examples/gods_of_the_arena/sim.nim +++ b/examples/gods_of_the_arena/sim.nim @@ -67,6 +67,8 @@ type HeroVm* = ref object output*: PrintProc + prepareDecision*: proc(tick: int32) {.closure.} + pollRequests*: proc(): bool {.closure.} runtime*: Runtime limits*: Limits ready*: bool diff --git a/examples/inference/chat.bas b/examples/inference/chat.bas new file mode 100644 index 00000000..2b21d0c1 --- /dev/null +++ b/examples/inference/chat.bas @@ -0,0 +1,15 @@ +' Set COGAME_LLM_MODEL on the host or supply a model ID to llmAsk. +if request > 0 then + status = llmPoll(request) + if status = 1 then + answer$ = llmText$(request) + print answer$ + request = 0 + elseif status = -1 then + print llmError$(request) + request = 0 + end if +end if +if llmReady() = 0 then + request = llmAsk("", "Give one short tactical suggestion.") +end if diff --git a/examples/inference/jev.bas b/examples/inference/jev.bas new file mode 100644 index 00000000..99ce4ab6 --- /dev/null +++ b/examples/inference/jev.bas @@ -0,0 +1,23 @@ +' Jev returns typed judgments through the SystemOne endpoint. +if request > 0 then + status = oraclePoll(request) + if status > 0 then + print "Hold probability in thousandths:", oracleAnswer(request, "hold") + print "Choice index:", oracleAnswer(request, "mode") + request = 0 + elseif status = -1 then + print llmError$(request) + request = 0 + end if +end if +if oracleReady() = 0 then + oracleState("self.hp", 3) + oracleStateText("objective", "Keep the checkpoint.") + oracleQuestion("hold", 0, "Should we hold position?") + oracleCriterion("hold", "true", "Staying protects our objective.") + oracleCriterion("hold", "false", "Leaving is necessary to survive.") + oracleQuestion("mode", 2, "Choose a tactical posture.") + oracleCriterion("mode", "hold", "Defend the checkpoint.") + oracleCriterion("mode", "advance", "Move toward the next checkpoint.") + request = oracleAsk() +end if diff --git a/examples/inference/mailbox_llm.bas b/examples/inference/mailbox_llm.bas new file mode 100644 index 00000000..f4b01017 --- /dev/null +++ b/examples/inference/mailbox_llm.bas @@ -0,0 +1,23 @@ +' Answer direct messages with a normal LLM response. +' While the LLM is busy, unread messages remain queued. +if request > 0 then + status = llmPoll(request) + if status = 1 then + answer$ = llmText$(request) + sendChat(replyTo, left$(answer$, 1024)) + request = 0 + elseif status = -1 then + print llmError$(request) + request = 0 + end if +end if +if llmReady() = 0 then + while request = 0 and mailboxCount() > 0 + question$ = pullMailbox$() + if mailboxId() >= 0 and mailboxId() <> mailboxSelf() then + replyTo = mailboxId() + prompt$ = "Answer briefly in plain text: " + question$ + request = llmAsk("", prompt$) + end if + wend +end if diff --git a/examples/inference/request.bas b/examples/inference/request.bas new file mode 100644 index 00000000..55ecab3b --- /dev/null +++ b/examples/inference/request.bas @@ -0,0 +1,22 @@ +' Supply any model and Chat Completions fields in the JSON body. +' Replace the model ID with one enabled by your sidecar. +if request > 0 then + status = llmPoll(request) + if status = 1 then + response$ = llmResponse$(request) + answer$ = jsonGet$(response$, "/choices/0/message/content") + print answer$ + request = 0 + elseif status = -1 then + print llmStatus(request), llmError$(request) + request = 0 + end if +end if +if llmReady() = 0 then + model$ = "your-provider/your-model" + prompt$ = "Return a JSON object with a tactical suggestion." + body$ = "{""model"":" + jsonQuote$(model$) + body$ = body$ + ",""messages"":[{""role"":""user"",""content"":" + body$ = body$ + jsonQuote$(prompt$) + "}],""response_format"":{""type"":""json_object""}}" + request = llmRequest("POST", "/v1/chat/completions", body$) +end if diff --git a/examples/light_vs_dark/bots.nim b/examples/light_vs_dark/bots.nim index b111e436..3ba8c645 100644 --- a/examples/light_vs_dark/bots.nim +++ b/examples/light_vs_dark/bots.nim @@ -12,7 +12,7 @@ import bassy, - polyworld/[mailboxes, bodies, metrics, profiles], + polyworld/[llms, mailboxes, bodies, metrics, profiles], content, sim @@ -243,6 +243,8 @@ proc overlordLimits*(): Limits = ## unit will exceed it and fail the script, which is the pressure that ## pushes authors onto `nearestEnemy` and friends. result = defaultLimits() + result.maxStrings = 1024 + result.maxStringLength = 64 * 1024 result.maxStringBytes = 256 * 1024 result.maxSourceBytes = 256 * 1024 result.maxCodeInstructions = 100_000 @@ -286,7 +288,7 @@ proc sendChat*( if game.inboxes[recipient].push(id, text): inc result -proc buildOverlordHost*(playerId: int32): Host = +proc buildOverlordHost*(playerId: int32, llm: LlmClient = nil): Host = ## Builds the complete world-query and command interface for one player. ## ## The same builder makes both the compile-time schema and each player's @@ -299,6 +301,8 @@ proc buildOverlordHost*(playerId: int32): Host = ## cost far more than their own cycles, so a script's budget prices its ## demand on the simulation rather than only its own arithmetic. result = initHost() + let services = if llm == nil: newLlmClient(0, LlmConfig()) else: llm + services.addFunctions(result) let sendChatProc: NumericHostProc = proc(args: openArray[Value]): Value = ## Sends script text through the game's routing rules. let player = int(playerId) @@ -607,6 +611,7 @@ proc loadBots*( when not defined(coworld): if sources[player].len == 0: continue + let llm = newLlmClient(int(player)) let source = sources[player] let program = when defined(coworld): @@ -614,9 +619,12 @@ proc loadBots*( else: compile(source, schema, limits) game.brains[player] = OverlordVm( - runtime: initRuntime(program, buildOverlordHost(player), limits), + runtime: initRuntime(program, buildOverlordHost(player, llm), limits), ready: true, + prepareDecision: llm.decisionCallback(), + pollRequests: llm.requestPoller() ) + llm.bindRuntime(game.brains[player].runtime) if not bound: bindOverlordData(program) bound = true @@ -642,6 +650,8 @@ proc runDecision(game: Game, player: int32) = try: game.brains[player].runtime.restart() + if game.brains[player].prepareDecision != nil: + game.brains[player].prepareDecision(game.world.tick) let economy = addr game.world.players[player] ids = overlordDataIds diff --git a/examples/light_vs_dark/game.nim b/examples/light_vs_dark/game.nim index a8e1e232..b3e52a1c 100644 --- a/examples/light_vs_dark/game.nim +++ b/examples/light_vs_dark/game.nim @@ -7,7 +7,7 @@ import std/[os, strformat, strutils, times], - polyworld/[cli, controllers, metrics, profiles, tapes], + polyworld/[cli, controllers, metrics, profiles, tapes, timings], content, maps as mapgen, sim, @@ -37,6 +37,8 @@ Light vs Dark, a small real-time strategy match between two BASIC overlords. --speed N Graphical start speed: 1, 2, 4, or 16. --windowSize WxH Graphical window, such as 800x400. --vsync:off Unlock the frame rate (default on). + --headless-tick-rate N Wall-clock ticks per second, 0 is unlimited. + --llm-mode:async|barrier Wait for all requests between headless ticks. --help Show this message. Compile with -d:headless for a command-line match. @@ -215,13 +217,22 @@ proc describeResult*(): string = else: &"draw ({light} to {dark})" proc runHeadless*() = - ## Runs a whole match with no renderer, as fast as the machine allows. + ## Runs fixed simulation ticks with optional pacing and request barriers. startGameProfile() defer: finishGameProfile() let started = epochTime() + var + pacer = initTickPacer(options.headlessTickRate) + pollers: seq[RequestPoll] + if options.waitForLlm and not run.replayMode: + for vm in run.brains: + if vm != nil: + pollers.add vm.pollRequests while run.world.tick < run.maximumTicks and not run.world.over: advanceGame() + waitForRequests(pollers) + pacer.pace() if profileShouldDump(run.world.tick): finishGameProfile() let diff --git a/examples/light_vs_dark/sim.nim b/examples/light_vs_dark/sim.nim index cfb3f898..038d488e 100644 --- a/examples/light_vs_dark/sim.nim +++ b/examples/light_vs_dark/sim.nim @@ -133,6 +133,8 @@ type explored*: array[PlayerCount, seq[uint8]] # HASH: derived OverlordVm* = ref object output*: PrintProc + prepareDecision*: proc(tick: int32) {.closure.} + pollRequests*: proc(): bool {.closure.} ## One compiled BASIC program for a player. Not simulation state. runtime*: Runtime ready*: bool diff --git a/nimby.lock b/nimby.lock index 50ca4a84..cddad08f 100644 --- a/nimby.lock +++ b/nimby.lock @@ -1,4 +1,6 @@ bassy 0.1.0 https://github.com/treeform/bassy 669a7c4b94e3d5b0a38dc9557608a7e58c2a764d +curly 1.1.1 https://github.com/guzba/curly 4f36f01ac6ca881fff4bbf0f8daa362d4b9f0668 +libcurl 1.0.0 https://github.com/Araq/libcurl 7a420498f60a31d99fc8513886ce36c4e8c3a4ae fixxy 0.1.0 https://github.com/treeform/fixxy 05e5446dffb70093056cebb0c57721a60deaf52a silky 0.2.0 https://github.com/treeform/silky fb9b13910edd66cf1751056784c2f7d2932a59fc pixie 6.1.0 https://github.com/treeform/pixie 87cecced5c4c6f311c658a5f3ca0c9b43edb6aa7 diff --git a/src/polyworld/cli.nim b/src/polyworld/cli.nim index 1b674e30..17fee3d3 100644 --- a/src/polyworld/cli.nim +++ b/src/polyworld/cli.nim @@ -43,6 +43,10 @@ type ## Graphical window waits for the display when true. playerSlot*: int32 ## One-based human controller slot. Zero means bots fill every slot. + headlessTickRate*: int32 + ## Zero runs at unlimited speed; positive values pace wall-clock ticks. + waitForLlm*: bool + ## Waits for the whole tick's LLM request batch before advancing. proc fail*(message: string) {.noreturn.} = ## Prints one command-line error and exits. @@ -228,6 +232,30 @@ proc takeCommonFlag*( argument: string ): bool = ## Handles one shared flag. Returns false when the game should try it. + if argument.startsWith("--headless-tick-rate:") or + argument.startsWith("--headless-tick-rate=") or + argument == "--headless-tick-rate": + let text = + if argument == "--headless-tick-rate": + arguments.argumentValue(index, argument) + else: + argument[21 .. ^1] + options.headlessTickRate = parseInt32(text, "--headless-tick-rate") + if options.headlessTickRate < 0 or options.headlessTickRate > 10000: + fail("--headless-tick-rate must be 0 .. 10000") + return true + if argument.startsWith("--llm-mode:") or + argument.startsWith("--llm-mode=") or argument == "--llm-mode": + let text = + if argument == "--llm-mode": + arguments.argumentValue(index, argument) + else: + argument[11 .. ^1] + case text.toLowerAscii() + of "async": options.waitForLlm = false + of "barrier": options.waitForLlm = true + else: fail("--llm-mode must be async or barrier") + return true if argument.startsWith("--bot:"): options.botGroups.addBotSpec(argument[6 .. ^1]) return true diff --git a/src/polyworld/configs.nim b/src/polyworld/configs.nim index a7a66f54..9a635c43 100644 --- a/src/polyworld/configs.nim +++ b/src/polyworld/configs.nim @@ -22,6 +22,8 @@ type spawnIntervalTicks*: int32 = DefaultSpawnIntervalTicks playerSlot*: int32 dayCount*: int32 + headlessTickRate*: int32 + waitForLlm*: bool when Preset isnot void: mapPreset*: Preset @@ -50,7 +52,9 @@ proc gameConfig*[Preset](config: MatchConfig[Preset]): GameConfig = maxTicks: config.maxTicks, spawnIntervalTicks: config.spawnIntervalTicks, playerSlot: config.playerSlot, - dayCount: config.dayCount + dayCount: config.dayCount, + headlessTickRate: config.headlessTickRate, + waitForLlm: config.waitForLlm ) proc withMapPreset*[Preset]( @@ -64,6 +68,8 @@ proc withMapPreset*[Preset]( spawnIntervalTicks: config.spawnIntervalTicks, playerSlot: config.playerSlot, dayCount: config.dayCount, + headlessTickRate: config.headlessTickRate, + waitForLlm: config.waitForLlm, mapPreset: preset ) diff --git a/src/polyworld/controllers.nim b/src/polyworld/controllers.nim index 9e75316b..73f9479f 100644 --- a/src/polyworld/controllers.nim +++ b/src/polyworld/controllers.nim @@ -39,6 +39,8 @@ proc localGameConfig*(options: GameOptions, slotCount: int): GameConfig = maxTicks: options.maximumTicks, spawnIntervalTicks: options.spawnIntervalTicks, playerSlot: options.playerSlot, + headlessTickRate: options.headlessTickRate, + waitForLlm: options.waitForLlm, players: unnamedPlayers(slotCount) ) var next = 0 diff --git a/src/polyworld/coworld.nim b/src/polyworld/coworld.nim index e73c8b03..479c404c 100644 --- a/src/polyworld/coworld.nim +++ b/src/polyworld/coworld.nim @@ -287,6 +287,8 @@ proc coworldOptions*(slotCount: int): GameOptions = maximumTicks: config.maxTicks, seconds: config.maxTicks div SharedTickRate, spawnIntervalTicks: config.spawnIntervalTicks, + headlessTickRate: config.headlessTickRate, + waitForLlm: config.waitForLlm, recordPath: replayPath, speed: 1, windowWidth: 1920, diff --git a/src/polyworld/llms.nim b/src/polyworld/llms.nim new file mode 100644 index 00000000..bf029606 --- /dev/null +++ b/src/polyworld/llms.nim @@ -0,0 +1,478 @@ +import + std/[json, monotimes, options, os, strutils, tables, times, uri], + bassy, jsony, + oracles, timings + +const + NativeRequests* = not defined(emscripten) and not defined(js) + MaxRequestBytes* = 64 * 1024 + MaxResponseBytes* = 256 * 1024 + MaxHeaderBytes = 16 * 1024 + DefaultOracleModel* = "typesafe/jev-1.13" + +when NativeRequests: + import curly + + type LlmConnection = object + curl: Curly + inFlight: bool + + proc close(connection: var LlmConnection) {.raises: [].} = + ## Drains the outstanding request before releasing Curly's worker. + if connection.curl != nil: + if connection.inFlight: + discard connection.curl.waitForResponse() + connection.curl.close() + connection.curl = nil + connection.inFlight = false + + proc `=destroy`(connection: var LlmConnection) = + ## Releases the HTTP worker when its player is destroyed. + connection.close() + +type + LlmError* = object of CatchableError + LlmConfig* = object + baseUrl*, key*, model*, oracleModel*: string + sidecar*: bool + interval*: int32 + timeoutMs*: int + LlmClient* = ref object + runtime {.cursor.}: Runtime + oracle*: Oracle + config*: LlmConfig + slot*: int + tick, lastAsk, nextId, pending: int32 + asked: bool + when NativeRequests: + connection: LlmConnection + started: MonoTime + completed, httpStatus: int32 + body, failure: string + +proc parseDocument*(body: string): JsonNode {.raises: [LlmError].} = + ## Parses bounded LLM JSON with jsony and reports LlmError on failure. + if body.len > MaxResponseBytes: + raise newException(LlmError, "LLM JSON exceeds the byte limit") + var + depth = 0 + quoted, escaped: bool + for character in body: + if quoted: + if escaped: + escaped = false + elif character == '\\': + escaped = true + elif character == '"': + quoted = false + else: + case character + of '"': + quoted = true + of '{', '[': + inc depth + if depth > 64: + raise newException(LlmError, "LLM JSON nesting exceeds 64 levels") + of '}', ']': + dec depth + else: + discard + try: + result = body.fromJson(JsonNode) + except ValueError: + raise newException(LlmError, "Invalid LLM JSON: " & getCurrentExceptionMsg()) + +proc environmentInt(name: string, fallback, maximum: int): int = + ## Reads a bounded host setting without silently ignoring bad values. + try: + result = parseInt(getEnv(name, $fallback)) + except ValueError: + raise newException(LlmError, name & " must be an integer") + if result < 1 or result > maximum: + raise newException(LlmError, name & " is outside the supported range") + +proc llmConfig*(): LlmConfig = + ## Uses the platform sidecar first, or explicit local OpenRouter access. + if not NativeRequests or getEnv("COGAME_LLM").toLowerAscii == "off": + return + result.baseUrl = getEnv("AWS_ENDPOINT_URL_BEDROCK_RUNTIME").strip() + result.sidecar = result.baseUrl.len > 0 + if not result.sidecar: + result.key = getEnv("COGAME_LLM_KEY", getEnv("OPENROUTER_API_KEY")) + result.baseUrl = getEnv("COGAME_LLM_BASE_URL") + if result.baseUrl.len == 0 and result.key.len > 0: + result.baseUrl = "https://openrouter.ai/api" + if result.baseUrl.len == 0: + return + result.baseUrl = result.baseUrl.strip(trailing = true, chars = {'/'}) + var url: Uri + try: + url = parseUri(result.baseUrl) + except ValueError: + raise newException(LlmError, "Invalid host LLM base URL") + if url.hostname.len == 0 or url.username.len > 0 or + url.password.len > 0 or url.query.len > 0 or url.anchor.len > 0 or + (url.scheme != "https" and not (result.sidecar and url.scheme == "http")): + raise newException(LlmError, "Invalid host LLM base URL") + result.model = getEnv("COGAME_LLM_MODEL") + result.oracleModel = getEnv("COGAME_ORACLE_MODEL", DefaultOracleModel) + result.interval = int32(environmentInt("COGAME_LLM_INTERVAL", 1, 100000)) + result.timeoutMs = environmentInt("COGAME_LLM_TIMEOUT_MS", 30000, 120000) + +proc newLlmClient*(slot: int, config: LlmConfig): LlmClient = + ## Creates one isolated seat without opening a network connection. + if slot < 0 or config.interval < 0 or config.timeoutMs < 0: + raise newException(LlmError, "Invalid LLM client configuration") + LlmClient(config: config, slot: slot, tick: -1, oracle: newOracle()) + +proc newLlmClient*(slot: int): LlmClient = + ## Reads host settings once when constructing a player's LLM client. + result = newLlmClient(slot, llmConfig()) + result.oracle.enabled = getEnv("COGAME_ORACLE").toLowerAscii != "off" + +proc close*(client: LlmClient) {.raises: [].} = + ## Waits for outstanding HTTP work, releases Curly, and forgets answers. + if client == nil: + return + when NativeRequests: + client.connection.close() + client.pending = 0 + client.completed = 0 + client.body.setLen(0) + client.failure.setLen(0) + client.oracle.reset() + +proc available*(client: LlmClient): bool {.raises: [].} = + ## Reports whether the host configured a native inference endpoint. + NativeRequests and client != nil and client.config.baseUrl.len > 0 + +proc hasPending*(client: LlmClient): bool {.raises: [].} = + ## Reports requests that must settle before a barrier may advance. + client != nil and client.pending != 0 + +proc ready*(client: LlmClient): int32 {.raises: [].} = + ## Returns zero when ready, remaining spacing ticks, or minus one. + if not client.available or client.pending != 0: + return -1 + when NativeRequests: + if client.connection.inFlight: + return -1 + if client.asked: + return int32(max(0'i64, + int64(client.config.interval) - (int64(client.tick) - client.lastAsk))) + +proc beginTick*(client: LlmClient, tick: int32) = + ## Polls ready network work at the decision boundary and detects resets. + if tick < client.tick: + client.pending = 0 + client.completed = 0 + client.body.setLen(0) + client.failure.setLen(0) + client.asked = false + client.tick = tick + client.oracle.beginTick(tick) + when NativeRequests: + if not client.connection.inFlight: + return + var + completed = client.connection.curl.pollForResponse() + status: int32 + body, failure: string + if completed.isSome: + client.connection.inFlight = false + if client.pending == 0: + return + var received = move(completed.get()) + status = int32(received.response.code) + failure = move(received.error) + var headerBytes = 0 + for (name, value) in received.response.headers: + headerBytes += name.len + value.len + 4 + if received.response.body.len > MaxResponseBytes: + failure = "LLM response exceeds the byte limit" + elif headerBytes > MaxHeaderBytes: + failure = "LLM response headers exceed the byte limit" + else: + body = move(received.response.body) + if (getMonoTime() - client.started).inMilliseconds >= + client.config.timeoutMs: + failure = "LLM request timed out" + if failure.len == 0 and status notin 200 .. 299: + failure = "LLM HTTP " & $status + elif client.pending != 0 and + (getMonoTime() - client.started).inMilliseconds >= client.config.timeoutMs: + failure = "LLM request timed out" + else: + return + if client.pending == client.oracle.pending: + var document: JsonNode + if failure.len == 0: + try: + document = parseDocument(body) + except LlmError: + discard # Malformed JEV responses settle as failed requests. + client.oracle.complete(client.pending, document) + client.completed = client.pending + client.httpStatus = status + client.body = move(body) + client.failure = move(failure) + client.pending = 0 + +proc ask*(client: LlmClient, verb, path, body: string): int32 = + ## Forwards an inference API body unchanged and returns its request ID. + if client.ready != 0 or body.len > MaxRequestBytes: + return 0 + if verb notin ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"] or + not path.startsWith("/v1/") or path.len > 2048 or + path.find({'\x00' .. '\x1f', '\x7f', '\\', '#'}) >= 0: + raise newException(LlmError, "Invalid LLM method or API path") + let route = path.split('?', 1)[0] + for segment in route.split('/'): + if segment in [".", ".."] or segment.contains('%'): + raise newException(LlmError, "LLM path must stay under /v1/") + if client.nextId == int32.high: + raise newException(LlmError, "LLM request ID limit reached") + var headers = @[("Content-Type", "application/json")] + if client.config.sidecar: + headers.add ("X-Coworld-Player-Slot", $client.slot) + elif client.config.key.len > 0: + headers.add ("Authorization", "Bearer " & client.config.key) + when NativeRequests: + if client.config.timeoutMs <= 0: + raise newException(LlmError, "LLM timeout must be positive") + if client.connection.curl == nil: + client.connection.curl = newCurly(maxInFlight = 1) + client.started = getMonoTime() + client.connection.curl.startRequest( + verb, + client.config.baseUrl & path, + headers = headers, + body = body, + timeout = max(1, (client.config.timeoutMs + 999) div 1000) + ) + client.connection.inFlight = true + inc client.nextId + client.pending = client.nextId + client.lastAsk = client.tick + client.asked = true + client.nextId + +proc chat*(client: LlmClient, model, prompt: string): int32 = + ## Sends one ordinary user message using a caller or host selected model. + let selected = if model.len > 0: model else: client.config.model + if selected.len == 0: + raise newException(LlmError, "LLM model is required") + client.ask("POST", "/v1/chat/completions", $(%*{ + "model": selected, "messages": [{"role": "user", "content": prompt}] + })) + +proc poll*(client: LlmClient, id: int32): int32 = + ## Returns zero while pending, one on success, or minus one on failure. + if id > 0 and id == client.pending: + return 0 + if id > 0 and id == client.completed and client.failure.len == 0: + return 1 + -1 + +proc status*(client: LlmClient, id: int32): int32 = + ## Returns the latest response's HTTP status, or zero for another ID. + if id > 0 and id == client.completed: + client.httpStatus + else: + 0 + +proc error*(client: LlmClient, id: int32): string = + ## Returns the latest request error, or reports an unavailable result. + if id > 0 and id == client.completed: + client.failure + else: + "Unknown or expired LLM request" + +proc response*(client: LlmClient, id: int32): string = + ## Returns the latest completed JSON or SSE response body. + if id > 0 and id == client.completed: + client.body + else: + "" + +proc contentText(content: JsonNode): string = + ## Reads text content while leaving non-text data in the raw response. + if content == nil: + return + case content.kind + of JString: + result = content.getStr() + of JArray: + for part in content: + if part.kind == JObject and part.hasKey("text") and + part["text"].kind == JString: + result.add part["text"].getStr() + else: + discard + +proc documentText(document: JsonNode): string = + ## Reads Chat Completions or Responses API text from a JSON document. + let choices = document{"choices"} + if choices != nil and choices.kind == JArray and choices.len > 0: + result = contentText(choices[0]{"message", "content"}) + if result.len == 0: + result = contentText(choices[0]{"delta", "content"}) + let output = document{"output"} + if output != nil and output.kind == JArray: + for item in output: + result.add contentText(item{"content"}) + if document{"type"}.getStr() == "response.output_text.delta": + result.add document{"delta"}.getStr() + +proc text*(client: LlmClient, id: int32): string = + ## Extracts ordinary text from a completed JSON or SSE response. + let body = client.response(id) + if body.strip().startsWith("{"): + return documentText(parseDocument(body)) + for line in body.splitLines(): + if line.startsWith("data:"): + let data = line[5 .. ^1].strip() + if data.len == 0 or data == "[DONE]": + continue + try: + result.add documentText(parseDocument(data)) + except LlmError: + discard # Ignore malformed SSE data events. + +proc askOracle(client: LlmClient): int32 = + ## Sends a JEV draft through this player's shared LLM client. + defer: + client.oracle.submit(result) + if not client.oracle.enabled or client.ready != 0: + return 0 + let body = client.oracle.requestBody(client.config.oracleModel) + if body.len > 0: + result = client.ask("POST", "/v1/systemone", body) + +proc bindRuntime*(client: LlmClient, runtime: Runtime) = + ## Borrows the runtime that owns these callbacks, avoiding a ref cycle. + client.runtime = runtime + +proc requestPoller*(client: LlmClient): RequestPoll = + ## Polls this seat during the shared barrier without rerunning BASIC. + result = proc(): bool = + ## Delivers completed replies while keeping the simulation tick fixed. + client.beginTick(client.tick) + client.hasPending() + +proc decisionCallback*(client: LlmClient): proc(tick: int32) = + ## Keeps inference state outside the deterministic simulation modules. + result = proc(tick: int32) = + ## Advances the state belonging to this VM only. + client.beginTick(tick) + +proc jsonGet(document: JsonNode, path: string): string = + ## Reads an RFC 6901 pointer as text or serialized JSON for non-strings. + var current = document + if path.len > 0: + if path[0] != '/': + raise newException(LlmError, "JSON pointer must start with a slash") + for part in path[1 .. ^1].split('/'): + if current == nil: + return "" + let key = part.replace("~1", "/").replace("~0", "~") + case current.kind + of JObject: + current = current{key} + of JArray: + var index: int + try: + index = parseInt(key) + except ValueError: + return "" + if index < 0 or index >= current.len: + return "" + current = current[index] + else: + return "" + if current == nil: + return "" + if current.kind == JString: current.getStr() else: $current + +proc addFunctions*(client: LlmClient, host: var Host) = + ## Registers the BASIC calls directly on this player's LLM client. + template register(name: string, arity: int, operation: untyped) = + ## Shares BASIC value conversion and error handling between callbacks. + block: + let binding: NumericHostProc = proc(arguments: openArray[Value]): Value = + ## Runs one LLM operation with values from this BASIC runtime. + template text(index: int): string {.inject.} = + ## Reads a BASIC string argument. + client.runtime.getString(arguments[index]) + template integer(index: int): int32 {.inject.} = + ## Reads an integer argument, rejecting fractional values. + arguments[index].asInt() + template output(text: string): Value {.inject.} = + ## Copies a string result into BASIC storage. + client.runtime.putString(text) + var value {.inject.}: Value + try: + operation + except LlmError, OracleError: + raise newException(BasicError, getCurrentExceptionMsg()) + value + discard host.addFunction(name, arity, binding, 256) + register("llmAvailable", 0): + value = int32(client.available) + register("llmReady", 0): + value = client.ready + register("llmAsk", 2): + value = client.chat(text(0), text(1)) + register("llmRequest", 3): + value = client.ask(text(0), text(1), text(2)) + register("llmPoll", 1): + value = client.poll(integer(0)) + register("llmStatus", 1): + value = client.status(integer(0)) + register("llmResponse$", 1): + value = output(client.response(integer(0))) + register("llmRead$", 3): + let + body = client.response(integer(0)) + offset = integer(1) + count = integer(2) + if offset < 0 or count < 0: + raise newException(LlmError, "LLM slice must be nonnegative") + let start = min(int(offset), body.len) + value = output(body[start ..< start + min(int(count), body.len - start)]) + register("llmText$", 1): + value = output(client.text(integer(0))) + register("llmError$", 1): + value = output(client.error(integer(0))) + register("jsonQuote$", 1): + value = output(text(0).toJson()) + register("jsonGet$", 2): + value = output(jsonGet(parseDocument(text(0)), text(1))) + register("oracleAvailable", 0): + value = int32(client.oracle.enabled and client.available) + register("oracleReady", 0): + value = if client.oracle.enabled: client.ready else: -1 + register("oracleState", 2): + value = client.oracle.state(text(0), %integer(1)) + register("oracleStateText", 2): + value = client.oracle.state(text(0), %text(1)) + register("oracleNote", 1): + value = client.oracle.note(text(0)) + register("oracleQuestion", 3): + value = client.oracle.question(text(0), integer(1), text(2)) + register("oracleCriterion", 3): + value = client.oracle.criterion(text(0), text(1), text(2)) + register("oracleCriterionField", 4): + value = client.oracle.criterionField(text(0), text(1), text(2), text(3)) + register("oracleAsk", 0): + value = client.askOracle() + register("oraclePoll", 1): + value = client.oracle.poll(integer(0)) + register("oracleAnswer", 2): + value = client.oracle.answer(integer(0), text(1)).value + register("oracleConfidence", 2): + value = client.oracle.answer(integer(0), text(1)).confidence + register("oracleProbability", 3): + value = client.oracle.answer(integer(0), text(1)).probabilities.getOrDefault( + text(2), -1'i32 + ) diff --git a/src/polyworld/oracles.nim b/src/polyworld/oracles.nim new file mode 100644 index 00000000..6bad3ce8 --- /dev/null +++ b/src/polyworld/oracles.nim @@ -0,0 +1,314 @@ +import std/[json, math, tables] + +const + MaxStateFields* = 256 + MaxNotes* = 16 + MaxQuestions* = 64 + MaxCriteria* = 16 + MaxKeyLength* = 64 + MaxOracleBytes* = 32 * 1024 + MaxStateNodes = 4096 + +type + OracleError* = object of CatchableError + QuestionKind* = enum + NoulQuestion, ScoreQuestion, ChoiceQuestion + Criterion = object + label, text: string + fields: seq[(string, string)] + Question = object + key, instructions: string + kind: QuestionKind + criteria: seq[Criterion] + OracleDraft = object + state: JsonNode + fields, notes, bytes, nodes: int + questions: seq[Question] + OracleAnswer* = object + key*: string + value*, confidence*: int32 + probabilities*: Table[string, int32] + Oracle* = ref object + enabled*: bool + draft: OracleDraft + pending*: int32 + tick: int32 + questions: seq[Question] + completed: int32 + answers: seq[OracleAnswer] + +proc newOracle*(enabled = true): Oracle = + ## Creates structured JEV state without an HTTP client or BASIC runtime. + Oracle(enabled: enabled, tick: -1) + +proc reset*(oracle: Oracle) = + ## Clears drafts, pending questions, and retained answers. + oracle.draft = OracleDraft() + oracle.pending = 0 + oracle.questions.setLen(0) + oracle.completed = 0 + oracle.answers.setLen(0) + oracle.tick = -1 + +proc validKey(key: string): bool = + ## Bounds names before storing or expanding them into JSON paths. + key.len > 0 and key.len <= MaxKeyLength + +proc reserve(draft: var OracleDraft, bytes: int): bool = + ## Bounds retained draft strings even before JSON serialization. + if bytes > MaxOracleBytes - draft.bytes: + return false + draft.bytes += bytes + true + +proc setPath( + node: JsonNode, key: string, value: JsonNode, nodes: var int +) = + ## Expands dotted keys and bounded array indices into structured state. + var + current = node + position = 0 + while position < key.len: + var + name: string + index = -1 + if key[position] == '[': + inc position + index = 0 + let start = position + while position < key.len and key[position] in {'0' .. '9'}: + index = index * 10 + ord(key[position]) - ord('0') + if index >= MaxStateFields: + raise newException(OracleError, "Oracle array index is too large") + inc position + if position == start or position >= key.len or key[position] != ']': + raise newException(OracleError, "Invalid oracle field path") + inc position + else: + while position < key.len and key[position] notin {'.', '['}: + name.add key[position] + inc position + if name.len == 0: + raise newException(OracleError, "Invalid oracle field path") + let final = position == key.len + if not final and key[position] == '.': + inc position + if position == key.len: + raise newException(OracleError, "Invalid oracle field path") + let child = + if final: value + elif key[position] == '[': newJArray() + else: newJObject() + inc nodes + if nodes > MaxStateNodes: + raise newException(OracleError, "Oracle state node limit exceeded") + if index >= 0: + if current.kind != JArray: + raise newException(OracleError, "Conflicting oracle field paths") + while current.len <= index: + inc nodes + if nodes > MaxStateNodes: + raise newException(OracleError, "Oracle state node limit exceeded") + current.add newJNull() + if final or current[index].kind == JNull: + current.elems[index] = child + current = current[index] + else: + if current.kind != JObject: + raise newException(OracleError, "Conflicting oracle field paths") + if final or not current.hasKey(name): + current[name] = child + current = current[name] + +proc state*(oracle: Oracle, key: string, value: JsonNode): int32 = + ## Adds a bounded fact to this decision's request draft. + if not validKey(key) or oracle.draft.fields >= MaxStateFields: + return 0 + if not oracle.draft.reserve(key.len + ($value).len + 8): + return 0 + if oracle.draft.state == nil: + oracle.draft.state = newJObject() + oracle.draft.state.setPath(key, value, oracle.draft.nodes) + inc oracle.draft.fields + 1 + +proc note*(oracle: Oracle, text: string): int32 = + ## Adds one explanatory note to the state. + if oracle.draft.notes >= MaxNotes: + return 0 + let key = "notes[" & $oracle.draft.notes & "]" + result = oracle.state(key, %text) + if result == 1: + inc oracle.draft.notes + +proc question*( + oracle: Oracle, key: string, kind: int32, instructions: string +): int32 = + ## Creates or replaces a typed question in the current draft. + if not validKey(key) or kind notin 0 .. 2: + return 0 + if not oracle.draft.reserve(key.len + instructions.len + 64): + return 0 + let question = Question( + key: key, kind: QuestionKind(kind), instructions: instructions + ) + for item in oracle.draft.questions.mitems: + if item.key == key: + item = question + return 1 + if oracle.draft.questions.len >= MaxQuestions: + return 0 + oracle.draft.questions.add question + 1 + +proc criterion*(oracle: Oracle, key, label, text: string): int32 = + ## Adds one choice label or ordered score criterion. + for item in oracle.draft.questions.mitems: + if item.key != key: + continue + if item.criteria.len >= MaxCriteria: + return 0 + if item.kind != ScoreQuestion: + if not validKey(label): + return 0 + for existing in item.criteria: + if existing.label == label: + return 0 + if not oracle.draft.reserve(label.len + text.len + 16): + return 0 + item.criteria.add Criterion(label: label, text: text) + return 1 + +proc criterionField*( + oracle: Oracle, key, label, field, text: string +): int32 = + ## Adds bounded descriptive fields to a choice or yes/no criterion. + if not validKey(field) or field == "what": + return 0 + for item in oracle.draft.questions.mitems: + if item.key != key or item.kind == ScoreQuestion: + continue + for criterion in item.criteria.mitems: + if criterion.label == label and criterion.fields.len < MaxCriteria: + if not oracle.draft.reserve(field.len + text.len + 16): + return 0 + criterion.fields.add (field, text) + return 1 + +proc requestBody*(oracle: Oracle, model: string): string = + ## Encodes Jev's state and typed questions without a chat prompt wrapper. + if oracle.draft.questions.len == 0: + return + var questions = newJObject() + for item in oracle.draft.questions: + var + question = %*{"instructions": item.instructions} + criteria = if item.kind == ScoreQuestion: newJArray() else: newJObject() + question["type"] = %(case item.kind + of NoulQuestion: "noul" + of ScoreQuestion: "score" + of ChoiceQuestion: "choice") + for criterion in item.criteria: + if item.kind == ScoreQuestion: + criteria.add %criterion.text + else: + var value = %criterion.text + if criterion.fields.len > 0: + value = %*{"what": criterion.text} + for (name, text) in criterion.fields: + if not value.hasKey(name): + value[name] = %text + else: + if value[name].kind != JArray: + value[name] = %*[value[name]] + value[name].add %text + criteria[criterion.label] = value + question["criteria"] = criteria + questions[item.key] = question + result = $(%*{ + "model": model, + "state": (if oracle.draft.state == nil: newJObject() + else: oracle.draft.state), + "questions": questions + }) + if result.len > MaxOracleBytes: + result.setLen(0) + +proc submit*(oracle: Oracle, id: int32) = + ## Saves accepted question definitions and clears the submitted draft. + if id > 0: + oracle.pending = id + oracle.questions = oracle.draft.questions + oracle.draft = OracleDraft() + +proc thousandths(node: JsonNode): int32 = + ## Converts finite API numbers to BASIC's integer thousandths. + if node == nil or node.kind notin {JInt, JFloat}: + return -1 + let value = node.getFloat() * 1000 + if classify(value) in {fcNan, fcInf, fcNegInf}: + return -1 + int32(clamp(round(value), -1_000_000_000.0, 1_000_000_000.0)) + +proc flatten(oracle: Oracle, document: JsonNode): seq[OracleAnswer] = + ## Converts only answers matching the submitted question definitions. + let answers = document{"answers"} + if answers == nil or answers.kind != JObject: + return + for question in oracle.questions: + let node = answers{question.key} + if node == nil or node.kind != JObject: + continue + var answer = OracleAnswer( + key: question.key, value: -1, + confidence: thousandths(node{"confidence"}) + ) + case question.kind + of NoulQuestion: + answer.value = thousandths(node{"noul"}) + of ScoreQuestion: + answer.value = thousandths(node{"score"}) + of ChoiceQuestion: + let selected = node{"choice"}.getStr() + for index, criterion in question.criteria: + if criterion.label == selected: + answer.value = int32(index) + let probability = thousandths(node{"probabilities", criterion.label}) + if probability >= 0: + answer.probabilities[criterion.label] = probability + if answer.value >= 0: + result.add answer + +proc beginTick*(oracle: Oracle, tick: int32) = + ## Clears the draft for this decision and resets answers when rewinding. + if tick < oracle.tick: + oracle.reset() + oracle.tick = tick + oracle.draft = OracleDraft() + +proc complete*(oracle: Oracle, id: int32, document: JsonNode) = + ## Stores structured answers, using nil for failed or malformed responses. + if id <= 0 or id != oracle.pending: + return + oracle.completed = id + oracle.answers.setLen(0) + if document != nil: + oracle.answers = oracle.flatten(document) + oracle.pending = 0 + oracle.questions.setLen(0) + +proc poll*(oracle: Oracle, id: int32): int32 = + ## Returns the answer count, zero while pending, or minus one on failure. + if id > 0 and id == oracle.pending: + return 0 + if id > 0 and id == oracle.completed and oracle.answers.len > 0: + return int32(oracle.answers.len) + -1 + +proc answer*(oracle: Oracle, id: int32, key: string): OracleAnswer = + ## Reads a named judgment, returning missing values as minus one. + if id > 0 and id == oracle.completed: + for answer in oracle.answers: + if answer.key == key: + return answer + OracleAnswer(value: -1, confidence: -1) diff --git a/src/polyworld/timings.nim b/src/polyworld/timings.nim new file mode 100644 index 00000000..5be8f395 --- /dev/null +++ b/src/polyworld/timings.nim @@ -0,0 +1,40 @@ +import std/[monotimes, os, times] + +type + TimingError* = object of CatchableError + RequestPoll* = proc(): bool {.closure.} + TickPacer* = object + period: Duration + deadline: MonoTime + enabled: bool + +proc initTickPacer*(rate: int32): TickPacer = + ## Selects unlimited speed at zero or a wall-clock tick frequency. + if rate < 0 or rate > 10000: + raise newException(TimingError, "Headless tick rate must be 0 .. 10000") + if rate > 0: + result.enabled = true + result.period = initDuration(nanoseconds = 1_000_000_000 div int64(rate)) + result.deadline = getMonoTime() + result.period + +proc pace*(pacer: var TickPacer) = + ## Limits tick speed without changing simulation time or catching up bursts. + if not pacer.enabled: + return + var now = getMonoTime() + while now < pacer.deadline: + let remaining = (pacer.deadline - now).inMilliseconds + sleep(int(clamp(remaining, 1'i64, 50'i64))) + now = getMonoTime() + pacer.deadline = max(pacer.deadline + pacer.period, now + pacer.period) + +proc waitForRequests*(pollers: openArray[RequestPoll]) = + ## Polls every seat together until the whole tick's request batch settles. + while true: + var pending = false + for poll in pollers: + if poll != nil and poll(): + pending = true + if not pending: + return + sleep(1) diff --git a/tests/fixtures/jev_request.bas b/tests/fixtures/jev_request.bas new file mode 100644 index 00000000..9e245985 --- /dev/null +++ b/tests/fixtures/jev_request.bas @@ -0,0 +1,20 @@ +if request = 0 then + ok = oracleStateText("game", "Gods of the Arena, a team lane battle") + ok = oracleStateText("situation", "Early game at 120 seconds. Our warrior has 90% health, level 3, and little gold. Allied towers are healthy. Mid has safe enemy creeps to farm and a nearby ally. Two healthy enemies are visible in top lane. No enemy hero is visible in bottom lane.") + ok = oracleQuestion("strategy", 2, "Choose our next macro strategy.") + ok = oracleCriterion("strategy", "farm", "Kill lane creeps for gold and experience.") + ok = oracleCriterion("strategy", "gank", "Join an ally to attack an enemy hero.") + ok = oracleCriterion("strategy", "push", "Advance to damage enemy structures.") + ok = oracleCriterion("strategy", "defend", "Protect our threatened base.") + ok = oracleCriterion("strategy", "regroup", "Move toward nearby teammates.") + ok = oracleQuestion("lane", 2, "Choose the lane for this strategy.") + ok = oracleCriterion("lane", "top", "Top lane with two visible enemies.") + ok = oracleCriterion("lane", "mid", "Mid lane with safe creeps and an ally.") + ok = oracleCriterion("lane", "bottom", "Bottom lane with no visible enemy heroes.") + request = oracleAsk() +else + answers = oraclePoll(request) + strategy = oracleAnswer(request, "strategy") + lane = oracleAnswer(request, "lane") + model$ = jsonGet$(llmResponse$(request), "/model") +end if diff --git a/tests/fixtures/jev_response.json b/tests/fixtures/jev_response.json new file mode 100644 index 00000000..df67bfae --- /dev/null +++ b/tests/fixtures/jev_response.json @@ -0,0 +1,34 @@ +{ + "model": "typesafe/jev-1.13-20260917", + "answers": { + "strategy": { + "type": "choice", + "choice": "farm", + "probabilities": { + "push": 0.01, + "defend": 0, + "regroup": 0.03, + "gank": 0.21, + "farm": 0.75 + }, + "confidence": 0.69 + }, + "lane": { + "type": "choice", + "choice": "mid", + "probabilities": { + "top": 0.02, + "bottom": 0.27, + "mid": 0.71 + }, + "confidence": 0.57 + } + }, + "usage": { + "input_tokens": 542, + "output_tokens": 90, + "cost": 2.2764e-05 + }, + "id": "gen-dec-1790274068-NTDKtRQcsA3VcFt1fc2a", + "provider": "TypeSafe" +} diff --git a/tests/manual_jev.nim b/tests/manual_jev.nim new file mode 100644 index 00000000..ca0bdba8 --- /dev/null +++ b/tests/manual_jev.nim @@ -0,0 +1,46 @@ +import + std/[os, strutils], + bassy, + polyworld/[llms, timings] + +const JevRequest = staticRead("fixtures/jev_request.bas") + +echo "Manual paid test: one BASIC request directly to OpenRouter JEV" +block: + let key = getEnv("OPENROUTER_API_KEY") + doAssert key.len > 0, "Set OPENROUTER_API_KEY before running this test" + let scriptClient = newLlmClient(0, LlmConfig( + baseUrl: "https://openrouter.ai/api", + key: key, + oracleModel: DefaultOracleModel, + interval: 1, + timeoutMs: 30_000 + )) + defer: + scriptClient.close() + var host = initHost() + scriptClient.addFunctions(host) + let program = compile(JevRequest, host) + var runtime = initRuntime(program, host) + scriptClient.bindRuntime(runtime) + scriptClient.beginTick(0) + discard runtime.run() + let id = runtime.getGlobal("request") + doAssert id > 0, "BASIC did not submit its JEV request" + waitForRequests([scriptClient.requestPoller()]) + let body = scriptClient.response(id) + echo "HTTP status: ", scriptClient.status(id) + doAssert scriptClient.status(id) == 200, scriptClient.error(id) & " " & body + doAssert scriptClient.error(id).len == 0, scriptClient.error(id) + runtime.restart() + scriptClient.beginTick(1) + discard runtime.run() + let + strategy = runtime.getGlobal("strategy") + lane = runtime.getGlobal("lane") + model = runtime.getString(runtime.getGlobalValue("model$")) + doAssert runtime.getGlobal("answers") == 2, body + doAssert strategy in 0 .. 4 and lane in 0 .. 2, body + doAssert model.startsWith("typesafe/jev-"), body + echo "BASIC strategy index: ", strategy, ", lane index: ", lane + echo "OpenRouter response: ", body diff --git a/tests/test_cli.nim b/tests/test_cli.nim index a66dceb9..5334fea5 100644 --- a/tests/test_cli.nim +++ b/tests/test_cli.nim @@ -18,6 +18,22 @@ proc parseCommon( inc index result.validateGameOptions(liveBotCount, liveBotMessage) +echo "Testing independent pacing and barrier controls" +block: + let defaults = parseCommon(@[]) + doAssert not defaults.waitForLlm + doAssert defaults.headlessTickRate == 0 + for flags in [ + @["--headless-tick-rate:24", "--llm-mode:barrier"], + @["--headless-tick-rate=24", "--llm-mode=barrier"], + @["--headless-tick-rate", "24", "--llm-mode", "barrier"] + ]: + let options = parseCommon(flags) + doAssert options.waitForLlm + doAssert options.headlessTickRate == 24 + let disabled = parseCommon(@["--llm-mode:async"]) + doAssert not disabled.waitForLlm + echo "Testing bot path and count parsing" block: var groups: seq[BotGroup] diff --git a/tests/test_controllers.nim b/tests/test_controllers.nim index e27714f7..2f7f2077 100644 --- a/tests/test_controllers.nim +++ b/tests/test_controllers.nim @@ -97,3 +97,22 @@ block: doAssert config.players[1].displayName(1) == "Player 2" echo "Controller tests passed" + +echo "Testing hosted and replay inference settings round trip" +block: + let config = """{ + "headless_tick_rate": 12, "wait_for_llm": true + }""".fromJson(GameConfig) + doAssert config.waitForLlm + doAssert config.headlessTickRate == 12 + let restored = config.toJson().fromJson(GameConfig) + doAssert restored == config + let defaults = "{}".fromJson(GameConfig) + doAssert not defaults.waitForLlm + doAssert defaults.headlessTickRate == 0 + let options = GameOptions( + headlessTickRate: 12, waitForLlm: true + ) + let local = options.localGameConfig(0) + doAssert local.waitForLlm + doAssert local.headlessTickRate == 12 diff --git a/tests/test_llm_hosts.nim b/tests/test_llm_hosts.nim new file mode 100644 index 00000000..e617042e --- /dev/null +++ b/tests/test_llm_hosts.nim @@ -0,0 +1,75 @@ +import + std/[os, strutils, tempfiles], + bassy, + polyworld/cli + +when defined(llmCta): + import ../examples/call_to_adventure/[bots, content, sim] +elif defined(llmLvd): + import ../examples/light_vs_dark/[bots, content, maps, sim] +else: + import ../examples/gods_of_the_arena/[bots, maps, replays, sim] + +const Program = """ +remoteAvailable = llmAvailable() +quoted$ = jsonQuote$("hello") +text$ = jsonGet$(quoted$, "") +sent = sendChat(-2, "global hello") +message$ = pullMailbox$() +from = mailboxId() +while mailboxCount() > 0 + ignored$ = pullMailbox$() +wend +""" + +echo "Testing LLM and mailbox functions through the game's actual BASIC hosts and loaders" +block: + let + directory = createTempDir("polyworld-llm-hosts-", "") + path = directory / "player.bas" + hadSetting = existsEnv("COGAME_LLM") + setting = getEnv("COGAME_LLM") + putEnv("COGAME_LLM", "off") + defer: + removeDir(directory) + if hadSetting: + putEnv("COGAME_LLM", setting) + else: + delEnv("COGAME_LLM") + writeFile(path, Program) + when defined(llmCta): + let game = newGame(2026) + elif defined(llmLvd): + let game = newGame(generateMap(DefaultSeed), 240) + else: + let game = newGame(generateMap(54), 240, 10, false, ReplayData(), + drafting = false) + + when defined(llmLvd): + game.loadBots([Program, Program]) + elif defined(llmCta): + game.loadBots([BotGroup(path: path, count: PartySize)]) + else: + game.loadBots([BotGroup(path: path, count: 10)]) + for tick in 1 .. 2: + game.world.tick = int32(tick) + when defined(llmCta): + for slot in 0'i32 ..< PartySize: + game.runBotDecisions(slot) + else: + game.runBotDecisions() + when defined(llmLvd): + let vms = game.brains + else: + let vms = game.heroVms + for vm in vms: + doAssert vm != nil and not vm.failed, vm.lastError + doAssert vm.runtime.getGlobal("remoteAvailable") == 0 + doAssert vm.runtime.getGlobal("sent") == vms.len + doAssert vm.runtime.getGlobal("from") == -2 + doAssert vm.runtime.getString(vm.runtime.getGlobalValue("message$")) == + "global hello" + doAssert vm.runtime.getString(vm.runtime.getGlobalValue("text$")) == "hello" + doAssert vm.pollRequests != nil and not vm.pollRequests() + let large = vm.runtime.putString(repeat('x', 64 * 1024)) + doAssert vm.runtime.getString(large).len == 64 * 1024 diff --git a/tests/test_llms.nim b/tests/test_llms.nim new file mode 100644 index 00000000..94abcedb --- /dev/null +++ b/tests/test_llms.nim @@ -0,0 +1,508 @@ +import + std/[json, monotimes, net, os, strutils, tempfiles, times], + bassy, fixxy, + polyworld/[cli, llms, mailboxes, oracles, timings], + ../examples/gods_of_the_arena/[bots, maps, replays, sim] + +const + JevRequest = staticRead("fixtures/jev_request.bas") + JevResponse = staticRead("fixtures/jev_response.json") + +var + ports: Channel[int] + received: Channel[string] + +proc sendReply(socket: Socket, body: string, status = "200 OK") = + ## Sends a fixed-length response from the local mock sidecar. + socket.send("HTTP/1.1 " & status & "\r\nContent-Type: application/json\r\n" & + "Content-Length: " & $body.len & "\r\nConnection: close\r\n\r\n" & body) + +proc serve() {.thread.} = + ## Emulates inference routes without making any external API calls. + let server = newSocket() + server.bindAddr(Port(0), "127.0.0.1") + server.listen() + ports.send(int(server.getLocalAddr()[1])) + var + paired: Socket + strategyRequests = 0 + while true: + var socket: Socket + server.accept(socket) + let first = socket.recvLine(timeout = 3000) + if first.contains("/stop"): + socket.close() + break + var + headers = first & "\n" + length = 0 + while true: + let line = socket.recvLine(timeout = 3000) + if line.len == 0 or line == "\r\n": + break + headers.add line & "\n" + if line.toLowerAscii.startsWith("content-length:"): + length = parseInt(line.split(':', 1)[1].strip()) + var body: string + while body.len < length: + body.add socket.recv(length - body.len, timeout = 3000) + received.send(headers & body) + try: + if first.contains("/pair"): + if paired == nil: + paired = socket + continue + paired.sendReply("{}") + paired.close() + paired = nil + socket.sendReply("{}") + elif first.contains("/systemone"): + if body.contains("Gods of the Arena, a team lane battle"): + socket.sendReply(JevResponse) + elif body.contains("\"strategy\""): + inc strategyRequests + if strategyRequests <= 5: + let + strategy = ["gank", "defend", "push", "regroup", "farm"][ + strategyRequests - 1] + lane = ["bottom", "top", "mid", "bottom", "top"][ + strategyRequests - 1] + socket.sendReply("{\"answers\":{\"strategy\":{\"choice\":\"" & + strategy & "\"},\"lane\":{\"choice\":\"" & lane & "\"}}}") + elif strategyRequests == 6: + socket.sendReply("""{"answers":{"strategy":{"choice":"gank"}}}""") + else: + socket.sendReply("{}", "429 Too Many Requests") + else: + socket.sendReply("""{"answers":{ + "guard":{"noul":0.8}, + "mode":{"choice":"hold","confidence":0.7, + "probabilities":{"hold":0.7,"push":0.3}}, + "risk":{"score":1.5}}}""") + elif first.contains("/slow"): + sleep(180) + socket.sendReply("{}") + elif first.contains("/huge"): + socket.sendReply(repeat('x', MaxResponseBytes + 100)) + elif first.contains("/failure"): + socket.sendReply("{\"error\":\"spend limit\"}", "429 Too Many Requests") + elif first.contains("/stream"): + let stream = "data: {\"choices\":[{\"delta\":{\"content\":\"hel\"}}]}\n\n" & + "data: {\"choices\":[{\"delta\":{\"content\":\"lo\"}}]}\n\n" & + "data: [DONE]\n\n" + socket.sendReply(stream) + else: + socket.sendReply("""{"choices":[{"message":{"content":"hello", + "tool_calls":[{"id":"call-1","function":{"name":"move", + "arguments":"{}"}}]}}],"usage":{"total_tokens":12}}""") + except OSError: + discard # Oversize and timed-out requests deliberately close early. + socket.close() + if paired != nil: + paired.close() + server.close() + +proc settle(client: LlmClient, id: int32, tick: int32) = + ## Bounds the test wait independently of the request's own deadline. + let deadline = getMonoTime() + initDuration(seconds = 3) + while client.poll(id) == 0: + doAssert getMonoTime() < deadline, "request remained pending" + client.beginTick(tick) + sleep(1) + +echo "Testing the native sidecar transport and complete OpenRouter payloads" +block: + ports.open(1) + received.open(32) + var thread: Thread[void] + createThread(thread, serve) + let port = ports.recv() + defer: + let stop = newSocket() + stop.connect("127.0.0.1", Port(port)) + stop.send("GET /stop HTTP/1.1\r\n\r\n") + stop.close() + joinThread(thread) + ports.close() + received.close() + let config = LlmConfig( + baseUrl: "http://127.0.0.1:" & $port, + key: "must-not-be-sent", sidecar: true, + model: "test/model", oracleModel: DefaultOracleModel, + interval: 2, timeoutMs: 1000 + ) + let client = newLlmClient(3, config) + client.beginTick(0) + let body = """{"model":"test/model","messages":[{"role":"user", + "content":[{"type":"text","text":"hello"}]}], + "tools":[{"type":"function","function":{"name":"move"}}], + "response_format":{"type":"json_object"},"provider":{"order":["test"]}}""" + let id = client.ask("POST", "/v1/chat/completions", body) + doAssert id == 1 + doAssert client.ask("POST", "/v1/chat/completions", body) == 0 + client.settle(id, 1) + doAssert client.poll(id) == 1 + doAssert client.text(id) == "hello" + doAssert client.response(id).contains("tool_calls") + let request = received.recv() + doAssert request.endsWith(body) + doAssert request.contains("X-Coworld-Player-Slot: 3") + doAssert not request.toLowerAscii.contains("authorization:") + doAssert not request.contains("must-not-be-sent") + doAssert client.ready == 1 + + echo "Testing normal strings through the published BASIC chat example" + let chatClient = newLlmClient(4, config) + var chatHost = initHost() + chatClient.addFunctions(chatHost) + let chatSource = readFile( + currentSourcePath().parentDir / "../examples/inference/chat.bas" + ) + var chatRuntime = initRuntime(compile(chatSource, chatHost), chatHost) + chatClient.bindRuntime(chatRuntime) + chatClient.beginTick(0) + discard chatRuntime.run() + waitForRequests([chatClient.requestPoller()]) + doAssert received.recv().contains("test/model") + chatRuntime.restart() + chatClient.beginTick(1) + discard chatRuntime.run() + doAssert chatRuntime.getString(chatRuntime.getGlobalValue("answer$")) == + "hello" + + echo "Testing a mailbox DM becomes an LLM reply to the original sender" + block: + let + directory = createTempDir("polyworld-mailbox-llm-", "") + idle = directory / "idle.bas" + responder = currentSourcePath().parentDir / + "../examples/inference/mailbox_llm.bas" + settings = [ + ("AWS_ENDPOINT_URL_BEDROCK_RUNTIME", config.baseUrl), + ("COGAME_LLM", "on"), + ("COGAME_LLM_MODEL", "test/model") + ] + var saved: seq[(string, bool, string)] + for (name, value) in settings: + saved.add (name, existsEnv(name), getEnv(name)) + putEnv(name, value) + defer: + removeDir(directory) + for (name, existed, value) in saved: + if existed: + putEnv(name, value) + else: + delEnv(name) + writeFile(idle, "idle = 0") + let game = newGame(generateMap(54), 240, 10, false, ReplayData(), + drafting = false) + game.loadBots([ + BotGroup(path: idle, count: 1), + BotGroup(path: responder, count: 1), + BotGroup(path: idle, count: 8) + ]) + doAssert game.sendChat(0, 1, "What should we do?") == 1 + game.world.tick = 1 + game.runBotDecisions() + let vm = game.heroVms[1] + doAssert not vm.failed, vm.lastError + waitForRequests([vm.pollRequests]) + doAssert received.recv().contains("What should we do?") + game.world.tick = 2 + game.runBotDecisions() + doAssert not vm.failed, vm.lastError + let inbox = game.inboxes[0] + doAssert inbox.count == 1 + doAssert inbox.messages[inbox.first] == "hello" + doAssert inbox.pop() == 1 + doAssert game.inboxes[1].count == 0 + + echo "Testing raw streaming replies, HTTP failures, and response bounds" + client.beginTick(2) + let stream = client.ask("POST", "/v1/stream", "{\"stream\":true}") + doAssert client.response(id).contains("tool_calls") + client.settle(stream, 3) + doAssert client.poll(id) == -1 + doAssert client.status(id) == 0 + doAssert client.response(id) == "" + doAssert client.error(id).contains("expired") + doAssert client.text(stream) == "hello" + doAssert client.response(stream).endsWith("data: [DONE]\n\n") + discard received.recv() + client.beginTick(4) + let failed = client.ask("POST", "/v1/failure", "{}") + client.settle(failed, 5) + doAssert client.poll(failed) == -1 + doAssert client.status(failed) == 429 + doAssert client.response(failed).contains("spend limit") + discard received.recv() + client.beginTick(6) + let huge = client.ask("POST", "/v1/huge", "{}") + client.settle(huge, 7) + doAssert client.poll(huge) == -1 + doAssert client.response(huge).len <= MaxResponseBytes + discard received.recv() + + echo "Testing native Jev flattening and structured BASIC strings" + let scriptClient = newLlmClient(5, config) + var host = initHost() + scriptClient.addFunctions(host) + let program = compile(""" +if request = 0 then + oracleState("candidates[0].hp", 3) + oracleQuestion("guard", 0, "Guard the objective?") + oracleCriterion("guard", "true", "We are hurt.") + oracleCriterion("guard", "false", "We can safely advance.") + oracleQuestion("mode", 2, "Choose an objective.") + oracleCriterion("mode", "hold", "Hold.") + oracleCriterion("mode", "push", "Advance.") + oracleQuestion("risk", 1, "How risky?") + oracleCriterion("risk", "", "Safe.") + oracleCriterion("risk", "", "Risky.") + request = oracleAsk() +else + status = oraclePoll(request) + guard = oracleAnswer(request, "guard") + mode = oracleAnswer(request, "mode") + risk = oracleAnswer(request, "risk") + probability = oracleProbability(request, "mode", "push") +end if +""", host) + var runtime = initRuntime(program, host) + scriptClient.bindRuntime(runtime) + scriptClient.beginTick(0) + discard runtime.run() + waitForRequests([scriptClient.requestPoller()]) + let sent = received.recv() + doAssert sent.contains("\"candidates\":[{\"hp\":3}]") + doAssert sent.contains(DefaultOracleModel) + runtime.restart() + scriptClient.beginTick(1) + discard runtime.run() + doAssert runtime.getGlobal("status") == 3 + doAssert runtime.getGlobal("guard") == 800 + doAssert runtime.getGlobal("mode") == 0 + doAssert runtime.getGlobal("risk") == 1500 + doAssert runtime.getGlobal("probability") == 300 + let answered = runtime.getGlobal("request") + doAssert scriptClient.oracle.poll(answered) == 3 + runtime.restart() + runtime.setGlobal("request", 0) + scriptClient.beginTick(2) + discard runtime.run() + let replacement = runtime.getGlobal("request") + doAssert replacement > answered + doAssert scriptClient.oracle.poll(answered) == 3 + waitForRequests([scriptClient.requestPoller()]) + discard received.recv() + doAssert scriptClient.oracle.poll(replacement) == 3 + doAssert scriptClient.oracle.poll(answered) == -1 + doAssert scriptClient.oracle.answer(answered, "guard").value == -1 + doAssert scriptClient.response(answered) == "" + scriptClient.close() + doAssert scriptClient.oracle.poll(replacement) == -1 + doAssert scriptClient.response(replacement) == "" + + echo "Testing BASIC readback of the captured live OpenRouter JEV response" + block: + let scriptClient = newLlmClient(0, config) + var host = initHost() + scriptClient.addFunctions(host) + let program = compile(JevRequest, host) + var runtime = initRuntime(program, host) + scriptClient.bindRuntime(runtime) + scriptClient.beginTick(0) + discard runtime.run() + doAssert runtime.getGlobal("request") > 0 + waitForRequests([scriptClient.requestPoller()]) + discard received.recv() + runtime.restart() + scriptClient.beginTick(1) + discard runtime.run() + doAssert runtime.getGlobal("answers") == 2 + doAssert runtime.getGlobal("strategy") == 0 + doAssert runtime.getGlobal("lane") == 1 + doAssert runtime.getString(runtime.getGlobalValue("model$")) == + "typesafe/jev-1.13-20260917" + + echo "Testing GotA's JEV bot follows strategy and lane advice periodically" + block: + let settings = [ + ("AWS_ENDPOINT_URL_BEDROCK_RUNTIME", config.baseUrl), + ("COGAME_LLM", "on"), + ("COGAME_ORACLE", "on"), + ("COGAME_LLM_INTERVAL", "1"), + ("COGAME_LLM_TIMEOUT_MS", "1000"), + ("COGAME_ORACLE_MODEL", DefaultOracleModel) + ] + var saved: seq[(string, bool, string)] + for (name, value) in settings: + saved.add (name, existsEnv(name), getEnv(name)) + putEnv(name, value) + defer: + for (name, existed, value) in saved: + if existed: + putEnv(name, value) + else: + delEnv(name) + let + root = currentSourcePath().parentDir / "../examples/gods_of_the_arena" + game = newGame(generateMap(54), 240, 10, false, ReplayData(), + drafting = false) + game.recorder = initReplayRecorder(game.currentSetup(16000), game.map.preset) + game.loadBots([ + BotGroup(path: root / "players/jev.bas", count: 1), + BotGroup(path: root / "players/rusher.bas", count: 9) + ]) + let vm = game.heroVms[0] + for round in 0 ..< 7: + let tick = int32([1, 361, 4321, 4681, 14401, 14761, 15121][round]) + game.world.tick = tick + game.runBotDecisions() + doAssert not vm.failed, vm.lastError + doAssert vm.runtime.getGlobal("request") > 0 + waitForRequests([vm.pollRequests]) + let + sent = received.recv() + body = parseDocument(sent[sent.find('{') .. ^1]) + doAssert sent.contains("X-Coworld-Player-Slot: 0") + let expectedStage = + if round < 2: "early laning and farming" + elif round < 4: "mid game rotations and objectives" + else: "late game and finishing the enemy god" + doAssert body["state"]["stage"].getStr == expectedStage + doAssert body["state"]["situation"].getStr.contains("HP:") + doAssert body["state"]["notes"][0].getStr.contains("allied heroes=") + doAssert body["questions"]["strategy"]["criteria"].len == 5 + doAssert body["questions"]["lane"]["criteria"].len == 3 + game.world.tick = tick + 1 + game.runBotDecisions() + doAssert not vm.failed, vm.lastError + doAssert vm.runtime.getGlobal("request") == 0 + let + expectedStrategy = [1, 3, 2, 4, 0, 0, 0][round] + expectedLane = [2, 0, 1, 2, 0, 0, 0][round] + doAssert vm.runtime.getGlobal("strategy") == expectedStrategy + doAssert vm.runtime.getGlobal("chosenLane") == expectedLane + if round == 0: + doAssert vm.runtime.getGlobalValue("goalX").asFixed > + fixed(int32(mapTiles() * 8 div 10)) + elif round == 1: + doAssert vm.runtime.getGlobalValue("goalX").asFixed == + vm.runtime.getGlobalValue("homeX").asFixed + game.world.tick = tick + 7 + game.runBotDecisions() + doAssert not vm.failed, vm.lastError + doAssert not vm.pollRequests() + doAssert not received.tryRecv().dataAvailable + doAssert game.recorder.data.actions.len > 0 + game.world.heroes[0].hp = 1 + game.world.tick += 6 + game.runBotDecisions() + doAssert not vm.failed, vm.lastError + doAssert vm.runtime.getGlobal("retreating") == 1 + var lastOwn: ReplayAction + for action in game.recorder.data.actions: + if action.heroId == game.world.heroes[0].id: + lastOwn = action + doAssert lastOwn.kind == ActionWalkTo + doAssert lastOwn.first == vm.runtime.getGlobalValue("spawnX").asFixed.toInt + + echo "Testing the barrier submits all seats before waiting for any seat" + let + first = newLlmClient(0, config) + second = newLlmClient(1, config) + first.beginTick(10) + second.beginTick(10) + let + firstId = first.ask("POST", "/v1/pair", "{}") + secondId = second.ask("POST", "/v1/pair", "{}") + waitForRequests([first.requestPoller(), second.requestPoller()]) + doAssert first.poll(firstId) == 1 + doAssert second.poll(secondId) == 1 + discard received.recv() + discard received.recv() + + echo "Testing timeouts settle a barrier instead of advancing forever" + var short = config + short.timeoutMs = 40 + let slow = newLlmClient(2, short) + slow.beginTick(10) + let slowId = slow.ask("POST", "/v1/slow", "{}") + waitForRequests([slow.requestPoller()]) + doAssert slow.poll(slowId) == -1 + doAssert slow.error(slowId).contains("timed out") + discard received.recv() + doAssert slow.ready == -1 + doAssert slow.ask("POST", "/v1/chat/completions", "{}") == 0 + let deadline = getMonoTime() + initDuration(seconds = 3) + while slow.ready < 0: + doAssert getMonoTime() < deadline, "late response was not drained" + slow.beginTick(12) + sleep(1) + doAssert slow.poll(slowId) == -1 + doAssert slow.response(slowId) == "" + let retry = slow.chat("", "after timeout") + doAssert retry > slowId + slow.settle(retry, 12) + doAssert slow.text(retry) == "hello" + discard received.recv() + slow.close() + client.close() + + echo "Testing resets discard old replies and closed clients can reopen" + client.beginTick(20) + let abandoned = client.chat("", "before reset") + doAssert abandoned > 0 + client.beginTick(0) + doAssert not client.hasPending() + let resetDeadline = getMonoTime() + initDuration(seconds = 3) + while client.ready < 0: + doAssert getMonoTime() < resetDeadline, "reset request was not drained" + client.beginTick(0) + sleep(1) + doAssert client.poll(abandoned) == -1 + doAssert client.response(abandoned) == "" + discard received.recv() + let reopened = client.chat("", "after reset") + client.settle(reopened, 0) + doAssert client.text(reopened) == "hello" + discard received.recv() + client.close() + +echo "Testing invalid JSON and API paths fail at the library boundary" +block: + for body in ["{broken}", repeat('[', 65) & "0" & repeat(']', 65)]: + var rejected = false + try: + discard parseDocument(body) + except LlmError: + rejected = true + doAssert rejected + let client = newLlmClient(0, LlmConfig( + baseUrl: "http://127.0.0.1:1", sidecar: true, timeoutMs: 10 + )) + for path in ["/v1/../admin", "/v1/%2e%2e/admin", "/v1/a\x00b"]: + var rejected = false + try: + discard client.ask("POST", path, "{}") + except LlmError: + rejected = true + doAssert rejected + let scriptClient = newLlmClient(0, LlmConfig()) + var host = initHost() + scriptClient.addFunctions(host) + for name in ["chat", "jev", "request"]: + let source = readFile(currentSourcePath().parentDir / + "../examples/inference" / (name & ".bas")) + discard compile(source, host) + +echo "Testing wall-clock pacing and unlimited speed" +block: + let start = getMonoTime() + var pacer = initTickPacer(100) + for i in 0 ..< 4: + pacer.pace() + doAssert (getMonoTime() - start).inMilliseconds >= 35 + var unlimited = initTickPacer(0) + unlimited.pace() diff --git a/tests/tests.nim b/tests/tests.nim index 0ad97e4e..fd71f26d 100644 --- a/tests/tests.nim +++ b/tests/tests.nim @@ -68,8 +68,10 @@ import test_lvd_maps, test_lvd_replays, test_lvd_sim, + test_llms, test_mailboxes, test_chats, + test_llm_hosts, test_metrics, test_stats, test_nav,