From d46242f32231879f7a263387b865095fda519231 Mon Sep 17 00:00:00 2001 From: David Amos Date: Thu, 1 Oct 2026 15:06:31 -0500 Subject: [PATCH 1/5] Add draft retail delivery agent template What changed: - Define a pinned PyRel graph-to-prescriptive delivery package and synthetic sources. - Add safe source scripts, independent oracle, offline tests, evaluation materials, and gated agent handoff artifacts. Why: - Give reviewers a verifiable upstream foundation without inventing live Snowflake outputs or access. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- v1/retail-delivery-agent/README.md | 238 +++++++++++++ v1/retail-delivery-agent/data/demo_state.csv | 2 + v1/retail-delivery-agent/data/depots.csv | 3 + v1/retail-delivery-agent/data/orders.csv | 5 + v1/retail-delivery-agent/data/roads.csv | 6 + v1/retail-delivery-agent/data/stores.csv | 4 + v1/retail-delivery-agent/evaluation/arms.sql | 7 + .../evaluation/check_oracle.py | 218 ++++++++++++ .../evaluation/generate_cases.py | 88 +++++ .../evaluation/questions.json | 10 + .../evaluation/scorecard.csv | 1 + v1/retail-delivery-agent/model/__init__.py | 29 ++ v1/retail-delivery-agent/model/schema.py | 216 ++++++++++++ v1/retail-delivery-agent/model/sources.py | 63 ++++ v1/retail-delivery-agent/pyproject.toml | 18 + .../raiconfig.example.yaml | 35 ++ v1/retail-delivery-agent/runbook.md | 133 ++++++++ .../scripts/_snowflake.py | 42 +++ .../scripts/change_state.py | 64 ++++ .../scripts/check_results.py | 152 +++++++++ .../scripts/load_fixture.py | 85 +++++ .../scripts/teardown_sources.py | 47 +++ .../sql/00_create_sources.sql | 34 ++ .../sql/10_semantic_view.sql | 9 + .../sql/20_create_agent.sql | 5 + v1/retail-delivery-agent/sql/30_grants.sql | 9 + .../sql/40_close_north.sql | 10 + .../sql/41_reopen_north.sql | 10 + v1/retail-delivery-agent/sql/90_teardown.sql | 7 + v1/retail-delivery-agent/template-docs.yaml | 204 +++++++++++ .../tests/test_fixture.py | 323 ++++++++++++++++++ .../tests/test_live_chain.py | 85 +++++ 32 files changed, 2162 insertions(+) create mode 100644 v1/retail-delivery-agent/README.md create mode 100644 v1/retail-delivery-agent/data/demo_state.csv create mode 100644 v1/retail-delivery-agent/data/depots.csv create mode 100644 v1/retail-delivery-agent/data/orders.csv create mode 100644 v1/retail-delivery-agent/data/roads.csv create mode 100644 v1/retail-delivery-agent/data/stores.csv create mode 100644 v1/retail-delivery-agent/evaluation/arms.sql create mode 100644 v1/retail-delivery-agent/evaluation/check_oracle.py create mode 100644 v1/retail-delivery-agent/evaluation/generate_cases.py create mode 100644 v1/retail-delivery-agent/evaluation/questions.json create mode 100644 v1/retail-delivery-agent/evaluation/scorecard.csv create mode 100644 v1/retail-delivery-agent/model/__init__.py create mode 100644 v1/retail-delivery-agent/model/schema.py create mode 100644 v1/retail-delivery-agent/model/sources.py create mode 100644 v1/retail-delivery-agent/pyproject.toml create mode 100644 v1/retail-delivery-agent/raiconfig.example.yaml create mode 100644 v1/retail-delivery-agent/runbook.md create mode 100644 v1/retail-delivery-agent/scripts/_snowflake.py create mode 100644 v1/retail-delivery-agent/scripts/change_state.py create mode 100644 v1/retail-delivery-agent/scripts/check_results.py create mode 100644 v1/retail-delivery-agent/scripts/load_fixture.py create mode 100644 v1/retail-delivery-agent/scripts/teardown_sources.py create mode 100644 v1/retail-delivery-agent/sql/00_create_sources.sql create mode 100644 v1/retail-delivery-agent/sql/10_semantic_view.sql create mode 100644 v1/retail-delivery-agent/sql/20_create_agent.sql create mode 100644 v1/retail-delivery-agent/sql/30_grants.sql create mode 100644 v1/retail-delivery-agent/sql/40_close_north.sql create mode 100644 v1/retail-delivery-agent/sql/41_reopen_north.sql create mode 100644 v1/retail-delivery-agent/sql/90_teardown.sql create mode 100644 v1/retail-delivery-agent/template-docs.yaml create mode 100644 v1/retail-delivery-agent/tests/test_fixture.py create mode 100644 v1/retail-delivery-agent/tests/test_live_chain.py diff --git a/v1/retail-delivery-agent/README.md b/v1/retail-delivery-agent/README.md new file mode 100644 index 0000000..5677375 --- /dev/null +++ b/v1/retail-delivery-agent/README.md @@ -0,0 +1,238 @@ +--- +title: "Delivery planning with a model-backed agent" +description: "Explore directed reachability and whole-order delivery planning in a synthetic PyRel model; Snowflake agent deployment awaits account verification." +private: false +experience_level: advanced +industry: Supply Chain & Logistics +reasoning_types: + - Graph + - Prescriptive +tags: + - delivery + - optimization + - agent + - evaluation +--- + +## What this template is for + +Delivery planners need to distinguish a store with **no directed path from an +open depot** from a store whose orders are reachable but not selected for the +best stock-constrained plan. This synthetic example defines graph routes and a +maximum-*whole-order* decision problem in PyRel, then proposes a narrow +Snowflake Cortex Analyst/Cortex Agent/CoWork handoff over published outputs. +The agent reads selected, refreshed results, **not** arbitrary PyRel code. + +**Draft, not a public runnable integration.** The offline fixture/oracle, +declarative model package, and source scripts are available for review. The +combined graph-to-scheduled-solver deployment, output schema, semantic view, +agent, grants, and CoWork access have **not** been tested in an approved live +account. SQL files `10`, `20`, `30`, and `90` are intentionally non-runnable +gates, not a substitute for verified account-specific SQL. Do not publish the +agent or use its recommendations until every gate in [runbook.md](runbook.md) +is cleared. Model deployment is Public Preview; prescriptive access must be +approved separately. Do not treat this as a production decision workflow. + +## Who this is for + +- A Python/Snowflake model author runs offline checks and, after approval, + deploys and validates the PyRel model in a dedicated demo environment. +- An account administrator provisions least-privileged access and reviews the + later semantic view/agent setup. +- A delivery SME signs off on the business meanings and observed answers. +- An authorized planner eventually uses only CoWork once the live gate passes. + +## What you'll build + +Five source tables encode two depots, three stores, directed roads through a +hub, four orders, and one increasing source revision. `model/schema.py` owns +the declarations, weighted directed shortest paths, on-time and full-stock +candidate pairs, a named binary `Problem`, and derived status/decision/summary +objects. `model/sources.py` binds Snowflake tables at import without reading +CSV data or solving. `model/__init__.py` loads schema then bindings and exports +`model`, `Depot`, `Store`, and `Order` for other processes. + +The solver maximizes **number of complete orders**, not units: an order has at +most one depot; each depot's selected orders consume no more than its stock. +An on-time route alone does not promise stock or selection. The sample makes +unlimited separate dispatches and one SKU explicit simplifications; there is +no driver, vehicle, split shipment, or arbitrary what-if service. For larger +cases equally optimal assignment sets can exist even when the count is unique. +Only `OPTIMAL` permits a recommendation; timeout/failure means stop. + +## What's included + +- **Schema and sources**: [`model/`](model/) with all business rules in + `schema.py`, Snowflake bindings in `sources.py`, and ordered package exports. +- **Fixture**: five synthetic [`data/`](data/) CSV files; no customer data. +- **Source changes**: guarded `scripts/load_fixture.py` and + `scripts/change_state.py`; SQL `00`, `40`, and `41` document their scope. +- **Verification**: an independent Dijkstra/brute-force oracle in + `evaluation/check_oracle.py`, fail-closed snapshot comparator + `scripts/check_results.py`, and offline unit tests. +- **Account-gated handoff**: `raiconfig.example.yaml`, deliberately + non-executable semantic-view/agent/grant/teardown SQL drafts, and a + [runbook](runbook.md). `evaluation/` contains paired prompts, a blank + scorecard, bounded held-out cases, and a separate gold key. + +## Prerequisites + +- Python 3.12, a clean environment, and exact `relationalai==1.33.0` from + `pyproject.toml`. This version's package import and directory loader have + been exercised **offline**; the combined deployment has not. +- For the **later, gated** account trial: an isolated existing Snowflake + database and a saved Snowflake Connector connection profile authenticated + outside this folder; Native App and graph access; by-request prescriptive + solver enablement; approved refresh identity, secret object and external + access integration; Cortex Analyst/Agent/CoWork regional entitlement and + administrator-reviewed grants. Do not copy secret material into a config, + ZIP, example command, output log, or PR. +- A delivery SME must approve the reachability/stock/deadline definitions, + the output wording, and the observed open/closed plan. + +## Quickstart + +1. **Offline review (no account needed):** from this template directory, + install the pinned runtime and connector into a fresh environment. + + ```bash + python3.12 -m venv .venv + .venv/bin/python -m pip install -e '.[fixture]' + .venv/bin/python -m unittest discover -s tests -p 'test_*.py' + .venv/bin/python evaluation/check_oracle.py + ``` + + The oracle independently checks the proposed fixture: when North is open, + all stores are reachable and the maximum whole-order count is three + (North→A1, South→C1/C2); when North is closed, A is unreachable and the + count is two (South→C1/C2). North→B takes 20 minutes and South→B/C takes + 25 minutes; B remains reachable but unselected. **These are offline + expectations, not observed PyRel/Snowflake output.** The tests also check + zero/one/all closed depots, reversed roads, late deliveries, inventory + competition, ties, failed solver status, and stale/mixed revisions. + +2. **Stop here unless the account and feature owner have approved a live + spike.** Have an admin inspect the exact source and output schema names and + privilege boundaries in [runbook.md](runbook.md). Complete a local + `raiconfig.yaml` from `raiconfig.example.yaml`; keep that local file and all + connection credentials **outside the downloadable/committed files**. + `model.path: model/` loads the *package directory*; import and CLI + processes each construct their own model, not a shared in-memory object. + +3. **Account-gated source setup:** with an existing Connector profile and + only after confirming the database is dedicated, load a *new* source + schema (the loader refuses any existing schema): + + ```bash + .venv/bin/python -m scripts.load_fixture \ + --connection YOUR_SAVED_PROFILE --database YOUR_DEMO_DATABASE + ``` + + The loader checks IDs, road weights, source keys and counts, writes only + the five tables under `DELIVERY_SOURCES`, and marks schema ownership. + Source DDL may commit before the data transaction; on failure, inspect + the partially created marked schema rather than blindly rerunning. No CSV + is loaded on `import model`. + +4. **Live gate, not a copy/paste deploy recipe yet:** in the approved + environment test directory loading and graph→prescriptive dependencies; + verify the actual output names/keys and the latest `REFRESH_STATUS()` after + `rai models deploy --wait` on `1.33.0`. The example config uses one manual + schedule (`interval_s: 0`, deploy-time initial run, `type: table`); the + refresh task is not an instant freshness guarantee. A periodic schedule + is a later adaptation. Do not run a comparison or agent on stale, + non-optimal, missing, or mixed-revision outputs. + +5. **Only after a live schema/permissions review:** replace the gated SQL + drafts with a tested complete `CREATE SEMANTIC VIEW`, `SEMANTIC_VIEW(...)` + checks, Cortex Analyst tool/agent definition, and scoped grants. Verify + SQL rows and tool traces for both prompts in + [`evaluation/questions.json`](evaluation/questions.json), then test as a + least-privileged CoWork user. The consumer needs a default role and + warehouse plus access to the view **and its underlying published tables**; + view access alone does not suffice. + +6. **Later source-change trial:** while all other writes are paused, use + `.venv/bin/python -m scripts.change_state close --connection + YOUR_SAVED_PROFILE --database YOUR_DEMO_DATABASE`, trigger the *same* + manual refresh task, then verify a **later** successful run and a newer + revision on **both** public result sets. Re-ask the **identical** two + prompts. Reopening North uses `reopen` and increments the revision again, + never resets it. `scripts/check_results.py` accepts an actual, + account-exported JSON snapshot normalized to the documented columns; + its snapshot exporter and the exact column mapping remain a **live + blocker**. It refuses failed solver status and inconsistent results. + +## Sample data + +| Source | Representative synthetic facts | +| --- | --- | +| Depots and roads | North (stock 2)→A 10, →B 20; South (stock 4)→Hub 10; Hub→B/C 15. All edges are directed. | +| Orders | A1 needs 2, B1 needs 4, C1/C2 need 2 each; every deadline is 60 minutes from dispatch. | +| State | Revision starts at 1 and advances when a guarded source change commits. | + +Closing North removes both its routes and its stock. Geographical +reachability ignores stock; eligibility for a full on-time order does not. + +## Model overview + +`Node`, `Depot`, `Store`, `Road`, and `Order` are base facts from dedicated +tables. The directed weighted `Graph` finds shortest travel times from +currently open depots. The configured `Problem` chooses whole orders from +eligible open-depot routes within stock. `StoreRouteStatus` includes every +store, including unreachable stores; `OrderDecision` includes every order +only after an optimal solve. Unselected decisions intentionally lack a chosen +depot and arrival; reachable but late and insufficient-stock states are +distinct. `PlanSummary` carries solver status and revision; its count should +be trusted only with an optimal solve. + +## How it works + +```text +source tables → model/sources.py → model/schema.py graph/solver + → deployed outputs [LIVE GATE] + → reviewed semantic view [LIVE GATE] + → Cortex Analyst in Agent → CoWork [LIVE GATE] +``` + +`raiconfig.example.yaml` names a separate operational meta schema; never +attach that schema to Analyst. The published output schema is not itself +a Snowflake semantic view or an agent. Even after one refresh succeeds, +users querying during a later refresh can see eventually consistent data; +this example does not enforce a globally atomic freshness barrier. + +## Troubleshooting and cleanup + +- If import fails, use the pinned environment and run + `.venv/bin/python -c 'from model import model, Depot, Store, Order'`; + this imports definitions only, not a live solver result. +- If deployment, configured solve, or refresh fails, **stop**. Do not label + feasible/timed-out assignments optimal; inspect the verified refresh + status/solver diagnostics and retry only after the owner approves. +- If an Analyst query is empty or denied, check view and underlying-table + grants separately. An inaccessible-tool policy does not cover every + query-time error; do not let the agent answer from guesses. +- The default agent answers the *current published world*, not a hypothetical + depot closure before a new source change and completed refresh. +- Before cleanup, verify ownership. Have an admin remove only the reviewed + agent/view; use `rai models teardown` for managed deployment objects. + `.venv/bin/python -m scripts.teardown_sources --connection + YOUR_SAVED_PROFILE --database YOUR_DEMO_DATABASE --confirm + YOUR_DEMO_DATABASE.DELIVERY_SOURCES` removes only the specifically marked + five-table source schema with no unexpected objects; never drop shared data. + +## Evaluation + +Three matched arms are specified in [`evaluation/arms.sql`](evaluation/arms.sql) +as a **non-runnable protocol**, not three active agents: (A) minimally +described raw-data semantic view, (B) the same raw data plus reviewed +definitions, and (C) the same definitions plus derived outputs. All arms +need equally scoped evaluation data access, the same prompts and settings, +and a separately held gold key. Generate bounded additional cases with +`python -m evaluation.generate_cases --cases-output +--gold-output ` and do not give the gold file +to any agent. Repeat each question per state/arm, record tool/SQL evidence, +correctness, calibrated abstention, latency/cost, and fixed/broken/net per +question in the blank [`scorecard.csv`](evaluation/scorecard.csv). The tiny +fixture is an inspectable demo, **not** an accuracy benchmark. Cases above +the brute-force bound are ungraded for optimality. diff --git a/v1/retail-delivery-agent/data/demo_state.csv b/v1/retail-delivery-agent/data/demo_state.csv new file mode 100644 index 0000000..67bc220 --- /dev/null +++ b/v1/retail-delivery-agent/data/demo_state.csv @@ -0,0 +1,2 @@ +revision +1 diff --git a/v1/retail-delivery-agent/data/depots.csv b/v1/retail-delivery-agent/data/depots.csv new file mode 100644 index 0000000..049f9c7 --- /dev/null +++ b/v1/retail-delivery-agent/data/depots.csv @@ -0,0 +1,3 @@ +depot_id,node_id,is_open,stock_units +North,NORTH,true,2 +South,SOUTH,true,4 diff --git a/v1/retail-delivery-agent/data/orders.csv b/v1/retail-delivery-agent/data/orders.csv new file mode 100644 index 0000000..e498996 --- /dev/null +++ b/v1/retail-delivery-agent/data/orders.csv @@ -0,0 +1,5 @@ +order_id,store_id,units,deadline_minutes +A1,A,2,60 +B1,B,4,60 +C1,C,2,60 +C2,C,2,60 diff --git a/v1/retail-delivery-agent/data/roads.csv b/v1/retail-delivery-agent/data/roads.csv new file mode 100644 index 0000000..14e8871 --- /dev/null +++ b/v1/retail-delivery-agent/data/roads.csv @@ -0,0 +1,6 @@ +from_node_id,to_node_id,travel_minutes +NORTH,A,10 +NORTH,B,20 +SOUTH,HUB,10 +HUB,B,15 +HUB,C,15 diff --git a/v1/retail-delivery-agent/data/stores.csv b/v1/retail-delivery-agent/data/stores.csv new file mode 100644 index 0000000..8c5322c --- /dev/null +++ b/v1/retail-delivery-agent/data/stores.csv @@ -0,0 +1,4 @@ +store_id,node_id +A,A +B,B +C,C diff --git a/v1/retail-delivery-agent/evaluation/arms.sql b/v1/retail-delivery-agent/evaluation/arms.sql new file mode 100644 index 0000000..3a06e47 --- /dev/null +++ b/v1/retail-delivery-agent/evaluation/arms.sql @@ -0,0 +1,7 @@ +-- DRAFT: not executable. Requires an approved Snowflake account, reviewed +-- semantic-view DDL, actual published table names, and equal grants. +-- A: minimal descriptions over raw depot/road/stock/order sources. +-- B: same raw sources with SME-reviewed joins/terms/constraint descriptions. +-- C: same descriptions plus *verified* model-derived statuses/decisions. +-- Freeze questions.json, fixture revision, model/settings, and access for all +-- arms. Keep gold answer keys out of views and agent instructions. diff --git a/v1/retail-delivery-agent/evaluation/check_oracle.py b/v1/retail-delivery-agent/evaluation/check_oracle.py new file mode 100644 index 0000000..7dcd253 --- /dev/null +++ b/v1/retail-delivery-agent/evaluation/check_oracle.py @@ -0,0 +1,218 @@ +"""Independent, bounded fixture oracle; does not import PyRel or model code.""" + +from __future__ import annotations + +import csv +import heapq +import itertools +from dataclasses import dataclass +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +MAX_ORDERS = 10 +MAX_DEPOTS = 4 +MAX_ASSIGNMENTS = 100_000 + + +@dataclass(frozen=True) +class Expected: + revision: int + routes: dict[tuple[str, str], float] + store_states: dict[str, str] + order_states: dict[str, str] + optimal_count: int + optimal_plans: frozenset[tuple[tuple[str, str], ...]] + + +def read_fixture(directory: Path = ROOT / "data") -> dict[str, list[dict[str, str]]]: + """Read only the five expected local CSVs; reject empty or malformed files.""" + sources = {} + for name, required in { + "depots": {"depot_id", "node_id", "is_open", "stock_units"}, + "stores": {"store_id", "node_id"}, + "roads": {"from_node_id", "to_node_id", "travel_minutes"}, + "orders": {"order_id", "store_id", "units", "deadline_minutes"}, + "demo_state": {"revision"}, + }.items(): + with (directory / f"{name}.csv").open(newline="", encoding="utf-8") as stream: + reader = csv.DictReader(stream) + if reader.fieldnames is None or set(reader.fieldnames) != required: + raise ValueError(f"{name}: expected columns {sorted(required)}") + rows = list(reader) + if not rows or any( + value is None or value == "" for row in rows for value in row.values() + ): + raise ValueError(f"{name}: empty or incomplete rows") + sources[name] = rows + validate_sources(sources) + return sources + + +def validate_sources(sources: dict[str, list[dict[str, str]]]) -> None: + """Validate identifiers, foreign keys and nonnegative inputs before loading.""" + required = {"depots", "stores", "roads", "orders", "demo_state"} + if set(sources) != required: + raise ValueError(f"Expected exactly five source tables: {sorted(required)}") + if len(sources["demo_state"]) != 1 or int(sources["demo_state"][0]["revision"]) < 1: + raise ValueError("DEMO_STATE must have exactly one positive revision") + for name, key in (("depots", "depot_id"), ("stores", "store_id"), ("orders", "order_id")): + values = [row[key] for row in sources[name]] + if len(set(values)) != len(values): + raise ValueError(f"{name}: duplicate {key}") + depot_nodes = [row["node_id"] for row in sources["depots"]] + store_nodes = [row["node_id"] for row in sources["stores"]] + if len(set(depot_nodes + store_nodes)) != len(depot_nodes + store_nodes): + raise ValueError("Depot and store node IDs must be unique") + stores = {row["store_id"] for row in sources["stores"]} + roads = set() + for row in sources["roads"]: + pair = (row["from_node_id"], row["to_node_id"]) + if pair in roads or float(row["travel_minutes"]) < 0: + raise ValueError("Roads must be unique and have nonnegative weights") + roads.add(pair) + for row in sources["depots"]: + if row["is_open"].lower() not in {"true", "false"} or int(row["stock_units"]) < 0: + raise ValueError("Depots need boolean availability and nonnegative stock") + for row in sources["orders"]: + if ( + row["store_id"] not in stores + or int(row["units"]) <= 0 + or float(row["deadline_minutes"]) < 0 + ): + raise ValueError("Orders need an existing store, positive units, and a deadline") + + +def shortest_paths(start: str, roads: list[dict[str, str]]) -> dict[str, float]: + """Dijkstra over directed, nonnegative roads; source code is independent of PyRel.""" + edges: dict[str, list[tuple[float, str]]] = {} + for road in roads: + edges.setdefault(road["from_node_id"], []).append( + (float(road["travel_minutes"]), road["to_node_id"]) + ) + distances = {start: 0.0} + pending = [(0.0, start)] + while pending: + minutes, node = heapq.heappop(pending) + if minutes > distances[node]: + continue + for travel, neighbor in edges.get(node, []): + candidate = minutes + travel + if candidate < distances.get(neighbor, float("inf")): + distances[neighbor] = candidate + heapq.heappush(pending, (candidate, neighbor)) + return distances + + +def evaluate(sources: dict[str, list[dict[str, str]]]) -> Expected: + """Enumerate all whole-order assignments; reject cases beyond the declared bound.""" + validate_sources(sources) + depots = [row for row in sources["depots"] if row["is_open"].lower() == "true"] + stores = {row["store_id"]: row["node_id"] for row in sources["stores"]} + orders = sorted(sources["orders"], key=lambda row: row["order_id"]) + if len(orders) > MAX_ORDERS or len(depots) > MAX_DEPOTS: + raise ValueError("Optimal-count grading is unbounded for this case; mark it ungraded") + distances = { + depot["depot_id"]: shortest_paths(depot["node_id"], sources["roads"]) + for depot in depots + } + routes = { + (depot["depot_id"], store_id): distances[depot["depot_id"]][node] + for depot in depots + for store_id, node in stores.items() + if node in distances[depot["depot_id"]] + } + store_states = { + store_id: ( + "REACHABLE" if any((depot["depot_id"], store_id) in routes for depot in depots) + else "NO_ROUTE" + ) + for store_id in stores + } + candidates: dict[str, list[str]] = {} + order_states = {} + for order in orders: + oid, sid = order["order_id"], order["store_id"] + reachable = [depot for depot in depots if (depot["depot_id"], sid) in routes] + on_time = [ + depot for depot in reachable + if routes[depot["depot_id"], sid] <= float(order["deadline_minutes"]) + ] + candidates[oid] = [ + depot["depot_id"] for depot in on_time + if int(depot["stock_units"]) >= int(order["units"]) + ] + order_states[oid] = ( + "NO_ROUTE" if not reachable else + "LATE_ONLY" if not on_time else + "INSUFFICIENT_STOCK" if not candidates[oid] else + "REACHABLE_NOT_SELECTED" + ) + combinations = 1 + for order in orders: + combinations *= 1 + len(candidates[order["order_id"]]) + if combinations > MAX_ASSIGNMENTS: + raise ValueError("Oracle search exceeds 100,000 plans; mark optimal-count grading ungraded") + stock = {depot["depot_id"]: int(depot["stock_units"]) for depot in depots} + best = -1 + plans: set[tuple[tuple[str, str], ...]] = set() + for choice in itertools.product(*([None, *candidates[row["order_id"]]] for row in orders)): + assignments = tuple( + (depot_id, order["order_id"]) + for order, depot_id in zip(orders, choice, strict=True) + if depot_id is not None + ) + assignments = tuple(sorted(assignments)) + used = { + depot_id: sum( + int(order["units"]) + for order, chosen in zip(orders, choice, strict=True) + if chosen == depot_id + ) + for depot_id in stock + } + if any(used[depot_id] > available for depot_id, available in stock.items()): + continue + if len(assignments) > best: + best, plans = len(assignments), {assignments} + elif len(assignments) == best: + plans.add(assignments) + if best < 0: + raise AssertionError("The empty assignment must always be feasible") + for order in orders: + order_id = order["order_id"] + if all(any(oid == order_id for _, oid in plan) for plan in plans): + order_states[order_id] = "SELECTED" + elif any(any(oid == order_id for _, oid in plan) for plan in plans): + order_states[order_id] = "OPTIONALLY_SELECTED" + return Expected( + revision=int(sources["demo_state"][0]["revision"]), + routes=routes, + store_states=store_states, + order_states=order_states, + optimal_count=best, + optimal_plans=frozenset(plans), + ) + + +if __name__ == "__main__": + baseline = read_fixture() + for name, case in ( + ("open", baseline), + ( + "closed", + { + **baseline, + "depots": [ + {**depot, "is_open": "false"} if depot["depot_id"] == "North" else depot + for depot in baseline["depots"] + ], + "demo_state": [{"revision": "2"}], + }, + ), + ): + expected = evaluate(case) + print( + f"{name}: revision={expected.revision}; " + f"unreachable={sorted(k for k, v in expected.store_states.items() if v == 'NO_ROUTE')}; " + f"maximum complete orders={expected.optimal_count}" + ) diff --git a/v1/retail-delivery-agent/evaluation/generate_cases.py b/v1/retail-delivery-agent/evaluation/generate_cases.py new file mode 100644 index 0000000..81f42e8 --- /dev/null +++ b/v1/retail-delivery-agent/evaluation/generate_cases.py @@ -0,0 +1,88 @@ +"""Generate bounded held-out synthetic cases; no agent input includes the gold key.""" + +from __future__ import annotations + +import argparse +import json +import random +from pathlib import Path + +from evaluation.check_oracle import evaluate + + +def generate(seed: int, count: int = 12) -> tuple[list[dict], list[dict]]: + if not 1 <= count <= 100: + raise ValueError("Generate 1..100 bounded cases") + rng = random.Random(seed) + cases, gold = [], [] + for index in range(count): + depots = [ + { + "depot_id": f"D{depot}", + "node_id": f"D{depot}", + "is_open": "true" if depot == 0 or rng.randrange(4) else "false", + "stock_units": str(rng.randrange(1, 7)), + } + for depot in range(3) + ] + stores = [{"store_id": f"S{i}", "node_id": f"S{i}"} for i in range(4)] + roads = [] + for source in ["D0", "D1", "D2", "HUB"]: + for target in ["HUB", "S0", "S1", "S2", "S3"]: + if source != target and rng.randrange(3): + roads.append( + { + "from_node_id": source, "to_node_id": target, + "travel_minutes": str(rng.randrange(2, 31)), + } + ) + orders = [ + { + "order_id": f"O{i}", + "store_id": f"S{rng.randrange(4)}", + "units": str(rng.randrange(1, 5)), + "deadline_minutes": str(rng.randrange(5, 45)), + } + for i in range(6) + ] + sources = { + "depots": depots, "stores": stores, "roads": roads, + "orders": orders, "demo_state": [{"revision": "1"}], + } + expected = evaluate(sources) + case_id = f"synthetic-{seed}-{index:03d}" + cases.append({"case_id": case_id, "sources": sources}) + gold.append( + { + "case_id": case_id, + "store_states": expected.store_states, + "optimal_count": expected.optimal_count, + "optimal_plans": [ + [list(pair) for pair in plan] for plan in sorted(expected.optimal_plans) + ], + } + ) + return cases, gold + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--seed", type=int, default=27631) + parser.add_argument("--count", type=int, default=12) + parser.add_argument("--cases-output", type=Path, required=True) + parser.add_argument("--gold-output", type=Path, required=True) + args = parser.parse_args() + if ( + args.cases_output == args.gold_output + or args.cases_output.exists() + or args.gold_output.exists() + ): + raise FileExistsError("Use distinct unused paths; refusing to overwrite case data or gold") + cases, gold = generate(args.seed, args.count) + args.cases_output.write_text(json.dumps(cases, indent=2) + "\n") + args.gold_output.write_text(json.dumps(gold, indent=2) + "\n") + print(f"Wrote {args.count} cases and a separate gold key. Keep gold away from agents.") + + +if __name__ == "__main__": + main() diff --git a/v1/retail-delivery-agent/evaluation/questions.json b/v1/retail-delivery-agent/evaluation/questions.json new file mode 100644 index 0000000..55afd17 --- /dev/null +++ b/v1/retail-delivery-agent/evaluation/questions.json @@ -0,0 +1,10 @@ +[ + { + "id": "unreachable-stores", + "prompt": "Which stores cannot get a delivery from any depot that is open right now?" + }, + { + "id": "max-complete-orders", + "prompt": "What should we send from the depots still open to complete the most orders on time?" + } +] diff --git a/v1/retail-delivery-agent/evaluation/scorecard.csv b/v1/retail-delivery-agent/evaluation/scorecard.csv new file mode 100644 index 0000000..ebc558e --- /dev/null +++ b/v1/retail-delivery-agent/evaluation/scorecard.csv @@ -0,0 +1 @@ +case_id,source_revision,state,arm,question_id,run,answer_correct,tool_evidence,abstention_appropriate,source_revision_checked,solver_optimal_checked,latency_ms,cost,sql_trace_reference,error_notes diff --git a/v1/retail-delivery-agent/model/__init__.py b/v1/retail-delivery-agent/model/__init__.py new file mode 100644 index 0000000..f35567a --- /dev/null +++ b/v1/retail-delivery-agent/model/__init__.py @@ -0,0 +1,29 @@ +"""Import the schema first, then register source bindings on the same model.""" + +from .schema import ( + DemoState, + Depot, + Order, + OrderDecision, + PlanSummary, + Road, + Store, + StoreRouteStatus, + model, +) + +# isort: split +from . import sources as sources + +__all__ = [ + "DemoState", + "Depot", + "Order", + "OrderDecision", + "PlanSummary", + "Road", + "Store", + "StoreRouteStatus", + "model", + "sources", +] diff --git a/v1/retail-delivery-agent/model/schema.py b/v1/retail-delivery-agent/model/schema.py new file mode 100644 index 0000000..5b14d90 --- /dev/null +++ b/v1/retail-delivery-agent/model/schema.py @@ -0,0 +1,216 @@ +"""Delivery ontology, directed routes, and the maximum whole-order plan. + +This module declares the model without connecting to Snowflake or solving it. +The public output contracts still require a live deployment/refresh test. +""" + +from relationalai.semantics import Boolean, Float, Integer, Model, String, not_, sum +from relationalai.semantics.reasoners.graph import Graph +from relationalai.semantics.reasoners.prescriptive import Problem +from relationalai.semantics.std.aggregates import count + +model = Model("retail_delivery_agent") + +Node = model.Concept("Node", identify_by={"id": String}) + +Depot = model.Concept("Depot", identify_by={"id": String}) +Depot.node = model.Relationship(f"{Depot} is at {Node:node}") +Depot.is_open = model.Property(f"{Depot} is open {Boolean:is_open}") +Depot.stock_units = model.Property(f"{Depot} has {Integer:stock_units} units") + +Store = model.Concept("Store", identify_by={"id": String}) +Store.node = model.Relationship(f"{Store} is at {Node:node}") + +Road = model.Concept("Road", identify_by={"from_id": String, "to_id": String}) +Road.source = model.Relationship(f"{Road} starts at {Node:source}") +Road.destination = model.Relationship(f"{Road} ends at {Node:destination}") +Road.travel_minutes = model.Property(f"{Road} takes {Float:travel_minutes} minutes") + +Order = model.Concept("Order", identify_by={"id": String}) +Order.store = model.Relationship(f"{Order} goes to {Store:store}") +Order.units = model.Property(f"{Order} needs {Integer:units} units") +Order.deadline_minutes = model.Property(f"{Order} is due in {Float:deadline_minutes} minutes") + +DemoState = model.Concept("DemoState", identify_by={"id": String}) +DemoState.revision = model.Property(f"{DemoState} has {Integer:revision}") + +graph = Graph(model, directed=True, weighted=True, node_concept=Node) +model.define( + graph.Edge.new( + src=Road.source, + dst=Road.destination, + weight=Road.travel_minutes, + ) +) +distance = graph.distance(full=True) + +DepotStoreTime = model.Relationship( + f"{Depot:depot} can reach {Store:store} in {Float:minutes} minutes" +) +origin, destination = Node.ref(), Node.ref() +depot, store = Depot.ref(), Store.ref() +model.where( + depot.node(origin), + store.node(destination), + depot.is_open == True, + distance(origin, destination, Float), +).define(DepotStoreTime(depot, store, Float)) + +ReachableStore = model.Relationship(f"{Store} is reachable from an open depot") +model.where(DepotStoreTime(Depot, Store, Float)).define(ReachableStore(Store)) + +OnTimeRoute = model.Relationship( + f"{Depot:depot} can reach {Order:order} on time in {Float:minutes} minutes" +) +depot, order = Depot.ref(), Order.ref() +model.where( + DepotStoreTime(depot, order.store, Float), + Float <= order.deadline_minutes, +).define(OnTimeRoute(depot, order, Float)) + +EligiblePair = model.Relationship(f"{Depot:depot} can complete {Order:order} in full") +depot, order = Depot.ref(), Order.ref() +model.where( + OnTimeRoute(depot, order, Float), + depot.stock_units >= order.units, +).define(EligiblePair(depot, order)) + +Assignment = model.Relationship( + f"{Depot:depot} sends {Order:order} with {Integer:assigned}" +) +problem = Problem(model, Integer, name="delivery_plan") +depot, order, chosen = Depot.ref(), Order.ref(), Integer.ref() +problem.solve_for( + Assignment(depot, order, chosen), + where=[EligiblePair(depot, order)], + type="bin", + lower=0, + upper=1, + name=["send", depot.id, order.id], +) +depot, order, chosen = Depot.ref(), Order.ref(), Integer.ref() +problem.satisfy( + model.where(Assignment(depot, order, chosen)).require( + sum(chosen).per(order) <= 1 + ) +) +depot, order, chosen = Depot.ref(), Order.ref(), Integer.ref() +problem.satisfy( + model.where(Assignment(depot, order, chosen)).require( + sum(order.units * chosen).per(depot) <= depot.stock_units + ) +) +depot, order, chosen = Depot.ref(), Order.ref(), Integer.ref() +problem.maximize( + sum(chosen).where(Assignment(depot, order, chosen)), + name="completed_whole_orders", +) +problem.configure("highs", time_limit_sec=120) + +Selected = model.Relationship(f"{Depot:depot} is selected for {Order:order}") +model.where( + problem.termination_status()("OPTIMAL"), + Assignment(Depot, Order, 1), +).define(Selected(Depot, Order)) +HasSelectedOrder = model.Relationship(f"{Order} has a selected depot") +model.where(Selected(Depot, Order)).define(HasSelectedOrder(Order)) +HasOnTimeRoute = model.Relationship(f"{Order} has an on-time route") +model.where(OnTimeRoute(Depot, Order, Float)).define(HasOnTimeRoute(Order)) +HasEligiblePair = model.Relationship(f"{Order} has a full-stock on-time depot") +model.where(EligiblePair(Depot, Order)).define(HasEligiblePair(Order)) + +StoreRouteStatus = model.Concept("StoreRouteStatus", identify_by={"store_id": String}) +StoreRouteStatus.state = model.Property(f"{StoreRouteStatus} has {String:state}") +StoreRouteStatus.revision = model.Property( + f"{StoreRouteStatus} has {Integer:revision}" +) +model.define(StoreRouteStatus.new(store_id=Store.id)) +model.where(ReachableStore(Store)).define( + StoreRouteStatus.state(Store.id, "REACHABLE") +) +model.where(Store, not_(ReachableStore(Store))).define( + StoreRouteStatus.state(Store.id, "NO_ROUTE") +) +model.where(StoreRouteStatus, DemoState.id == "current").define( + StoreRouteStatus.revision(StoreRouteStatus, DemoState.revision) +) + +OrderDecision = model.Concept("OrderDecision", identify_by={"order_id": String}) +OrderDecision.store_id = model.Property(f"{OrderDecision} goes to {String:store_id}") +OrderDecision.units = model.Property(f"{OrderDecision} needs {Integer:units} units") +OrderDecision.deadline_minutes = model.Property( + f"{OrderDecision} is due in {Float:deadline_minutes} minutes" +) +OrderDecision.state = model.Property(f"{OrderDecision} has {String:state}") +OrderDecision.depot_id = model.Property(f"{OrderDecision} uses {String:depot_id}") +OrderDecision.arrival_minutes = model.Property( + f"{OrderDecision} arrives in {Float:arrival_minutes} minutes" +) +OrderDecision.revision = model.Property(f"{OrderDecision} has {Integer:revision}") + +model.where(Order, problem.termination_status()("OPTIMAL")).define( + decision := OrderDecision.new(order_id=Order.id), + decision.store_id(Order.store.id), + decision.units(Order.units), + decision.deadline_minutes(Order.deadline_minutes), +) +depot, order = Depot.ref(), Order.ref() +model.where(Selected(depot, order)).define( + OrderDecision.state(order.id, "SELECTED"), + OrderDecision.depot_id(order.id, depot.id), +) +model.where(Selected(depot, order), OnTimeRoute(depot, order, Float)).define( + OrderDecision.arrival_minutes(order.id, Float) +) +order = Order.ref() +model.where( + order, + problem.termination_status()("OPTIMAL"), + not_(ReachableStore(order.store)), +).define(OrderDecision.state(order.id, "NO_ROUTE")) +order = Order.ref() +model.where( + order, + problem.termination_status()("OPTIMAL"), + ReachableStore(order.store), + not_(HasOnTimeRoute(order)), +).define(OrderDecision.state(order.id, "LATE_ONLY")) +order = Order.ref() +model.where( + order, + problem.termination_status()("OPTIMAL"), + HasOnTimeRoute(order), + not_(HasEligiblePair(order)), +).define(OrderDecision.state(order.id, "INSUFFICIENT_STOCK")) +order = Order.ref() +model.where( + order, + problem.termination_status()("OPTIMAL"), + HasEligiblePair(order), + not_(HasSelectedOrder(order)), +).define(OrderDecision.state(order.id, "REACHABLE_NOT_SELECTED")) +model.where( + OrderDecision, DemoState.id == "current", problem.termination_status()("OPTIMAL") +).define( + OrderDecision.revision(OrderDecision, DemoState.revision) +) + +PlanSummary = model.Concept("PlanSummary", identify_by={"id": String}) +PlanSummary.revision = model.Property(f"{PlanSummary} has {Integer:revision}") +PlanSummary.solver_status = model.Property(f"{PlanSummary} has {String:solver_status}") +PlanSummary.completed_orders = model.Property( + f"{PlanSummary} has {Integer:completed_orders} completed orders" +) +model.define(PlanSummary.new(id="current")) +status = String.ref() +model.where(problem.termination_status()(status)).define( + PlanSummary.solver_status("current", status) +) +model.where(DemoState.id == "current").define( + PlanSummary.revision("current", DemoState.revision) +) +model.where(problem.termination_status()("OPTIMAL")).define( + PlanSummary.completed_orders( + "current", count(Order).where(Selected(Depot, Order)) | 0 + ) +) diff --git a/v1/retail-delivery-agent/model/sources.py b/v1/retail-delivery-agent/model/sources.py new file mode 100644 index 0000000..dc22e1e --- /dev/null +++ b/v1/retail-delivery-agent/model/sources.py @@ -0,0 +1,63 @@ +"""Bind Snowflake source tables to schema facts; never load local CSV on import.""" + +from relationalai.semantics import Boolean, Float, Integer, String + +from .schema import DemoState, Depot, Node, Order, Road, Store, model + +depots = model.Table( + "DEPOTS", + schema={ + "DEPOT_ID": String, + "NODE_ID": String, + "IS_OPEN": Boolean, + "STOCK_UNITS": Integer, + }, +) +stores = model.Table("STORES", schema={"STORE_ID": String, "NODE_ID": String}) +roads = model.Table( + "ROADS", + schema={"FROM_NODE_ID": String, "TO_NODE_ID": String, "TRAVEL_MINUTES": Float}, +) +orders = model.Table( + "ORDERS", + schema={ + "ORDER_ID": String, + "STORE_ID": String, + "UNITS": Integer, + "DEADLINE_MINUTES": Float, + }, +) +state = model.Table("DEMO_STATE", schema={"REVISION": Integer}) + +model.define( + Node.new(id=depots.NODE_ID), + Node.new(id=stores.NODE_ID), + Node.new(id=roads.FROM_NODE_ID), + Node.new(id=roads.TO_NODE_ID), +) +model.define( + depot := Depot.new(id=depots.DEPOT_ID), + depot.node(Node.lookup(id=depots.NODE_ID)), + depot.is_open(depots.IS_OPEN), + depot.stock_units(depots.STOCK_UNITS), +) +model.define( + store := Store.new(id=stores.STORE_ID), + store.node(Node.lookup(id=stores.NODE_ID)), +) +model.define( + road := Road.new(from_id=roads.FROM_NODE_ID, to_id=roads.TO_NODE_ID), + road.source(Node.lookup(id=roads.FROM_NODE_ID)), + road.destination(Node.lookup(id=roads.TO_NODE_ID)), + road.travel_minutes(roads.TRAVEL_MINUTES), +) +model.define( + order := Order.new(id=orders.ORDER_ID), + order.store(Store.lookup(id=orders.STORE_ID)), + order.units(orders.UNITS), + order.deadline_minutes(orders.DEADLINE_MINUTES), +) +model.define( + demo := DemoState.new(id="current"), + demo.revision(state.REVISION), +) diff --git a/v1/retail-delivery-agent/pyproject.toml b/v1/retail-delivery-agent/pyproject.toml new file mode 100644 index 0000000..c452244 --- /dev/null +++ b/v1/retail-delivery-agent/pyproject.toml @@ -0,0 +1,18 @@ +[build-system] +requires = ["setuptools>=64", "wheel"] +build-backend = "setuptools.build_meta" + +[project] +name = "rai-retail-delivery-agent" +version = "0.1.0" +description = "Synthetic delivery planning with PyRel and a governed Snowflake agent" +requires-python = ">=3.12" +dependencies = [ + "relationalai==1.33.0", +] + +[project.optional-dependencies] +fixture = ["snowflake-connector-python==3.17.4"] + +[tool.setuptools] +packages = ["model"] diff --git a/v1/retail-delivery-agent/raiconfig.example.yaml b/v1/retail-delivery-agent/raiconfig.example.yaml new file mode 100644 index 0000000..e5f24f0 --- /dev/null +++ b/v1/retail-delivery-agent/raiconfig.example.yaml @@ -0,0 +1,35 @@ +# DRAFT ONLY: complete with an approved account administrator, save locally as +# raiconfig.yaml, then verify these fields against the installed 1.33.0 release. +# Do not place credential values or this local config in a shared ZIP/commit. +default_connection: sf +connections: + sf: + type: snowflake + account: + user: + warehouse: + role: + authenticator: externalbrowser + +tables: + default_schema: .DELIVERY_SOURCES + +model: + path: model/ + implicit_properties: false + +deployment: + schema: .DELIVERY_OUTPUTS + meta_schema: .DELIVERY_OUTPUTS_META + role: + pat_secret: + pat_eai: + pat_user: + pat_warehouse: + schedules: + standard: + interval_s: 0 + run_on_deploy: true + outputs: + type: table + schedule: standard diff --git a/v1/retail-delivery-agent/runbook.md b/v1/retail-delivery-agent/runbook.md new file mode 100644 index 0000000..6e93bd0 --- /dev/null +++ b/v1/retail-delivery-agent/runbook.md @@ -0,0 +1,133 @@ +# Delivery-agent live verification and safe operation (draft) + +The Python package and oracle can be inspected offline. **No live combined +graph→prescriptive deployment, Snowflake semantic view, Cortex Agent, grants, +or CoWork conversation is claimed.** This is a review checklist until an +approved evaluation account is available. Do not remove the SQL gates, +advertise a runnable ZIP, or merge the upstream PR on offline evidence. + +## Roles and preflight + +1. **Feature owner / admin:** reconcile deployed prescriptive access for + pinned `relationalai==1.33.0` with the older "internal only" catalog entry. + Obtain account-specific approval and verify Graph, solver backend, + Native App, model deployment, Cortex Analyst/Agent and CoWork entitlement + in the target region. Check where the PAT **secret object** and EAI are + provisioned; never place a secret value in this folder or logs. +2. **Admin:** reserve three isolated schemas in an existing demo database: + `DELIVERY_SOURCES`, `DELIVERY_OUTPUTS`, and + `DELIVERY_OUTPUTS_META`. Check role ownership, source-table change + tracking, refresh/PAT user warehouse, `EXECUTE TASK`, relevant inference + privileges, and separate grants to build, refresh and consume. +3. **SME:** agree that reachability ignores stock, travel is shortest + *directed* minutes, all deadlines share a dispatch clock, each order is + whole and uses one depot, and there are unlimited independent dispatches. + Agree what a solver failure and unsupported hypothetical should say. + +## Offline check before spending account resources + +From the template root, run the README's pinned install/test/oracle commands. +`from model import model, Depot, Store, Order` constructs declarations and +source bindings in one Python process without a fixture load, write, or solve. +The pinned PyRel directory loader separately loads `model/`; no shared +Python object persists between the CLI and a standalone interpreter. +The oracle's North-open/closed assignments are **proposed ground truth**, +not PyRel observations. Keep the evaluator's gold file away from every +agent, view, tool description, and retrieval source. + +## Live model and source validation gate + +1. Confirm `scripts/load_fixture.py` targets a **new** schema in a database + explicitly approved for the demo. Its `00_create_sources.sql` DDL + autocommits separately from the five data inserts. Inspect data counts and + keys if any step fails; do not overwrite or drop an unmarked schema. +2. Complete `raiconfig.example.yaml` locally as `raiconfig.yaml` using the + configuration builder and account owner guidance. Check `model.path: + model/`, source default schema, published and meta schemas, solver + identity/warehouse, manual schedule with deploy-time refresh, and table + materialization. Do not commit local config or credentials. +3. In the **approved** account run `rai models deploy --wait` with pinned + release. Capture the graph route result, actual materialized object names, + output columns and case, PKs, solver optimality, and the dependency + sequence from graph to configured `Problem` to public decisions. One + prescriptive-only test is not evidence of this combined chain. +4. Directly query the actual objects: one `StoreRouteStatus` row per store, + one `OrderDecision` row per order, one `PlanSummary`; match both + revisions, plan assignments and optimum counts against the independent + oracle. Verify North→A=10, North→B=20, South→B/C=25, no false + route to A after closure, and B reachable but unselected. Include a + reachable-but-late case and failed solver/empty candidate cases. Do not + present a non-optimal plan. +5. With other source writes paused, note the previous refresh run evidence. + Call `scripts.change_state close` (guarded transaction); check the revision + increments once. Trigger the same `REFRESH_STANDARD` task, wait for a + **newer** successful `REFRESH_STATUS()` entry, then check revision and facts + in both public outputs. Repeat close→reopen for revision 3, and repeat + refresh without a source change to confirm no stale/no-op false success. + Verify actual status field names before automating the check. Refreshes + are eventually consistent, not globally snapshot-isolated. +6. Define the actual snapshot exporter/column mapping for + `scripts/check_results.py` from observed Snowflake result columns: + `sources` (all five normalized CSV-shaped tables), `routes` with + `depot_id,store_id,minutes`, `store_route_status` with + `store_id,state,revision`, `order_decision` with + `order_id,store_id,units,deadline_minutes,state,depot_id,arrival_minutes, + revision`, and `plan_summary` with + `id,revision,solver_status,completed_orders`. Null chosen depot/arrival + must be JSON `null`. The script deliberately cannot assert a live + `REFRESH_STATUS` success without verified fields and run identity: + preserve the separate refresh trace. Never generate the snapshot from + the oracle and call it observed data. + The opt-in `tests/test_live_chain.py` consumes two real snapshot files + plus normalized refresh-plan/status evidence after an approved run. + Set `RAI_LIVE_APPROVED=1`, `RAI_LIVE_OPEN_SNAPSHOT`, + `RAI_LIVE_CLOSED_SNAPSHOT`, and + `RAI_LIVE_PLAN_AND_REFRESH_EVIDENCE` only when those artifacts exist. + Its normalized `plan` entries require `id`, `kind` and + `dependency_ids`; `refreshes` require run IDs, status, source revision, + start/end times and `source_mutation_at`. This file is skipped offline; + it does not deploy, query an account, or substitute for inspecting the + live SQL/agent evidence. + +## Semantic interface and browser handoff gate + +1. Replace `sql/10_semantic_view.sql` comments **only after** recording the + real table names/PKs/case. Avoid a fanout between store status and order + decisions; verify both questions with direct `SEMANTIC_VIEW(...)` SQL + under a consumer-equivalent role. The SME signs off descriptions, + count aggregation, and reachable-versus-selected language. +2. Replace `sql/20_create_agent.sql` comments with a tested named + Cortex Analyst tool bound to that view, evidence-only instructions, and + verified `orchestration.tool_not_accessible: reject` placement. The + agent must refuse business-data answers if a tool or solver fails. + Inspect tool invocation, generated SQL, and actual rows for both + questions; test a hypothetical depot closure **without** source + mutation to ensure the agent abstains. +3. Replace `sql/30_grants.sql` comments with reviewed, least-privilege + statements. Test a role denied the view and a separate role denied its + underlying published tables. CoWork users need default role/warehouse, + agent access, and Analyst/read access to the published outputs; a + missing-tool reject setting cannot catch all underlying-table failures. + Have the admin expose the tested agent to CoWork and confirm end-user + wording/UI in that account. A playground-only answer is insufficient. + +## Evaluation, recovery and teardown + +Use identical prompts in `evaluation/questions.json` before and after the +North source change in all three arms. Freeze settings, permissions and +grading **before** runs; repeat each pair and record per-question +correctness/abstention, SQL trace, source revision and solver status. +Raw-data arms must have tools/data too. Differences between B and C include +**new computed facts**; they cannot be described as pure semantic-lift +measurement. Report ties, uncertainty and failures, not an unverified lift. + +If load, solve, refresh, Analyst, permissions, or CoWork handoff fails, pause +the agent demonstration. Do not fill missing rows with hard-coded SQL/Python +answers or trust a `SUCCEEDED` status from a *previous* run. Investigate +only the owned source/target/meta objects with an admin. + +For teardown, identify and remove **only** the test agent and reviewed +semantic view using verified account-specific DDL; run `rai models teardown` +with the same local config for PyRel-managed deployment objects. Only then +use `scripts/teardown_sources.py` with a precise typed confirmation. It +rejects unmarked schemas or any objects beyond its five source tables. diff --git a/v1/retail-delivery-agent/scripts/_snowflake.py b/v1/retail-delivery-agent/scripts/_snowflake.py new file mode 100644 index 0000000..ca0f827 --- /dev/null +++ b/v1/retail-delivery-agent/scripts/_snowflake.py @@ -0,0 +1,42 @@ +"""Shared fail-closed connection and ownership checks for demo source scripts.""" + +from __future__ import annotations + +import re +from typing import Any + +OWNER_MARKER = "rai-retail-delivery-agent-owned:1" +IDENTIFIER = re.compile(r"[A-Za-z][A-Za-z0-9_]*\Z") + + +def identifier(value: str) -> str: + if not IDENTIFIER.fullmatch(value): + raise ValueError("Use an unquoted Snowflake identifier (letters, digits, underscore)") + return value.upper() + + +def connect(profile: str) -> Any: + from snowflake.connector import connect as snowflake_connect + + if not profile or profile.lower() == "default": + raise ValueError("Pass an explicit Snowflake connector connection name") + return snowflake_connect(connection_name=profile) + + +def schema_comment(cursor: Any, database: str, schema: str) -> str | None: + database, schema = identifier(database), identifier(schema) + cursor.execute(f"SHOW SCHEMAS IN DATABASE {database}") + fields = [column[0].lower() for column in cursor.description] + for raw in cursor.fetchall(): + row = dict(zip(fields, raw, strict=True)) + if str(row["name"]).upper() == schema: + return str(row.get("comment") or "") + return None + + +def require_owned_schema(cursor: Any, database: str, schema: str) -> None: + if schema_comment(cursor, database, schema) != OWNER_MARKER: + raise RuntimeError( + f"{database}.{schema} is missing or lacks the demo ownership marker; " + "no objects were changed" + ) diff --git a/v1/retail-delivery-agent/scripts/change_state.py b/v1/retail-delivery-agent/scripts/change_state.py new file mode 100644 index 0000000..88fd513 --- /dev/null +++ b/v1/retail-delivery-agent/scripts/change_state.py @@ -0,0 +1,64 @@ +"""Change only North's availability and increment the source revision atomically.""" + +from __future__ import annotations + +import argparse + +from scripts._snowflake import connect, identifier, require_owned_schema + + +def change_state(connection: object, database: str, schema: str, *, close: bool) -> int: + database, schema = identifier(database), identifier(schema) + prefix = f"{database}.{schema}" + with connection.cursor() as cursor: + require_owned_schema(cursor, database, schema) + cursor.execute( + f"SELECT DEPOT_ID, IS_OPEN FROM {prefix}.DEPOTS WHERE DEPOT_ID = %s", + ("North",), + ) + depot_rows = cursor.fetchall() + cursor.execute(f"SELECT REVISION FROM {prefix}.DEMO_STATE") + revision_rows = cursor.fetchall() + if len(depot_rows) != 1 or len(revision_rows) != 1: + raise RuntimeError("Expected exactly North and one revision; no changes were made") + if bool(depot_rows[0][1]) == (not close): + raise RuntimeError("North is already in the requested state; no revision was changed") + revision = int(revision_rows[0][0]) + try: + cursor.execute("BEGIN") + cursor.execute( + f"UPDATE {prefix}.DEPOTS SET IS_OPEN = %s " + "WHERE DEPOT_ID = %s AND IS_OPEN = %s", + (not close, "North", close), + ) + if cursor.rowcount != 1: + raise RuntimeError("North changed concurrently; refusing to advance revision") + cursor.execute( + f"UPDATE {prefix}.DEMO_STATE SET REVISION = REVISION + 1 " + "WHERE REVISION = %s", + (revision,), + ) + if cursor.rowcount != 1: + raise RuntimeError("Revision changed concurrently; rolling back North update") + cursor.execute("COMMIT") + except Exception: + cursor.execute("ROLLBACK") + raise + return revision + 1 + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("action", choices=["close", "reopen"]) + parser.add_argument("--connection", required=True) + parser.add_argument("--database", required=True) + parser.add_argument("--source-schema", default="DELIVERY_SOURCES") + args = parser.parse_args() + database, schema = identifier(args.database), identifier(args.source_schema) + with connect(args.connection) as connection: + revision = change_state(connection, database, schema, close=args.action == "close") + print(f"North is now {'closed' if args.action == 'close' else 'open'}; revision {revision}.") + + +if __name__ == "__main__": + main() diff --git a/v1/retail-delivery-agent/scripts/check_results.py b/v1/retail-delivery-agent/scripts/check_results.py new file mode 100644 index 0000000..63c5166 --- /dev/null +++ b/v1/retail-delivery-agent/scripts/check_results.py @@ -0,0 +1,152 @@ +"""Fail-closed comparison of a *real exported snapshot* with the independent oracle. + +Snapshot collection/column mapping is an account-gated step, not simulated here. +This validator does not call PyRel's query engine or compute its output. +""" + +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from evaluation.check_oracle import evaluate, read_fixture +from model import Depot, Order, Store, model + +EXPECTED_SECTIONS = { + "sources", + "routes", + "store_route_status", + "order_decision", + "plan_summary", +} +ROUTE_COLUMNS = {"depot_id", "store_id", "minutes"} +STORE_COLUMNS = {"store_id", "state", "revision"} +ORDER_COLUMNS = { + "order_id", "store_id", "units", "deadline_minutes", "state", + "depot_id", "arrival_minutes", "revision", +} +SUMMARY_COLUMNS = {"id", "revision", "solver_status", "completed_orders"} + + +def _rows(snapshot: dict, section: str, columns: set[str], key: str | None) -> list[dict]: + rows = snapshot[section] + if not isinstance(rows, list) or any( + not isinstance(row, dict) or set(row) != columns for row in rows + ): + raise ValueError(f"{section}: expected rows with exactly {sorted(columns)}") + if key is not None and len({row[key] for row in rows}) != len(rows): + raise ValueError(f"{section}: duplicate {key}") + return rows + + +def compare_snapshot(snapshot: dict, *, closed: bool, prior_revision: int | None = None) -> None: + """Compare all observed rows/keys with an independent fixture computation.""" + if set(snapshot) != EXPECTED_SECTIONS: + raise ValueError(f"Snapshot requires exactly {sorted(EXPECTED_SECTIONS)}") + if model.name != "retail_delivery_agent" or any( + item is None for item in (Depot, Store, Order) + ): + raise AssertionError("The deployed model package did not load") + baseline = read_fixture() + expected_sources = { + **baseline, + "depots": [ + {**depot, "is_open": "false" if closed else "true"} + if depot["depot_id"] == "North" else depot + for depot in baseline["depots"] + ], + } + actual_sources = snapshot["sources"] + if not isinstance(actual_sources, dict) or set(actual_sources) != set(expected_sources): + raise ValueError("Snapshot must include all five source tables") + revisions = actual_sources["demo_state"] + if len(revisions) != 1 or set(revisions[0]) != {"revision"}: + raise ValueError("Source revision must be one row") + revision = int(revisions[0]["revision"]) + if prior_revision is None and (closed or revision != 1): + raise ValueError("Initial snapshot must be open at revision 1; supply prior revision after a change") + if prior_revision is not None and revision <= prior_revision: + raise ValueError("Source revision did not advance after the earlier refresh") + expected_sources["demo_state"] = [{"revision": str(revision)}] + for table, rows in expected_sources.items(): + if sorted(actual_sources[table], key=str) != sorted(rows, key=str): + raise ValueError(f"Source {table} differs from the approved fixture") + expected = evaluate(expected_sources) + + routes = _rows(snapshot, "routes", ROUTE_COLUMNS, None) + actual_routes = {} + for row in routes: + pair = row["depot_id"], row["store_id"] + if pair in actual_routes: + raise ValueError(f"Duplicate route {pair}") + actual_routes[pair] = float(row["minutes"]) + if actual_routes != expected.routes: + raise ValueError(f"Route pairs/times disagree: {actual_routes} != {expected.routes}") + + store_rows = _rows(snapshot, "store_route_status", STORE_COLUMNS, "store_id") + if {row["store_id"]: row["state"] for row in store_rows} != expected.store_states: + raise ValueError("Missing, extra, or incorrect store reachability rows") + if any(int(row["revision"]) != revision for row in store_rows): + raise ValueError("Store results do not all carry the current source revision") + + summary = _rows(snapshot, "plan_summary", SUMMARY_COLUMNS, "id") + if len(summary) != 1 or summary[0]["id"] != "current": + raise ValueError("Expected one PlanSummary row") + if summary[0]["solver_status"] != "OPTIMAL": + raise ValueError("Solver did not prove optimality: do not publish a recommendation") + if ( + int(summary[0]["revision"]) != revision + or int(summary[0]["completed_orders"]) != expected.optimal_count + ): + raise ValueError("PlanSummary count or revision is inconsistent with the sources") + + order_rows = _rows(snapshot, "order_decision", ORDER_COLUMNS, "order_id") + base_orders = {row["order_id"]: row for row in expected_sources["orders"]} + if {row["order_id"] for row in order_rows} != set(base_orders): + raise ValueError("OrderDecision must contain exactly one row for each source order") + selected = [] + for row in order_rows: + oid = row["order_id"] + original = base_orders[oid] + if ( + int(row["revision"]) != revision + or row["store_id"] != original["store_id"] + or int(row["units"]) != int(original["units"]) + or float(row["deadline_minutes"]) != float(original["deadline_minutes"]) + ): + raise ValueError(f"{oid}: order fields or revision disagree with sources") + if row["state"] == "SELECTED": + depot_id = row["depot_id"] + pair = depot_id, oid + travel = expected.routes.get((depot_id, row["store_id"])) + if travel is None or row["arrival_minutes"] is None: + raise ValueError(f"{oid}: selected order lacks a reachable route/arrival") + if float(row["arrival_minutes"]) != travel: + raise ValueError(f"{oid}: arrival differs from shortest directed path") + selected.append(pair) + elif row["state"] != ( + "REACHABLE_NOT_SELECTED" + if expected.order_states[oid] == "OPTIONALLY_SELECTED" + else expected.order_states[oid] + ): + raise ValueError(f"{oid}: incorrect unselected state {row['state']}") + elif row["depot_id"] is not None or row["arrival_minutes"] is not None: + raise ValueError(f"{oid}: unselected decision must have null depot/arrival") + if tuple(sorted(selected)) not in expected.optimal_plans: + raise ValueError("Selected assignments are not a maximum-count feasible whole-order plan") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--snapshot", required=True, type=Path, help="Actual Snowflake export in JSON") + parser.add_argument("--state", required=True, choices=["open", "closed"]) + parser.add_argument("--prior-revision", type=int, help="Require revision newer than prior state") + args = parser.parse_args() + snapshot = json.loads(args.snapshot.read_text(encoding="utf-8")) + compare_snapshot(snapshot, closed=args.state == "closed", prior_revision=args.prior_revision) + print(f"Snapshot matches fixture, optimum, and revision ({args.state}).") + + +if __name__ == "__main__": + main() diff --git a/v1/retail-delivery-agent/scripts/load_fixture.py b/v1/retail-delivery-agent/scripts/load_fixture.py new file mode 100644 index 0000000..cf3798d --- /dev/null +++ b/v1/retail-delivery-agent/scripts/load_fixture.py @@ -0,0 +1,85 @@ +"""Create a new, demo-owned source schema and load validated synthetic CSVs.""" + +from __future__ import annotations + +import argparse +from pathlib import Path + +from evaluation.check_oracle import read_fixture +from scripts._snowflake import OWNER_MARKER, connect, identifier, schema_comment + +ROOT = Path(__file__).resolve().parents[1] + + +def typed_rows(sources: dict[str, list[dict[str, str]]]) -> dict[str, list[tuple]]: + return { + "DEPOTS": [ + ( + row["depot_id"], row["node_id"], row["is_open"].lower() == "true", + int(row["stock_units"]), + ) + for row in sources["depots"] + ], + "STORES": [(row["store_id"], row["node_id"]) for row in sources["stores"]], + "ROADS": [ + (row["from_node_id"], row["to_node_id"], float(row["travel_minutes"])) + for row in sources["roads"] + ], + "ORDERS": [ + ( + row["order_id"], row["store_id"], int(row["units"]), + float(row["deadline_minutes"]), + ) + for row in sources["orders"] + ], + "DEMO_STATE": [(int(sources["demo_state"][0]["revision"]),)], + } + + +def load_fixture(connection: object, database: str, schema: str) -> None: + database, schema = identifier(database), identifier(schema) + sources = read_fixture() + rows = typed_rows(sources) + ddl = (ROOT / "sql/00_create_sources.sql").read_text(encoding="utf-8") + ddl = ddl.replace("{{DATABASE}}", database).replace("{{SOURCE_SCHEMA}}", schema) + with connection.cursor() as cursor: + if schema_comment(cursor, database, schema) is not None: + raise RuntimeError( + f"{database}.{schema} already exists; refusing to replace even demo data" + ) + for statement in ddl.split(";"): + if statement.strip(): + cursor.execute(statement) + if schema_comment(cursor, database, schema) != OWNER_MARKER: + raise RuntimeError("New schema lacks the expected ownership marker") + try: + cursor.execute("BEGIN") + for table, records in rows.items(): + placeholders = ", ".join(["%s"] * len(records[0])) + cursor.executemany( + f"INSERT INTO {database}.{schema}.{table} VALUES ({placeholders})", + records, + ) + cursor.execute(f"SELECT COUNT(*) FROM {database}.{schema}.{table}") + if cursor.fetchone()[0] != len(records): + raise RuntimeError(f"{table}: inserted row count does not match fixture") + cursor.execute("COMMIT") + except Exception: + cursor.execute("ROLLBACK") + raise + print(f"Loaded five synthetic source tables in {database}.{schema}, revision 1.") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--connection", required=True, help="Existing local Snowflake connection name") + parser.add_argument("--database", required=True, help="Existing dedicated database") + parser.add_argument("--source-schema", default="DELIVERY_SOURCES") + args = parser.parse_args() + database, schema = identifier(args.database), identifier(args.source_schema) + with connect(args.connection) as connection: + load_fixture(connection, database, schema) + + +if __name__ == "__main__": + main() diff --git a/v1/retail-delivery-agent/scripts/teardown_sources.py b/v1/retail-delivery-agent/scripts/teardown_sources.py new file mode 100644 index 0000000..bd667ac --- /dev/null +++ b/v1/retail-delivery-agent/scripts/teardown_sources.py @@ -0,0 +1,47 @@ +"""Remove only the exact, marked five-table demo source schema on explicit request.""" + +from __future__ import annotations + +import argparse + +from scripts._snowflake import connect, identifier, require_owned_schema + +EXPECTED = {"DEPOTS", "STORES", "ROADS", "ORDERS", "DEMO_STATE"} + + +def teardown(connection: object, database: str, schema: str, confirmation: str) -> None: + database, schema = identifier(database), identifier(schema) + target = f"{database}.{schema}" + if confirmation != target: + raise ValueError(f"Type the exact target {target} to confirm") + with connection.cursor() as cursor: + require_owned_schema(cursor, database, schema) + cursor.execute(f"SHOW OBJECTS IN SCHEMA {target}") + names = { + str(row[[col[0].lower() for col in cursor.description].index("name")]).upper() + for row in cursor.fetchall() + } + if names != EXPECTED: + raise RuntimeError( + f"Schema has unexpected objects {sorted(names ^ EXPECTED)}; no drop performed" + ) + cursor.execute(f"DROP SCHEMA {target} RESTRICT") + print(f"Removed only marked source schema {target}.") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--connection", required=True) + parser.add_argument("--database", required=True) + parser.add_argument("--source-schema", default="DELIVERY_SOURCES") + parser.add_argument("--confirm", required=True, help="Exact DATABASE.SCHEMA") + args = parser.parse_args() + with connect(args.connection) as connection: + teardown( + connection, identifier(args.database), identifier(args.source_schema), + args.confirm, + ) + + +if __name__ == "__main__": + main() diff --git a/v1/retail-delivery-agent/sql/00_create_sources.sql b/v1/retail-delivery-agent/sql/00_create_sources.sql new file mode 100644 index 0000000..1396965 --- /dev/null +++ b/v1/retail-delivery-agent/sql/00_create_sources.sql @@ -0,0 +1,34 @@ +-- Executed only by scripts/load_fixture.py after verifying the source schema +-- does not exist. {{DATABASE}} and {{SOURCE_SCHEMA}} are validated identifiers. +CREATE SCHEMA {{DATABASE}}.{{SOURCE_SCHEMA}} + COMMENT = 'rai-retail-delivery-agent-owned:1'; + +CREATE TABLE {{DATABASE}}.{{SOURCE_SCHEMA}}.DEPOTS ( + DEPOT_ID VARCHAR NOT NULL PRIMARY KEY, + NODE_ID VARCHAR NOT NULL, + IS_OPEN BOOLEAN NOT NULL, + STOCK_UNITS NUMBER(38,0) NOT NULL +) CHANGE_TRACKING = TRUE; + +CREATE TABLE {{DATABASE}}.{{SOURCE_SCHEMA}}.STORES ( + STORE_ID VARCHAR NOT NULL PRIMARY KEY, + NODE_ID VARCHAR NOT NULL +) CHANGE_TRACKING = TRUE; + +CREATE TABLE {{DATABASE}}.{{SOURCE_SCHEMA}}.ROADS ( + FROM_NODE_ID VARCHAR NOT NULL, + TO_NODE_ID VARCHAR NOT NULL, + TRAVEL_MINUTES FLOAT NOT NULL, + PRIMARY KEY (FROM_NODE_ID, TO_NODE_ID) +) CHANGE_TRACKING = TRUE; + +CREATE TABLE {{DATABASE}}.{{SOURCE_SCHEMA}}.ORDERS ( + ORDER_ID VARCHAR NOT NULL PRIMARY KEY, + STORE_ID VARCHAR NOT NULL, + UNITS NUMBER(38,0) NOT NULL, + DEADLINE_MINUTES FLOAT NOT NULL +) CHANGE_TRACKING = TRUE; + +CREATE TABLE {{DATABASE}}.{{SOURCE_SCHEMA}}.DEMO_STATE ( + REVISION NUMBER(38,0) NOT NULL PRIMARY KEY +) CHANGE_TRACKING = TRUE; diff --git a/v1/retail-delivery-agent/sql/10_semantic_view.sql b/v1/retail-delivery-agent/sql/10_semantic_view.sql new file mode 100644 index 0000000..2275c0e --- /dev/null +++ b/v1/retail-delivery-agent/sql/10_semantic_view.sql @@ -0,0 +1,9 @@ +-- DRAFT / NOT RUNNABLE. DO NOT replace these comments with guessed SQL. +-- Gate 1: capture live 1.33.0 deployed object names, actual columns/case and +-- PRIMARY KEYS for StoreRouteStatus, OrderDecision, and PlanSummary. +-- Gate 2: prove every store and order has one row at the same source revision, +-- no join fanout, and OPTIMAL solver status before exposing a completed count. +-- Gate 3: have an SME review "reachable regardless of stock", "on time", and +-- "whole order" in a full CREATE SEMANTIC VIEW definition, then execute it +-- and a SEMANTIC_VIEW(...) query under an approved role. Until then no +-- semantic-view SQL is represented as working. diff --git a/v1/retail-delivery-agent/sql/20_create_agent.sql b/v1/retail-delivery-agent/sql/20_create_agent.sql new file mode 100644 index 0000000..a0b9c4b --- /dev/null +++ b/v1/retail-delivery-agent/sql/20_create_agent.sql @@ -0,0 +1,5 @@ +-- DRAFT / NOT RUNNABLE. Account gate: verify current Cortex Agent DDL/API, +-- region, Analyst tool binding to the reviewed semantic view, supported +-- orchestration.tool_not_accessible: reject placement, evidence-only agent +-- instructions, and missing-tool failure behavior in the target account. +-- Model deployment itself does not register an agent or provide CoWork access. diff --git a/v1/retail-delivery-agent/sql/30_grants.sql b/v1/retail-delivery-agent/sql/30_grants.sql new file mode 100644 index 0000000..582f138 --- /dev/null +++ b/v1/retail-delivery-agent/sql/30_grants.sql @@ -0,0 +1,9 @@ +-- ADMIN-REVIEW CHECKLIST / NOT RUNNABLE. Do not paste guessed GRANT syntax. +-- Builder: only dedicated source/output/meta creation privileges; refresh +-- identity: SELECT on the five demo source tables, task/procedure/solver +-- privileges and account-specific approved PAT/secret/EAI configuration. +-- Consumer: default role + warehouse, USAGE on the agent/semantic-view +-- namespace, SELECT on the semantic view AND underlying published output +-- tables, plus verified Cortex Agent/Analyst access. No source/meta grants to +-- the consumer just to make Analyst work. Test separately with a role denied +-- the view and a role denied underlying published tables; fail closed. diff --git a/v1/retail-delivery-agent/sql/40_close_north.sql b/v1/retail-delivery-agent/sql/40_close_north.sql new file mode 100644 index 0000000..626153a --- /dev/null +++ b/v1/retail-delivery-agent/sql/40_close_north.sql @@ -0,0 +1,10 @@ +-- Operation contract: scripts/change_state.py close --connection +-- --database executes the following guarded DML transaction. +-- Do not execute these illustrative statements directly: the Python runner +-- verifies schema ownership and both affected-row counts before commit. +-- BEGIN; +-- UPDATE ..DEPOTS SET IS_OPEN = FALSE +-- WHERE DEPOT_ID = 'North' AND IS_OPEN = TRUE; +-- UPDATE ..DEMO_STATE SET REVISION = REVISION + 1 +-- WHERE REVISION = ; +-- COMMIT; diff --git a/v1/retail-delivery-agent/sql/41_reopen_north.sql b/v1/retail-delivery-agent/sql/41_reopen_north.sql new file mode 100644 index 0000000..09716f0 --- /dev/null +++ b/v1/retail-delivery-agent/sql/41_reopen_north.sql @@ -0,0 +1,10 @@ +-- Operation contract: scripts/change_state.py reopen --connection +-- --database executes the following guarded DML transaction. +-- Do not execute these illustrative statements directly: the Python runner +-- verifies schema ownership and both affected-row counts before commit. +-- BEGIN; +-- UPDATE ..DEPOTS SET IS_OPEN = TRUE +-- WHERE DEPOT_ID = 'North' AND IS_OPEN = FALSE; +-- UPDATE ..DEMO_STATE SET REVISION = REVISION + 1 +-- WHERE REVISION = ; +-- COMMIT; diff --git a/v1/retail-delivery-agent/sql/90_teardown.sql b/v1/retail-delivery-agent/sql/90_teardown.sql new file mode 100644 index 0000000..b2b4fe2 --- /dev/null +++ b/v1/retail-delivery-agent/sql/90_teardown.sql @@ -0,0 +1,7 @@ +-- ADMIN-REVIEW CHECKLIST / NOT RUNNABLE. +-- 1. Stop consumer access and explicitly drop ONLY the verified demo-owned +-- Cortex Agent and reviewed semantic view after inspecting their ownership. +-- 2. From a matching local raiconfig.yaml, run `rai models teardown` for the +-- managed deployment; do not manually sweep the output or meta schemas. +-- 3. scripts/teardown_sources.py removes the *marked* source schema only +-- after its exact five-table inventory is verified. diff --git a/v1/retail-delivery-agent/template-docs.yaml b/v1/retail-delivery-agent/template-docs.yaml new file mode 100644 index 0000000..91794bc --- /dev/null +++ b/v1/retail-delivery-agent/template-docs.yaml @@ -0,0 +1,204 @@ +version: 1 + +page: + lead_from_readme: + - what-this-template-is-for + - what-youll-build + omit_from_readme: [] + +model_explorer: + title: Delivery model vocabulary + description: >- + Open depots and directed weighted roads determine reachability; a + configured whole-order problem determines selected decisions. CSVs are + setup fixtures; the runtime model binds Snowflake tables separately. + concepts: + - symbol: Node + description: A depot, store, or transfer-hub location in the directed road network. + - symbol: Depot + description: A source of available stock when open. + - symbol: Store + description: A destination whose geographic reachability is checked separately from stock. + - symbol: Road + description: A one-way road carrying a nonnegative travel time. + - symbol: Order + description: A whole single-product order with an on-time deadline. + - symbol: StoreRouteStatus + description: A result row for every store, including stores with no open-depot path. + - symbol: OrderDecision + description: A result row for every order after a proven optimal solve. + - symbol: PlanSummary + description: The solver status and optimum count for the current source revision. + relationships: + - symbol: Depot.node + label: at + reading: Depot is at Node + description: Associates each depot with its road-network node. + - symbol: Store.node + label: at + reading: Store is at Node + description: Associates each store with its road-network node. + - symbol: Road.source + label: starts at + reading: Road starts at Node + description: Source endpoint of a directed road. + - symbol: Road.destination + label: ends at + reading: Road ends at Node + description: Destination endpoint of a directed road. + - symbol: Order.store + label: delivered to + reading: Order is delivered to Store + description: Destination store for the entire order. + # The CSVs are setup fixtures, not direct in-model data sources. The model + # binds Snowflake tables in sources.py instead of reading these files. + sources: [] + +guided_walkthrough: + title: From open depots to a checked recommendation + description: >- + Review the source binding, the graph-to-solver chain, and the + fail-closed result contract before any live agent setup. + steps: + - id: declare-delivery-entities + title: Declare the delivery vocabulary + explanation: >- + Depot, Store, Road, and Order separate the road network from + stock and whole-order business constraints. + focus: + file: model/schema.py + symbols: + - Depot + - Store + - Road + - Order + references: + Depot: + symbol: Depot + Store: + symbol: Store + Road: + symbol: Road + Order: + symbol: Order + links: + - title: Declare concepts + href: /build/guides/modeling/declare-concepts/ + - id: bind-snowflake-sources + title: Bind sources without fixture side effects + explanation: >- + depots and roads bind to Snowflake source tables; local CSV loading + remains a separate, explicit administrative operation. + focus: + file: model/sources.py + symbols: + - depots + - roads + references: + depots: + symbol: depots + roads: + symbol: roads + links: + - title: Declare data sources + href: /build/guides/modeling/declare-data-sources/ + - id: derive-directed-routes + title: Find open-depot travel times + explanation: >- + graph computes directed shortest paths; DepotStoreTime restricts + those paths to depots currently open. + focus: + file: model/schema.py + symbols: + - graph + - DepotStoreTime + references: + graph: + symbol: graph + DepotStoreTime: + symbol: DepotStoreTime + links: + - title: Create a graph + href: /build/guides/reasoning/graph/create-a-graph/ + - title: Run a graph algorithm + href: /build/guides/reasoning/graph/run-an-algorithm/ + - id: plan-whole-orders + title: Maximize complete on-time orders + explanation: >- + problem uses binary Assignment variables over EligiblePair routes + and stock constraints; a reachable store need not be selected. + focus: + file: model/schema.py + symbols: + - EligiblePair + - Assignment + - problem + references: + EligiblePair: + symbol: EligiblePair + Assignment: + symbol: Assignment + problem: + symbol: problem + links: + - title: Define decision variables + href: /build/guides/reasoning/prescriptive/decision-problems/variables/ + - title: Add constraints + href: /build/guides/reasoning/prescriptive/decision-problems/constraints/ + - id: report-guarded-results + title: Preserve all stores and orders + explanation: >- + StoreRouteStatus includes unreachable destinations; OrderDecision + emits chosen depots only after an optimal solve, with PlanSummary + recording solver status and source revision for verification. + focus: + file: model/schema.py + symbols: + - StoreRouteStatus + - OrderDecision + - PlanSummary + references: + StoreRouteStatus: + symbol: StoreRouteStatus + OrderDecision: + symbol: OrderDecision + PlanSummary: + symbol: PlanSummary + links: + - title: Derive facts with logic + href: /build/guides/modeling/derive-facts-with-logic/ + +sample_data: + default_rows: 4 + tables: + - path: data/depots.csv + title: Depots + description: Open state and stock for the synthetic depots. + columns: + - depot_id + - node_id + - is_open + - stock_units + rows: 2 + - path: data/stores.csv + title: Stores + description: Every store must have a reachable/unreachable status row. + columns: + - store_id + - node_id + rows: 3 + - path: data/roads.csv + title: Directed roads + description: Minutes of travel in one direction only. + columns: + - from_node_id + - to_node_id + - travel_minutes + - path: data/orders.csv + title: Orders + description: Single-product orders completed whole or not at all. + columns: + - order_id + - store_id + - units + - deadline_minutes diff --git a/v1/retail-delivery-agent/tests/test_fixture.py b/v1/retail-delivery-agent/tests/test_fixture.py new file mode 100644 index 0000000..ec061a2 --- /dev/null +++ b/v1/retail-delivery-agent/tests/test_fixture.py @@ -0,0 +1,323 @@ +"""Offline contracts only; these tests do not claim a Snowflake/PyRel solve.""" + +from __future__ import annotations + +import copy +import subprocess +import sys +import unittest +from pathlib import Path + +from evaluation.check_oracle import evaluate, read_fixture +from evaluation.generate_cases import generate +from scripts.change_state import change_state +from scripts.check_results import compare_snapshot +from scripts.load_fixture import load_fixture +from scripts.teardown_sources import teardown + +ROOT = Path(__file__).resolve().parents[1] + + +def with_north_closed(sources: dict, revision: int = 2) -> dict: + result = copy.deepcopy(sources) + for depot in result["depots"]: + if depot["depot_id"] == "North": + depot["is_open"] = "false" + result["demo_state"] = [{"revision": str(revision)}] + return result + + +def snapshot_double(sources: dict) -> dict: + """Test-only synthetic rows for testing *rejection*, never PyRel output.""" + expected = evaluate(sources) + plan = sorted(expected.optimal_plans)[0] + assignment = {oid: depot for depot, oid in plan} + orders = [] + for order in sources["orders"]: + oid = order["order_id"] + depot = assignment.get(oid) + orders.append( + { + **order, + "state": "SELECTED" if depot else expected.order_states[oid], + "depot_id": depot, + "arrival_minutes": ( + expected.routes[depot, order["store_id"]] if depot else None + ), + "revision": expected.revision, + } + ) + return { + "sources": copy.deepcopy(sources), + "routes": [ + {"depot_id": did, "store_id": sid, "minutes": minutes} + for (did, sid), minutes in expected.routes.items() + ], + "store_route_status": [ + {"store_id": sid, "state": state, "revision": expected.revision} + for sid, state in expected.store_states.items() + ], + "order_decision": orders, + "plan_summary": [ + { + "id": "current", "revision": expected.revision, + "solver_status": "OPTIMAL", "completed_orders": expected.optimal_count, + } + ], + } + + +class TestFixtureOracle(unittest.TestCase): + def setUp(self) -> None: + self.base = read_fixture() + + def test_exact_questions_and_directed_shortest_times(self) -> None: + open_result = evaluate(self.base) + self.assertEqual(open_result.revision, 1) + self.assertEqual(set(open_result.store_states.values()), {"REACHABLE"}) + self.assertEqual(open_result.optimal_count, 3) + self.assertEqual( + open_result.optimal_plans, + {tuple(sorted((("North", "A1"), ("South", "C1"), ("South", "C2"))))}, + ) + self.assertEqual( + open_result.routes, + { + ("North", "A"): 10, ("North", "B"): 20, + ("South", "B"): 25, ("South", "C"): 25, + }, + ) + self.assertEqual(open_result.order_states["B1"], "REACHABLE_NOT_SELECTED") + + closed = evaluate(with_north_closed(self.base)) + self.assertEqual(closed.revision, 2) + self.assertEqual(closed.store_states, {"A": "NO_ROUTE", "B": "REACHABLE", "C": "REACHABLE"}) + self.assertEqual(closed.routes, {("South", "B"): 25, ("South", "C"): 25}) + self.assertEqual(closed.optimal_count, 2) + self.assertEqual(closed.optimal_plans, {(("South", "C1"), ("South", "C2"))}) + self.assertEqual(closed.order_states["A1"], "NO_ROUTE") + self.assertEqual(closed.order_states["B1"], "REACHABLE_NOT_SELECTED") + + def test_zero_one_and_all_closed_depots(self) -> None: + case = copy.deepcopy(self.base) + case["depots"] = [] + self.assertEqual(evaluate(case).store_states, {"A": "NO_ROUTE", "B": "NO_ROUTE", "C": "NO_ROUTE"}) + self.assertEqual(evaluate(case).optimal_count, 0) + case["depots"] = [self.base["depots"][0]] + self.assertEqual(evaluate(case).optimal_count, 1) + case = copy.deepcopy(self.base) + for depot in case["depots"]: + depot["is_open"] = "false" + self.assertEqual(evaluate(case).optimal_count, 0) + self.assertFalse(evaluate(case).routes) + + def test_missing_reversed_road_and_late_orders(self) -> None: + case = copy.deepcopy(self.base) + case["roads"] = [ + row for row in case["roads"] + if (row["from_node_id"], row["to_node_id"]) != ("NORTH", "A") + ] + case["roads"].append({"from_node_id": "A", "to_node_id": "NORTH", "travel_minutes": "1"}) + self.assertEqual(evaluate(case).store_states["A"], "NO_ROUTE") + self.assertNotIn(("North", "A"), evaluate(case).routes) + case = copy.deepcopy(self.base) + case["orders"][0]["deadline_minutes"] = "9" + self.assertEqual(evaluate(case).order_states["A1"], "LATE_ONLY") + self.assertEqual(evaluate(case).optimal_count, 2) + + def test_competing_stock_non_split_and_ties(self) -> None: + case = copy.deepcopy(self.base) + case["orders"].append( + {"order_id": "C3", "store_id": "C", "units": "2", "deadline_minutes": "60"} + ) + expected = evaluate(case) + self.assertEqual(expected.optimal_count, 3) + self.assertEqual(len(expected.optimal_plans), 3) + case["orders"].append( + {"order_id": "HUGE", "store_id": "B", "units": "8", "deadline_minutes": "60"} + ) + self.assertEqual(evaluate(case).order_states["HUGE"], "INSUFFICIENT_STOCK") + self.assertEqual(evaluate(case).optimal_count, 3) + + def test_source_validation_rejects_invalid_rows(self) -> None: + bad = copy.deepcopy(self.base) + bad["roads"][0]["travel_minutes"] = "-1" + with self.assertRaisesRegex(ValueError, "nonnegative"): + evaluate(bad) + bad = copy.deepcopy(self.base) + bad["orders"][1]["store_id"] = "UNKNOWN" + with self.assertRaisesRegex(ValueError, "existing store"): + evaluate(bad) + bad = copy.deepcopy(self.base) + bad["demo_state"].append({"revision": "2"}) + with self.assertRaisesRegex(ValueError, "exactly one"): + evaluate(bad) + bad = copy.deepcopy(self.base) + bad["orders"] = [ + {**bad["orders"][index % 4], "order_id": f"ORDER_{index}"} + for index in range(11) + ] + with self.assertRaisesRegex(ValueError, "unbounded"): + evaluate(bad) + + def test_held_out_generator_is_bounded_reproducible_and_separates_gold(self) -> None: + cases, key = generate(672, 3) + self.assertEqual((cases, key), generate(672, 3)) + self.assertEqual(len(cases), 3) + self.assertNotIn("gold", cases[0]) + self.assertIn("optimal_count", key[0]) + self.assertTrue(all(evaluate(case["sources"]).optimal_count >= 0 for case in cases)) + with self.assertRaises(ValueError): + generate(672, 101) + + +class TestModelPackage(unittest.TestCase): + def test_release_imports_declarations_sources_without_io_or_solve(self) -> None: + code = """ +from unittest.mock import patch +import snowflake.connector +from relationalai.semantics.reasoners.prescriptive import Problem +with patch.object(Problem, 'solve', side_effect=AssertionError('eager solve')), \\ + patch.object(snowflake.connector, 'connect', side_effect=AssertionError('network')): + from model import model, Depot, Store, Order, sources + from model.schema import model as schema_model + assert model is schema_model + assert model.name == 'retail_delivery_agent' + assert all(item is not None for item in (Depot, Store, Order)) + assert {t for t in ('DEPOTS', 'STORES', 'ROADS', 'ORDERS', 'DEMO_STATE')} == { + sources.depots._name, sources.stores._name, sources.roads._name, + sources.orders._name, sources.state._name, + } +""" + result = subprocess.run( + [sys.executable, "-c", code], cwd=ROOT, capture_output=True, text=True + ) + self.assertEqual(result.returncode, 0, result.stderr) + + def test_pinned_cli_directory_loader_registers_one_model(self) -> None: + result = subprocess.run( + [ + sys.executable, "-c", + "from relationalai.oplog.loader import load_model; " + "m=load_model('model', None); " + "assert m.name == 'retail_delivery_agent'; " + "assert len(m._reasoner_bindings) == 1; " + "assert m._reasoner_bindings[0].name == 'delivery_plan'", + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + + +class FakeCursor: + def __init__(self, *, open_: bool = True, revision: int = 1) -> None: + self.open, self.revision = open_, revision + self.description = [] + self.rowcount = 0 + self.result = [] + self.calls = [] + self.pending = None + self.fail_revision = False + + def __enter__(self): + return self + + def __exit__(self, *_): + return False + + def execute(self, query, parameters=()): + self.calls.append(query) + if query.startswith("SHOW SCHEMAS"): + self.description = [("name",), ("comment",)] + self.result = [("DELIVERY_SOURCES", "rai-retail-delivery-agent-owned:1")] + elif query.startswith("SELECT DEPOT_ID"): + self.result = [("North", self.open)] + elif query.startswith("SELECT REVISION"): + self.result = [(self.revision,)] + elif query == "BEGIN": + self.pending = (self.open, self.revision) + elif query.startswith("UPDATE") and ".DEPOTS" in query: + self.rowcount = int(self.open == parameters[2]) + if self.rowcount: + self.open = parameters[0] + elif query.startswith("UPDATE") and ".DEMO_STATE" in query: + self.rowcount = int(self.revision == parameters[0] and not self.fail_revision) + if self.rowcount: + self.revision += 1 + elif query == "ROLLBACK": + self.open, self.revision = self.pending + elif query.startswith("DROP") or query.startswith("CREATE"): + raise AssertionError("mutation did not require DDL") + return self + + def fetchall(self): + return self.result + + +class FakeConnection: + def __init__(self, cursor: FakeCursor) -> None: + self.fake = cursor + + def cursor(self): + return self.fake + + +class TestMutationAndSnapshot(unittest.TestCase): + def test_close_reopen_repeat_and_failed_revision_rolls_back(self) -> None: + cursor = FakeCursor() + connection = FakeConnection(cursor) + self.assertEqual(change_state(connection, "DEMO", "DELIVERY_SOURCES", close=True), 2) + self.assertFalse(cursor.open) + with self.assertRaisesRegex(RuntimeError, "already"): + change_state(connection, "DEMO", "DELIVERY_SOURCES", close=True) + self.assertEqual(cursor.revision, 2) + cursor.fail_revision = True + with self.assertRaisesRegex(RuntimeError, "Revision changed"): + change_state(connection, "DEMO", "DELIVERY_SOURCES", close=False) + self.assertEqual((cursor.open, cursor.revision), (False, 2)) + cursor.fail_revision = False + self.assertEqual(change_state(connection, "DEMO", "DELIVERY_SOURCES", close=False), 3) + self.assertEqual((cursor.open, cursor.revision), (True, 3)) + + def test_loader_fails_on_existing_target_without_inserts(self) -> None: + cursor = FakeCursor() + with self.assertRaisesRegex(RuntimeError, "already exists"): + load_fixture(FakeConnection(cursor), "DEMO", "DELIVERY_SOURCES") + self.assertFalse(any(q.startswith("INSERT") for q in cursor.calls)) + with self.assertRaisesRegex(ValueError, "unquoted Snowflake identifier"): + load_fixture(FakeConnection(cursor), "DEMO; DROP DATABASE DEMO", "DELIVERY_SOURCES") + self.assertFalse(any(q.startswith("DROP") for q in cursor.calls)) + with self.assertRaisesRegex(ValueError, "exact target"): + teardown(FakeConnection(cursor), "DEMO", "DELIVERY_SOURCES", "OTHER.SCHEMA") + self.assertFalse(any(q.startswith("DROP") for q in cursor.calls)) + + def test_snapshot_rejects_missing_rows_bad_status_and_stale_refresh(self) -> None: + base = read_fixture() + snapshot = snapshot_double(base) + compare_snapshot(snapshot, closed=False) + bad = copy.deepcopy(snapshot) + bad["plan_summary"][0]["solver_status"] = "TIME_LIMIT" + with self.assertRaisesRegex(ValueError, "optimality"): + compare_snapshot(bad, closed=False) + bad = copy.deepcopy(snapshot) + bad["store_route_status"].pop() + with self.assertRaisesRegex(ValueError, "reachability"): + compare_snapshot(bad, closed=False) + bad = copy.deepcopy(snapshot) + bad["order_decision"][1]["depot_id"] = "South" + with self.assertRaisesRegex(ValueError, "null"): + compare_snapshot(bad, closed=False) + closed = snapshot_double(with_north_closed(base)) + compare_snapshot(closed, closed=True, prior_revision=1) + with self.assertRaisesRegex(ValueError, "did not advance"): + compare_snapshot(closed, closed=True, prior_revision=2) + bad = copy.deepcopy(closed) + bad["order_decision"][0]["revision"] = 1 + with self.assertRaisesRegex(ValueError, "revision"): + compare_snapshot(bad, closed=True, prior_revision=1) + + +if __name__ == "__main__": + unittest.main() diff --git a/v1/retail-delivery-agent/tests/test_live_chain.py b/v1/retail-delivery-agent/tests/test_live_chain.py new file mode 100644 index 0000000..a8d650c --- /dev/null +++ b/v1/retail-delivery-agent/tests/test_live_chain.py @@ -0,0 +1,85 @@ +"""Opt-in contract on *exported live evidence*, not an offline simulated solve. + +The approved operator must export real rows and a refresh-plan trace from the +pinned release in an account enabled for the combined graph/solver workflow. +""" + +from __future__ import annotations + +import json +import os +import unittest +from datetime import datetime +from pathlib import Path + +from scripts.check_results import compare_snapshot + + +def _read_env_json(name: str) -> dict: + path = os.environ.get(name) + if not path: + raise RuntimeError(f"{name} required for approved live chain validation") + return json.loads(Path(path).read_text(encoding="utf-8")) + + +@unittest.skipUnless(os.environ.get("RAI_LIVE_APPROVED") == "1", "No approved live account/evidence") +class TestApprovedLiveChain(unittest.TestCase): + def test_graph_solver_public_rows_and_later_refresh(self) -> None: + open_snapshot = _read_env_json("RAI_LIVE_OPEN_SNAPSHOT") + closed_snapshot = _read_env_json("RAI_LIVE_CLOSED_SNAPSHOT") + evidence = _read_env_json("RAI_LIVE_PLAN_AND_REFRESH_EVIDENCE") + + compare_snapshot(open_snapshot, closed=False) + compare_snapshot(closed_snapshot, closed=True, prior_revision=1) + if set(evidence) != {"plan", "refreshes", "source_mutation_at"}: + self.fail("Need live plan, two refreshes, and source mutation timestamp") + plan = evidence["plan"] + entries = {str(item["id"]): item for item in plan} + if len(entries) != len(plan): + self.fail("Duplicate refresh-plan entry IDs") + graph_entries = [item for item in plan if item["kind"] == "graph"] + if not graph_entries: + self.fail("No graph computation in the live refresh plan") + for kind in ("prescriptive", "public_decisions"): + if sum(item["kind"] == kind for item in plan) != 1: + self.fail(f"Expected exactly one {kind} plan entry") + solver = next(item for item in plan if item["kind"] == "prescriptive") + decisions = next(item for item in plan if item["kind"] == "public_decisions") + + def depends_on(entry: dict, target_id: str) -> bool: + pending = [str(value) for value in entry["dependency_ids"]] + seen = set() + while pending: + dependency = pending.pop() + if dependency == target_id: + return True + if dependency not in seen: + seen.add(dependency) + if dependency not in entries: + self.fail(f"Missing dependency entry {dependency}") + pending.extend(str(value) for value in entries[dependency]["dependency_ids"]) + return False + + if not any(depends_on(solver, str(graph["id"])) for graph in graph_entries): + self.fail("No verified graph→prescriptive refresh dependency") + if not depends_on(decisions, str(solver["id"])): + self.fail("No verified prescriptive→public-decision refresh dependency") + first, second = evidence["refreshes"] + if ( + first["status"] != "SUCCEEDED" + or second["status"] != "SUCCEEDED" + or first["run_id"] == second["run_id"] + or first["source_revision"] != 1 + or second["source_revision"] != 2 + or not ( + datetime.fromisoformat(first["completed_at"]) + < datetime.fromisoformat(evidence["source_mutation_at"]) + <= datetime.fromisoformat(second["started_at"]) + < datetime.fromisoformat(second["completed_at"]) + ) + ): + self.fail("Refresh trace is stale, incomplete, or not later than source mutation") + + +if __name__ == "__main__": + unittest.main() From ab5208f089440e8c3de7e4d46dfcac536a095272 Mon Sep 17 00:00:00 2001 From: David Amos Date: Thu, 1 Oct 2026 15:08:22 -0500 Subject: [PATCH 2/5] Gate repeated refresh with distinct live evidence What changed: - Require a third observed snapshot and distinct later successful refresh run on unchanged source revision. - Document the evidence needed to run the account-gated test. Why: - A previous successful run cannot prove that a repeated NEW refresh and its public rows are current. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- v1/retail-delivery-agent/runbook.md | 9 ++++--- .../tests/test_live_chain.py | 25 ++++++++++++++++--- 2 files changed, 26 insertions(+), 8 deletions(-) diff --git a/v1/retail-delivery-agent/runbook.md b/v1/retail-delivery-agent/runbook.md index 6e93bd0..6ab57f1 100644 --- a/v1/retail-delivery-agent/runbook.md +++ b/v1/retail-delivery-agent/runbook.md @@ -78,14 +78,15 @@ agent, view, tool description, and retrieval source. `REFRESH_STATUS` success without verified fields and run identity: preserve the separate refresh trace. Never generate the snapshot from the oracle and call it observed data. - The opt-in `tests/test_live_chain.py` consumes two real snapshot files + The opt-in `tests/test_live_chain.py` consumes three real snapshot files + (initial, post-closure and repeated NEW refresh on the unchanged revision) plus normalized refresh-plan/status evidence after an approved run. Set `RAI_LIVE_APPROVED=1`, `RAI_LIVE_OPEN_SNAPSHOT`, - `RAI_LIVE_CLOSED_SNAPSHOT`, and + `RAI_LIVE_CLOSED_SNAPSHOT`, `RAI_LIVE_REPEAT_SNAPSHOT`, and `RAI_LIVE_PLAN_AND_REFRESH_EVIDENCE` only when those artifacts exist. Its normalized `plan` entries require `id`, `kind` and - `dependency_ids`; `refreshes` require run IDs, status, source revision, - start/end times and `source_mutation_at`. This file is skipped offline; + `dependency_ids`; three `refreshes` require distinct run IDs, status, + source revision, start/end times and `source_mutation_at`. This file is skipped offline; it does not deploy, query an account, or substitute for inspecting the live SQL/agent evidence. diff --git a/v1/retail-delivery-agent/tests/test_live_chain.py b/v1/retail-delivery-agent/tests/test_live_chain.py index a8d650c..f410ec7 100644 --- a/v1/retail-delivery-agent/tests/test_live_chain.py +++ b/v1/retail-delivery-agent/tests/test_live_chain.py @@ -27,12 +27,14 @@ class TestApprovedLiveChain(unittest.TestCase): def test_graph_solver_public_rows_and_later_refresh(self) -> None: open_snapshot = _read_env_json("RAI_LIVE_OPEN_SNAPSHOT") closed_snapshot = _read_env_json("RAI_LIVE_CLOSED_SNAPSHOT") + repeat_snapshot = _read_env_json("RAI_LIVE_REPEAT_SNAPSHOT") evidence = _read_env_json("RAI_LIVE_PLAN_AND_REFRESH_EVIDENCE") compare_snapshot(open_snapshot, closed=False) compare_snapshot(closed_snapshot, closed=True, prior_revision=1) + compare_snapshot(repeat_snapshot, closed=True, prior_revision=1) if set(evidence) != {"plan", "refreshes", "source_mutation_at"}: - self.fail("Need live plan, two refreshes, and source mutation timestamp") + self.fail("Need live plan, three refreshes, and source mutation timestamp") plan = evidence["plan"] entries = {str(item["id"]): item for item in plan} if len(entries) != len(plan): @@ -64,21 +66,36 @@ def depends_on(entry: dict, target_id: str) -> bool: self.fail("No verified graph→prescriptive refresh dependency") if not depends_on(decisions, str(solver["id"])): self.fail("No verified prescriptive→public-decision refresh dependency") - first, second = evidence["refreshes"] + if len(evidence["refreshes"]) != 3: + self.fail("Need initial, post-closure, and repeated NEW refresh evidence") + first, second, repeated = evidence["refreshes"] + if any( + int(snapshot["sources"]["demo_state"][0]["revision"]) != refresh["source_revision"] + for snapshot, refresh in zip( + (open_snapshot, closed_snapshot, repeat_snapshot), + (first, second, repeated), + strict=True, + ) + ): + self.fail("A snapshot revision differs from its corresponding refresh trace") if ( first["status"] != "SUCCEEDED" or second["status"] != "SUCCEEDED" - or first["run_id"] == second["run_id"] + or repeated["status"] != "SUCCEEDED" + or len({first["run_id"], second["run_id"], repeated["run_id"]}) != 3 or first["source_revision"] != 1 or second["source_revision"] != 2 + or repeated["source_revision"] != 2 or not ( datetime.fromisoformat(first["completed_at"]) < datetime.fromisoformat(evidence["source_mutation_at"]) <= datetime.fromisoformat(second["started_at"]) < datetime.fromisoformat(second["completed_at"]) + < datetime.fromisoformat(repeated["started_at"]) + < datetime.fromisoformat(repeated["completed_at"]) ) ): - self.fail("Refresh trace is stale, incomplete, or not later than source mutation") + self.fail("Refresh trace is stale, incomplete, or lacks a later repeated NEW run") if __name__ == "__main__": From 8d863a17a56d55c384cb554f1183da6ba570e2a1 Mon Sep 17 00:00:00 2001 From: David Amos Date: Thu, 1 Oct 2026 15:11:30 -0500 Subject: [PATCH 3/5] Move retail delivery agent template to repository root What changed: - Relocate the complete delivery template to the root-level source layout. - Regenerate the template index and assert the root link in policy tests. Why: - Match the current upstream contribution contract and flattened docs sync while preserving v1 public URLs. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- README.md | 3 ++- {v1/retail-delivery-agent => retail-delivery-agent}/README.md | 0 .../data/demo_state.csv | 0 .../data/depots.csv | 0 .../data/orders.csv | 0 .../data/roads.csv | 0 .../data/stores.csv | 0 .../evaluation/arms.sql | 0 .../evaluation/check_oracle.py | 0 .../evaluation/generate_cases.py | 0 .../evaluation/questions.json | 0 .../evaluation/scorecard.csv | 0 .../model/__init__.py | 0 .../model/schema.py | 0 .../model/sources.py | 0 .../pyproject.toml | 0 .../raiconfig.example.yaml | 0 .../runbook.md | 0 .../scripts/_snowflake.py | 0 .../scripts/change_state.py | 0 .../scripts/check_results.py | 0 .../scripts/load_fixture.py | 0 .../scripts/teardown_sources.py | 0 .../sql/00_create_sources.sql | 0 .../sql/10_semantic_view.sql | 0 .../sql/20_create_agent.sql | 0 .../sql/30_grants.sql | 0 .../sql/40_close_north.sql | 0 .../sql/41_reopen_north.sql | 0 .../sql/90_teardown.sql | 0 .../template-docs.yaml | 0 .../tests/test_fixture.py | 0 .../tests/test_live_chain.py | 0 tests/test_generate_template_index.py | 4 +++- 34 files changed, 5 insertions(+), 2 deletions(-) rename {v1/retail-delivery-agent => retail-delivery-agent}/README.md (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/data/demo_state.csv (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/data/depots.csv (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/data/orders.csv (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/data/roads.csv (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/data/stores.csv (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/evaluation/arms.sql (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/evaluation/check_oracle.py (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/evaluation/generate_cases.py (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/evaluation/questions.json (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/evaluation/scorecard.csv (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/model/__init__.py (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/model/schema.py (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/model/sources.py (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/pyproject.toml (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/raiconfig.example.yaml (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/runbook.md (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/scripts/_snowflake.py (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/scripts/change_state.py (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/scripts/check_results.py (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/scripts/load_fixture.py (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/scripts/teardown_sources.py (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/sql/00_create_sources.sql (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/sql/10_semantic_view.sql (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/sql/20_create_agent.sql (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/sql/30_grants.sql (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/sql/40_close_north.sql (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/sql/41_reopen_north.sql (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/sql/90_teardown.sql (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/template-docs.yaml (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/tests/test_fixture.py (100%) rename {v1/retail-delivery-agent => retail-delivery-agent}/tests/test_live_chain.py (100%) diff --git a/README.md b/README.md index 2bd9d02..820b876 100644 --- a/README.md +++ b/README.md @@ -98,13 +98,14 @@ The index below covers the current templates. Expand an industry to see its temp
-Supply Chain & Logistics (9) +Supply Chain & Logistics (10) | Template | Reasoners | Description | | --- | --- | --- | | [demand_planning_temporal](demand_planning_temporal/) | Prescriptive | Plan weekly production and inventory across sites over a date-filtered planning horizon to minimize total cost while meeting demand. | | [humanitarian-aid-supply-chain](humanitarian-aid-supply-chain/) | Graph | Analyze a humanitarian aid supply-chain network with PageRank and weighted degree centrality to optimize resource distribution. | | [network_flow_planning](network_flow_planning/) | Prescriptive | Plan a multi-tier distribution flow that decides which fulfillment centers to open and how much to ship on every lane to satisfy customer demand at minimum cost. | +| [retail-delivery-agent](retail-delivery-agent/) | Graph, Prescriptive | Explore directed reachability and whole-order delivery planning in a synthetic PyRel model; Snowflake agent deployment awaits account verification. | | [shipment_compliance](shipment_compliance/) | Rules-based | Derived classifications for shipment compliance, sourcing risk, and demand escalation. | | [supplier_reliability](supplier_reliability/) | Prescriptive | Select suppliers to meet product demand at minimum cost, with sensitivity marginals and supplier-disruption scenario analysis. | | [supply_chain_resilience](supply_chain_resilience/) | Graph, Rules-based, Prescriptive | Chain blast-radius reachability, network analysis, and rule-based risk classification into a risk-adjusted minimum-cost network flow for supply-chain routing. | diff --git a/v1/retail-delivery-agent/README.md b/retail-delivery-agent/README.md similarity index 100% rename from v1/retail-delivery-agent/README.md rename to retail-delivery-agent/README.md diff --git a/v1/retail-delivery-agent/data/demo_state.csv b/retail-delivery-agent/data/demo_state.csv similarity index 100% rename from v1/retail-delivery-agent/data/demo_state.csv rename to retail-delivery-agent/data/demo_state.csv diff --git a/v1/retail-delivery-agent/data/depots.csv b/retail-delivery-agent/data/depots.csv similarity index 100% rename from v1/retail-delivery-agent/data/depots.csv rename to retail-delivery-agent/data/depots.csv diff --git a/v1/retail-delivery-agent/data/orders.csv b/retail-delivery-agent/data/orders.csv similarity index 100% rename from v1/retail-delivery-agent/data/orders.csv rename to retail-delivery-agent/data/orders.csv diff --git a/v1/retail-delivery-agent/data/roads.csv b/retail-delivery-agent/data/roads.csv similarity index 100% rename from v1/retail-delivery-agent/data/roads.csv rename to retail-delivery-agent/data/roads.csv diff --git a/v1/retail-delivery-agent/data/stores.csv b/retail-delivery-agent/data/stores.csv similarity index 100% rename from v1/retail-delivery-agent/data/stores.csv rename to retail-delivery-agent/data/stores.csv diff --git a/v1/retail-delivery-agent/evaluation/arms.sql b/retail-delivery-agent/evaluation/arms.sql similarity index 100% rename from v1/retail-delivery-agent/evaluation/arms.sql rename to retail-delivery-agent/evaluation/arms.sql diff --git a/v1/retail-delivery-agent/evaluation/check_oracle.py b/retail-delivery-agent/evaluation/check_oracle.py similarity index 100% rename from v1/retail-delivery-agent/evaluation/check_oracle.py rename to retail-delivery-agent/evaluation/check_oracle.py diff --git a/v1/retail-delivery-agent/evaluation/generate_cases.py b/retail-delivery-agent/evaluation/generate_cases.py similarity index 100% rename from v1/retail-delivery-agent/evaluation/generate_cases.py rename to retail-delivery-agent/evaluation/generate_cases.py diff --git a/v1/retail-delivery-agent/evaluation/questions.json b/retail-delivery-agent/evaluation/questions.json similarity index 100% rename from v1/retail-delivery-agent/evaluation/questions.json rename to retail-delivery-agent/evaluation/questions.json diff --git a/v1/retail-delivery-agent/evaluation/scorecard.csv b/retail-delivery-agent/evaluation/scorecard.csv similarity index 100% rename from v1/retail-delivery-agent/evaluation/scorecard.csv rename to retail-delivery-agent/evaluation/scorecard.csv diff --git a/v1/retail-delivery-agent/model/__init__.py b/retail-delivery-agent/model/__init__.py similarity index 100% rename from v1/retail-delivery-agent/model/__init__.py rename to retail-delivery-agent/model/__init__.py diff --git a/v1/retail-delivery-agent/model/schema.py b/retail-delivery-agent/model/schema.py similarity index 100% rename from v1/retail-delivery-agent/model/schema.py rename to retail-delivery-agent/model/schema.py diff --git a/v1/retail-delivery-agent/model/sources.py b/retail-delivery-agent/model/sources.py similarity index 100% rename from v1/retail-delivery-agent/model/sources.py rename to retail-delivery-agent/model/sources.py diff --git a/v1/retail-delivery-agent/pyproject.toml b/retail-delivery-agent/pyproject.toml similarity index 100% rename from v1/retail-delivery-agent/pyproject.toml rename to retail-delivery-agent/pyproject.toml diff --git a/v1/retail-delivery-agent/raiconfig.example.yaml b/retail-delivery-agent/raiconfig.example.yaml similarity index 100% rename from v1/retail-delivery-agent/raiconfig.example.yaml rename to retail-delivery-agent/raiconfig.example.yaml diff --git a/v1/retail-delivery-agent/runbook.md b/retail-delivery-agent/runbook.md similarity index 100% rename from v1/retail-delivery-agent/runbook.md rename to retail-delivery-agent/runbook.md diff --git a/v1/retail-delivery-agent/scripts/_snowflake.py b/retail-delivery-agent/scripts/_snowflake.py similarity index 100% rename from v1/retail-delivery-agent/scripts/_snowflake.py rename to retail-delivery-agent/scripts/_snowflake.py diff --git a/v1/retail-delivery-agent/scripts/change_state.py b/retail-delivery-agent/scripts/change_state.py similarity index 100% rename from v1/retail-delivery-agent/scripts/change_state.py rename to retail-delivery-agent/scripts/change_state.py diff --git a/v1/retail-delivery-agent/scripts/check_results.py b/retail-delivery-agent/scripts/check_results.py similarity index 100% rename from v1/retail-delivery-agent/scripts/check_results.py rename to retail-delivery-agent/scripts/check_results.py diff --git a/v1/retail-delivery-agent/scripts/load_fixture.py b/retail-delivery-agent/scripts/load_fixture.py similarity index 100% rename from v1/retail-delivery-agent/scripts/load_fixture.py rename to retail-delivery-agent/scripts/load_fixture.py diff --git a/v1/retail-delivery-agent/scripts/teardown_sources.py b/retail-delivery-agent/scripts/teardown_sources.py similarity index 100% rename from v1/retail-delivery-agent/scripts/teardown_sources.py rename to retail-delivery-agent/scripts/teardown_sources.py diff --git a/v1/retail-delivery-agent/sql/00_create_sources.sql b/retail-delivery-agent/sql/00_create_sources.sql similarity index 100% rename from v1/retail-delivery-agent/sql/00_create_sources.sql rename to retail-delivery-agent/sql/00_create_sources.sql diff --git a/v1/retail-delivery-agent/sql/10_semantic_view.sql b/retail-delivery-agent/sql/10_semantic_view.sql similarity index 100% rename from v1/retail-delivery-agent/sql/10_semantic_view.sql rename to retail-delivery-agent/sql/10_semantic_view.sql diff --git a/v1/retail-delivery-agent/sql/20_create_agent.sql b/retail-delivery-agent/sql/20_create_agent.sql similarity index 100% rename from v1/retail-delivery-agent/sql/20_create_agent.sql rename to retail-delivery-agent/sql/20_create_agent.sql diff --git a/v1/retail-delivery-agent/sql/30_grants.sql b/retail-delivery-agent/sql/30_grants.sql similarity index 100% rename from v1/retail-delivery-agent/sql/30_grants.sql rename to retail-delivery-agent/sql/30_grants.sql diff --git a/v1/retail-delivery-agent/sql/40_close_north.sql b/retail-delivery-agent/sql/40_close_north.sql similarity index 100% rename from v1/retail-delivery-agent/sql/40_close_north.sql rename to retail-delivery-agent/sql/40_close_north.sql diff --git a/v1/retail-delivery-agent/sql/41_reopen_north.sql b/retail-delivery-agent/sql/41_reopen_north.sql similarity index 100% rename from v1/retail-delivery-agent/sql/41_reopen_north.sql rename to retail-delivery-agent/sql/41_reopen_north.sql diff --git a/v1/retail-delivery-agent/sql/90_teardown.sql b/retail-delivery-agent/sql/90_teardown.sql similarity index 100% rename from v1/retail-delivery-agent/sql/90_teardown.sql rename to retail-delivery-agent/sql/90_teardown.sql diff --git a/v1/retail-delivery-agent/template-docs.yaml b/retail-delivery-agent/template-docs.yaml similarity index 100% rename from v1/retail-delivery-agent/template-docs.yaml rename to retail-delivery-agent/template-docs.yaml diff --git a/v1/retail-delivery-agent/tests/test_fixture.py b/retail-delivery-agent/tests/test_fixture.py similarity index 100% rename from v1/retail-delivery-agent/tests/test_fixture.py rename to retail-delivery-agent/tests/test_fixture.py diff --git a/v1/retail-delivery-agent/tests/test_live_chain.py b/retail-delivery-agent/tests/test_live_chain.py similarity index 100% rename from v1/retail-delivery-agent/tests/test_live_chain.py rename to retail-delivery-agent/tests/test_live_chain.py diff --git a/tests/test_generate_template_index.py b/tests/test_generate_template_index.py index c7a8fa7..fe8f2d1 100644 --- a/tests/test_generate_template_index.py +++ b/tests/test_generate_template_index.py @@ -15,7 +15,8 @@ def test_collects_only_root_template_directories(self) -> None: entries = collect_templates(REPO_ROOT) names = {entry.name for entry in entries} - self.assertEqual(len(entries), 53) + self.assertEqual(len(entries), 54) + self.assertIn("retail-delivery-agent", names) self.assertIn("simple-start", names) self.assertNotIn("sample-template", names) self.assertNotIn("scripts", names) @@ -26,6 +27,7 @@ def test_root_index_uses_flat_template_links(self) -> None: index = build_index(entries, "") self.assertIn("[simple-start](simple-start/)", index) + self.assertIn("[retail-delivery-agent](retail-delivery-agent/)", index) self.assertNotIn("(v1/", index) From 0d65f36a8e212062b56c354f113abaa78942ee36 Mon Sep 17 00:00:00 2001 From: David Amos Date: Thu, 1 Oct 2026 15:19:34 -0500 Subject: [PATCH 4/5] Simplify delivery model explorer road labels What changed: - Remove two redundant Road-to-Node diagram edges while retaining the directed source-to-destination explanation in the Road concept. Why: - Avoid a wide-layout relationship-label collision that blocks generated template page rendering. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- retail-delivery-agent/template-docs.yaml | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/retail-delivery-agent/template-docs.yaml b/retail-delivery-agent/template-docs.yaml index 91794bc..f350068 100644 --- a/retail-delivery-agent/template-docs.yaml +++ b/retail-delivery-agent/template-docs.yaml @@ -20,7 +20,7 @@ model_explorer: - symbol: Store description: A destination whose geographic reachability is checked separately from stock. - symbol: Road - description: A one-way road carrying a nonnegative travel time. + description: A one-way road from a source to a destination node with nonnegative travel time. - symbol: Order description: A whole single-product order with an on-time deadline. - symbol: StoreRouteStatus @@ -38,14 +38,6 @@ model_explorer: label: at reading: Store is at Node description: Associates each store with its road-network node. - - symbol: Road.source - label: starts at - reading: Road starts at Node - description: Source endpoint of a directed road. - - symbol: Road.destination - label: ends at - reading: Road ends at Node - description: Destination endpoint of a directed road. - symbol: Order.store label: delivered to reading: Order is delivered to Store From 966d3dd6762cbdbcaf7757325e56dacca68b5fef Mon Sep 17 00:00:00 2001 From: David Amos Date: Thu, 1 Oct 2026 15:23:03 -0500 Subject: [PATCH 5/5] Align delivery walkthrough prose with source order What changed: - Introduce EligiblePair, Assignment, and problem in source order in the planning step explanation. Why: - Allow the generated walkthrough to pass build-time reference-order validation without changing the model or lesson. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- retail-delivery-agent/template-docs.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/retail-delivery-agent/template-docs.yaml b/retail-delivery-agent/template-docs.yaml index f350068..001b361 100644 --- a/retail-delivery-agent/template-docs.yaml +++ b/retail-delivery-agent/template-docs.yaml @@ -117,8 +117,10 @@ guided_walkthrough: - id: plan-whole-orders title: Maximize complete on-time orders explanation: >- - problem uses binary Assignment variables over EligiblePair routes - and stock constraints; a reachable store need not be selected. + EligiblePair identifies on-time routes that can cover a whole order. + Binary Assignment choices in the named problem maximize complete + orders under depot stock constraints; a reachable store need not + be selected. focus: file: model/schema.py symbols: