From f0da32aa4e22e3183939e5b5b3ce4e5d8b299b16 Mon Sep 17 00:00:00 2001 From: Yifan Lu Date: Sat, 31 Oct 2026 17:44:36 -0500 Subject: [PATCH] Add a scheduled price sync: every other day, open a PR when a price changes .github/workflows/price-sync.yml runs scripts/pricing/sync_openrouter.py at 06:17 UTC on odd days of the month (and on demand). When a price changed, it opens or updates one pull request (branch price-sync/auto) with the new price table, the sync report, and a preview of the leaderboard at the new prices from reprice.py (report only). Nothing is merged or re-priced automatically. Creating the PR needs the repository setting that lets Actions create pull requests, or a PRICE_SYNC_TOKEN secret; with the secret the PR's checks run. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/price-sync.yml | 97 ++++++++++++++++++++++++++++++++ model_cost/model_profiles.yaml | 5 +- 2 files changed, 100 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/price-sync.yml diff --git a/.github/workflows/price-sync.yml b/.github/workflows/price-sync.yml new file mode 100644 index 00000000..74e6dabe --- /dev/null +++ b/.github/workflows/price-sync.yml @@ -0,0 +1,97 @@ +name: Price sync + +# Every other day, sync model prices with OpenRouter (scripts/pricing/sync_openrouter.py) +# and, when a price changed, open or update one pull request with the new price table, the +# sync report and what re-pricing the leaderboard would do. Nothing is merged or re-priced +# automatically: a maintainer reviews the PR, and re-prices the leaderboard as a separate +# step (scripts/pricing/reprice.py --all --write). +# +# Creating the PR needs either the repository setting "Allow GitHub Actions to create and +# approve pull requests", or a secret PRICE_SYNC_TOKEN (a fine-grained token with +# Contents: write and Pull requests: write on this repository). With the secret, the PR's +# own checks run as for any other PR; with the built-in token they do not. + +on: + schedule: + - cron: "17 6 */2 * *" # 06:17 UTC on odd days of the month + workflow_dispatch: {} + +permissions: + contents: write + pull-requests: write + +concurrency: + group: price-sync + cancel-in-progress: false + +jobs: + sync: + runs-on: ubuntu-latest + timeout-minutes: 30 + steps: + - uses: actions/checkout@v5 + + - name: Set up Python + uses: actions/setup-python@v6 + with: + python-version: "3.11" + + - name: Install uv + run: | + set -euo pipefail + python -m pip install --upgrade pip + python -m pip install uv + + - name: Sync prices with OpenRouter + id: sync + run: | + set -euo pipefail + today=$(date -u +%F) + echo "date=$today" >> "$GITHUB_OUTPUT" + uv run python scripts/pricing/sync_openrouter.py --date "$today" \ + --report "$RUNNER_TEMP/sync_report.md" + if [[ -n "$(git status --porcelain -- model_cost)" ]]; then + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + + - name: Preview the leaderboard at the new prices + if: ${{ steps.sync.outputs.changed == 'true' }} + # Report only: no prediction file or README is written. The dataset is not + # downloaded, so the optimality columns are left out of the preview. + run: | + set -euo pipefail + uv run python scripts/pricing/reprice.py --all \ + --report "$RUNNER_TEMP/reprice_report.md" + + - name: Write the PR description + if: ${{ steps.sync.outputs.changed == 'true' }} + run: | + set -euo pipefail + { + echo "The scheduled price sync found prices that changed on OpenRouter." + echo "Each model is priced at its standard price (the lab's own price, promotions undone); see \`scripts/pricing/sync_openrouter.py\`." + echo + cat "$RUNNER_TEMP/sync_report.md" + echo + echo "### Leaderboard at these prices (not applied)" + echo + cat "$RUNNER_TEMP/reprice_report.md" + echo + echo "Merging updates the price table only. To bring the leaderboard to these prices, run \`python scripts/pricing/reprice.py --all --write\` on \`main\` and open a PR with the result." + echo + echo "*Opened by the [price-sync workflow](${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}).*" + } > "$RUNNER_TEMP/pr_body.md" + + - name: Open or update the pull request + if: ${{ steps.sync.outputs.changed == 'true' }} + uses: peter-evans/create-pull-request@v7 + with: + token: ${{ secrets.PRICE_SYNC_TOKEN || github.token }} + branch: price-sync/auto + delete-branch: true + add-paths: model_cost/ + commit-message: "Price sync: OpenRouter standard prices of ${{ steps.sync.outputs.date }}" + title: "Price sync: OpenRouter standard prices of ${{ steps.sync.outputs.date }}" + body-path: ${{ runner.temp }}/pr_body.md diff --git a/model_cost/model_profiles.yaml b/model_cost/model_profiles.yaml index 4d280dce..975491ff 100644 --- a/model_cost/model_profiles.yaml +++ b/model_cost/model_profiles.yaml @@ -19,8 +19,9 @@ # Profiles keyed by an OpenRouter id (or with openrouter_id) take OpenRouter's list price # when a maintainer runs: # python scripts/pricing/sync_openrouter.py -# which also records the prices used in model_cost/price_snapshots/.json. A model -# name that matches no id or alias is charged the highest price in this file. +# which also records the prices used in model_cost/price_snapshots/.json. The +# price-sync workflow runs it every other day and opens a PR when a price changes. A +# model name that matches no id or alias is charged the highest price in this file. models: "agnes-2.0-flash":