update-data.yml

name: Update WisdomTree ETF data

on:
  schedule:
    - cron: '0 0 * * 0'
  workflow_dispatch:
    inputs:
      max_fetches:
        description: "Batch size; continue after the saved ticker cursor; 0 means a full pass over every catalog fund; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      request_sleep:
        description: "Seconds between outgoing request starts; keep requests polite to WisdomTree, SEC and Yahoo; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      concurrency:
        description: "Parallel fund workers (keep low); blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      tickers:
        description: "Only update these tickers, separated by spaces or commas; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      aum:
        description: "AUM min:max; bounds may be amounts or K/M/B/T suffixes, or nano/micro/small/mid/large; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      ter:
        description: "Net expense ratio range in percent: min:max; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      dividend_yield:
        description: "Catalog trailing-yield percentage range; colon required; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      sec_yield:
        description: "30-day SEC yield percentage range read from the product page: min:max; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      history_page_size:
        description: "Rows in each generated history JSON page; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      max_retries:
        description: "Retries after the initial request (integer, at least 1); blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      history_range:
        description: "Yahoo history window (max, ytd, 1d, 5d, 1mo, 3mo, 6mo, 1y, 2y, 5y or 10y); blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      edgar_fallback:
        description: "Use SEC EDGAR Form N-PORT-P for full holdings (true or false); blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      skip_yahoo:
        description: "Keep previous history and distributions while refreshing catalog and holdings (true or false); blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      skip_wisdomtree:
        description: "Keep the previously published official catalog (true or false); blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      performance_ytd:
        description: "Annualized YTD adjusted-close return range: min:max; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      performance_1y:
        description: "Annualized 1Y adjusted-close return range: min:max; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      performance_3y:
        description: "Annualized 3Y adjusted-close return range: min:max; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      performance_5y:
        description: "Annualized 5Y adjusted-close return range: min:max; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      performance_10y:
        description: "Annualized 10Y adjusted-close return range: min:max; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      total_return_ytd:
        description: "Cumulative YTD adjusted-close return range: min:max; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      total_return_1y:
        description: "Cumulative 1Y adjusted-close return range: min:max; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      total_return_3y:
        description: "Cumulative 3Y adjusted-close return range: min:max; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      total_return_5y:
        description: "Cumulative 5Y adjusted-close return range: min:max; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      total_return_10y:
        description: "Cumulative 10Y adjusted-close return range: min:max; blank inherits scripts/update-data.config.json"
        required: false
        default: ""
        type: string
      advanced:
        description: 'JSON object of supported control overrides not exposed above (keys are UPPER_CASE control names, values are scalars); nonblank individual inputs win'
        required: false
        default: '{}'
        type: string

permissions:
  contents: write

concurrency:
  group: update-data
  cancel-in-progress: false

jobs:
  update-data:
    runs-on: ubuntu-latest
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@v7
        with:
          persist-credentials: false
      - name: Setup bun
        uses: oven-sh/setup-bun@v2
        with:
          bun-version: latest
      - name: Install dependencies
        run: bun install --frozen-lockfile
      - name: Run updater unit tests
        run: bun test
      - name: Resolve file defaults and manual overrides
        env:
          DISPATCH_INPUTS: ${{ toJSON(inputs) }}
          PROTECTED_SEC_UA: ${{ vars.SEC_UA }}
        run: |
          bun --eval '
            import { resolveControls } from "./scripts/update-data.ts";
            import { appendFile } from "node:fs/promises";
            // schedule runs have no dispatch inputs: file defaults apply as-is
            const inputs = JSON.parse(process.env.DISPATCH_INPUTS || "{}") || {};
            const advanced = JSON.parse(inputs.advanced || "{}");
            const individual = Object.fromEntries(Object.entries(inputs)
              .filter(([key]) => key !== "advanced")
              .map(([key, value]) => [key.toUpperCase(), value]));
            const file = await Bun.file("scripts/update-data.config.json").json();
            // Protected repository Actions variables: highest precedence, applied only when nonblank (never printed).
            const protectedVars = {};
            if ((process.env.PROTECTED_SEC_UA ?? "").trim()) protectedVars.SEC_UA = process.env.PROTECTED_SEC_UA.trim();
            const controls = resolveControls(file, advanced, individual, protectedVars);
            await appendFile(process.env.GITHUB_ENV,
              Object.entries(controls).map(([key, value]) => `${key}=${value}\n`).join(""));
          '
      - name: Generate api/wisdomtree static data
        run: bun ./scripts/update-data.ts
      - name: Commit updated data
        if: ${{ !cancelled() }}
        env:
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          git add api/wisdomtree
          if git diff --cached --quiet -- api/wisdomtree; then
            echo "api/wisdomtree is already up to date"
            exit 0
          fi
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git commit -m "chore(data): update WisdomTree ETF data"
          # Runtime-only auth; no token/header stored in URL, file or Git config.
          push() { git -c credential.helper= -c credential.helper='!f() { if test "$1" = get; then printf "username=x-access-token\npassword=%s\n" "$GITHUB_TOKEN"; fi; }; f' push; }
          # main may have moved while the updater ran (merges, other pushes): rebase the data commit and retry
          for attempt in 1 2 3 4; do
            push && exit 0
            echo "push rejected: rebasing onto origin/${GITHUB_REF_NAME} (attempt ${attempt})"
            git pull --rebase origin "${GITHUB_REF_NAME}" || exit 1
          done
          exit 1