example_tests.yml

name: Example tests

on:
  push:
    branches: ["pull-request/[0-9]+"]
    # NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used
  schedule:
    - cron: "0 0 * * *" # Nightly
  workflow_dispatch:
    # On-demand

concurrency:
  # Cancel previous runs if new commit is pushed to the same PR
  group: ${{ github.workflow }}-${{ startsWith(github.ref, 'refs/heads/pull-request/') && github.ref || github.sha }}
  cancel-in-progress: true

# Every job passes the repo variable ALLOW_FAILURE_EXAMPLE_TESTS to the runner as `allow_failure_examples`:
# a comma-separated list of example names whose test failures should be non-blocking, e.g. "torch_trt,llm_qat"
#
# An `example` of several names separated by spaces runs them one after the other in one job, each in its own
# step, which saves the container start and the base install (3-5 min each) of every job after the first. Only
# small examples with the same image and extras are grouped; each example installs its own requirements just
# before its tests, so a group lists first the examples that leave the environment alone and last the ones that
# pin or upgrade packages. An example that changes torch (torch_trt downgrades it) must not be grouped: the
# extension cache key is the group name plus the image, with no torch version, and a cache built against another
# torch fails with undefined symbols.

jobs:
  # One changed-files pass decides which lanes run. Lane granularity, not per-example: a job's
  # `if` cannot read `matrix`, so gating an individual example would need a job per example.
  pr-gate:
    runs-on: ubuntu-24.04
    permissions:
      checks: read
      contents: read
    outputs:
      any_changed: ${{ steps.lanes.outputs.any_changed }}
      stacked: ${{ steps.base.outputs.stacked == 'true' }}
      torch: ${{ steps.lanes.outputs.torch }}
      vllm: ${{ steps.lanes.outputs.vllm }}
      trtllm: ${{ steps.lanes.outputs.trtllm }}
      megatron: ${{ steps.lanes.outputs.megatron }}
      diffusers: ${{ steps.lanes.outputs.diffusers }}
      onnx: ${{ steps.lanes.outputs.onnx }}
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0
      - if: startsWith(github.ref, 'refs/heads/pull-request/')
        id: base
        uses: ./.github/actions/pr-merge-base
      - if: startsWith(github.ref, 'refs/heads/pull-request/')
        id: changed
        uses: step-security/changed-files@2e07db73e5ccdb319b9a6c7766bd46d39d304bad # v47.0.5
        with:
          base_sha: ${{ steps.base.outputs.merge_base }}
          sha: ${{ steps.base.outputs.head_sha }}
          fail_on_initial_diff_error: true
          # `common` runs every lane. Each lane lists the example directories it covers, plus any
          # extra directory those tests reach into (e.g. hf_ptq's script runs lm_eval from
          # ../llm_eval, and the speculative_decoding test drives hf_ptq).
          files_yaml: |
            common:
              - .github/actions/**
              - .github/workflows/_example_tests_runner.yml
              - .github/workflows/example_tests.yml
              - modelopt/**
              - modelopt_recipes/**
              - pyproject.toml
              - tests/_test_utils/**
              - tests/conftest.py
              - tests/examples/conftest.py
              - "!**/*.{md,rst,png,ipynb}"
            torch:
              - "{examples,tests/examples}/{llm_distill,llm_qat,llm_sparsity,specdec_bench,speculative_decoding}/**"
              - examples/dataset/** # data prep for llm_qat and speculative_decoding
              - examples/hf_ptq/** # the speculative_decoding test drives hf_ptq
              - "!**/*.{md,rst,png,ipynb}"
            vllm:
              - "{examples,tests/examples}/vllm_serve/**"
              - examples/hf_ptq/convert_ggml_to_int8.py # tests/examples/vllm_serve serves its output
              - "!**/*.{md,rst,png,ipynb}"
            trtllm:
              - "{examples,tests/examples}/{gpt-oss,hf_ptq,llm_eval}/**"
              - examples/dataset/** # data prep for gpt-oss
              - "!**/*.{md,rst,png,ipynb}"
            megatron:
              - "{examples,tests/examples}/megatron_bridge/**"
              - "!**/*.{md,rst,png,ipynb}"
            diffusers:
              - "{examples,tests/examples}/diffusers/**"
              - "!**/*.{md,rst,png,ipynb}"
            onnx:
              - "{examples,tests/examples}/{torch_onnx,torch_trt}/**"
              - examples/onnx_ptq/** # torch_trt reuses onnx_ptq
              - "!**/*.{md,rst,png,ipynb}"
      - id: lanes
        env:
          # Nightly and on-demand runs have no diff to inspect, so they run everything.
          NON_PR: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }}
          # Stacked PRs (base is another PR's branch) skip GPU tests; only the bottom PR runs them.
          STACKED: ${{ steps.base.outputs.stacked }}
          COMMON: ${{ steps.changed.outputs.common_any_modified }}
          TORCH: ${{ steps.changed.outputs.torch_any_modified }}
          VLLM: ${{ steps.changed.outputs.vllm_any_modified }}
          TRTLLM: ${{ steps.changed.outputs.trtllm_any_modified }}
          MEGATRON: ${{ steps.changed.outputs.megatron_any_modified }}
          DIFFUSERS: ${{ steps.changed.outputs.diffusers_any_modified }}
          ONNX: ${{ steps.changed.outputs.onnx_any_modified }}
        run: |
          any=false
          for lane in TORCH VLLM TRTLLM MEGATRON DIFFUSERS ONNX; do
            if [[ "$STACKED" != "true" && ( "$NON_PR" == "true" || "$COMMON" == "true" || "${!lane}" == "true" ) ]]; then
              run=true
              any=true
            else
              run=false
            fi
            echo "${lane,,}=$run" | tee --append "$GITHUB_OUTPUT"
          done
          echo "any_changed=$any" | tee --append "$GITHUB_OUTPUT"
      - name: Wait for unit tests before spending GPU runners
        if: ${{ steps.lanes.outputs.any_changed == 'true' && startsWith(github.ref, 'refs/heads/pull-request/') }}
        uses: poseidon/wait-for-status-checks@e7e24423722c5b8e97a6dd5848827d2a412f812d # v0.7.0
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          match_pattern: "^linux$" # Wait for Unit tests / linux
          delay: 300s

  ##### PyTorch Example Tests #####
  torch:
    needs: [pr-gate]
    if: needs.pr-gate.outputs.torch == 'true'
    strategy:
      fail-fast: false
      matrix:
        # llm_qat and speculative_decoding take 9-16 min and stay on their own jobs; the three small ones share one.
        # llm_distill pins trl and torchao, so it goes last.
        example: ["specdec_bench llm_sparsity llm_distill", llm_qat, speculative_decoding]
    uses: ./.github/workflows/_example_tests_runner.yml
    permissions:
      contents: read
    secrets: inherit
    with:
      docker_image: "nvcr.io/nvidia/pytorch:26.09-py3"
      example: ${{ matrix.example }}
      timeout_minutes: 30
      pip_install_extras: "[hf,dev-test]"
      runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
      allow_failure_examples: ${{ vars.ALLOW_FAILURE_EXAMPLE_TESTS }}

  ##### vLLM Example Tests #####
  vllm:
    needs: [pr-gate]
    if: needs.pr-gate.outputs.vllm == 'true'
    strategy:
      fail-fast: false
      matrix:
        example: [vllm_serve]
    uses: ./.github/workflows/_example_tests_runner.yml
    permissions:
      contents: read
    secrets: inherit
    with:
      docker_image: "vllm/vllm-openai:v0.30.0@sha256:8a69ffad015f138d7170c4ddc429e230a3bc1c1719f67e14324749df200a4b90"
      example: ${{ matrix.example }}
      timeout_minutes: 30
      pip_install_extras: "[hf,dev-test]"
      runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
      allow_failure_examples: ${{ vars.ALLOW_FAILURE_EXAMPLE_TESTS }}

  ##### TensorRT-LLM Example Tests #####
  trtllm:
    needs: [pr-gate]
    if: needs.pr-gate.outputs.trtllm == 'true'
    strategy:
      fail-fast: false
      matrix:
        # hf_ptq is the longest and stays alone. gpt-oss upgrades datasets and trl, so it goes after llm_eval.
        example: ["llm_eval gpt-oss", hf_ptq]
    uses: ./.github/workflows/_example_tests_runner.yml
    permissions:
      contents: read
    secrets: inherit
    with:
      # Newer releases break quickstart_multimodal.py (test_qwen_vl): rc21-rc23 import MultimodalConfig,
      # which tensorrt_llm.llmapi does not export yet, and rc24+ import `fla`, which the image lacks.
      docker_image: "nvcr.io/nvidia/tensorrt-llm/release:1.3.0rc20"
      example: ${{ matrix.example }}
      pip_install_extras: "[hf,dev-test]"
      runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
      allow_failure_examples: ${{ vars.ALLOW_FAILURE_EXAMPLE_TESTS }}

  ##### Megatron Example Tests #####
  megatron:
    needs: [pr-gate]
    if: needs.pr-gate.outputs.megatron == 'true'
    uses: ./.github/workflows/_example_tests_runner.yml
    permissions:
      contents: read
    secrets: inherit
    with:
      docker_image: "nvcr.io/nvidia/nemo:26.08"
      example: megatron_bridge
      # PR runs single-GPU (~11 min); the nightly runs every test multi-GPU (~58 min).
      timeout_minutes: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 30 || 75 }}
      pip_install_extras: "[hf,puzzletron,dev-test]"
      runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
      allow_failure_examples: ${{ vars.ALLOW_FAILURE_EXAMPLE_TESTS }}

  ##### Diffusers Example Tests #####
  diffusers:
    needs: [pr-gate]
    if: needs.pr-gate.outputs.diffusers == 'true'
    uses: ./.github/workflows/_example_tests_runner.yml
    permissions:
      contents: read
    secrets: inherit
    with:
      # Retain the existing environment while retiring the ONNX/TensorRT workflow.
      docker_image: "nvcr.io/nvidia/tensorrt:26.05-py3"
      example: diffusers
      timeout_minutes: 60
      pip_install_extras: "[onnx,hf,dev-test]"
      runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
      allow_failure_examples: ${{ vars.ALLOW_FAILURE_EXAMPLE_TESTS }}

  ##### ONNX/TensorRT Example Tests #####
  onnx:
    needs: [pr-gate]
    if: needs.pr-gate.outputs.onnx == 'true'
    strategy:
      fail-fast: false
      matrix:
        example: [torch_onnx, torch_trt]
    uses: ./.github/workflows/_example_tests_runner.yml
    permissions:
      contents: read
    secrets: inherit
    with:
      # Pinned to 26.05 (TensorRT 10): torch-tensorrt is capped at <2.13 (== 2.12.1),
      # which needs libnvinfer.so.10; newer tensorrt containers drop it. Bump only once
      # a torch-tensorrt build for the newer TensorRT is available.
      docker_image: "nvcr.io/nvidia/tensorrt:26.05-py3"
      example: ${{ matrix.example }}
      timeout_minutes: 45
      pip_install_extras: "[onnx,hf,dev-test]"
      runner: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && 'linux-amd64-gpu-rtxpro6000-latest-1' || 'linux-amd64-gpu-rtxpro6000-latest-2' }}
      allow_failure_examples: ${{ vars.ALLOW_FAILURE_EXAMPLE_TESTS }}

  ##### Required Check for PR #####
  example-pr-required-check:
    # Run even if example tests are skipped
    if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }}
    needs: [pr-gate, torch, vllm, trtllm, megatron, diffusers, onnx]
    runs-on: ubuntu-24.04
    steps:
      - name: Stacked PR, example tests deferred
        if: needs.pr-gate.outputs.stacked == 'true'
        run: |
          echo "::error::This PR targets another PR's branch, so example tests are deferred until it targets main. After retargeting, push or use 'Re-run all jobs'."
          exit 1
      - name: Required example tests did not succeed
        if: |
          needs.pr-gate.result != 'success' ||
          (needs.pr-gate.outputs.any_changed == 'true' && (
            !contains(fromJSON('["success", "skipped"]'), needs.torch.result) ||
            !contains(fromJSON('["success", "skipped"]'), needs.vllm.result) ||
            !contains(fromJSON('["success", "skipped"]'), needs.trtllm.result) ||
            !contains(fromJSON('["success", "skipped"]'), needs.megatron.result) ||
            !contains(fromJSON('["success", "skipped"]'), needs.diffusers.result) ||
            !contains(fromJSON('["success", "skipped"]'), needs.onnx.result)
          ))
        run: exit 1