code-foundry/.github/workflows/eval.yml at main · 0xPlayerOne/code-foundry · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
name: Code Foundry Eval

on:
  workflow_call:
    inputs:
      runtime-repository:
        description: Repository containing the Code Foundry runtime.
        required: false
        type: string
        default: 0xPlayerOne/code-foundry
      runtime-ref:
        description: Code Foundry runtime tag or ref.
        required: false
        type: string
        default: v1.0.5
      runner:
        description: >-
          Runner used by the eval harness. Browser evals need a
          Chrome-capable runner such as ubuntu-latest; ubuntu-slim does not
          ship a browser.
        required: false
        type: string
        default: ubuntu-latest
      artifact-prefix:
        description: Prefix for task receipts; use a unique value when calling this workflow more than once.
        required: false
        type: string
        default: task-result

permissions:
  contents: read

env:
  TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
  TURBO_TEAM: ${{ vars.TURBO_TEAM }}
  REPO_FOUNDRY_PROFILE: ${{ vars.REPO_FOUNDRY_PROFILE }}
  REPO_FOUNDRY_LANGUAGES: ${{ vars.REPO_FOUNDRY_LANGUAGES }}
  REPO_FOUNDRY_FEATURES: ${{ vars.REPO_FOUNDRY_FEATURES }}
  REPO_FOUNDRY_PACKAGE_MANAGER: ${{ vars.REPO_FOUNDRY_PACKAGE_MANAGER }}
  REPO_FOUNDRY_RUNNER: ${{ vars.REPO_FOUNDRY_RUNNER }}
  REPO_FOUNDRY_CACHE_PACKAGES: ${{ vars.REPO_FOUNDRY_CACHE_PACKAGES || 'auto' }}
  REPO_FOUNDRY_CACHE_BUILD: ${{ vars.REPO_FOUNDRY_CACHE_BUILD || 'auto' }}

concurrency:
  group: code-foundry-eval-${{ github.event_name }}-${{ github.event.pull_request.head.repo.full_name || github.repository }}-${{ github.event.pull_request.head.ref || github.ref_name }}
  # Superseded pull-request runs are cancelled so the newest head owns the
  # group; push and scheduled runs queue instead, because cancelling a main
  # push can drop default-branch analysis or release-adjacent work.
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

jobs:
  eval:
    name: Eval
    if: vars.CI_BILLING_PAUSED != 'true'
    runs-on: ${{ inputs.runner }}
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false
      - name: Runtime
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
        with:
          persist-credentials: false
          repository: ${{ inputs.runtime-repository }}
          ref: ${{ inputs.runtime-ref }}
          path: .github/.code-foundry
          sparse-checkout: |
            .github/actions
            src/lib
            src/runtime-core.mjs
            src/runtime.mjs
      - name: Install runtime
        run: |
          mkdir -p .github/actions
          mv .github/.code-foundry "$RUNNER_TEMP/code-foundry"
          cp -R "$RUNNER_TEMP/code-foundry/.github/actions/." .github/actions/
      - name: Detect
        id: applicability
        env:
          EVENT_NAME: ${{ github.event_name }}
          BASE_REF: ${{ github.event.pull_request.base.ref }}
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
          BEFORE_SHA: ${{ github.event.before }}
        run: node "$RUNNER_TEMP/code-foundry/src/runtime.mjs" ci should_run eval >> "$GITHUB_OUTPUT"
      - name: Setup
        if: steps.applicability.outputs.applicable == 'true' && steps.applicability.outputs.affected != 'false'
        uses: ./.github/actions/setup
        with:
          # Eval harnesses run through the repository's own package scripts and
          # need their dependencies installed before the task executes.
          install: 'true'
          cache-build: 'true'
          task: eval
      - name: Eval
        id: execute
        if: steps.applicability.outputs.applicable == 'true' && steps.applicability.outputs.affected != 'false'
        run: node "$RUNNER_TEMP/code-foundry/src/runtime.mjs" ci eval
      - name: Retain task receipt
        if: >-
          always() &&
          (steps.execute.outcome == 'success' || steps.execute.outcome == 'failure' ||
          steps.applicability.outputs.applicable == 'false' || steps.applicability.outputs.affected == 'false') &&
          hashFiles('.code-foundry/results/eval.json') != ''
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: ${{ inputs.artifact-prefix }}-${{ github.run_id }}-${{ github.run_attempt }}-eval
          path: .code-foundry/results/eval.json
          include-hidden-files: true
          if-no-files-found: error
          retention-days: 14
      - name: Retain eval report
        if: >-
          always() &&
          steps.execute.outcome == 'failure' &&
          hashFiles('eval-results/result.json') != ''
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7
        with:
          name: ${{ inputs.artifact-prefix }}-${{ github.run_id }}-${{ github.run_attempt }}-eval-report
          path: |
            eval-results/result.json
            eval-results/summary.json
          if-no-files-found: error
          retention-days: 14