coder_eval/.github/workflows/harbor-e2e.yml at main · UiPath/coder_eval · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
name: Harbor E2E

# Required on every PR: exercises real Docker builds, a real `harbor` install,
# and (for the llm_judge scenario) a real model call, so it is the one gate
# that actually round-trips a task through Harbor's own export/agent/verifier
# contract rather than coder-eval's in-process tests. workflow_dispatch lets a
# maintainer run it on demand; the nightly schedule catches drift between
# coder-eval's own release and Harbor's own upstream releases even when no PR
# touched anything.
on:
  workflow_dispatch:
  schedule:
    - cron: "17 5 * * *" # nightly, off the hour to avoid GitHub's peak-load pile-up
  push:
    branches: [main]
  pull_request:
    branches: [main]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  TELEMETRY_ENABLED: "false"
  # Route through Bedrock, mirroring pr-checks.yml's smoke-pass job -- keeps
  # Anthropic-credit spend off this path; DirectRoute is exercised elsewhere.
  API_BACKEND: "bedrock"
  CLAUDE_CODE_USE_BEDROCK: "1"

jobs:
  harbor-e2e:
    name: Harbor export + CoderEvalAgent round trip
    # A fork PR's own workflow, scripts, uv.lock and Dockerfiles all run for a
    # `pull_request` event -- and this job does two `docker build`s plus a real
    # Bedrock model call. Fork PRs get no secrets (below) so the job cannot pass
    # for one anyway; skip rather than run untrusted code on the shared pool and
    # leave a required check permanently red for every external contributor.
    if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository
    runs-on: uipath-ubuntu-latest
    timeout-minutes: 20

    steps:
      - name: Checkout code
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2

      - name: Set up Python 3.13
        uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: "3.13"

      - name: Set up Node.js 20
        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020  # v4.4.0
        with:
          node-version: "20"

      - name: Install Claude CLI
        run: npm install -g @anthropic-ai/claude-code

      - name: Cache dependencies
        uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae  # v5.0.5
        with:
          path: |
            ~/.cache/uv
            ~/.cache/pip
          key: ${{ runner.os }}-py3.13-harbor-e2e-${{ hashFiles('pyproject.toml', 'uv.lock') }}
          restore-keys: |
            ${{ runner.os }}-py3.13-harbor-e2e-

      - name: Install uv
        run: |
          python -m pip install --upgrade "pip>=26.2"
          pip install uv

      - name: Install project dependencies (hash-verified from uv.lock)
        # --extra harbor installs `harbor` into the SAME venv as coder-eval:
        # CoderEvalAgent is resolved by `hb run -a
        # coder_eval.harbor.agent:CoderEvalAgent` on the HOST process, so
        # `harbor` and `coder_eval` must be importable from the same
        # interpreter (see harbor/agent.py's module docstring). The version is
        # pinned once, in pyproject.toml's `harbor` extra -- bump it there.
        run: uv sync --frozen --extra dev --extra harbor

      - name: Put the project venv on PATH
        run: echo "${{ github.workspace }}/.venv/bin" >> "$GITHUB_PATH"

      - name: Build coder-eval-agent base Docker image
        run: make docker-image

      - name: Build BYOD template Docker image
        run: docker build -t byod-custom-image:0.1.0 templates/byod_smoke_test/

      - name: Run Harbor E2E scenarios
        # Bedrock credentials are scoped to this one step, not the workflow-level
        # env: above (where earlier revisions left them) -- so npm/pip/docker build
        # steps that fetch third-party code never see them in their environment.
        env:
          AWS_BEARER_TOKEN_BEDROCK: ${{ secrets.AWS_BEARER_TOKEN_BEDROCK }}
          AWS_REGION: ${{ secrets.AWS_REGION }}
          BEDROCK_MODEL: ${{ secrets.BEDROCK_MODEL }}
        run: python .github/scripts/harbor_e2e.py

      - name: Upload failing scenario artifacts
        # A failing scenario's full export/ + jobs/ tree (docker/agent logs,
        # every task.json/trajectory.json, artifacts/ workspaces) zipped by the
        # script above -- so a failure can be inspected after the fact instead
        # of guessed at from stdout. Empty on a fully green run; `if: failure()`
        # still uploads whatever any failing scenario left behind.
        if: failure()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
        with:
          name: harbor-e2e-failure-artifacts
          path: tmp/harbor_e2e_failures/
          retention-days: 14
          if-no-files-found: ignore