lib-crawlee/.github/workflows/test-e2e.yml at master · code/lib-crawlee · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
name: E2E tests

on:
    workflow_dispatch:
    schedule:
        # Runs at 2 am every day
        -   cron: '0 2 * * *'

concurrency:
    group: ${{ github.workflow }}-${{ github.ref }}
    cancel-in-progress: true

jobs:
    build_and_test:
        name: Build & Test
        runs-on: ubuntu-22.04

        strategy:
            fail-fast: false
            matrix:
                # TODO(v4): re-enable LOCAL once @apify/storage-local supports v4, see https://github.com/apify/crawlee/issues/3406
                storage: [ MEMORY, PLATFORM ]

        steps:
            -   name: Checkout repository
                uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1

            -   name: Use Node.js 24
                uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
                with:
                    node-version: 24
                    package-manager-cache: false

            -   name: Turbo cache
                id: turbo-cache
                uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
                with:
                    path: .turbo
                    key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
                    restore-keys: |
                        turbo-${{ github.job }}-${{ github.ref_name }}-


            -   name: Setup Apify CLI
                uses: apify/setup-apify-cli-action@main
                with:
                    version: 'beta'
                    token: ${{ secrets.APIFY_SCRAPER_TESTS_API_TOKEN }}

            -   name: Add Apify secrets for E2E tests
                run: apify secrets add anthropicApiKey ${{ secrets.ANTHROPIC_API_KEY }}

            -   uses: apify/workflows/pnpm-install@main

            -   name: Cache Playwright browsers
                if: (matrix.storage != 'PLATFORM')
                uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
                with:
                    path: ~/.cache/ms-playwright
                    key: playwright-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }}
                    restore-keys: |
                        playwright-${{ runner.os }}-

            -   name: Install Playwright Dependencies
                if: (matrix.storage != 'PLATFORM')
                timeout-minutes: 10
                run: pnpm exec playwright install --with-deps

            -   name: Build
                run: pnpm ci:build

            -   name: Test with storage ${{ matrix.storage }}
                run: pnpm test:e2e
                env:
                    STORAGE_IMPLEMENTATION: ${{ matrix.storage }}
                    # Provided explicitly: the CLI may store the token in the OS keyring, where the tests can't read it.
                    APIFY_TOKEN: ${{ secrets.APIFY_SCRAPER_TESTS_API_TOKEN }}
                    APIFY_HTTPBIN_TOKEN: ${{ secrets.APIFY_HTTPBIN_TOKEN }}
                    ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
                    GITHUB_TOKEN: ${{ github.token }}