diff --git a/.github/actions/integration-tests/action.yml b/.github/actions/integration-tests/action.yml index 2a009a6ff46faf..58be31aa4d5349 100644 --- a/.github/actions/integration-tests/action.yml +++ b/.github/actions/integration-tests/action.yml @@ -27,12 +27,6 @@ inputs: test_build_ref: description: "The build ref of the test run. Used in the IDE integration tests." required: false - integration_test_username: - description: "The username for integration test" - required: true - integration_test_usertoken: - description: "The username for integration test" - required: true identity_provider: description: "GCP workload identity provider" required: true @@ -83,8 +77,6 @@ runs: shell: bash env: ROBOQUAT_TOKEN: ${{ inputs.github_token }} - INTEGRATION_TEST_USERNAME: ${{ inputs.integration_test_username }} - INTEGRATION_TEST_USER_TOKEN: ${{ inputs.integration_test_usertoken }} PREVIEW_NAME: ${{ inputs.preview_name }} TEST_USE_LATEST_VERSION: ${{ inputs.latest_ide_version }} TEST_BUILD_ID: ${{ inputs.test_build_id }} @@ -122,6 +114,11 @@ runs: paths: "test/**/TEST-*.xml" show: "all" if: always() + - name: Explain skipped IDE coverage + if: ${{ always() && contains(fromJSON('["ide", "jetbrains", "vscode", "ssh", "all", ""]'), inputs.test_suite) }} + shell: bash + run: | + printf '%s\n' 'GitHub-backed IDE tests skip in CI because no GitHub test-user credentials are supplied. They remain available for manually configured runs. Skipped tests do not validate IDE or SSH gateway functionality. See test/README.md.' >> "$GITHUB_STEP_SUMMARY" - name: Slack Notification uses: rtCamp/action-slack-notify@v2 if: ${{ (success() || failure()) && inputs.notify_slack_webhook != '' }} diff --git a/.github/workflows/branch-build.yml b/.github/workflows/branch-build.yml index 29bbb573b6ea88..b09b3b7c80b3ca 100644 --- a/.github/workflows/branch-build.yml +++ b/.github/workflows/branch-build.yml @@ -576,8 +576,6 @@ jobs: identity_provider: ${{ secrets.DEV_PREVIEW_PROVIDER }} service_account: ${{ secrets.DEV_PREVIEW_SA }} leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }} - integration_test_username: ${{ secrets.WORKSPACE_INTEGRATION_TEST_USERNAME }} - integration_test_usertoken: ${{ secrets.WORKSPACE_INTEGRATION_TEST_USER_TOKEN }} workspace-integration-tests-main: name: "Run workspace integration tests on main branch" diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ceb284534475a1..276991baa5713b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -614,8 +614,6 @@ jobs: identity_provider: ${{ secrets.DEV_PREVIEW_PROVIDER }} service_account: ${{ secrets.DEV_PREVIEW_SA }} leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }} - integration_test_username: ${{ secrets.WORKSPACE_INTEGRATION_TEST_USERNAME }} - integration_test_usertoken: ${{ secrets.WORKSPACE_INTEGRATION_TEST_USER_TOKEN }} workspace-integration-tests-main: name: "Run workspace integration tests on main branch" diff --git a/.github/workflows/ide-integration-tests.yml b/.github/workflows/ide-integration-tests.yml index 5c1da72397fc19..2f72dcbe95b0e8 100644 --- a/.github/workflows/ide-integration-tests.yml +++ b/.github/workflows/ide-integration-tests.yml @@ -145,8 +145,6 @@ jobs: shell: bash env: ROBOQUAT_TOKEN: ${{ secrets.GITHUB_TOKEN }} - USERNAME: ${{ secrets.IDE_INTEGRATION_TEST_USERNAME }} - USER_TOKEN: ${{ secrets.IDE_INTEGRATION_TEST_USER_TOKEN }} PREVIEW_NAME: ${{ needs.configuration.outputs.name }} TEST_BUILD_ID: ${{ github.run_id }} TEST_BUILD_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} @@ -165,7 +163,6 @@ jobs: args=() args+=( "-kubeconfig=$HOME/.kube/config" ) args+=( "-namespace=default" ) - [[ "$USERNAME" != "" ]] && args+=( "-username=$USERNAME" ) args+=( "-timeout=60m" ) IDE_TESTS_DIR="$GITHUB_WORKSPACE/test/tests/ide" @@ -201,13 +198,18 @@ jobs: with: paths: "test/tests/**/TEST-*.xml" if: always() + - name: Explain skipped IDE coverage + if: always() + shell: bash + run: | + printf '%s\n' 'The 13 IDE integration tests skip in this workflow because no GitHub test-user credentials are supplied. They remain available for manually configured runs. Deployment/readiness and skipped-test reports provide no functional IDE or SSH gateway coverage.' >> "$GITHUB_STEP_SUMMARY" - name: Slack Notification uses: rtCamp/action-slack-notify@cdf0a2130cbcdfd82ba5fcac8e076370bf381b36 # pin@v2 if: success() || failure() env: SLACK_WEBHOOK: ${{ secrets.IDE_SLACK_WEBHOOK }} SLACK_COLOR: ${{ job.status }} - SLACK_MESSAGE: ${{ steps.test_summary.outputs.passed }}/${{ steps.test_summary.outputs.total }} tests passed + SLACK_MESSAGE: "GitHub-backed IDE tests skip in CI (no test-user credentials). ${{ steps.test_summary.outputs.passed }} passed, ${{ steps.test_summary.outputs.failed }} failed, ${{ steps.test_summary.outputs.skipped }} skipped." SLACK_FOOTER: "" delete: diff --git a/.github/workflows/preview-env-check-regressions.yml b/.github/workflows/preview-env-check-regressions.yml index 824692f0c4c4d9..52dfafedfea41a 100644 --- a/.github/workflows/preview-env-check-regressions.yml +++ b/.github/workflows/preview-env-check-regressions.yml @@ -110,8 +110,6 @@ jobs: shell: bash env: ROBOQUAT_TOKEN: ${{ secrets.GITHUB_TOKEN }} - USERNAME: ${{ secrets.IDE_INTEGRATION_TEST_USERNAME }} - USER_TOKEN: ${{ secrets.IDE_INTEGRATION_TEST_USER_TOKEN }} PREVIEW_NAME: ${{ needs.configuration.outputs.name }} run: | set -euo pipefail @@ -126,7 +124,6 @@ jobs: args=() args+=( "-kubeconfig=/home/gitpod/.kube/config" ) args+=( "-namespace=default" ) - [[ "$USERNAME" != "" ]] && args+=( "-username=$USERNAME" ) args+=( "-timeout=60m" ) TESTS_DIR="$GITHUB_WORKSPACE/test/tests/smoke-test" @@ -150,6 +147,11 @@ jobs: with: paths: "test/tests/**/TEST.xml" if: always() + - name: Explain skipped workspace smoke coverage + if: always() + shell: bash + run: | + printf '%s\n' 'The workspace creation/image-build smoke test skips in CI because no GitHub test-user credentials are supplied; it remains available manually. Gitpod API smoke tests require explicit opt-in and separate Gitpod credentials, which this workflow does not supply. Skipped-only runs provide no functional smoke coverage.' >> "$GITHUB_STEP_SUMMARY" - id: auth if: failure() uses: google-github-actions/auth@955352c3b43196640b567e4646256d2fbb4aa1c7 # pin@v1 diff --git a/.github/workflows/workspace-integration-tests.yml b/.github/workflows/workspace-integration-tests.yml index 3d61ac2737a3cd..5415b1869bf389 100644 --- a/.github/workflows/workspace-integration-tests.yml +++ b/.github/workflows/workspace-integration-tests.yml @@ -174,8 +174,6 @@ jobs: identity_provider: ${{ secrets.DEV_PREVIEW_PROVIDER }} service_account: ${{ secrets.DEV_PREVIEW_SA }} leeway_segment_key: ${{ secrets.LEEWAY_SEGMENT_KEY }} - integration_test_username: ${{ secrets.WORKSPACE_INTEGRATION_TEST_USERNAME }} - integration_test_usertoken: ${{ secrets.WORKSPACE_INTEGRATION_TEST_USER_TOKEN }} delete: name: Delete preview environment diff --git a/test/README.md b/test/README.md index 7612c89e9aadb3..045bbe1dd172e4 100644 --- a/test/README.md +++ b/test/README.md @@ -22,17 +22,19 @@ Such tests are for example: ## Automatically at Gitpod -You can opt-in to run the integrations tests as part of the build job. that runs the integration tests against preview environments. +The **Branch Build** workflow runs webapp tests when the PR description selects: - > For tests that require an existing user the framework tries to automatically select one from the DB. - > - On preview envs make sure to create one before running tests against it! - > - If it's important to use a certain user (with fixed settings, for example) pass the additional `username` parameter. +```markdown +- [x] with-integration-tests=webapp +``` -Example command: +This builds and deploys the branch to a large preview and runs the server/database +suite. The **Workspace integration tests** workflow always runs `workspace`. -```console -werft job run github -a with-preview=true -a with-integration-tests=webapp -f -``` +CI does not supply GitHub test-user credentials. Tests requiring them skip; +other workspace, component, and webapp tests continue to run. Default IDE and +workspace-creation smoke runs have no functional coverage when all tests skip. +The implementations and manual entry points remain available. ## Manually @@ -69,9 +71,27 @@ If you want to run an entire test suite, the easiest is to use `./test/run.sh`: If you're iterating on a single test, the easiest is to use `go test` directly. -If your integration tests depends on having having a user token available, then you'll have to set `USER_NAME` and `USER_TOKEN` environment variables. This can be done a couple ways: -1. Get credentials persisted as secrets (either in Github Actions, or GCP Secret Manager via the `core-dev` project), which vary by job that trigger tests. Refer to `run.sh` for details. -2. In your Gitpod (preview) environment, log into the preview environment, set `USER_NAME` to the user you logged in with, and set `USER_TOKEN` to any (does not have to be valid). +For GitHub-backed tests, explicitly supply `USER_NAME` (or `-username`) and +`USER_TOKEN`, where `USER_TOKEN` is a **GitHub user token**. The runner preserves +these manual inputs and no longer loads credentials from CI environment aliases +or the Kubernetes test-user secret. Use a preview with a working GitHub auth +provider. Disk tests require the selected user to already have a usable GitHub +identity/token in the preview database; they skip when no username is supplied. + +```sh +export USER_NAME='' +export USER_TOKEN='' +./test/run.sh -s workspace +``` + +Without these variables, credential-dependent tests skip and the remaining tests +use their builtin or temporary user paths. IDE tests retain their additional +setup requirements; see [JetBrains manual instructions](../dev/jetbrains-test/README.md). + +The opt-in collaborator smoke tests use `USER_TOKEN` for a different purpose: a +**Gitpod PAT or session cookie**, with `TEST_COLLABORATOR=true`. Temporary-token +smoke tests use `INSTALLATION_ADMIN_PAT` / `MEMBER_USER_PAT` and +`TEST_CREATE_TMP_TOKEN=true`. Those interfaces are unchanged. ```console cd test diff --git a/test/pkg/integration/disk-client.go b/test/pkg/integration/disk-client.go index a5c1ebda50c8af..72a90567a58f71 100644 --- a/test/pkg/integration/disk-client.go +++ b/test/pkg/integration/disk-client.go @@ -36,7 +36,7 @@ func (d DiskClient) Fallocate(testFilePath string, spaceToAllocate string) error return fmt.Errorf("returned returned rc: %d err: %v", resp.ExitCode, resp.Stderr) } if strings.Contains(resp.Stdout, NoSpaceErrorMsg) { - return fmt.Errorf(resp.Stdout) + return fmt.Errorf("%s", resp.Stdout) } return nil diff --git a/test/pkg/integration/setup.go b/test/pkg/integration/setup.go index 6b31e30ac3146f..6a1f94f8fe26cb 100644 --- a/test/pkg/integration/setup.go +++ b/test/pkg/integration/setup.go @@ -249,7 +249,7 @@ func logGitpodStatus(t *testing.T, client klient.Client, namespace string) { } } tw.Flush() - t.Logf("Gitpod components status:\n" + buf.String()) + t.Logf("Gitpod components status:\n%s", buf.String()) } func isPreviewReady(client klient.Client, namespace string) (ready bool, reason string, err error) { diff --git a/test/run.sh b/test/run.sh index a869d27fcb13b7..422904e6c6eeaf 100755 --- a/test/run.sh +++ b/test/run.sh @@ -83,21 +83,8 @@ args+=( "-kubeconfig=${KUBECONFIG:-/home/gitpod/.kube/config}" ) args+=( "-namespace=${NAMESPACE:-default}" ) args+=( "-timeout=120m" ) -if [[ "${GITPOD_REPO_ROOT:-}" != "" ]]; then - echo "Running in Gitpod workspace. Fetching USER_NAME and USER_TOKEN" - USER_NAME="$(kubectl --context=dev -n werft get secret integration-test-user -o jsonpath='{.data.username}' | base64 -d)" - USER_TOKEN="$(kubectl --context=dev -n werft get secret integration-test-user -o jsonpath='{.data.token}' | base64 -d)" - export USER_NAME - export USER_TOKEN -else - echo "Using INTEGRATION_TEST_USERNAME and INTEGRATION_TEST_USER_TOKEN for USER_NAME and USER_TOKEN" - USER_NAME="${INTEGRATION_TEST_USERNAME}" - USER_TOKEN="${INTEGRATION_TEST_USER_TOKEN}" - export USER_NAME - export USER_TOKEN -fi - -[[ "$USER_NAME" != "" ]] && args+=( "-username=$USER_NAME" ) +# Tests use builtin or temporary Gitpod users unless one is explicitly selected. +[[ -n "${USER_NAME:-}" ]] && args+=( "-username=$USER_NAME" ) go install github.com/jstemmer/go-junit-report/v2@latest diff --git a/test/tests/workspace/disk_test.go b/test/tests/workspace/disk_test.go index f14e327c7bee30..c742f08e3ee773 100644 --- a/test/tests/workspace/disk_test.go +++ b/test/tests/workspace/disk_test.go @@ -28,6 +28,8 @@ type DiskTest struct { } func TestDiskActions(t *testing.T) { + integration.SkipWithoutUsername(t, username) + tests := []DiskTest{ { Name: "xfs-quota-is_exceeded", diff --git a/test/tests/workspace/process_limit_test.go b/test/tests/workspace/process_limit_test.go index 251019de9d07ac..e3b8c441f968bf 100644 --- a/test/tests/workspace/process_limit_test.go +++ b/test/tests/workspace/process_limit_test.go @@ -103,8 +103,9 @@ func TestProcessLimit(t *testing.T) { } t.Logf("checking output for fork errors due to process limiting") - if !strings.Contains(res.Stdout, "bash: fork: retry: Resource temporarily unavailable") { - t.Errorf("expected fork error (Resource temporarily unavailable), but got none (%d): %s", res.ExitCode, res.Stdout) + // Exec captures stderr separately, including bash's fork diagnostics. + if !strings.Contains(res.Stdout+res.Stderr, "bash: fork: retry: Resource temporarily unavailable") { + t.Errorf("expected fork error (Resource temporarily unavailable), but got none (%d): stdout: %s\nstderr: %s", res.ExitCode, res.Stdout, res.Stderr) } return testCtx