diff --git a/.github/workflows/qa-android-critical-flow-tests.yml b/.github/workflows/qa-android-critical-flow-tests.yml index 0e35bd755f..4db3d7e16f 100644 --- a/.github/workflows/qa-android-critical-flow-tests.yml +++ b/.github/workflows/qa-android-critical-flow-tests.yml @@ -23,13 +23,13 @@ on: type: string isUpgrade: - description: "Upgrade test? If true, oldBuildNumber is REQUIRED." + description: "Upgrade test? If true, oldBuildNumber is optional and defaults to the previous APK." required: true default: false type: boolean oldBuildNumber: - description: "For upgrade runs: type the old build number here" + description: "For upgrade runs: optional old build number. Leave empty to auto-pick the previous APK." required: false default: "" type: string @@ -98,7 +98,7 @@ jobs: outputs: resolvedAppBuildNumber: ${{ steps.resolve_run_inputs.outputs.appBuildNumber }} - resolvedIsUpgrade: ${{ steps.resolve_run_inputs.outputs.isUpgrade }} + resolvedIsUpgrade: ${{ steps.resolve_run_inputs.outputs.isUpgrade == 'true' || steps.resolve_selector.outputs.category == 'upgrade' }} resolvedOldBuildNumber: ${{ steps.resolve_run_inputs.outputs.oldBuildNumber }} resolvedEnforceAppInstall: ${{ steps.resolve_run_inputs.outputs.enforceAppInstall }} resolvedFlavor: ${{ steps.resolve_run_inputs.outputs.flavor }} @@ -138,7 +138,7 @@ jobs: { echo "appBuildNumber=latest" - echo "isUpgrade=false" + echo "isUpgrade=true" echo "oldBuildNumber=" echo "enforceAppInstall=false" echo "flavor=internal release candidate" @@ -179,6 +179,7 @@ jobs: env: WORKFLOW_REF: ${{ github.ref_name }} FLAVOR_INPUT: ${{ steps.resolve_run_inputs.outputs.flavor }} + IS_UPGRADE: ${{ steps.resolve_run_inputs.outputs.isUpgrade == 'true' || steps.resolve_selector.outputs.category == 'upgrade' }} RESOLVED_TESTCASE_ID: ${{ steps.resolve_selector.outputs.testCaseId }} RESOLVED_CATEGORY: ${{ steps.resolve_selector.outputs.category }} TESTINY_RUN_NAME: ${{ steps.resolve_run_inputs.outputs.testinyRunName }} @@ -297,7 +298,53 @@ jobs: RERUN_FAILED_ENABLED: ${{ needs.validate-and-resolve-inputs.outputs.resolvedRerunFailedEnabled }} RERUN_FAILED_COUNT: ${{ needs.validate-and-resolve-inputs.outputs.resolvedRerunFailedCount }} ALLURE_RESULTS_ROOT: ${{ runner.temp }}/allure-results - run: bash scripts/qa_android_ui_tests/run_ui_tests.sh + run: | + if [[ "${IS_UPGRADE}" == "true" && ( "${RESOLVED_CATEGORY}" == "upgrade" || -n "${RESOLVED_TESTCASE_ID}" ) ]]; then + # Single upgrade-tagged runs do not need the normal phase. They only + # pin execution to one device and install the old APK before launch. + upgrade_device="${DEVICE_LIST%% *}" + DEVICE_LIST="${upgrade_device}" \ + DEVICE_COUNT="1" \ + APP_APK_BEFORE_ATTEMPT="${OLD_APK_DEVICE_PATH:-/data/local/tmp/Wire.old.apk}" \ + RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-upgrade" \ + bash scripts/qa_android_ui_tests/run_ui_tests.sh + elif [[ "${IS_UPGRADE}" == "true" && -z "${RESOLVED_TESTCASE_ID}" ]]; then + # For broad upgrade runs, run the selected normal tests on the new APK first. + # Then run only upgrade tests afterward, starting them from the old APK. + upgrade_device="${DEVICE_LIST%% *}" + normal_category="${RESOLVED_CATEGORY}" + normal_status=0 + upgrade_status=0 + + echo "Running normal tests first on the new APK, excluding upgrade tests." + RESOLVED_CATEGORY="${normal_category}" \ + EXCLUDE_CATEGORY="upgrade" \ + RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-normal" \ + ALLURE_RESULTS_ROOT="${RUNNER_TEMP}/allure-results/normal" \ + bash scripts/qa_android_ui_tests/run_ui_tests.sh || normal_status=$? + + echo "Running upgrade tests last with old APK installed before each attempt." + DEVICE_LIST="${upgrade_device}" \ + DEVICE_COUNT="1" \ + RESOLVED_CATEGORY="upgrade" \ + APP_APK_BEFORE_ATTEMPT="${OLD_APK_DEVICE_PATH:-/data/local/tmp/Wire.old.apk}" \ + RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-upgrade" \ + ALLURE_RESULTS_ROOT="${RUNNER_TEMP}/allure-results/upgrade" \ + bash scripts/qa_android_ui_tests/run_ui_tests.sh || upgrade_status=$? + + RETRY_STATE_DIRS="${RUNNER_TEMP}/retry-state-normal ${RUNNER_TEMP}/retry-state-upgrade" \ + COMBINED_RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-combined" \ + bash scripts/qa_android_ui_tests/reporting.sh combine-retry-state + + if (( normal_status != 0 )); then + exit "${normal_status}" + fi + if (( upgrade_status != 0 )); then + exit "${upgrade_status}" + fi + else + bash scripts/qa_android_ui_tests/run_ui_tests.sh + fi # Export one standard artifact so future manual deflake runs can reuse # this workflow's exact run context and leftover failed test list. @@ -320,7 +367,7 @@ jobs: RESOLVED_CATEGORY: ${{ needs.validate-and-resolve-inputs.outputs.resolvedCategory }} APP_BUILD_NUMBER_INPUT: ${{ needs.validate-and-resolve-inputs.outputs.resolvedAppBuildNumber }} IS_UPGRADE: ${{ needs.validate-and-resolve-inputs.outputs.resolvedIsUpgrade }} - OLD_BUILD_NUMBER: ${{ needs.validate-and-resolve-inputs.outputs.resolvedOldBuildNumber }} + OLD_BUILD_NUMBER: ${{ steps.download_apks.outputs.OLD_BUILD_NUMBER }} ENFORCE_APP_INSTALL: ${{ needs.validate-and-resolve-inputs.outputs.resolvedEnforceAppInstall }} TESTINY_RUN_NAME: ${{ needs.validate-and-resolve-inputs.outputs.resolvedTestinyRunName }} ANDROID_DEVICE_ID: ${{ needs.validate-and-resolve-inputs.outputs.resolvedAndroidDeviceId }} diff --git a/.github/workflows/qa-android-ui-test-manual-deflake.yml b/.github/workflows/qa-android-ui-test-manual-deflake.yml index 57a8a54e1c..bb24c062d8 100644 --- a/.github/workflows/qa-android-ui-test-manual-deflake.yml +++ b/.github/workflows/qa-android-ui-test-manual-deflake.yml @@ -166,7 +166,51 @@ jobs: RERUN_FAILED_ENABLED: ${{ steps.validate_deflake_input.outputs.rerunFailedEnabled }} RERUN_FAILED_COUNT: ${{ steps.validate_deflake_input.outputs.rerunFailedCount }} ALLURE_RESULTS_ROOT: ${{ runner.temp }}/allure-results - run: bash scripts/qa_android_ui_tests/run_ui_tests.sh + run: | + if [[ "${IS_UPGRADE}" == "true" ]]; then + upgrade_failed_file="${RUNNER_TEMP}/deflake-upgrade-failed-tests.txt" + normal_failed_file="${RUNNER_TEMP}/deflake-normal-failed-tests.txt" + normal_status=0 + upgrade_status=0 + retry_state_dirs="" + grep 'tests\.UpgradeVersion#' "${INITIAL_FAILED_TESTS_FILE}" > "${upgrade_failed_file}" || true + grep -v 'tests\.UpgradeVersion#' "${INITIAL_FAILED_TESTS_FILE}" > "${normal_failed_file}" || true + + if [[ -s "${normal_failed_file}" ]]; then + INITIAL_FAILED_TESTS_FILE="${normal_failed_file}" \ + RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-normal-deflake" \ + ALLURE_RESULTS_ROOT="${RUNNER_TEMP}/allure-results/normal" \ + bash scripts/qa_android_ui_tests/run_ui_tests.sh || normal_status=$? + retry_state_dirs="${retry_state_dirs} ${RUNNER_TEMP}/retry-state-normal-deflake" + fi + + if [[ -s "${upgrade_failed_file}" ]]; then + upgrade_device="${DEVICE_LIST%% *}" + DEVICE_LIST="${upgrade_device}" \ + DEVICE_COUNT="1" \ + INITIAL_FAILED_TESTS_FILE="${upgrade_failed_file}" \ + APP_APK_BEFORE_ATTEMPT="${OLD_APK_DEVICE_PATH:-/data/local/tmp/Wire.old.apk}" \ + RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-upgrade-deflake" \ + ALLURE_RESULTS_ROOT="${RUNNER_TEMP}/allure-results/upgrade" \ + bash scripts/qa_android_ui_tests/run_ui_tests.sh || upgrade_status=$? + retry_state_dirs="${retry_state_dirs} ${RUNNER_TEMP}/retry-state-upgrade-deflake" + fi + + if [[ -n "${retry_state_dirs// /}" ]]; then + RETRY_STATE_DIRS="${retry_state_dirs}" \ + COMBINED_RETRY_STATE_DIR="${RUNNER_TEMP}/retry-state-deflake-combined" \ + bash scripts/qa_android_ui_tests/reporting.sh combine-retry-state + fi + + if (( normal_status != 0 )); then + exit "${normal_status}" + fi + if (( upgrade_status != 0 )); then + exit "${upgrade_status}" + fi + else + bash scripts/qa_android_ui_tests/run_ui_tests.sh + fi # Publish a fresh deflake bundle so a later run can deflake this manual # deflake execution without going back to qa-android-critical-flow-tests.yml. @@ -189,7 +233,7 @@ jobs: RESOLVED_CATEGORY: ${{ steps.validate_deflake_input.outputs.selectorType == 'category' && steps.validate_deflake_input.outputs.selectorValue || '' }} APP_BUILD_NUMBER_INPUT: ${{ steps.validate_deflake_input.outputs.resolvedBuildNumber || steps.validate_deflake_input.outputs.appBuildNumberInput }} IS_UPGRADE: ${{ steps.validate_deflake_input.outputs.isUpgrade }} - OLD_BUILD_NUMBER: ${{ steps.validate_deflake_input.outputs.oldBuildNumber }} + OLD_BUILD_NUMBER: ${{ steps.download_apks.outputs.OLD_BUILD_NUMBER }} ENFORCE_APP_INSTALL: ${{ steps.validate_deflake_input.outputs.enforceAppInstall }} TESTINY_RUN_NAME: ${{ steps.validate_deflake_input.outputs.effectiveTestinyRunName }} ANDROID_DEVICE_ID: ${{ steps.validate_deflake_input.outputs.androidDeviceId }} diff --git a/scripts/qa_android_ui_tests/execution_setup.sh b/scripts/qa_android_ui_tests/execution_setup.sh index 05ca3921d2..319977ee0d 100755 --- a/scripts/qa_android_ui_tests/execution_setup.sh +++ b/scripts/qa_android_ui_tests/execution_setup.sh @@ -92,6 +92,8 @@ download_apks() { local new_s3_key="" local old_s3_key="" + local new_apk_name="" + local old_apk_name="" while IFS= read -r line || [[ -n "${line}" ]]; do [[ -z "${line}" ]] && continue if [[ "${line}" != *=* ]]; then @@ -119,6 +121,12 @@ download_apks() { OLD_S3_KEY) old_s3_key="${value}" ;; + NEW_APK_NAME) + new_apk_name="${value}" + ;; + OLD_APK_NAME) + old_apk_name="${value}" + ;; esac done < "${apk_env_file}" @@ -133,6 +141,9 @@ download_apks() { test -s "${new_apk_path}" if [[ "${IS_UPGRADE:-}" == "true" ]]; then + echo "OLD_APK_NAME=${old_apk_name}" + echo "NEW_APK_NAME=${new_apk_name}" + if [[ -z "${old_s3_key}" ]]; then echo "ERROR: Missing OLD_S3_KEY for upgrade flow" exit 1 @@ -198,6 +209,8 @@ install_apks_on_devices() { : "${NEW_APK_PATH:?NEW_APK_PATH missing}" : "${GITHUB_ENV:?GITHUB_ENV not set}" + # Export stable device paths so upgrade tests can install the new APK during + # the in-test upgrade step. CI also keeps the old APK available for setup. local new_apk_device_path="/data/local/tmp/Wire.new.apk" local old_apk_device_path="/data/local/tmp/Wire.old.apk" echo "NEW_APK_DEVICE_PATH=${new_apk_device_path}" >> "$GITHUB_ENV" @@ -228,15 +241,15 @@ install_apks_on_devices() { if [[ "${IS_UPGRADE:-}" == "true" ]]; then : "${OLD_APK_PATH:?OLD_APK_PATH missing for upgrade}" - # Upgrade tests need both APKs on the device because instrumentation - # receives those paths and performs the in-test upgrade flow itself. + # Keep both APK files available on the device for the in-test upgrade + # flow. Remove stale copies first so retries do not reuse an older file. ${adb_cmd} shell rm -f "${new_apk_device_path}" "${old_apk_device_path}" || true ${adb_cmd} push "${OLD_APK_PATH}" "${old_apk_device_path}" >/dev/null ${adb_cmd} push "${NEW_APK_PATH}" "${new_apk_device_path}" >/dev/null - ${adb_cmd} install ${install_flags} "${OLD_APK_PATH}" - else - ${adb_cmd} install ${install_flags} "${NEW_APK_PATH}" fi + # Always install the selected new APK during general setup. The upgrade + # phase explicitly reinstalls the old device-side APK before instrumentation. + ${adb_cmd} install ${install_flags} "${NEW_APK_PATH}" if ! ${adb_cmd} shell pm list packages | grep -qx "package:${APP_ID}"; then echo "ERROR: '${APP_ID}' not installed on ${serial}." diff --git a/scripts/qa_android_ui_tests/merge_allure_results.py b/scripts/qa_android_ui_tests/merge_allure_results.py index 767c56b4ff..6a946eb856 100755 --- a/scripts/qa_android_ui_tests/merge_allure_results.py +++ b/scripts/qa_android_ui_tests/merge_allure_results.py @@ -39,9 +39,10 @@ def contains_result_files(device_dir: Path) -> bool: src_dir = src_candidate if src_candidate.is_dir() else device_dir return src_dir.is_dir() and any(src_dir.glob("*-result.json")) - # Discover retry-aware layout first: OUT_DIR/attempt-N//... + # Discover retry-aware layout first. Upgrade runs can keep phase results + # under OUT_DIR//attempt-N//..., so search recursively. attempt_dirs = [] - for candidate in sorted(base_dir.iterdir()): + for candidate in sorted(base_dir.rglob("attempt-*")): if not candidate.is_dir(): continue if not candidate.name.startswith("attempt-"): diff --git a/scripts/qa_android_ui_tests/reporting.sh b/scripts/qa_android_ui_tests/reporting.sh index df31b5a7ce..a4868fa803 100755 --- a/scripts/qa_android_ui_tests/reporting.sh +++ b/scripts/qa_android_ui_tests/reporting.sh @@ -4,7 +4,7 @@ set -euo pipefail # Reporting and publication utilities for QA Android UI test workflows. usage() { - echo "Usage: $0 {remove-runtime-secrets|pull-allure-results|prepare-deflake-bundle|merge-allure-results|summarize-allure-results|generate-allure-report|publish-allure-report|cleanup-workspace}" >&2 + echo "Usage: $0 {remove-runtime-secrets|pull-allure-results|combine-retry-state|prepare-deflake-bundle|merge-allure-results|summarize-allure-results|generate-allure-report|publish-allure-report|cleanup-workspace}" >&2 exit 2 } @@ -22,12 +22,10 @@ pull_allure_results() { mkdir -p "${out_dir}" # Retry-aware runs already persist per-attempt results during execution. - if compgen -G "${out_dir}/attempt-*" >/dev/null; then - if find "${out_dir}"/attempt-* -type f -name '*-result.json' -print -quit | grep -q .; then - echo "Per-attempt Allure results already present under ${out_dir}; skipping fallback pull." - return - fi - echo "Attempt folders exist but no result files found yet; running fallback pull." + # Upgrade runs may store phase results one level deeper under this root. + if find "${out_dir}" -type f -name '*-result.json' -print -quit | grep -q .; then + echo "Per-attempt Allure results already present under ${out_dir}; skipping fallback pull." + return fi if [[ -z "${DEVICE_LIST:-}" ]]; then @@ -47,6 +45,58 @@ pull_allure_results() { done } +combine_retry_state() { + : "${RETRY_STATE_DIRS:?RETRY_STATE_DIRS not set}" + : "${COMBINED_RETRY_STATE_DIR:?COMBINED_RETRY_STATE_DIR not set}" + + # Upgrade runs can produce separate retry states for the normal and upgrade + # phases. Merge them back into one contract for the deflake artifact. + mkdir -p "${COMBINED_RETRY_STATE_DIR}" + local first_failed_file="${COMBINED_RETRY_STATE_DIR}/first-attempt-failed.txt" + local final_failed_file="${COMBINED_RETRY_STATE_DIR}/final-failed.txt" + : > "${first_failed_file}" + : > "${final_failed_file}" + + read -ra STATE_DIRS <<< "${RETRY_STATE_DIRS}" + for state_dir in "${STATE_DIRS[@]}"; do + [[ -d "${state_dir}" ]] || continue + + if [[ -s "${state_dir}/first-attempt-failed.txt" ]]; then + cat "${state_dir}/first-attempt-failed.txt" >> "${first_failed_file}" + fi + + local latest_failed_file="" + while IFS= read -r candidate; do + latest_failed_file="${candidate}" + done < <(find "${state_dir}" -maxdepth 1 -type f -name 'attempt-*-failed.txt' | sort -V) + + if [[ -n "${latest_failed_file}" && -s "${latest_failed_file}" ]]; then + cat "${latest_failed_file}" >> "${final_failed_file}" + fi + done + + sort -u "${first_failed_file}" -o "${first_failed_file}" + sort -u "${final_failed_file}" -o "${final_failed_file}" + + local first_failed_count + local final_failed_count + local passed_on_rerun_count=0 + first_failed_count="$(grep -cve '^[[:space:]]*$' "${first_failed_file}" || true)" + final_failed_count="$(grep -cve '^[[:space:]]*$' "${final_failed_file}" || true)" + if (( first_failed_count > final_failed_count )); then + passed_on_rerun_count=$((first_failed_count - final_failed_count)) + fi + + { + echo "RETRY_STATE_DIR=${COMBINED_RETRY_STATE_DIR}" + echo "FIRST_FAILED_TESTS_FILE=${first_failed_file}" + echo "FINAL_FAILED_TESTS_FILE=${final_failed_file}" + echo "FIRST_FAILED_TESTS_COUNT=${first_failed_count}" + echo "FINAL_FAILED_TESTS_COUNT=${final_failed_count}" + echo "PASSED_ON_RERUN_COUNT=${passed_on_rerun_count}" + } >> "${GITHUB_ENV}" +} + prepare_deflake_bundle() { : "${DEFLAKE_BUNDLE_DIR:?DEFLAKE_BUNDLE_DIR not set}" @@ -237,6 +287,14 @@ cleanup_workspace() { rm -f "secrets.json" "${RUNNER_TEMP}/secrets.json" || true rm -f "${RUNNER_TEMP}/Wire.apk" "${RUNNER_TEMP}/Wire.old.apk" || true + if [[ -n "${DEVICE_LIST:-}" ]]; then + # Remove APK copies pushed for upgrade runs; each new job pushes fresh files. + read -ra DEVICES <<< "${DEVICE_LIST}" + for serial in "${DEVICES[@]}"; do + adb -s "${serial}" shell rm -f /data/local/tmp/Wire.old.apk /data/local/tmp/Wire.new.apk || true + done + fi + rm -rf "${ALLURE_RESULTS_DIR}" || true rm -rf "${ALLURE_RESULTS_MERGED_DIR}" || true rm -rf "${ALLURE_REPORT_DIR}" || true @@ -255,6 +313,9 @@ case "${1:-}" in pull-allure-results) pull_allure_results ;; + combine-retry-state) + combine_retry_state + ;; prepare-deflake-bundle) prepare_deflake_bundle ;; diff --git a/scripts/qa_android_ui_tests/run_ui_tests.sh b/scripts/qa_android_ui_tests/run_ui_tests.sh index a31c8b824f..adbf96836a 100755 --- a/scripts/qa_android_ui_tests/run_ui_tests.sh +++ b/scripts/qa_android_ui_tests/run_ui_tests.sh @@ -58,7 +58,7 @@ PULL_BASE_DELAY_SEC="$((10#${ALLURE_PULL_BASE_DELAY_SEC}))" INLINE_PART_MAX_CHARS="$((10#${RERUN_INLINE_PART_MAX_CHARS}))" LOG_DIR="${RUNNER_TEMP}/instrumentation-logs" -STATE_DIR="${RUNNER_TEMP}/retry-state" +STATE_DIR="${RETRY_STATE_DIR:-${RUNNER_TEMP}/retry-state}" mkdir -p "${LOG_DIR}" "${STATE_DIR}" "${ALLURE_RESULTS_ROOT}" read -ra DEVICES <<< "${DEVICE_LIST}" @@ -172,6 +172,41 @@ count_tests_in_list_file() { wc -l < "${list_file}" | tr -d ' ' } +install_app_before_attempt_if_needed() { + local attempt="$1" + # The first argument is the attempt number and the remaining arguments are device serials. + # shift removes the first argument, so "$@" only contains device serials for the install loop. + shift + local devices=("$@") + local apk_path="${APP_APK_BEFORE_ATTEMPT:-}" + + if [[ -z "${apk_path}" ]]; then + return + fi + + echo "Installing app APK before attempt ${attempt}: ${apk_path}" + for serial in "${devices[@]}"; do + adb -s "${serial}" wait-for-device + # Upgrade attempts must start from the old APK. Uninstall first because + # Android rejects installing an older version over an already-newer app. + adb -s "${serial}" uninstall "${APP_ID}" >/dev/null 2>&1 || true + if [[ "${apk_path}" == /data/local/tmp/* ]]; then + local output + output="$(adb -s "${serial}" shell pm install -r -g "${apk_path}" 2>&1 | tr -d '\r' || true)" + if [[ "${output}" != *"Success"* ]]; then + echo "ERROR: Failed to install app APK on ${serial} from ${apk_path}. Output: ${output:-}" + exit 1 + fi + else + if [[ ! -s "${apk_path}" ]]; then + echo "ERROR: APP_APK_BEFORE_ATTEMPT does not exist or is empty: ${apk_path}" + exit 1 + fi + adb -s "${serial}" install -r "${apk_path}" >/dev/null + fi + done +} + rerun_list_file_for_device() { local attempt="$1" local serial="$2" @@ -380,6 +415,9 @@ run_attempt_on_devices() { if [[ -n "${RESOLVED_CATEGORY:-}" ]]; then args+=(-e category "${RESOLVED_CATEGORY}") fi + if [[ -n "${EXCLUDE_CATEGORY:-}" ]]; then + args+=(-e excludeCategory "${EXCLUDE_CATEGORY}") + fi else local retry_list_file local retry_test_count @@ -412,7 +450,6 @@ run_attempt_on_devices() { if [[ "${IS_UPGRADE:-}" == "true" ]]; then args+=(-e newApkPath "${NEW_APK_DEVICE_PATH}") - args+=(-e oldApkPath "${OLD_APK_DEVICE_PATH}") fi local log_file="${LOG_DIR}/attempt-${attempt}-instrument-${serial}.log" @@ -506,6 +543,7 @@ while true; do fi echo "=== Attempt ${attempt} ===" + install_app_before_attempt_if_needed "${attempt}" "${attempt_devices[@]}" attempt_worker_failed=0 if ! run_attempt_on_devices "${attempt}" "${attempt_num_shards}" "${attempt_devices[@]}"; then attempt_worker_failed=1 diff --git a/scripts/qa_android_ui_tests/select_apks.py b/scripts/qa_android_ui_tests/select_apks.py index 8a82cb000f..72722d1948 100755 --- a/scripts/qa_android_ui_tests/select_apks.py +++ b/scripts/qa_android_ui_tests/select_apks.py @@ -127,10 +127,7 @@ def normalize_direct(value: str): else: new_key = pick_by_substring(app_build) if is_upgrade: - if not old_input: - print("ERROR: isUpgrade=true but oldBuildNumber is empty.", file=sys.stderr) - sys.exit(1) - old_key = pick_by_substring(old_input) + old_key = pick_by_substring(old_input) if old_input else second_latest_key if not new_key: print(f"ERROR: Could not resolve NEW apk for appBuildNumber='{app_build}'", file=sys.stderr) diff --git a/scripts/qa_android_ui_tests/validation.sh b/scripts/qa_android_ui_tests/validation.sh index 34a7f4cdcb..96c5cea625 100755 --- a/scripts/qa_android_ui_tests/validation.sh +++ b/scripts/qa_android_ui_tests/validation.sh @@ -14,12 +14,9 @@ trim() { } validate_upgrade_inputs() { - # Upgrade mode is only valid when the workflow also knows which old build to - # install before the in-test upgrade flow begins. - if [[ "${IS_UPGRADE:-}" == "true" && -z "${OLD_BUILD_NUMBER:-}" ]]; then - echo "ERROR: oldBuildNumber is REQUIRED when isUpgrade=true" - exit 1 - fi + # oldBuildNumber is optional for upgrade runs. When empty, APK selection uses + # the previous APK as the old version and the selected appBuildNumber as new. + : } validate_rerun_inputs() { @@ -90,6 +87,7 @@ resolve_selector_from_tags() { print_resolved_values() { echo "workflowRef=${WORKFLOW_REF:-}" echo "flavor=${FLAVOR_INPUT:-}" + echo "resolvedIsUpgrade=${IS_UPGRADE:-}" echo "resolvedTestCaseId=${RESOLVED_TESTCASE_ID:-}" echo "resolvedCategory=${RESOLVED_CATEGORY:-}" echo "testinyRunName=${TESTINY_RUN_NAME:-}" diff --git a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/UiAutomatorSetup.kt b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/UiAutomatorSetup.kt index ab3a882dab..341c644528 100644 --- a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/UiAutomatorSetup.kt +++ b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/UiAutomatorSetup.kt @@ -116,19 +116,13 @@ object UiAutomatorSetup { reportUpgradeLog("Upgrading Wire using APK: $apkPath") val output = device.executeShellCommand("pm install -r -d -g $apkPath").trim() - - if (!output.contains("Success")) { - val installOutput = output.ifBlank { "" } - throw IllegalStateException( - "Failed to upgrade Wire using APK from '$apkPath'. Output: $installOutput" - ) - } - val versionAfterUpgrade = getInstalledWireVersion() reportUpgradeLog("Installed Wire after upgrade: $versionAfterUpgrade") if (versionAfterUpgrade.versionCode <= versionBeforeUpgrade.versionCode) { + val installOutput = output.ifBlank { "" } throw IllegalStateException( - "Wire was not upgraded. Before: $versionBeforeUpgrade. After: $versionAfterUpgrade" + "Wire was not upgraded using APK from '$apkPath'. " + + "Before: $versionBeforeUpgrade. After: $versionAfterUpgrade. Install output: $installOutput" ) } diff --git a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedFilter.kt b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedFilter.kt index 63a06b6409..52d0c69e5d 100644 --- a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedFilter.kt +++ b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedFilter.kt @@ -33,14 +33,16 @@ import java.io.File /** * JUnit filter used by AndroidJUnitRunner / AllureAndroidJUnitRunner. * - * Supports normal selector args (@TestCaseId, @Category, @Tag), - * and rerun selector args (Class#method). + * Supports normal selector args (@TestCaseId, @Category, @Tag, excludeCategory), + * and rerun selector args (Class#method). excludeCategory skips tests that also + * have that category, for example normal test phases skip upgrade tests. */ class TaggedFilter : Filter() { private val args = InstrumentationRegistry.getArguments() private val filterTestCaseId: String? = args.getString("testCaseId") private val filterCategory: String? = args.getString("category") + private val excludeCategory: String? = args.getString("excludeCategory") private val filterTagKey: String? = args.getString("tagKey") private val filterTagValue: String? = args.getString("tagValue") @@ -64,6 +66,7 @@ class TaggedFilter : Filter() { // No filters -> include everything if (filterTestCaseId == null && filterCategory == null && + excludeCategory == null && filterTagKey == null && filterTagValue == null ) { @@ -155,11 +158,19 @@ class TaggedFilter : Filter() { } private fun matchesFilters(description: Description): Boolean { - val annos = description.annotations + val annotations = description.annotations + val categories = annotations.filterIsInstance() + + excludeCategory?.let { excludedCat -> + val matchesExcludedCat = categories.any { catAnno -> + catAnno.value.contains(excludedCat) + } + if (matchesExcludedCat) return false + } // 1) TestCaseId filterTestCaseId?.let { wantedId -> - val testCaseAnno = annos.filterIsInstance().firstOrNull() + val testCaseAnno = annotations.filterIsInstance().firstOrNull() if (testCaseAnno == null || testCaseAnno.value != wantedId) { return false } @@ -167,10 +178,9 @@ class TaggedFilter : Filter() { // 2) Category (Category(vararg val value: String)) filterCategory?.let { wantedCat -> - val cats = annos.filterIsInstance() - if (cats.isEmpty()) return false + if (categories.isEmpty()) return false - val matchesCat = cats.any { catAnno -> + val matchesCat = categories.any { catAnno -> catAnno.value.contains(wantedCat) } if (!matchesCat) return false @@ -178,7 +188,7 @@ class TaggedFilter : Filter() { // 3) Tag (key + value) if (filterTagKey != null || filterTagValue != null) { - val tags = annos.filterIsInstance() + val tags = annotations.filterIsInstance() if (tags.isEmpty()) return false val matchesTag = tags.any { tag -> @@ -194,7 +204,8 @@ class TaggedFilter : Filter() { override fun describe(): String { return "TaggedFilter(testCaseId=$filterTestCaseId, " + - "category=$filterCategory, tagKey=$filterTagKey, tagValue=$filterTagValue, " + + "category=$filterCategory, excludeCategory=$excludeCategory, " + + "tagKey=$filterTagKey, tagValue=$filterTagValue, " + "rerunModeEnabled=$rerunModeEnabled, rerunAttempt=$rerunAttempt)" } } diff --git a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedTestRunner.kt b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedTestRunner.kt index c1fc6c8043..c04c5a5f29 100644 --- a/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedTestRunner.kt +++ b/tests/testsSupport/src/androidTest/kotlin/com/wire/android/tests/support/suite/TaggedTestRunner.kt @@ -34,6 +34,7 @@ class TaggedTestRunner : AllureAndroidJUnitRunner() { override fun onCreate(arguments: Bundle) { val filterId = arguments.getString("testCaseId") val category = arguments.getString("category") + val excludeCategory = arguments.getString("excludeCategory") val tagKey = arguments.getString("tagKey") val tagValue = arguments.getString("tagValue") val rerunMode = arguments.getString(RetryContract.ARG_ENABLE_RERUN_MODE) @@ -48,7 +49,8 @@ class TaggedTestRunner : AllureAndroidJUnitRunner() { Log.i( "TaggedTestRunner", "onCreate called. " + - "testCaseId=$filterId, category=$category, tagKey=$tagKey, tagValue=$tagValue, " + + "testCaseId=$filterId, category=$category, excludeCategory=$excludeCategory, " + + "tagKey=$tagKey, tagValue=$tagValue, " + "rerunMode=$rerunMode, rerunAttempt=$rerunAttempt, " + "rerunListPath=$rerunListPath, rerunListInlineLength=${rerunListInline?.length ?: 0}, " + "rerunListInlinePartCount=$rerunListInlinePartCount"