diff --git a/.github/scripts/expand-ci-module-roots.sh b/.github/scripts/expand-ci-module-roots.sh new file mode 100644 index 000000000000..179e9af2ce32 --- /dev/null +++ b/.github/scripts/expand-ci-module-roots.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +group_roots="${GROUP_ROOTS:-}" +modules=() + +add_module() { + local module="$1" + local existing + + for existing in "${modules[@]}"; do + if [[ "${existing}" == "${module}" ]]; then + return + fi + done + + modules+=("${module}") +} + +IFS=',' read -ra roots <<< "${group_roots}" +for root in "${roots[@]}"; do + [[ -n "${root}" ]] || continue + + if [[ ! -f "${root}/pom.xml" ]]; then + echo "Maven module root is missing: ${root}" >&2 + exit 1 + fi + + if grep -q "" "${root}/pom.xml"; then + while IFS= read -r pom; do + add_module "$(dirname "${pom}")" + done < <(find "${root}" -name pom.xml | sort) + else + add_module "${root}" + fi +done + +if [[ "${#modules[@]}" -eq 0 ]]; then + echo "No Maven modules were resolved from GROUP_ROOTS." >&2 + exit 1 +fi + +IFS=, +printf '%s\n' "${modules[*]}" diff --git a/.github/scripts/resolve-ci-modules.sh b/.github/scripts/resolve-ci-modules.sh new file mode 100644 index 000000000000..5c505bc9b0c6 --- /dev/null +++ b/.github/scripts/resolve-ci-modules.sh @@ -0,0 +1,116 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +changed_files_json="${CHANGED_FILES_JSON:-[]}" +max_modules="${MAX_CI_MODULES:-8}" +has_code_changes=false +full_build_required=false +modules=() + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to resolve changed Maven modules." >&2 + exit 1 +fi + +add_module() { + local module="$1" + local existing + + for existing in "${modules[@]}"; do + if [[ "${existing}" == "${module}" ]]; then + return + fi + done + + modules+=("${module}") +} + +find_module() { + local path="$1" + local dir + + if [[ -d "${path}" ]]; then + dir="${path}" + else + dir="$(dirname "${path}")" + fi + + while [[ "${dir}" != "." && "${dir}" != "/" ]]; do + if [[ -f "${dir}/pom.xml" ]]; then + printf '%s\n' "${dir#./}" + return + fi + dir="$(dirname "${dir}")" + done +} + +is_ignored_change() { + local file="$1" + + case "${file}" in + .github/*|*.md|*.txt|resources/static/*|.asf.yaml|.gitignore|.licenserc.yaml|LICENSE|NOTICE|*/LICENSE|*/NOTICE) + return 0 + ;; + esac + + return 1 +} + +while IFS= read -r file; do + [[ -n "${file}" ]] || continue + + if is_ignored_change "${file}"; then + continue + fi + + has_code_changes=true + + case "${file}" in + pom.xml|mvnw|mvnw.cmd|.mvn/*|actions/*) + full_build_required=true + ;; + esac + + if [[ "$(basename "${file}")" == "pom.xml" && -f "${file}" ]] && grep -q "" "${file}"; then + full_build_required=true + fi + + if [[ "${file}" == *.java || "${file}" == *.xml || "${file}" == *.yml || "${file}" == *.yaml || "${file}" == *.properties || "${file}" == *.sh ]]; then + module="$(find_module "${file}")" + if [[ -n "${module:-}" ]]; then + add_module "${module}" + fi + fi +done < <(printf '%s' "${changed_files_json}" | jq -r '.[]') + +if [[ "${has_code_changes}" == "true" && ("${#modules[@]}" -eq 0 || "${#modules[@]}" -gt "${max_modules}") ]]; then + full_build_required=true +fi + +modules_csv="$(IFS=,; printf '%s' "${modules[*]}")" + +{ + echo "has_code_changes=${has_code_changes}" + echo "modules=${modules_csv}" + echo "full_build_required=${full_build_required}" +} >> "${GITHUB_OUTPUT}" + +echo "Has code changes: ${has_code_changes}" +echo "Resolved modules: ${modules_csv:-}" +echo "Full build required: ${full_build_required}" diff --git a/.github/scripts/resolve-test-case-matrix.sh b/.github/scripts/resolve-test-case-matrix.sh new file mode 100644 index 000000000000..92edbdd907cd --- /dev/null +++ b/.github/scripts/resolve-test-case-matrix.sh @@ -0,0 +1,343 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +mode="${CI_CASE_MODE:-${1:-e2e}}" +changed_files_json="${CHANGED_FILES_JSON:-[]}" +full_required=false +storage_cases=() +e2e_cases=() +integration_cases=() +run_k8s_ingress=false +run_k8s_examples=false + +if ! command -v jq >/dev/null 2>&1; then + echo "jq is required to resolve test case matrices." >&2 + exit 1 +fi + +add_unique() { + local target="$1" + local value="$2" + local existing + local -n values="${target}" + + for existing in "${values[@]}"; do + if [[ "${existing}" == "${value}" ]]; then + return + fi + done + + values+=("${value}") +} + +add_storage_all() { + add_unique storage_cases "e2e-h2-compose" + add_unique storage_cases "e2e-mysql-compose" + add_unique storage_cases "e2e-postgres-compose" + add_unique storage_cases "e2e-opengauss-compose" +} + +add_e2e_all() { + add_storage_all + add_unique e2e_cases "e2e-http-sync-compose" + add_unique e2e_cases "e2e-springcloud-sync-compose" + add_unique e2e_cases "e2e-apache-dubbo-sync-compose" + add_unique e2e_cases "e2e-grpc-sync-compose" + add_unique e2e_cases "e2e-websocket-sync-compose" + add_unique e2e_cases "e2e-logging-rocketmq-compose" +} + +add_integration_all() { + add_unique integration_cases "shenyu-integrated-test-apache-dubbo" + add_unique integration_cases "shenyu-integrated-test-grpc" + add_unique integration_cases "shenyu-integrated-test-http" + add_unique integration_cases "shenyu-integrated-test-https" + add_unique integration_cases "shenyu-integrated-test-spring-cloud" + add_unique integration_cases "shenyu-integrated-test-websocket" + add_unique integration_cases "shenyu-integrated-test-rewrite" + add_unique integration_cases "shenyu-integrated-test-combination" + add_unique integration_cases "shenyu-integrated-test-sdk-apache-dubbo" + add_unique integration_cases "shenyu-integrated-test-sdk-http" +} + +is_ignored_change() { + local file="$1" + + case "${file}" in + .github/*|*.md|*.txt|resources/static/*|.asf.yaml|.gitignore|.licenserc.yaml|LICENSE|NOTICE) + return 0 + ;; + esac + + return 1 +} + +resolve_k8s_change() { + local file="$1" + + if [[ "${mode}" == "k8s-ingress" ]]; then + case "${file}" in + shenyu-integrated-test-k8s-ingress*/*|shenyu-*/*|pom.xml|*/pom.xml|shenyu-examples/*) + run_k8s_ingress=true + ;; + esac + return 0 + fi + + if [[ "${mode}" == "k8s-examples-http" ]]; then + case "${file}" in + shenyu-examples/*|shenyu-*/*|pom.xml|*/pom.xml) + run_k8s_examples=true + ;; + esac + return 0 + fi + + return 1 +} + +mark_full_if_shared() { + local file="$1" + + case "${file}" in + pom.xml|mvnw|mvnw.cmd|.mvn/*|actions/*) + full_required=true + ;; + shenyu-common/*|shenyu-web/*|shenyu-bootstrap/*|shenyu-admin/*|shenyu-admin-listener/*|shenyu-sync-data-center/*) + full_required=true + ;; + shenyu-plugin/pom.xml|shenyu-plugin/shenyu-plugin-api/*|shenyu-plugin/shenyu-plugin-base/*) + full_required=true + ;; + shenyu-spring-boot-starter/pom.xml|shenyu-spring-boot-starter/shenyu-spring-boot-starter-plugin/pom.xml) + full_required=true + ;; + esac +} + +map_direct_case_path() { + local file="$1" + + if [[ "${mode}" == "e2e" ]]; then + case "${file}" in + shenyu-e2e/pom.xml|shenyu-e2e/shenyu-e2e-common/*|shenyu-e2e/shenyu-e2e-client/*|shenyu-e2e/shenyu-e2e-engine/*|shenyu-e2e/shenyu-e2e-case/pom.xml|shenyu-e2e/shenyu-e2e-case/compose/*|shenyu-e2e/shenyu-e2e-case/k8s/*) + full_required=true + ;; + shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-storage/*) + add_storage_all + ;; + shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-http/*) + add_unique e2e_cases "e2e-http-sync-compose" + ;; + shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-spring-cloud/*) + add_unique e2e_cases "e2e-springcloud-sync-compose" + ;; + shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-apache-dubbo/*) + add_unique e2e_cases "e2e-apache-dubbo-sync-compose" + ;; + shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-grpc/*) + add_unique e2e_cases "e2e-grpc-sync-compose" + ;; + shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-websocket/*) + add_unique e2e_cases "e2e-websocket-sync-compose" + ;; + shenyu-e2e/shenyu-e2e-case/shenyu-e2e-case-logging-rocketmq/*) + add_unique e2e_cases "e2e-logging-rocketmq-compose" + ;; + esac + else + case "${file}" in + shenyu-integrated-test/pom.xml|shenyu-integrated-test/shenyu-integrated-test-common/*) + full_required=true + ;; + shenyu-integrated-test/shenyu-integrated-test-apache-dubbo/*) + add_unique integration_cases "shenyu-integrated-test-apache-dubbo" + ;; + shenyu-integrated-test/shenyu-integrated-test-grpc/*) + add_unique integration_cases "shenyu-integrated-test-grpc" + ;; + shenyu-integrated-test/shenyu-integrated-test-http/*) + add_unique integration_cases "shenyu-integrated-test-http" + ;; + shenyu-integrated-test/shenyu-integrated-test-https/*) + add_unique integration_cases "shenyu-integrated-test-https" + ;; + shenyu-integrated-test/shenyu-integrated-test-spring-cloud/*) + add_unique integration_cases "shenyu-integrated-test-spring-cloud" + ;; + shenyu-integrated-test/shenyu-integrated-test-websocket/*) + add_unique integration_cases "shenyu-integrated-test-websocket" + ;; + shenyu-integrated-test/shenyu-integrated-test-rewrite/*) + add_unique integration_cases "shenyu-integrated-test-rewrite" + ;; + shenyu-integrated-test/shenyu-integrated-test-combination/*) + add_unique integration_cases "shenyu-integrated-test-combination" + ;; + shenyu-integrated-test/shenyu-integrated-test-sdk-apache-dubbo/*) + add_unique integration_cases "shenyu-integrated-test-sdk-apache-dubbo" + ;; + shenyu-integrated-test/shenyu-integrated-test-sdk-http/*) + add_unique integration_cases "shenyu-integrated-test-sdk-http" + ;; + esac + fi +} + +map_domain_path() { + local file="$1" + + case "${file}" in + db/*) + if [[ "${mode}" == "e2e" ]]; then + add_storage_all + else + add_unique integration_cases "shenyu-integrated-test-http" + fi + ;; + *apache-dubbo*|*shenyu-plugin-dubbo*|*shenyu-client-dubbo*|*shenyu-examples-dubbo*) + if [[ "${mode}" == "e2e" ]]; then + add_unique e2e_cases "e2e-apache-dubbo-sync-compose" + else + add_unique integration_cases "shenyu-integrated-test-apache-dubbo" + add_unique integration_cases "shenyu-integrated-test-sdk-apache-dubbo" + fi + ;; + *grpc*) + if [[ "${mode}" == "e2e" ]]; then + add_unique e2e_cases "e2e-grpc-sync-compose" + else + add_unique integration_cases "shenyu-integrated-test-grpc" + fi + ;; + *spring-cloud*|*springcloud*) + if [[ "${mode}" == "e2e" ]]; then + add_unique e2e_cases "e2e-springcloud-sync-compose" + else + add_unique integration_cases "shenyu-integrated-test-spring-cloud" + fi + ;; + *websocket*) + if [[ "${mode}" == "e2e" ]]; then + add_unique e2e_cases "e2e-websocket-sync-compose" + else + add_unique integration_cases "shenyu-integrated-test-websocket" + fi + ;; + *rocketmq*) + if [[ "${mode}" == "e2e" ]]; then + add_unique e2e_cases "e2e-logging-rocketmq-compose" + else + add_unique integration_cases "shenyu-integrated-test-http" + fi + ;; + *logging*) + if [[ "${mode}" == "e2e" ]]; then + add_unique e2e_cases "e2e-logging-rocketmq-compose" + else + add_unique integration_cases "shenyu-integrated-test-http" + fi + ;; + *rewrite*) + if [[ "${mode}" == "e2e" ]]; then + add_unique e2e_cases "e2e-http-sync-compose" + else + add_unique integration_cases "shenyu-integrated-test-rewrite" + fi + ;; + *https*) + if [[ "${mode}" == "e2e" ]]; then + add_unique e2e_cases "e2e-http-sync-compose" + else + add_unique integration_cases "shenyu-integrated-test-https" + fi + ;; + *http*|*springmvc*|*divide*) + if [[ "${mode}" == "e2e" ]]; then + add_unique e2e_cases "e2e-http-sync-compose" + else + add_unique integration_cases "shenyu-integrated-test-http" + add_unique integration_cases "shenyu-integrated-test-sdk-http" + fi + ;; + shenyu-sdk/*|shenyu-examples/shenyu-examples-sdk/*) + if [[ "${mode}" == "integration" ]]; then + add_unique integration_cases "shenyu-integrated-test-sdk-http" + add_unique integration_cases "shenyu-integrated-test-sdk-apache-dubbo" + fi + ;; + esac +} + +while IFS= read -r file; do + [[ -n "${file}" ]] || continue + + if is_ignored_change "${file}"; then + continue + fi + + if resolve_k8s_change "${file}"; then + continue + fi + + if [[ "$(basename "${file}")" == "pom.xml" && -f "${file}" ]] && grep -q "" "${file}"; then + full_required=true + fi + + mark_full_if_shared "${file}" + map_direct_case_path "${file}" + map_domain_path "${file}" +done < <(printf '%s' "${changed_files_json}" | jq -r '.[]') + +if [[ "${full_required}" == "true" ]]; then + if [[ "${mode}" == "e2e" ]]; then + add_e2e_all + else + add_integration_all + fi +fi + +storage_matrix="$(printf '%s\n' "${storage_cases[@]}" | jq -R . | jq -cs '{include: map(select(length > 0) | {case:"shenyu-e2e-case-storage", script:.})}')" +e2e_matrix="$(printf '%s\n' "${e2e_cases[@]}" | jq -R . | jq -cs '{include: map(select(length > 0) | {case:(if . == "e2e-http-sync-compose" then "shenyu-e2e-case-http" elif . == "e2e-springcloud-sync-compose" then "shenyu-e2e-case-spring-cloud" elif . == "e2e-apache-dubbo-sync-compose" then "shenyu-e2e-case-apache-dubbo" elif . == "e2e-grpc-sync-compose" then "shenyu-e2e-case-grpc" elif . == "e2e-websocket-sync-compose" then "shenyu-e2e-case-websocket" else "shenyu-e2e-case-logging-rocketmq" end), script:.})}')" +integration_matrix="$(printf '%s\n' "${integration_cases[@]}" | jq -R . | jq -cs '{include: map(select(length > 0) | {case:.})}')" + +run_storage=$([[ "${#storage_cases[@]}" -gt 0 ]] && echo true || echo false) +run_e2e_cases=$([[ "${#e2e_cases[@]}" -gt 0 ]] && echo true || echo false) +run_e2e=$([[ "${run_storage}" == "true" || "${run_e2e_cases}" == "true" ]] && echo true || echo false) +run_integration=$([[ "${#integration_cases[@]}" -gt 0 ]] && echo true || echo false) + +{ + echo "run_storage=${run_storage}" + echo "run_e2e_cases=${run_e2e_cases}" + echo "run_e2e=${run_e2e}" + echo "storage_matrix=${storage_matrix}" + echo "e2e_matrix=${e2e_matrix}" + echo "run_integration=${run_integration}" + echo "integration_matrix=${integration_matrix}" + echo "full_required=${full_required}" + echo "run_k8s_ingress=${run_k8s_ingress}" + echo "run_k8s_examples=${run_k8s_examples}" +} >> "${GITHUB_OUTPUT}" + +echo "Full required: ${full_required}" +echo "Storage matrix: ${storage_matrix}" +echo "E2E matrix: ${e2e_matrix}" +echo "Integration matrix: ${integration_matrix}" +echo "Run k8s ingress: ${run_k8s_ingress}" +echo "Run k8s examples: ${run_k8s_examples}" diff --git a/.github/scripts/restore-shenyu-local-repo.sh b/.github/scripts/restore-shenyu-local-repo.sh new file mode 100644 index 000000000000..d89c1eca62ee --- /dev/null +++ b/.github/scripts/restore-shenyu-local-repo.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -euo pipefail + +artifact_dir="${1:-/tmp/shenyu-m2}" +target_dir="${HOME}/.m2/repository/org/apache" + +if [[ ! -d "${artifact_dir}/org/apache/shenyu" ]]; then + echo "ShenYu local repository artifact is missing: ${artifact_dir}/org/apache/shenyu" >&2 + exit 1 +fi + +mkdir -p "${target_dir}" +if [[ -d "${target_dir}/shenyu" ]]; then + mv "${target_dir}/shenyu" "${target_dir}/shenyu.backup.${GITHUB_RUN_ID:-$$}" +fi +cp -R "${artifact_dir}/org/apache/shenyu" "${target_dir}/shenyu" + +echo "Restored ShenYu local Maven repository." diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7cbc12856e79..94521ec13652 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,14 +30,16 @@ jobs: if: github.repository == 'apache/shenyu' runs-on: ubuntu-latest outputs: - code: ${{ steps.filter.outputs.code }} + code: ${{ steps.modules.outputs.has_code_changes }} code_files: ${{ steps.filter.outputs.code_files }} + modules: ${{ steps.modules.outputs.modules }} + full_build_required: ${{ steps.modules.outputs.full_build_required }} steps: - uses: actions/checkout@v3 - uses: ./actions/paths-filter id: filter with: - list-files: "csv" + list-files: "json" filters: | code: - '!**/*.md' @@ -63,15 +65,18 @@ jobs: - 'mvnw.cmd' - '.mvn/**' - '.github/workflows/**' + - '.github/scripts/**' + - name: Resolve changed Maven modules + id: modules + if: steps.filter.outputs.code == 'true' + env: + CHANGED_FILES_JSON: ${{ steps.filter.outputs.code_files }} + run: bash .github/scripts/resolve-ci-modules.sh - build: + pr_build: needs: changes - strategy: - matrix: - java: [17, 21] - os: [ubuntu-latest] - runs-on: ${{ matrix.os }} - if: github.repository == 'apache/shenyu' && needs.changes.outputs.code == 'true' + runs-on: ubuntu-latest + if: github.repository == 'apache/shenyu' && github.event_name == 'pull_request' && needs.changes.outputs.code == 'true' steps: - uses: actions/checkout@v3 with: @@ -79,6 +84,8 @@ jobs: - name: Debug - Show matched files run: | echo "Matched files: ${{ needs.changes.outputs.code_files }}" + echo "Resolved modules: ${{ needs.changes.outputs.modules }}" + echo "Full build required: ${{ needs.changes.outputs.full_build_required }}" - name: Restore ShenYu Maven Repos id: restore-maven-cache uses: actions/cache/restore@v3 @@ -89,7 +96,7 @@ jobs: ${{ runner.os }}-maven- - uses: ./actions/setup-java-with-retry with: - java-version: ${{ matrix.java }} + java-version: "17" distribution: "temurin" - name: Install mvnd shell: bash @@ -109,16 +116,163 @@ jobs: - name: Build with Maven shell: bash run: | + MAVEN_ARGS="-B -ntp -T 1C" + TEST_ARGS="-Prelease -DskipRemoteResources=true -Djacoco.skip=true" + MODULE_ARGS="" + + if [[ "${{ needs.changes.outputs.full_build_required }}" != "true" && -n "${{ needs.changes.outputs.modules }}" ]]; then + MODULE_ARGS="-pl ${{ needs.changes.outputs.modules }} -am -amd" + fi + if mvnd --version > /dev/null 2>&1; then echo "Using mvnd for build" - mvnd -B clean test -Prelease -DskipRemoteResources=true + mvnd ${MAVEN_ARGS} ${MODULE_ARGS} test ${TEST_ARGS} else echo "Falling back to maven wrapper" - ./mvnw -B clean test -Prelease -DskipRemoteResources=true + ./mvnw ${MAVEN_ARGS} ${MODULE_ARGS} test ${TEST_ARGS} fi + + build_local_repo: + needs: changes + runs-on: ubuntu-latest + if: github.repository == 'apache/shenyu' && github.event_name == 'push' && needs.changes.outputs.code == 'true' + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Restore ShenYu Maven Repos + id: restore-maven-cache + uses: actions/cache/restore@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - uses: ./actions/setup-java-with-retry + with: + java-version: "17" + distribution: "temurin" + - name: Install mvnd + shell: bash + run: | + MVND_VERSION=1.0.6 + curl -sfL https://downloads.apache.org/maven/mvnd/${MVND_VERSION}/maven-mvnd-${MVND_VERSION}-linux-amd64.zip -o mvnd.zip + unzip -q mvnd.zip + mkdir -p $HOME/.local + mv maven-mvnd-${MVND_VERSION}-linux-amd64 $HOME/.local/mvnd + echo "$HOME/.local/mvnd/bin" >> $GITHUB_PATH + echo "MVND_HOME=$HOME/.local/mvnd" >> $GITHUB_ENV + + - name: Build ShenYu local Maven repository + shell: bash + run: | + MAVEN_ARGS="-B -ntp -T 1C" + BUILD_ARGS="-Prelease -DskipTests -DskipRemoteResources=true -Djacoco.skip=true -Dmaven.javadoc.skip=true -Drat.skip=true" + + if mvnd --version > /dev/null 2>&1; then + echo "Using mvnd for local repository build" + mvnd ${MAVEN_ARGS} install ${BUILD_ARGS} + else + echo "Falling back to maven wrapper" + ./mvnw ${MAVEN_ARGS} install ${BUILD_ARGS} + fi + + mkdir -p /tmp/shenyu-m2/org/apache + cp -R "${HOME}/.m2/repository/org/apache/shenyu" /tmp/shenyu-m2/org/apache/shenyu + - name: Upload ShenYu local Maven repository + uses: actions/upload-artifact@v4 + with: + name: shenyu-local-maven-repository + path: /tmp/shenyu-m2 + retention-days: 1 + + test_group: + needs: + - changes + - build_local_repo + runs-on: ubuntu-latest + if: github.repository == 'apache/shenyu' && github.event_name == 'push' && needs.changes.outputs.code == 'true' + strategy: + fail-fast: false + matrix: + include: + - name: core + modules: shenyu-common,shenyu-spi,shenyu-disruptor,shenyu-protocol,shenyu-loadbalancer + - name: gateway + modules: shenyu-web,shenyu-bootstrap,shenyu-spring-boot-starter + - name: admin-sync + modules: shenyu-admin,shenyu-admin-listener,shenyu-sync-data-center + - name: plugin + modules: shenyu-plugin + - name: client-register-registry + modules: shenyu-client,shenyu-register-center,shenyu-registry + - name: dist-extra + modules: shenyu-dist,shenyu-sdk,shenyu-alert,shenyu-kubernetes-controller,shenyu-infra + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Restore ShenYu Maven Repos + id: restore-maven-cache + uses: actions/cache/restore@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - uses: ./actions/setup-java-with-retry + with: + java-version: "17" + distribution: "temurin" + - uses: actions/download-artifact@v4 + with: + name: shenyu-local-maven-repository + path: /tmp/shenyu-m2 + - name: Restore ShenYu local Maven repository + run: bash .github/scripts/restore-shenyu-local-repo.sh /tmp/shenyu-m2 + - name: Run Maven tests + shell: bash + run: | + modules="$(GROUP_ROOTS="${{ matrix.modules }}" bash .github/scripts/expand-ci-module-roots.sh)" + echo "Testing Maven modules: ${modules}" + ./mvnw -B -ntp -T 1C -pl "${modules}" test -Prelease -DskipRemoteResources=true - uses: codecov/codecov-action@v1 with: token: 2760af6a-3405-4882-9e61-04c5176fecfa + flags: ${{ matrix.name }} + + compat_java21: + needs: + - changes + - build_local_repo + runs-on: ubuntu-latest + if: github.repository == 'apache/shenyu' && github.event_name == 'push' && needs.changes.outputs.code == 'true' + steps: + - uses: actions/checkout@v3 + with: + submodules: true + - name: Restore ShenYu Maven Repos + id: restore-maven-cache + uses: actions/cache/restore@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: | + ${{ runner.os }}-maven- + - uses: ./actions/setup-java-with-retry + with: + java-version: "21" + distribution: "temurin" + - uses: actions/download-artifact@v4 + with: + name: shenyu-local-maven-repository + path: /tmp/shenyu-m2 + - name: Restore ShenYu local Maven repository + run: bash .github/scripts/restore-shenyu-local-repo.sh /tmp/shenyu-m2 + - name: Run Java 21 compatibility tests + shell: bash + run: | + ./mvnw -B -ntp -T 1C test -Prelease -DskipRemoteResources=true -Djacoco.skip=true check-license-header: name: check-license-header @@ -139,10 +293,20 @@ jobs: if: (github.repository == 'apache/shenyu') && always() needs: - changes - - build + - pr_build + - build_local_repo + - test_group + - compat_java21 runs-on: ubuntu-latest steps: - name: checking job status run: | [[ "${{ needs.changes.result }}" == "success" ]] || exit -1 - [[ "${{ needs.changes.outputs.code }}" != "true" || "${{ needs.build.result }}" == "success" ]] || exit -1 + if [[ "${{ needs.changes.outputs.code }}" == "true" && "${{ github.event_name }}" == "pull_request" ]]; then + [[ "${{ needs.pr_build.result }}" == "success" ]] || exit -1 + fi + if [[ "${{ needs.changes.outputs.code }}" == "true" && "${{ github.event_name }}" == "push" ]]; then + [[ "${{ needs.build_local_repo.result }}" == "success" ]] || exit -1 + [[ "${{ needs.test_group.result }}" == "success" ]] || exit -1 + [[ "${{ needs.compat_java21.result }}" == "success" ]] || exit -1 + fi diff --git a/.github/workflows/e2e-k8s.yml b/.github/workflows/e2e-k8s.yml index b2da2b7b9508..108d97a69b75 100644 --- a/.github/workflows/e2e-k8s.yml +++ b/.github/workflows/e2e-k8s.yml @@ -30,6 +30,11 @@ jobs: runs-on: ubuntu-latest outputs: e2e: ${{ steps.filter.outputs.e2e }} + run_e2e: ${{ steps.cases.outputs.run_e2e }} + run_storage: ${{ steps.cases.outputs.run_storage }} + run_e2e_cases: ${{ steps.cases.outputs.run_e2e_cases }} + storage_matrix: ${{ steps.cases.outputs.storage_matrix }} + e2e_matrix: ${{ steps.cases.outputs.e2e_matrix }} steps: - uses: actions/checkout@v3 with: @@ -37,11 +42,13 @@ jobs: - uses: ./actions/paths-filter id: filter with: + list-files: "json" filters: | e2e: - 'shenyu-e2e/**' - '**/e2e/**' - 'shenyu-*/**' + - 'db/**' - 'pom.xml' - '**/pom.xml' - '!**/*.md' @@ -52,12 +59,20 @@ jobs: - '!.licenserc.yaml' - '!LICENSE' - '!NOTICE' + - '!.github/**' - '!.github/ISSUE_TEMPLATE/**' - '!.github/PULL_REQUEST_TEMPLATE' + - name: Resolve E2E cases + id: cases + if: steps.filter.outputs.e2e == 'true' + env: + CHANGED_FILES_JSON: ${{ steps.filter.outputs.e2e_files }} + CI_CASE_MODE: e2e + run: bash .github/scripts/resolve-test-case-matrix.sh build-docker-images: needs: changes - if: ${{ needs.changes.outputs.e2e == 'true' }} + if: ${{ needs.changes.outputs.run_e2e == 'true' }} runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 @@ -147,19 +162,10 @@ jobs: needs: - changes - build-docker-images - if: (github.repository == 'apache/shenyu' && ${{ needs.changes.outputs.e2e == 'true' }}) + if: github.repository == 'apache/shenyu' && needs.changes.outputs.run_storage == 'true' # if: (github.repository == 'apache/shenyu') strategy: - matrix: - include: - - case: shenyu-e2e-case-storage - script: e2e-h2-compose - - case: shenyu-e2e-case-storage - script: e2e-mysql-compose - - case: shenyu-e2e-case-storage - script: e2e-postgres-compose - - case: shenyu-e2e-case-storage - script: e2e-opengauss-compose + matrix: ${{ fromJSON(needs.changes.outputs.storage_matrix || '{"include":[]}') }} steps: - uses: actions/checkout@v2 with: @@ -231,29 +237,10 @@ jobs: needs: - changes - build-docker-images - if: (github.repository == 'apache/shenyu' && ${{ needs.changes.outputs.e2e == 'true' }}) + if: github.repository == 'apache/shenyu' && needs.changes.outputs.run_e2e_cases == 'true' # if: (github.repository == 'apache/shenyu') strategy: - matrix: - include: - - case: shenyu-e2e-case-http - script: e2e-http-sync-compose - - case: shenyu-e2e-case-spring-cloud - script: e2e-springcloud-sync-compose - - case: shenyu-e2e-case-apache-dubbo - script: e2e-apache-dubbo-sync-compose - - case: shenyu-e2e-case-grpc - script: e2e-grpc-sync-compose - - case: shenyu-e2e-case-websocket - script: e2e-websocket-sync-compose - # - case: shenyu-e2e-case-cluster - # script: e2e-cluster-jdbc-compose - # - case: shenyu-e2e-case-cluster - # script: e2e-cluster-zookeeper-compose - - case: shenyu-e2e-case-logging-rocketmq - script: e2e-logging-rocketmq-compose - # - case: shenyu-e2e-case-logging-kafka - # script: e2e-logging-kafka-compose + matrix: ${{ fromJSON(needs.changes.outputs.e2e_matrix || '{"include":[]}') }} steps: - uses: actions/checkout@v2 @@ -372,7 +359,7 @@ jobs: requirement: name: e2e - if: ${{ needs.changes.outputs.e2e == 'true' }} + if: ${{ always() && needs.changes.outputs.run_e2e == 'true' }} needs: - changes - e2e-storage @@ -381,5 +368,9 @@ jobs: steps: - name: checking job status run: | - [[ "${{ needs.e2e-storage.result }}" == "success" ]] || exit -1 - [[ "${{ needs.e2e-case.result }}" == "success" ]] || exit -1 + if [[ "${{ needs.changes.outputs.run_storage }}" == "true" ]]; then + [[ "${{ needs.e2e-storage.result }}" == "success" ]] || exit -1 + fi + if [[ "${{ needs.changes.outputs.run_e2e_cases }}" == "true" ]]; then + [[ "${{ needs.e2e-case.result }}" == "success" ]] || exit -1 + fi diff --git a/.github/workflows/integrated-test-k8s-ingress.yml b/.github/workflows/integrated-test-k8s-ingress.yml index f41e8a44f216..5b6792d2d18b 100644 --- a/.github/workflows/integrated-test-k8s-ingress.yml +++ b/.github/workflows/integrated-test-k8s-ingress.yml @@ -30,15 +30,15 @@ jobs: if: github.repository == 'apache/shenyu' runs-on: ubuntu-latest outputs: - k8s_ingress: ${{ steps.filter.outputs['k8s-ingress'] }} + k8s_ingress: ${{ steps.cases.outputs.run_k8s_ingress }} steps: - uses: actions/checkout@v3 - uses: ./actions/paths-filter id: filter with: + list-files: "json" filters: | k8s-ingress: - - '.github/workflows/integrated-test-k8s-ingress.yml' - 'shenyu-integrated-test-k8s-ingress*/**' - 'shenyu-*/**' - 'pom.xml' @@ -52,8 +52,16 @@ jobs: - '!.licenserc.yaml' - '!LICENSE' - '!NOTICE' + - '!.github/**' - '!.github/ISSUE_TEMPLATE/**' - '!.github/PULL_REQUEST_TEMPLATE' + - name: Resolve k8s ingress cases + id: cases + if: steps.filter.outputs['k8s-ingress'] == 'true' + env: + CHANGED_FILES_JSON: ${{ steps.filter.outputs['k8s-ingress_files'] }} + CI_CASE_MODE: k8s-ingress + run: bash .github/scripts/resolve-test-case-matrix.sh build: needs: changes diff --git a/.github/workflows/integrated-test.yml b/.github/workflows/integrated-test.yml index 12ca4165e7e2..ae23d0a6b9bf 100644 --- a/.github/workflows/integrated-test.yml +++ b/.github/workflows/integrated-test.yml @@ -31,16 +31,19 @@ jobs: runs-on: ubuntu-latest outputs: integration: ${{ steps.filter.outputs.integration }} + run_integration: ${{ steps.cases.outputs.run_integration }} + integration_matrix: ${{ steps.cases.outputs.integration_matrix }} steps: - uses: actions/checkout@v3 - uses: ./actions/paths-filter id: filter with: + list-files: "json" filters: | integration: - - '.github/workflows/integrated-test.yml' - 'shenyu-integrated-test/**' - 'shenyu-*/**' + - 'db/**' - 'pom.xml' - '**/pom.xml' - 'shenyu-examples/**' @@ -52,26 +55,23 @@ jobs: - '!.licenserc.yaml' - '!LICENSE' - '!NOTICE' + - '!.github/**' - '!.github/ISSUE_TEMPLATE/**' - '!.github/PULL_REQUEST_TEMPLATE' + - name: Resolve integrated test cases + id: cases + if: steps.filter.outputs.integration == 'true' + env: + CHANGED_FILES_JSON: ${{ steps.filter.outputs.integration_files }} + CI_CASE_MODE: integration + run: bash .github/scripts/resolve-test-case-matrix.sh build: needs: changes strategy: - matrix: - case: - - shenyu-integrated-test-apache-dubbo - - shenyu-integrated-test-grpc - - shenyu-integrated-test-http - - shenyu-integrated-test-https - - shenyu-integrated-test-spring-cloud - - shenyu-integrated-test-websocket - - shenyu-integrated-test-rewrite - - shenyu-integrated-test-combination - - shenyu-integrated-test-sdk-apache-dubbo - - shenyu-integrated-test-sdk-http + matrix: ${{ fromJSON(needs.changes.outputs.integration_matrix || '{"include":[]}') }} runs-on: ubuntu-latest - if: github.repository == 'apache/shenyu' && needs.changes.outputs.integration == 'true' + if: github.repository == 'apache/shenyu' && needs.changes.outputs.run_integration == 'true' steps: - uses: actions/checkout@v3 with: diff --git a/.github/workflows/k8s-examples-http.yml b/.github/workflows/k8s-examples-http.yml index 07493ec120e6..26abda660f53 100644 --- a/.github/workflows/k8s-examples-http.yml +++ b/.github/workflows/k8s-examples-http.yml @@ -36,6 +36,7 @@ jobs: - uses: ./actions/paths-filter id: filter with: + list-files: "json" filters: | k8s-examples: - 'shenyu-examples/**' @@ -50,10 +51,18 @@ jobs: - '!.licenserc.yaml' - '!LICENSE' - '!NOTICE' + - '!.github/**' - '!.github/ISSUE_TEMPLATE/**' - '!.github/PULL_REQUEST_TEMPLATE' + - name: Resolve k8s example cases + id: cases + if: steps.filter.outputs['k8s-examples'] == 'true' + env: + CHANGED_FILES_JSON: ${{ steps.filter.outputs['k8s-examples_files'] }} + CI_CASE_MODE: k8s-examples-http + run: bash .github/scripts/resolve-test-case-matrix.sh - name: Free disk space - if: steps.filter.outputs.k8s-examples == 'true' + if: steps.cases.outputs.run_k8s_examples == 'true' run: | df --human-readable sudo apt clean @@ -63,7 +72,7 @@ jobs: rm --recursive --force "$AGENT_TOOLSDIRECTORY" df --human-readable - name: Install k8s - if: steps.filter.outputs.k8s-examples == 'true' + if: steps.cases.outputs.run_k8s_examples == 'true' run: | install_k3s() { curl -sfL https://get.k3s.io | INSTALL_K3S_VERSION=v1.29.6+k3s2 K3S_KUBECONFIG_MODE=777 sh - @@ -85,7 +94,7 @@ jobs: mkdir -p ~/.kube cp /etc/rancher/k3s/k3s.yaml ~/.kube/config - name: Setup Java and Maven cache - if: steps.filter.outputs.k8s-examples == 'true' + if: steps.cases.outputs.run_k8s_examples == 'true' uses: ./actions/setup-java-with-retry with: java-version: 17 @@ -94,7 +103,7 @@ jobs: cache-dependency-path: | **/pom.xml - name: Install mvnd - if: steps.filter.outputs.k8s-examples == 'true' + if: steps.cases.outputs.run_k8s_examples == 'true' shell: bash run: | MVND_VERSION=1.0.6 @@ -105,7 +114,7 @@ jobs: echo "$HOME/.local/mvnd/bin" >> $GITHUB_PATH echo "MVND_HOME=$HOME/.local/mvnd" >> $GITHUB_ENV - name: Build with Maven - if: steps.filter.outputs.k8s-examples == 'true' + if: steps.cases.outputs.run_k8s_examples == 'true' shell: bash run: | if mvnd --version > /dev/null 2>&1; then @@ -116,7 +125,7 @@ jobs: ./mvnw -B -T 1C clean install -Prelease,docker -Dmaven.javadoc.skip=true -Dmaven.test.skip=true fi - name: Build examples - if: steps.filter.outputs.k8s-examples == 'true' + if: steps.cases.outputs.run_k8s_examples == 'true' shell: bash run: | if mvnd --version > /dev/null 2>&1; then @@ -127,7 +136,7 @@ jobs: ./mvnw -B -T 1C clean install -Pexample -Dmaven.javadoc.skip=true -Dmaven.test.skip=true -f ./shenyu-examples/pom.xml fi - name: Build k8s Cluster - if: steps.filter.outputs.k8s-examples == 'true' + if: steps.cases.outputs.run_k8s_examples == 'true' run: | docker save apache/shenyu-admin:latest apache/shenyu-bootstrap:latest shenyu-examples-http:latest | sudo k3s ctr images import - kubectl apply -f ./shenyu-examples/shenyu-examples-http/k8s/shenyu-deployment.yml @@ -135,11 +144,11 @@ jobs: kubectl apply -f ./shenyu-examples/shenyu-examples-http/k8s/shenyu-zookeeper.yml kubectl apply -f ./shenyu-examples/shenyu-examples-http/k8s/shenyu-examples-http.yml - name: Wait for k8s Cluster Start up - if: steps.filter.outputs.k8s-examples == 'true' + if: steps.cases.outputs.run_k8s_examples == 'true' run: | bash ./shenyu-examples/shenyu-examples-http/k8s/script/healthcheck.sh - name: Cluster Test after Healthcheck - if: steps.filter.outputs.k8s-examples == 'true' + if: steps.cases.outputs.run_k8s_examples == 'true' run: | kubectl get all kubectl get events --all-namespaces