diff --git a/.claude/docs/ci.md b/.claude/docs/ci.md index 1a64037a1c5..0ebd7ba3c14 100644 --- a/.claude/docs/ci.md +++ b/.claude/docs/ci.md @@ -35,5 +35,6 @@ The full Selenide UI suite takes ~1.5h per DB, so it does **not** run on every P - To force a specific UI IT to run on every PR, add `@Tag("smoke")` to that class (keep the list small - smoke must stay fast). - Shard-routing tags: `@Tag("sample")` sits on the `SampleProjectsIT` base (inherited by every sample-project IT); `@Tag("camel")` sits on each IT in `ui/tests/camel` (their `PredefinedProjectIT` base is shared with non-camel tests, so the base cannot carry it — tag new camel ITs individually). These route classes into the `samples` CI shard; everything else UI stays in the `ui` shard. - `@Tag("slow")` is the **fourth shard**, and it is a *balancing* tag, not a semantic one: it holds the long poles of both families (currently the api classes above ~55 s and the browser journeys above ~110 s), because without them `api` and `ui` are the critical path while `samples` idles. Membership is a judgement about measured CI time — re-check it when the shard times drift apart, and note that mistagging can only unbalance the shards, never drop an IT (`api` is the untagged complement). It does **not** affect the PR smoke gate: a `slow` api IT is still untagged-`ui`, so `!ui | smoke` still selects it. It cannot rebalance `api`, whose cost is uniform per class - see the budget note above. +- `@Tag("upgrade")` is outside the shards (#7641): `UpgradeFromPreviousReleaseIT` runs the release named in `tests/UPGRADE_FROM` as its published image on a Testcontainers PostgreSQL, lets it write a deployment (an intent project with rows, a running process, a fired job), then boots the build under test on that database and a copy of its repository folder. It needs Docker and pulls a ~1 GB image, so the `api` shards and the PR gate exclude it (`& !upgrade`) and the `upgrade-test` job is its only selector - nightly, and in `release.yml`, where `release-docker-images` needs it. The release job writes the version it released into `tests/UPGRADE_FROM` in its "for development" commit. A new tag that must stay out of the shards needs the same `& !upgrade`-style exclusion on the `api` expression of BOTH legs in `build.yml` and `nightly.yml`, or the untagged complement picks it up. `codeql.yml`, `release.yml` cover CodeQL and Maven Central release respectively. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4bdc9b88e07..23619a66441 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -190,7 +190,7 @@ jobs: matrix: shard: - name: api - groups: "!ui & !slow" + groups: "!ui & !slow & !upgrade" - name: ui groups: "ui & !slow & !sample & !camel" - name: samples @@ -293,7 +293,7 @@ jobs: matrix: shard: - name: api - groups: "!ui & !slow" + groups: "!ui & !slow & !upgrade" - name: ui groups: "ui & !slow & !sample & !camel" - name: samples diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index bfae36ff53d..e0008c76854 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -25,7 +25,7 @@ jobs: matrix: shard: - name: api - groups: "!ui & !slow" + groups: "!ui & !slow & !upgrade" - name: ui groups: "ui & !slow & !sample & !camel" - name: samples @@ -114,6 +114,69 @@ jobs: name: ${{ env.ARTIFACT_NAME }} path: tests/tests-integrations/build/reports/tests + # The release under test booted on a deployment the PREVIOUS release wrote (#7641) - every other + # IT boots on an empty database. UpgradeFromPreviousReleaseIT runs tests/UPGRADE_FROM as its + # published image on a PostgreSQL of its own (Testcontainers), so the job needs Docker but no + # database service, and it is the only selector of the `upgrade` tag the api shard excludes. + upgrade-test: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-maven- + + - name: Set up JDK Corretto 24 + uses: actions/setup-java@v4 + with: + distribution: 'corretto' + java-version: '24' + architecture: x64 + + - name: Install NodeJS + uses: actions/setup-node@v4 + with: + node-version: 22.x + + - name: Install TypeScript and esbuild + run: npm install -g typescript@5.9.3 esbuild + + - name: Install ttyd (prebuilt) + run: | + sudo apt update + sudo apt install -y ttyd + + - name: Upgrade from the previous release + run: | + echo "Upgrading from dirigiblelabs/dirigible:$(cat tests/UPGRADE_FROM)" + mvn clean install -P integration-tests -Dit.groups=upgrade + + - name: Upload integration test reports + uses: actions/upload-artifact@v4 + if: always() + with: + retention-days: 3 + name: failsafe-reports-upgrade-${{ github.run_attempt }} + path: | + tests/tests-integrations/target/failsafe-reports + tests/tests-integrations/target/surefire-reports + if-no-files-found: ignore + + - name: Assert the upgrade test ran + run: | + ran=$(ls tests/tests-integrations/target/failsafe-reports/TEST-*UpgradeFromPreviousReleaseIT.xml 2>/dev/null | wc -l) + if [ "$ran" -eq 0 ]; then + echo "::error::UpgradeFromPreviousReleaseIT did not run - the upgrade tag selected nothing (see #7215)" + exit 1 + fi + integration-tests-postgresql: runs-on: ubuntu-latest strategy: @@ -121,7 +184,7 @@ jobs: matrix: shard: - name: api - groups: "!ui & !slow" + groups: "!ui & !slow & !upgrade" - name: ui groups: "ui & !slow & !sample & !camel" - name: samples diff --git a/.github/workflows/pull-request.yml b/.github/workflows/pull-request.yml index e1ee73e825d..e21fd9ef2be 100644 --- a/.github/workflows/pull-request.yml +++ b/.github/workflows/pull-request.yml @@ -152,7 +152,7 @@ jobs: # (including two full clone -> generate -> validate app lifecycles). The heavy Selenide # suite (@Tag "ui") runs nightly and on master push - see nightly.yml and build.yml. - name: Smoke integration tests (H2) - run: mvn clean install -P integration-tests -Dit.groups="!ui | smoke" + run: mvn clean install -P integration-tests -Dit.groups="(!ui | smoke) & !upgrade" # Guards #7215: a tag expression that discovers nothing leaves the reports directory absent # and the build green, so what ran is asserted by count instead of by the job's colour. diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 6e8eb2ad572..a0356340ee2 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -26,7 +26,71 @@ permissions: id-token: write jobs: + # The release under test booted on a deployment the PREVIOUS release wrote (#7641) - every other + # IT boots on an empty database. UpgradeFromPreviousReleaseIT runs tests/UPGRADE_FROM as its + # published image on a PostgreSQL of its own (Testcontainers), so the job needs Docker but no + # database service, and it is the only selector of the `upgrade` tag the api shard excludes. + upgrade-test: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Cache local Maven repository + uses: actions/cache@v4 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-maven- + + - name: Set up JDK Corretto 24 + uses: actions/setup-java@v4 + with: + distribution: 'corretto' + java-version: '24' + architecture: x64 + + - name: Install NodeJS + uses: actions/setup-node@v4 + with: + node-version: 22.x + + - name: Install TypeScript and esbuild + run: npm install -g typescript@5.9.3 esbuild + + - name: Install ttyd (prebuilt) + run: | + sudo apt update + sudo apt install -y ttyd + + - name: Upgrade from the previous release + run: | + echo "Upgrading from dirigiblelabs/dirigible:$(cat tests/UPGRADE_FROM)" + mvn clean install -P integration-tests -Dit.groups=upgrade + + - name: Upload integration test reports + uses: actions/upload-artifact@v4 + if: always() + with: + retention-days: 3 + name: failsafe-reports-upgrade-${{ github.run_attempt }} + path: | + tests/tests-integrations/target/failsafe-reports + tests/tests-integrations/target/surefire-reports + if-no-files-found: ignore + + - name: Assert the upgrade test ran + run: | + ran=$(ls tests/tests-integrations/target/failsafe-reports/TEST-*UpgradeFromPreviousReleaseIT.xml 2>/dev/null | wc -l) + if [ "$ran" -eq 0 ]; then + echo "::error::UpgradeFromPreviousReleaseIT did not run - the upgrade tag selected nothing (see #7215)" + exit 1 + fi + release-docker-images: + needs: upgrade-test runs-on: ${{ matrix.runner }} strategy: matrix: @@ -204,8 +268,11 @@ jobs: run: mvn versions:set -DnewVersion=${{ github.event.inputs.snapshotVersion }} - name: "Git: Commit Snapshot Version" + # tests/UPGRADE_FROM names the release the upgrade test starts from: from here on, the one + # this run has just released. run: | - git add '**pom.xml' + echo "${{ github.event.inputs.releaseVersion }}" > tests/UPGRADE_FROM + git add '**pom.xml' tests/UPGRADE_FROM git commit -m "version set to ${{ github.event.inputs.snapshotVersion }} for development" #---------------Publish to Maven Central-----------------# diff --git a/tests/UPGRADE_FROM b/tests/UPGRADE_FROM new file mode 100644 index 00000000000..111baa92b54 --- /dev/null +++ b/tests/UPGRADE_FROM @@ -0,0 +1 @@ +14.74.0 diff --git a/tests/tests-framework/src/main/java/org/eclipse/dirigible/tests/framework/upgrade/PreviousRelease.java b/tests/tests-framework/src/main/java/org/eclipse/dirigible/tests/framework/upgrade/PreviousRelease.java new file mode 100644 index 00000000000..b6a52a551bb --- /dev/null +++ b/tests/tests-framework/src/main/java/org/eclipse/dirigible/tests/framework/upgrade/PreviousRelease.java @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2010-2026 Eclipse Dirigible contributors + * + * All rights reserved. This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v2.0 which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v20.html + * + * SPDX-FileCopyrightText: Eclipse Dirigible contributors SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.dirigible.tests.framework.upgrade; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; + +/** + * The release an upgrade test starts from. It is kept in ONE place, {@code tests/UPGRADE_FROM}, + * which the release workflow bumps to the version it has just released, so master is always tested + * against the release before it. The system property {@value #OVERRIDE_PROPERTY} overrides it - to + * try an upgrade from an older release, say. + */ +public final class PreviousRelease { + + /** The system property that overrides the file. */ + public static final String OVERRIDE_PROPERTY = "dirigible.upgrade.from"; + + /** The file that names the previous release. */ + static final String FILE_NAME = "UPGRADE_FROM"; + + private PreviousRelease() {} + + /** + * The tag of the previous release's {@code dirigiblelabs/dirigible} image. + * + * @return the tag, e.g. {@code 14.74.0} + */ + public static String tag() { + String override = System.getProperty(OVERRIDE_PROPERTY); + if (override != null && !override.isBlank()) { + return override.strip(); + } + return read(locate(Path.of("") + .toAbsolutePath())); + } + + /** + * Finds {@code tests/UPGRADE_FROM} from wherever the build runs: the tests module itself, or any + * folder below the repository root. + */ + static Path locate(Path start) { + for (Path folder = start; folder != null; folder = folder.getParent()) { + for (Path candidate : new Path[] {folder.resolve(FILE_NAME), folder.resolve("tests") + .resolve(FILE_NAME)}) { + if (Files.isRegularFile(candidate)) { + return candidate; + } + } + } + throw new IllegalStateException( + "No tests/" + FILE_NAME + " above [" + start + "] - it names the release an upgrade test starts from"); + } + + private static String read(Path file) { + try { + String tag = Files.readString(file, StandardCharsets.UTF_8) + .strip(); + if (tag.isEmpty()) { + throw new IllegalStateException("[" + file + "] is empty - it must name the release an upgrade test starts from"); + } + return tag; + } catch (IOException ex) { + throw new UncheckedIOException("Cannot read [" + file + "]", ex); + } + } +} diff --git a/tests/tests-framework/src/main/java/org/eclipse/dirigible/tests/framework/upgrade/PreviousReleaseClient.java b/tests/tests-framework/src/main/java/org/eclipse/dirigible/tests/framework/upgrade/PreviousReleaseClient.java new file mode 100644 index 00000000000..46396e5491b --- /dev/null +++ b/tests/tests-framework/src/main/java/org/eclipse/dirigible/tests/framework/upgrade/PreviousReleaseClient.java @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2010-2026 Eclipse Dirigible contributors + * + * All rights reserved. This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v2.0 which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v20.html + * + * SPDX-FileCopyrightText: Eclipse Dirigible contributors SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.dirigible.tests.framework.upgrade; + +import java.io.IOException; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Base64; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import java.util.function.Predicate; + +import org.awaitility.Awaitility; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; + +/** + * Drives a released Dirigible over its public HTTP surface, as the IDE does: the in-process test + * helpers ({@code ProjectDeployer}, the Selenide views) work on the application the test JVM runs, + * not on a release in a container. Every call authenticates as the default administrator. + */ +public final class PreviousReleaseClient { + + private static final String WORKSPACE = "workspace"; + + private static final Duration REQUEST_TIMEOUT = Duration.ofMinutes(2); + + private static final Gson GSON = new Gson(); + + private final String baseUrl; + + private final String authorization = "Basic " + Base64.getEncoder() + .encodeToString("admin:admin".getBytes(StandardCharsets.UTF_8)); + + private final HttpClient http = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(10)) + .build(); + + PreviousReleaseClient(String baseUrl) { + this.baseUrl = baseUrl; + } + + /** + * Creates a project in the administrator's workspace, and the workspace itself - a fresh instance + * has none until the IDE first lists it. + * + * @param project the project + */ + public void createProject(String project) { + expect(send("POST", "/services/ide/workspaces/" + WORKSPACE, null, null), 201, 304); + expect(send("POST", "/services/ide/workspaces/" + WORKSPACE + "/" + project, null, null), 201); + } + + /** + * Writes a new file into a workspace project. + * + * @param project the project + * @param path the project-relative path + * @param content the content + */ + public void writeFile(String project, String path, String content) { + expect(send("POST", "/services/ide/workspaces/" + WORKSPACE + "/" + project + "/" + path, "text/plain", content), 201); + } + + /** + * Runs the intent Generate - the model files and the code from them - and fails unless every code + * generation succeeded. + * + * @param project the project + * @param intentPath the project-relative path of the intent + */ + public void generateFromIntent(String project, String intentPath) { + HttpResponse response = send("POST", + "/services/ide/intent/generate?workspace=" + WORKSPACE + "&project=" + project + "&path=" + intentPath, null, null); + expect(response, 200); + Map result = GSON.fromJson(response.body(), new TypeToken>() {}.getType()); + Object generations = result.get("codeGenerations"); + if (!(generations instanceof List list) || list.isEmpty()) { + throw new IllegalStateException("The previous release generated no code from [" + intentPath + "]: " + response.body()); + } + for (Object generation : list) { + if (!(generation instanceof Map entry) || !Boolean.TRUE.equals(entry.get("generated"))) { + throw new IllegalStateException("The previous release failed a code generation: " + generation); + } + } + } + + /** + * Publishes a workspace project into the registry. + * + * @param project the project + */ + public void publish(String project) { + expect(send("POST", "/services/ide/publisher/" + WORKSPACE + "/" + project + "/", null, null), 200); + } + + /** + * Waits until a path answers 200 - a published controller, once the synchronizers compiled it. + * + * @param path the path + * @param timeout how long to wait + */ + public void awaitAvailable(String path, Duration timeout) { + Awaitility.await() + .pollInterval(2, TimeUnit.SECONDS) + .atMost(timeout) + .ignoreExceptions() + .until(() -> send("GET", path, null, null).statusCode() == 200); + } + + /** + * Creates a record through a JSON endpoint. + * + * @param path the endpoint + * @param record the record + * @return the created record as the endpoint answered it + */ + public Map create(String path, Map record) { + HttpResponse response = send("POST", path, "application/json", GSON.toJson(record)); + expect(response, 200); + return GSON.fromJson(response.body(), new TypeToken>() {}.getType()); + } + + /** + * Reads a JSON list. + * + * @param path the endpoint + * @return the list + */ + public List> list(String path) { + HttpResponse response = send("GET", path, null, null); + expect(response, 200); + return GSON.fromJson(response.body(), new TypeToken>>() {}.getType()); + } + + /** + * Waits until a JSON list satisfies a condition and returns it. + * + * @param path the endpoint + * @param condition the condition + * @param timeout how long to wait + * @return the list that satisfied it + */ + public List> awaitList(String path, Predicate>> condition, Duration timeout) { + return Awaitility.await() + .pollInterval(2, TimeUnit.SECONDS) + .atMost(timeout) + .until(() -> list(path), condition); + } + + private HttpResponse send(String method, String path, String contentType, String body) { + HttpRequest.Builder request = HttpRequest.newBuilder(URI.create(baseUrl + path)) + .timeout(REQUEST_TIMEOUT) + .header("Authorization", authorization) + .method(method, body == null ? HttpRequest.BodyPublishers.noBody() + : HttpRequest.BodyPublishers.ofString(body, StandardCharsets.UTF_8)); + if (contentType != null) { + request.header("Content-Type", contentType); + } + try { + return http.send(request.build(), HttpResponse.BodyHandlers.ofString()); + } catch (IOException ex) { + throw new IllegalStateException("Could not call the previous release: " + method + " " + path, ex); + } catch (InterruptedException ex) { + Thread.currentThread() + .interrupt(); + throw new IllegalStateException("Interrupted calling the previous release: " + method + " " + path, ex); + } + } + + private static void expect(HttpResponse response, int... statuses) { + for (int status : statuses) { + if (response.statusCode() == status) { + return; + } + } + throw new IllegalStateException(response.request() + .method() + + " " + response.uri() + " answered " + response.statusCode() + ": " + response.body()); + } +} diff --git a/tests/tests-framework/src/main/java/org/eclipse/dirigible/tests/framework/upgrade/PreviousReleaseEnvironment.java b/tests/tests-framework/src/main/java/org/eclipse/dirigible/tests/framework/upgrade/PreviousReleaseEnvironment.java new file mode 100644 index 00000000000..e887fa43409 --- /dev/null +++ b/tests/tests-framework/src/main/java/org/eclipse/dirigible/tests/framework/upgrade/PreviousReleaseEnvironment.java @@ -0,0 +1,239 @@ +/* + * Copyright (c) 2010-2026 Eclipse Dirigible contributors + * + * All rights reserved. This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v2.0 which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v20.html + * + * SPDX-FileCopyrightText: Eclipse Dirigible contributors SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.dirigible.tests.framework.upgrade; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Duration; +import java.util.Map; + +import org.apache.commons.compress.archivers.tar.TarArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.testcontainers.DockerClientFactory; +import org.testcontainers.containers.Container.ExecResult; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.Network; +import org.testcontainers.containers.output.Slf4jLogConsumer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * The state a previous release leaves behind, produced by that release itself: a PostgreSQL holding + * its DefaultDB and SystemDB, and the repository folder it wrote - together what a deployment's + * volume carries from one release to the next. The release runs as the published + * {@code dirigiblelabs/dirigible} image; the test JVM then boots the release under test on the same + * database and a copy of that folder. + * + *

+ * The folder is copied out of the stopped container rather than bind-mounted: the image runs as + * root, so on a Linux runner a bind mount would leave files the test JVM cannot write. + */ +public final class PreviousReleaseEnvironment implements AutoCloseable { + + private static final Logger LOGGER = LoggerFactory.getLogger(PreviousReleaseEnvironment.class); + + /** The DefaultDB database. */ + public static final String DEFAULT_DATABASE = "testdb"; + + /** The SystemDB database. */ + public static final String SYSTEM_DATABASE = "systemdb"; + + /** The database user. */ + public static final String USER = "testuser"; + + /** The database password. */ + public static final String PASSWORD = "testpass"; + + private static final String POSTGRES_ALIAS = "postgres"; + + /** + * The repository folder of the image: no WORKDIR, so the default {@code target} resolves under /. + */ + private static final String REPOSITORY_FOLDER = "/target/dirigible/repository"; + + private final String tag; + private final Network network = Network.newNetwork(); + private final GenericContainer postgres; + private final GenericContainer release; + + /** + * An environment for the given release, not started yet. + * + * @param tag the tag of the {@code dirigiblelabs/dirigible} image + */ + @SuppressWarnings("resource") + public PreviousReleaseEnvironment(String tag) { + this.tag = tag; + postgres = new GenericContainer<>(DockerImageName.parse("postgres:16")).withNetwork(network) + .withNetworkAliases(POSTGRES_ALIAS) + .withExposedPorts(5432) + .withEnv(Map.of("POSTGRES_DB", DEFAULT_DATABASE, + "POSTGRES_USER", USER, "POSTGRES_PASSWORD", + PASSWORD)) + .waitingFor(Wait.forLogMessage( + ".*database system is ready to accept connections.*\\s", + 2)); + release = new GenericContainer<>(DockerImageName.parse("dirigiblelabs/dirigible:" + tag)).withNetwork(network) + .withExposedPorts(8080) + .withEnv(releaseEnvironment()) + .withLogConsumer(new Slf4jLogConsumer( + LoggerFactory.getLogger( + "previous-release-" + + tag))) + .waitingFor(Wait.forHttp( + "/actuator/health/readiness") + .forPort(8080) + .forStatusCode(200) + .withStartupTimeout( + Duration.ofMinutes( + 10))); + } + + private static Map releaseEnvironment() { + String host = "jdbc:postgresql://" + POSTGRES_ALIAS + ":5432/"; + return Map.of("DIRIGIBLE_DATASOURCE_DEFAULT_DRIVER", "org.postgresql.Driver", // + "DIRIGIBLE_DATASOURCE_DEFAULT_URL", host + DEFAULT_DATABASE, // + "DIRIGIBLE_DATASOURCE_DEFAULT_USERNAME", USER, // + "DIRIGIBLE_DATASOURCE_DEFAULT_PASSWORD", PASSWORD, // + "DIRIGIBLE_DATABASE_SYSTEM_DRIVER", "org.postgresql.Driver", // + "DIRIGIBLE_DATABASE_SYSTEM_URL", host + SYSTEM_DATABASE, // + "DIRIGIBLE_DATABASE_SYSTEM_USERNAME", USER, // + "DIRIGIBLE_DATABASE_SYSTEM_PASSWORD", PASSWORD); + } + + /** + * Starts the database, creates the SystemDB in it, and starts the previous release on both. + * + * @return a client of the started release + */ + public PreviousReleaseClient start() { + postgres.start(); + createSystemDatabase(); + LOGGER.info("Starting the previous release [{}]...", tag); + release.start(); + LOGGER.info("The previous release [{}] is ready", tag); + return new PreviousReleaseClient("http://" + release.getHost() + ":" + release.getMappedPort(8080)); + } + + private void createSystemDatabase() { + psql(DEFAULT_DATABASE, "CREATE DATABASE " + SYSTEM_DATABASE); + } + + /** + * Stops the previous release the way an orchestrator does - SIGTERM and a grace period - so its + * shutdown hooks run as they would before an upgrade, and copies the repository folder it wrote. + * + * @param repositoryFolder where the repository goes: the {@code dirigible/repository} folder of the + * release under test, absent or empty + */ + public void stopReleaseAndCopyRepository(Path repositoryFolder) { + String containerId = release.getContainerId(); + DockerClientFactory.instance() + .client() + .stopContainerCmd(containerId) + .withTimeout(60) + .exec(); + LOGGER.info("Stopped the previous release [{}]; copying its repository into [{}]", tag, repositoryFolder); + try (InputStream archive = DockerClientFactory.instance() + .client() + .copyArchiveFromContainerCmd(containerId, REPOSITORY_FOLDER) + .exec(); + TarArchiveInputStream tar = new TarArchiveInputStream(archive)) { + extract(tar, repositoryFolder); + } catch (IOException ex) { + throw new UncheckedIOException("Could not copy the repository of the previous release", ex); + } + } + + /** The archive's entries are rooted at the copied folder's own name, {@code repository/...}. */ + private static void extract(TarArchiveInputStream tar, Path repositoryFolder) throws IOException { + Path root = repositoryFolder.toAbsolutePath() + .normalize(); + Files.createDirectories(root); + TarArchiveEntry entry; + while ((entry = tar.getNextEntry()) != null) { + String name = entry.getName(); + int slash = name.indexOf('/'); + if (slash < 0 || slash == name.length() - 1) { + continue; + } + Path target = root.resolve(name.substring(slash + 1)) + .normalize(); + if (!target.startsWith(root)) { + throw new IOException("The archive entry [" + name + "] leaves the repository folder"); + } + if (entry.isDirectory()) { + Files.createDirectories(target); + } else if (entry.isFile()) { + Files.createDirectories(target.getParent()); + Files.copy(tar, target, StandardCopyOption.REPLACE_EXISTING); + } + } + } + + /** + * Points the datasources the previous release recorded at the address the test JVM reaches the + * database by. A deployment keeps its database address across an upgrade, but here the two releases + * reach the same PostgreSQL by two names - the container by its network alias, the test JVM by the + * mapped port - and a datasource stores its URL as resolved when its file was parsed, which an + * unchanged file never is again. + */ + public void readdressRecordedDataSources() { + String from = "//" + POSTGRES_ALIAS + ":5432/"; + String to = "//" + postgres.getHost() + ":" + postgres.getMappedPort(5432) + "/"; + String sql = "UPDATE DIRIGIBLE_DATA_SOURCES SET DS_URL = replace(DS_URL, '" + from + "', '" + to + "') WHERE DS_URL LIKE '%" + from + + "%'"; + String output = psql(SYSTEM_DATABASE, sql); + if (!output.matches("(?s).*UPDATE [1-9]\\d*.*")) { + throw new IllegalStateException("The previous release recorded no datasource at [" + from + "]: " + output); + } + LOGGER.info("Readdressed the datasources the previous release recorded from [{}] to [{}]: {}", from, to, output.strip()); + } + + private String psql(String database, String sql) { + try { + ExecResult result = postgres.execInContainer("psql", "-v", "ON_ERROR_STOP=1", "-U", USER, "-d", database, "-c", sql); + if (result.getExitCode() != 0) { + throw new IllegalStateException("psql failed on [" + database + "]: " + result.getStderr()); + } + return result.getStdout(); + } catch (IOException ex) { + throw new UncheckedIOException("Could not run psql on [" + database + "]", ex); + } catch (InterruptedException ex) { + Thread.currentThread() + .interrupt(); + throw new IllegalStateException("Interrupted running psql on [" + database + "]", ex); + } + } + + /** + * The JDBC URL of one of the databases, as the test JVM reaches it. + * + * @param database {@link #DEFAULT_DATABASE} or {@link #SYSTEM_DATABASE} + * @return the URL + */ + public String jdbcUrl(String database) { + return "jdbc:postgresql://" + postgres.getHost() + ":" + postgres.getMappedPort(5432) + "/" + database; + } + + /** Stops and removes both containers and their network. */ + @Override + public void close() { + release.stop(); + postgres.stop(); + network.close(); + } +} diff --git a/tests/tests-framework/src/test/java/org/eclipse/dirigible/tests/framework/upgrade/PreviousReleaseTest.java b/tests/tests-framework/src/test/java/org/eclipse/dirigible/tests/framework/upgrade/PreviousReleaseTest.java new file mode 100644 index 00000000000..67f8b69c7d7 --- /dev/null +++ b/tests/tests-framework/src/test/java/org/eclipse/dirigible/tests/framework/upgrade/PreviousReleaseTest.java @@ -0,0 +1,52 @@ +/* + * Copyright (c) 2010-2026 Eclipse Dirigible contributors + * + * All rights reserved. This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v2.0 which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v20.html + * + * SPDX-FileCopyrightText: Eclipse Dirigible contributors SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.dirigible.tests.framework.upgrade; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** The previous release is named in one place, found from wherever the build runs. */ +class PreviousReleaseTest { + + @TempDir + Path repository; + + @Test + void theFileIsFoundFromAModuleBelowTheTestsFolder() throws IOException { + Path file = Files.writeString(Files.createDirectories(repository.resolve("tests")) + .resolve(PreviousRelease.FILE_NAME), + "14.74.0\n"); + Path module = Files.createDirectories(repository.resolve("tests/tests-integrations")); + + assertThat(PreviousRelease.locate(module)).isEqualTo(file); + } + + @Test + void theFileIsFoundFromTheRepositoryRoot() throws IOException { + Path file = Files.writeString(Files.createDirectories(repository.resolve("tests")) + .resolve(PreviousRelease.FILE_NAME), + "14.74.0\n"); + + assertThat(PreviousRelease.locate(repository)).isEqualTo(file); + } + + @Test + void aMissingFileIsNamedInTheFailure() { + assertThatThrownBy(() -> PreviousRelease.locate(repository)).isInstanceOf(IllegalStateException.class) + .hasMessageContaining("tests/UPGRADE_FROM"); + } +} diff --git a/tests/tests-integrations/src/main/java/org/eclipse/dirigible/integration/tests/api/UpgradeFromPreviousReleaseIT.java b/tests/tests-integrations/src/main/java/org/eclipse/dirigible/integration/tests/api/UpgradeFromPreviousReleaseIT.java new file mode 100644 index 00000000000..96238addc5c --- /dev/null +++ b/tests/tests-integrations/src/main/java/org/eclipse/dirigible/integration/tests/api/UpgradeFromPreviousReleaseIT.java @@ -0,0 +1,304 @@ +/* + * Copyright (c) 2010-2026 Eclipse Dirigible contributors + * + * All rights reserved. This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v2.0 which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v20.html + * + * SPDX-FileCopyrightText: Eclipse Dirigible contributors SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.dirigible.integration.tests.api; + +import static io.restassured.RestAssured.given; +import static org.assertj.core.api.Assertions.assertThat; +import static org.hamcrest.Matchers.equalTo; + +import java.io.IOException; +import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.sql.Connection; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; +import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; + +import org.eclipse.dirigible.commons.config.Configuration; +import org.eclipse.dirigible.commons.config.DirigibleConfig; +import org.eclipse.dirigible.components.base.readiness.PlatformReadiness; +import org.eclipse.dirigible.components.data.sources.manager.DataSourcesManager; +import org.eclipse.dirigible.tests.base.IntegrationTest; +import org.eclipse.dirigible.tests.framework.restassured.RestAssuredExecutor; +import org.eclipse.dirigible.tests.framework.upgrade.PreviousRelease; +import org.eclipse.dirigible.tests.framework.upgrade.PreviousReleaseClient; +import org.eclipse.dirigible.tests.framework.upgrade.PreviousReleaseEnvironment; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; +import org.springframework.beans.factory.annotation.Autowired; + +import com.google.gson.Gson; +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import com.google.gson.reflect.TypeToken; + +/** + * The release under test boots on a deployment the PREVIOUS release wrote (#7641) - every other IT + * boots on an empty database, and the upgrade failures that reached users (#7008, #7370, #7435, + * #7597, #7049) were exactly the ones only an existing database shows. + * + *

+ * Before the application context starts, the previous release ({@code tests/UPGRADE_FROM}) runs as + * its published image on a PostgreSQL of its own: it generates and publishes an intent project, + * takes rows in every entity, parks a process instance on a user task and fires a scheduled job; it + * is then stopped like an orchestrator stops it and its repository folder is copied. The release + * under test boots on that database and that folder, with + * {@code DIRIGIBLE_READINESS_REQUIRE_CLEAN_BOOT}, and has to: accept traffic, leave no artefact + * failed, run the system changelog forward, keep every row and the process instances, and fire the + * job again. + * + *

+ * Tagged {@code upgrade}: it pulls a release image and needs Docker, so it runs in its own job of + * the nightly and the release workflows, not in the shards or the PR gate. + */ +@Tag("upgrade") +class UpgradeFromPreviousReleaseIT extends IntegrationTest { + + private static final String PROJECT = "upgrade-it"; + private static final String INTENT = "app.intent"; + + private static final String API = "/services/java/" + PROJECT + "/gen/upgrade/api"; + private static final String CUSTOMERS = API + "/customer/CustomerController"; + private static final String TICKETS = API + "/ticket/TicketController"; + private static final String DIGESTS = API + "/ticketdigest/TicketDigestController"; + private static final String PROCESS_INSTANCES = "/services/bpm/bpm-processes/instances"; + + private static final String PROCESS = "TicketReview"; + + private static final Duration PUBLISH_TIMEOUT = Duration.ofMinutes(5); + private static final Duration JOB_TIMEOUT = Duration.ofMinutes(2); + private static final long BOOT_TIMEOUT_SECONDS = 600; + + private static final Gson GSON = new Gson(); + + private static PreviousReleaseEnvironment previousRelease; + + /** What the previous release left, read through its own API just before it stopped. */ + private static List> customers; + private static List> tickets; + private static List> digests; + private static Set processInstances; + + @Autowired + private RestAssuredExecutor restAssuredExecutor; + + @Autowired + private DataSourcesManager dataSourcesManager; + + /** + * Runs after the base class has wiped the Dirigible folder and before the application context + * starts - the release under test boots on what this leaves. + */ + @BeforeAll + static void letThePreviousReleaseWriteTheDeployment() { + previousRelease = new PreviousReleaseEnvironment(PreviousRelease.tag()); + PreviousReleaseClient client = previousRelease.start(); + + client.createProject(PROJECT); + client.writeFile(PROJECT, INTENT, resource(UpgradeFromPreviousReleaseIT.class.getSimpleName() + "/" + INTENT)); + client.generateFromIntent(PROJECT, INTENT); + client.publish(PROJECT); + client.awaitAvailable(CUSTOMERS, PUBLISH_TIMEOUT); + + int customer = id(client.create(CUSTOMERS, Map.of("Name", "Acme"))); + client.create(TICKETS, Map.of("Subject", "Printer jam", "Customer", customer)); + client.create(TICKETS, Map.of("Subject", "VPN down", "Customer", customer)); + + customers = client.list(CUSTOMERS); + // A process instance per ticket, its id stamped on the ticket once the trigger has run. + tickets = client.awaitList(TICKETS, rows -> rows.size() == 2 && rows.stream() + .allMatch(row -> row.get("ProcessId") != null), + PUBLISH_TIMEOUT); + digests = client.awaitList(DIGESTS, rows -> rows.size() == 2, JOB_TIMEOUT); + processInstances = processInstanceIds(client.list(PROCESS_INSTANCES)); + assertThat(processInstances).as("the previous release parks one review per ticket") + .hasSize(2); + + previousRelease.stopReleaseAndCopyRepository( + Path.of(DirigibleConfig.REPOSITORY_LOCAL_ROOT_FOLDER.getStringValue(), "dirigible", "repository")); + previousRelease.readdressRecordedDataSources(); + bootTheReleaseUnderTestOnTheSameDatabase(); + } + + private static void bootTheReleaseUnderTestOnTheSameDatabase() { + // Runtime values win over the environment, so this holds on a CI leg that exports its own. + Configuration.set("DIRIGIBLE_DATASOURCE_DEFAULT_DRIVER", "org.postgresql.Driver"); + Configuration.set("DIRIGIBLE_DATASOURCE_DEFAULT_URL", previousRelease.jdbcUrl(PreviousReleaseEnvironment.DEFAULT_DATABASE)); + Configuration.set("DIRIGIBLE_DATASOURCE_DEFAULT_USERNAME", PreviousReleaseEnvironment.USER); + Configuration.set("DIRIGIBLE_DATASOURCE_DEFAULT_PASSWORD", PreviousReleaseEnvironment.PASSWORD); + Configuration.set("DIRIGIBLE_DATABASE_SYSTEM_DRIVER", "org.postgresql.Driver"); + Configuration.set("DIRIGIBLE_DATABASE_SYSTEM_URL", previousRelease.jdbcUrl(PreviousReleaseEnvironment.SYSTEM_DATABASE)); + Configuration.set("DIRIGIBLE_DATABASE_SYSTEM_USERNAME", PreviousReleaseEnvironment.USER); + Configuration.set("DIRIGIBLE_DATABASE_SYSTEM_PASSWORD", PreviousReleaseEnvironment.PASSWORD); + DirigibleConfig.READINESS_AVAILABILITY_BRIDGE_ENABLED.setBooleanValue(true); + DirigibleConfig.READINESS_REQUIRE_CLEAN_BOOT.setBooleanValue(true); + } + + @AfterAll + static void removeThePreviousRelease() { + if (previousRelease != null) { + previousRelease.close(); + } + } + + @Test + void theReleaseUnderTestTakesOverWhatThePreviousReleaseLeft() throws SQLException { + restAssuredExecutor.execute(() -> given().when() + .get("/actuator/health/readiness") + .then() + .statusCode(200) + .body("status", equalTo("UP")), + BOOT_TIMEOUT_SECONDS); + assertThat(PlatformReadiness.getInstance() + .isCleanBoot()).as("a clean boot: no failed artefact, every AOT-listed class registered") + .isTrue(); + restAssuredExecutor.execute(() -> given().when() + .get("/actuator/health") + .then() + .statusCode(200) + .body("components.artefacts.details.failed", equalTo(0))); + + assertThat(appliedChangeSets()).as("the system changelog ran forward over the previous release's schema") + .containsAll(declaredChangeSets()); + + assertThat(list(CUSTOMERS)).as("customers") + .isEqualTo(customers); + assertThat(list(TICKETS)).as("tickets") + .isEqualTo(tickets); + assertThat(list(DIGESTS)).as("digests") + .isEqualTo(digests); + assertThat(processInstanceIds(list(PROCESS_INSTANCES))).as("the reviews still running") + .containsAll(processInstances); + + // The release under test serves the application, starts its process and fires its job. + int customer = id(customers.get(0)); + AtomicReference created = new AtomicReference<>(); + restAssuredExecutor.execute(() -> created.set(given().contentType("application/json") + .body(GSON.toJson(Map.of("Subject", "Disk full", "Customer", customer))) + .when() + .post(TICKETS) + .then() + .statusCode(200) + .extract() + .asString())); + int ticket = id(GSON.fromJson(created.get(), new TypeToken>() {}.getType())); + restAssuredExecutor.execute(() -> assertThat(list(DIGESTS)).as("the job fires again") + .anySatisfy( + digest -> assertThat(id(digest, "Ticket")).isEqualTo(ticket)), + JOB_TIMEOUT.toSeconds()); + restAssuredExecutor.execute(() -> assertThat(processInstanceIds(list(PROCESS_INSTANCES))).as("a new review starts") + .hasSize(3), + JOB_TIMEOUT.toSeconds()); + } + + private List> list(String path) { + AtomicReference body = new AtomicReference<>(); + restAssuredExecutor.execute(() -> body.set(given().when() + .get(path) + .then() + .statusCode(200) + .extract() + .asString())); + return GSON.fromJson(body.get(), new TypeToken>>() {}.getType()); + } + + private Set appliedChangeSets() throws SQLException { + Set applied = new HashSet<>(); + try (Connection connection = dataSourcesManager.getSystemDataSource() + .getConnection(); + Statement statement = connection.createStatement(); + ResultSet resultSet = statement.executeQuery("SELECT ID FROM DATABASECHANGELOG")) { + while (resultSet.next()) { + applied.add(resultSet.getString(1)); + } + } + return applied; + } + + /** The changesets of the system changelog the release under test ships that apply to PostgreSQL. */ + private static Set declaredChangeSets() { + JsonObject changelog = JsonParser.parseString(resource("db/changelog/dirigible-system.json")) + .getAsJsonObject(); + Set ids = new HashSet<>(); + for (JsonElement entry : changelog.getAsJsonArray("databaseChangeLog")) { + JsonObject changeSet = entry.getAsJsonObject() + .getAsJsonObject("changeSet"); + if (changeSet != null && appliesToPostgreSql(changeSet.get("dbms"))) { + ids.add(changeSet.get("id") + .getAsString()); + } + } + assertThat(ids).as("the system changelog declares changesets") + .isNotEmpty(); + return ids; + } + + /** + * Liquibase's {@code dbms} filter: absent or {@code all} applies everywhere, a list of names only + * there, a list of {@code !name} exclusions everywhere else. + */ + private static boolean appliesToPostgreSql(JsonElement dbms) { + if (dbms == null || dbms.isJsonNull()) { + return true; + } + Set names = Set.of(dbms.getAsString() + .replace(" ", "") + .split(",")); + if (names.contains("all") || names.contains("postgresql")) { + return true; + } + if (names.contains("!postgresql")) { + return false; + } + return names.stream() + .allMatch(name -> name.startsWith("!")); + } + + private static Set processInstanceIds(List> instances) { + return instances.stream() + .filter(instance -> Objects.equals(PROCESS, instance.get("processDefinitionKey"))) + .map(instance -> String.valueOf(instance.get("id"))) + .collect(Collectors.toSet()); + } + + /** Gson reads every JSON number as a double. */ + private static int id(Map record) { + return id(record, "Id"); + } + + private static int id(Map record, String property) { + return ((Number) record.get(property)).intValue(); + } + + private static String resource(String name) { + try (InputStream in = UpgradeFromPreviousReleaseIT.class.getClassLoader() + .getResourceAsStream(name)) { + if (in == null) { + throw new IllegalStateException("No resource [" + name + "] on the classpath"); + } + return new String(in.readAllBytes(), StandardCharsets.UTF_8); + } catch (IOException ex) { + throw new UncheckedIOException("Cannot read [" + name + "]", ex); + } + } +} diff --git a/tests/tests-integrations/src/main/resources/UpgradeFromPreviousReleaseIT/app.intent b/tests/tests-integrations/src/main/resources/UpgradeFromPreviousReleaseIT/app.intent new file mode 100644 index 00000000000..57253f05a46 --- /dev/null +++ b/tests/tests-integrations/src/main/resources/UpgradeFromPreviousReleaseIT/app.intent @@ -0,0 +1,39 @@ +# The application UpgradeFromPreviousReleaseIT generates and publishes on the PREVIOUS release, then +# boots the release under test on. Each part leaves state behind that an upgrade must carry over: +# rows in every entity, a process instance parked on a user task, a job that has fired. +name: upgrade +entities: + - name: Customer + fields: + - { name: id, type: integer, primaryKey: true, generated: true } + - { name: name, type: string, length: 100 } + - name: Ticket + fields: + - { name: id, type: integer, primaryKey: true, generated: true } + - { name: subject, type: string, length: 100 } + relations: + - { name: Customer, kind: manyToOne, to: Customer } + - name: TicketDigest + fields: + - { name: id, type: integer, primaryKey: true, generated: true } + relations: + - { name: Ticket, kind: manyToOne, to: Ticket } +processes: + # Started by every new ticket and left waiting on its review, so a running instance (and its task) + # spans the upgrade. + - name: TicketReview + trigger: { onCreate: Ticket } + steps: + - { name: review, kind: userTask, args: { assignee: agent, next: done } } + - { name: done, kind: end } +schedules: + # Every five seconds, one digest per ticket that has none yet: a job whose firing leaves rows, on + # either side of the upgrade. + - name: ticket-digest + cron: "0/5 * * * * ?" + entity: Ticket + generate: + to: TicketDigest + unique: [Ticket] + map: + Ticket: id