From 84d83eb2e44ac1528eba1d4e026c84d63fe2601b Mon Sep 17 00:00:00 2001 From: delchev Date: Mon, 5 Oct 2026 10:56:02 +0300 Subject: [PATCH] data-structures: a publish keeps a column its table no longer declares, drops it only when listed as dropped, and renames it in place when declared renamedFrom (#7635) TableAlterProcessor reconciled a published .table / .schema against the live table by ADDing missing columns and then DROPping every live column the model no longer named - in every tenant schema, with nothing in the log beyond the INFO "Processing Alter Table". A renamed intent field emits a new column name, so its data was gone on the first boot of the new image, and a rollback that only swaps the image could not bring it back. The alter now follows an expand/contract contract: - An undeclared column is KEPT with its data, logged at WARN with the table, the column and the artefact that owns the table, and counted as the orphanColumns detail of the artefacts health component (PlatformReadiness.recordOrphanColumns, keyed per catalog.schema.table so one tenant's copy does not overwrite another's). A kept NOT NULL column is relaxed to nullable, because the application no longer writes it and every insert would be refused otherwise; a dialect that cannot express that without the column's whole type (MySQL, MariaDB, MSSQL, HANA) leaves it and says so. - A column leaves only when the table lists it under `dropped`, or on an instance with DIRIGIBLE_DATABASE_DROP_UNDECLARED_COLUMNS=true (dev opt-in). - A column declaring `renamedFrom` renames the live one in place before the ADD pass (new ISqlDialect.renameColumn: ANSI by default, HANA's own form). Where a dialect has no in-place rename (MSSQL, MongoDB) the ADD pass adds the column nullable, copies the values and keeps the old one. - The incompatible-change refusals are unchanged. The intent carries both: a field's `renamedFrom: ` and an entity's `dropped: [names]` become the .model's dataRenamedFrom / dataDropped (the former COLUMN names, derived like dataName; comma-separated so they survive the scalar-only .edm), which the schema template emits as the column's renamedFrom and the table's dropped. The parser refuses either one naming a column the entity still declares (compared by column, so a relation counts), a rename from its own name or from a dropped name, and two renames from one name. Verified: TableAlterProcessorTest (keep + count + relaxed NOT NULL, declared drop, operator opt-in, in-place rename with a no-op second run), ColumnChangeStatementTest (default / MySQL / HANA statement shapes), PlatformReadinessTest, SchemaEvolutionValidationTest, EdmIntentGeneratorTest; the unit suites of every changed module; TableUndeclaredColumnIT on H2 and on a local PostgreSQL 16 (publish A,B + a row, republish A only: B and its data survive, the WARN is logged, an insert without B passes; republish with dropped: [B]: B is gone; the schema the template renders from a .model with dataRenamedFrom / dataDropped moves the data and drops the column), plus SchemaRepublishTypeToleranceIT, SchemaTemplateUniqueConstraintIT, SchemaTwoUniqueConstraintsIT and IntentEngineIT on H2; formatter:validate with the cache wiped; the release-profile javadoc build of the touched modules. Co-Authored-By: Claude Opus 5.5 --- .claude/docs/intent-dsl-features.md | 2 + .claude/docs/synchronizer-model.md | 2 + .../base/readiness/PlatformReadiness.java | 33 +++ .../base/readiness/PlatformReadinessTest.java | 14 + .../indicator/ArtefactsHealthIndicator.java | 9 +- .../data/structures/domain/Table.java | 37 ++- .../data/structures/domain/TableColumn.java | 30 ++- .../synchronizer/SchemasSynchronizer.java | 32 +++ .../table/TableAlterProcessor.java | 237 ++++++++++++++++- .../table/TableAlterProcessorTest.java | 109 ++++++++ .../generator/edm/EdmIntentGenerator.java | 18 ++ .../components/intent/model/EntityIntent.java | 15 ++ .../components/intent/model/FieldIntent.java | 15 ++ .../intent/parser/IntentParser.java | 66 +++++ .../main/resources/intent-assistant-guide.md | 22 ++ .../generator/edm/EdmIntentGeneratorTest.java | 37 +++ .../parser/SchemaEvolutionValidationTest.java | 94 +++++++ .../data/application.schema.template | 9 + .../commons/config/DirigibleConfig.java | 7 + .../sql/dialects/hana/HanaSqlDialect.java | 29 ++- .../hana/ColumnChangeStatementTest.java | 26 ++ .../dialects/mariadb/MariaDBSqlDialect.java | 12 + .../dialects/mongodb/MongoDBSqlDialect.java | 25 ++ .../sql/dialects/mssql/MSSQLSqlDialect.java | 26 ++ .../sql/dialects/mysql/MySQLSqlDialect.java | 12 + .../mysql/ColumnChangeStatementTest.java | 34 +++ .../dirigible/database/sql/ISqlDialect.java | 24 ++ .../sql/dialects/DefaultSqlDialect.java | 37 +++ .../dialects/ColumnChangeStatementTest.java | 38 +++ .../tests/api/TableUndeclaredColumnIT.java | 239 ++++++++++++++++++ 30 files changed, 1269 insertions(+), 21 deletions(-) create mode 100644 components/engine/engine-intent/src/test/java/org/eclipse/dirigible/components/intent/parser/SchemaEvolutionValidationTest.java create mode 100644 modules/database/database-sql-hana/src/test/java/org/eclipse/dirigible/database/sql/dialects/hana/ColumnChangeStatementTest.java create mode 100644 modules/database/database-sql-mysql/src/test/java/org/eclipse/dirigible/database/sql/dialects/mysql/ColumnChangeStatementTest.java create mode 100644 modules/database/database-sql/src/test/java/org/eclipse/dirigible/database/sql/dialects/ColumnChangeStatementTest.java create mode 100644 tests/tests-integrations/src/main/java/org/eclipse/dirigible/integration/tests/api/TableUndeclaredColumnIT.java diff --git a/.claude/docs/intent-dsl-features.md b/.claude/docs/intent-dsl-features.md index 319dd3dc776..828a7eacb82 100644 --- a/.claude/docs/intent-dsl-features.md +++ b/.claude/docs/intent-dsl-features.md @@ -18,6 +18,8 @@ Loaded on demand, not on every session: read this before changing a DSL construc **The declarative state machine (`lifecycle:`, [#6714](https://github.com/eclipse-dirigible/dirigible/issues/6714)):** an entity may declare the WHOLE set of legal status edges over its `function: EntityStatus` nomenclature — `lifecycle: { edges: [{ from: DRAFT, to: [ISSUED, CANCELLED] }, ...] }`, either side a seeded name or an id — and every status write is validated against it. Until then the status machinery was point constructs (`init:`, a `transitions:` button's own guard, a workflow `setRelationField`, a check's rejection) with nothing stating which moves were legal at all, so any writer that was not a transition button could move a document from any status to any other. Enforcement is in the generated **repository** — the one choke point every writer passes through (`update`, `updateWithoutEvent`, and `updateProperties`, which the transition controller's `updateProperty` and the workflow setters route through) — rejecting an unmodeled move with 400 and a message naming both statuses; with `init:` declared, a record cannot be CREATED mid-lifecycle either. At parse time the graph is what the other status sites are held to: a `transitions:` entry's `from`→`setStatus` pair must be a declared edge (a button is presentation over the graph), and a status written by a workflow step or forced by a check must be one some edge reaches — so a reject path transiting through an approved status fails when the intent is read. There is no `on:` key (the graph is always over the EntityStatus relation, and YAML reads a bare `on` as `true`, so it is refused rather than silently dropped), and a cross-model nomenclature is declared where it is seeded. Details in the engine-intent guide's state-machine bullet. +**Expand/contract on a live table (`renamedFrom:` / `dropped:`, [#7635](https://github.com/eclipse-dirigible/dirigible/issues/7635)):** a publish no longer drops a column its definition stops declaring - `TableAlterProcessor` keeps it with its data in every tenant, WARNs naming the owning artefact, relaxes a kept `NOT NULL` (dialect `dropNotNull`, `null` on MySQL/MariaDB/MSSQL/HANA, where it would need the whole type: then only the WARN), and counts it as the `orphanColumns` detail of the `artefacts` health component (`PlatformReadiness.recordOrphanColumns`, per catalog.schema.table so tenants do not overwrite each other). A field's `renamedFrom: ` becomes the `.model` property's `dataRenamedFrom` (the old COLUMN, derived like `dataName`) and the schema column's `renamedFrom`, and the alter renames the live column in place before the ADD pass (dialect `renameColumn`; on MSSQL/MongoDB it returns `null` and the ADD pass adds the column nullable, copies the values and keeps the old one); an entity's `dropped: [names]` becomes `dataDropped` (comma-separated - it must survive the scalar-only `.edm`) and the table's `dropped`, the only way besides the dev opt-in `DIRIGIBLE_DATABASE_DROP_UNDECLARED_COLUMNS=true` that a column leaves. Both act only while there is something to do, so they are harmless to leave in the intent. The parser refuses either one naming a column the entity still declares (compared by column, so a relation counts), a rename from its own name or from a dropped name, and two renames from one name. Unique constraints are still reconciled by drop - they hold no data. + **A field's label, and a label the TENANT'S COUNTRY resolves (`label:` / `countryLabels:`, [#6424](https://github.com/eclipse-dirigible/dirigible/issues/6424)):** a field may now declare its display `label:` - emitted as the property's own `widgetLabel`, which every generated surface renders and the en-US catalog is seeded from, so an acronym or a unit (`nationalId` as "National ID", not the humanized "National Id") is expressed in the intent instead of hand-edited into a catalog the next Generate overwrites. Alongside it, `countryLabels: { BG: ЕГН, DE: Steuer-ID }` declares variants resolved from the **tenant's country** (`DIRIGIBLE_APPLICATION_COUNTRY`, ISO 3166-1 alpha-2, tenant-overridable in the application shell's Tenant Configuration) rather than from the UI language - which term a national identifier goes by is a property of the company, so keying it off the language catalogs is wrong in both directions at once (the Bulgarian-reading user of a German company gets the local term; the English-reading accountant of a Bulgarian one gets the generic one). It also cannot live in a catalog mechanically: the shared `i18n.js` does not load catalogs at all in the default language. So the variants travel as a language-independent overlay - a structured `widgetCountryLabels` on the property (Map, hence `.model`-only), flattened at UI generation into the `countryLabels` object `config.js` carries, keyed by the very translation key the views bind, which `T()` consults ahead of both i18next and the baked fallback in every language. An app declaring no variant issues no extra request and generates byte-identically. A key that is not a country is refused at parse time (it could never match a tenant); report column labels are deliberately out of scope, a column alias being its SQL alias too. **Naming a rendered document (`fileName:`, [#6899](https://github.com/eclipse-dirigible/dirigible/issues/6899)):** the two server-side PDF renders — the snapshot copy a document mints on issue and the `attach: print` PDF a `notify` block sends — accept a **`fileName:`** pattern on the `function: Snapshot` child / inside the notify block: literals plus `{token}` interpolations, no expression language. `{field}` and one-hop `{relation.field}` use the same path vocabulary and **authored names** a notify subject does, `{field:pattern}` formats a `date`/`timestamp` through a `DateTimeFormatter` pattern, `{A|B}` renders the first non-blank operand, `{Version}` (snapshot only) places the copy's version and a pattern without it gets `_v` appended. Interpolated **values** are sanitized at run time by `sdk.print.FileNames` (trim, whitespace → one `_`, path/control characters stripped, non-ASCII deliberately kept); the literal separators are the author's. A pattern's relation loads are **merged into the notify block's own**, deduped by local, so a name may read a relation the message never mentions — but `attach: recordPrint` renders the anchor once, before the per-row loop, so only fields of the anchor are readable there. Absent a pattern, the snapshot default **changes** from the old primary-key form (`Order 42 v1.pdf`) to the same number-or-id expression the mail already used, plus the version — the deliberate fix for the two names disagreeing. An unresolvable path, an unbalanced brace, a format on a non-date field or a pattern that interpolates nothing are validation errors. Details in the engine-intent guide's fileName bullet. diff --git a/.claude/docs/synchronizer-model.md b/.claude/docs/synchronizer-model.md index b53bb9d3a3a..ccf926efb6e 100644 --- a/.claude/docs/synchronizer-model.md +++ b/.claude/docs/synchronizer-model.md @@ -19,3 +19,5 @@ This does **not** retire `RegistryMutationTracker`: a pass is now scheduled whil **A failure of a collaborator records `FAILED` and is retried; `FATAL` is not a synchronizer's answer to it** (#7248). A `.listener` whose subscription the embedded broker refused while it was still taking its store lease, a `.job` the scheduler could not schedule yet, a `.camel` route missing a bean, a `.schema` naming a `.datasource` published on a later pass - each of these is transient, and each of the four synchronizers used to promote the second failure to `FATAL`, after which `SynchronizationProcessor.parseDefinitions` strips the artefact from every later pass until the file's bytes change (a topic subscription lost to a boot race stayed lost, with every message discarded). The shape now, in all four: a failed start registers `FAILED` **with its cause** (never `CREATED` while nothing runs) and returns `false`, which hands the artefact to the in-pass cross-retry loop (`DIRIGIBLE_SYNCHRONIZER_CROSS_RETRY_COUNT` x `_INTERVAL_MILLIS`, so a permanently failing artefact costs that loop every pass - the same cost a `FAILED` view carries since #6942); the `START` phase retries it and heals it to `CREATED` on success. **A pass runs its phases only when it carries a `NEW`/`MODIFIED` artefact** (`isSynchronizationNeeded` answers "the registry changed", and the test framework waits on that answer for a quiet period - do not widen it), so the processor additionally gives every `FAILED` artefact ONE `START` attempt per pass on an idle instance every `DIRIGIBLE_SYNCHRONIZER_FAILED_RETRY_INTERVAL_SECONDS` (30) via a private gate (`isFailedRetryDue` / `retryFailed`) - no cross-retry loop, and a repeat of the same error logs at DEBUG rather than stack-tracing at ERROR on every attempt. Two multitenant details: `MultitenantBaseSynchronizer` completes one shared artefact once per tenant and resets only the lifecycle between tenants, so a retry gate must include `FAILED` and not rely on `running` alone (the first tenant to subscribe flips it and would skip the rest), and the managers' idempotence per tenant (`ListenersManager.LISTENERS.containsKey`, `JobsManager.scheduleJob`'s `checkExists`) is what makes the repeated attempt safe. `ListenersManager.startListener` throws for a missing handler rather than returning normally - a normal return read as CREATED and running with nothing subscribed. Artefacts already persisted as `FATAL` by an earlier version stay stripped until republished; `case BROKEN:` in `parseDefinitions` is the same philosophy one level down (a definition that failed to parse is re-parsed every pass). **A `.bpmn` DELETE retires its deployments, it never cascades (#7597).** `BpmnSynchronizer.removeFromProcessEngine` used to call Flowable's `deleteDeployment(id, true)`, which destroys every running instance of the definition together with its user tasks AND their history - so a `.bpmn` that was absent for one pass (the cleanup pass reaps any artefact whose source file is missing from the registry, which is what a fresh container sees before its module content has landed) took a month of approvals with it and left nothing to prove it. `BpmProviderFlowable.retireDeployment` now deletes a deployment only when none of its definitions has a running instance, non-cascading, so the finished instances stay in the history; one with running instances is kept with its definitions **suspended** (a WARN names the key, the deployment and the instance ids), so no new instance starts from it while the running ones complete. The artefact is still registered DELETED; should it come back, `deployProcess` is simply the next version of the same key (new instances start from it) and sweeps the retired version once its last instance has ended - as does the next DELETE. The explicit SDK `Deployer.undeployProcess` refuses with an `IllegalStateException` naming the running instances instead (its javadoc always promised not to terminate them). The history level is now set explicitly (`DIRIGIBLE_FLOWABLE_HISTORY_LEVEL`, default `audit`, Flowable's own default; `none` is accepted as an opt-out, logged at WARN, and shown on the monitoring shell's Processes page via `GET /services/bpm/bpm-processes/engine`), every instance deletion logs its id, definition and reason before the engine acts, and `BpmnDeploymentRetireIT` + `BpmProviderFlowableRetireDeploymentTest` pin the lifecycle. One Flowable detail the IT surfaced: `historic-instances?definitionKey=` is answered through a join on `ACT_RE_PROCDEF`, so the history of a definition whose deployments are all gone is listed unfiltered and by `businessKey`, never by `definitionKey`. + +**A table publish never drops a column on its own (#7635).** `TableAlterProcessor` (the alter path of both `.table` and `.schema`, so of every intent-generated table) keeps a live column its definition no longer declares, with its data, in every tenant - WARN, a relaxed `NOT NULL`, and the `orphanColumns` detail of the `artefacts` health component. A column leaves only when the table lists it under `dropped` (intent: `dropped:`), or on a dev instance with `DIRIGIBLE_DATABASE_DROP_UNDECLARED_COLUMNS=true`; a column's `renamedFrom` (intent: `renamedFrom:`) renames the live one in place. Do not reintroduce a silent DROP: a rename regenerated as add-new/drop-old lost the column's data on the first boot of a rolled image, and the rollback is image-only. diff --git a/components/core/core-base/src/main/java/org/eclipse/dirigible/components/base/readiness/PlatformReadiness.java b/components/core/core-base/src/main/java/org/eclipse/dirigible/components/base/readiness/PlatformReadiness.java index 0056c26e00c..d95acd1d893 100644 --- a/components/core/core-base/src/main/java/org/eclipse/dirigible/components/base/readiness/PlatformReadiness.java +++ b/components/core/core-base/src/main/java/org/eclipse/dirigible/components/base/readiness/PlatformReadiness.java @@ -11,7 +11,10 @@ import java.time.Instant; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReference; @@ -58,6 +61,7 @@ public enum State { private volatile Instant since = Instant.now(); private volatile ArtefactCensus artefacts = ArtefactCensus.NONE; private volatile CompiledModulesCensus compiledModules = null; + private final Map> orphanColumns = new ConcurrentHashMap<>(); private PlatformReadiness() {} @@ -127,6 +131,34 @@ public ArtefactCensus getArtefacts() { return artefacts; } + /** + * Records the columns a table keeps in the database although its published definition no longer + * declares them (#7635) - kept with their data instead of dropped. Each reconciliation of the table + * replaces its previous record, so a column that is declared again, or dropped, stops counting. + * + * @param table the table, qualified by its catalog and schema - one tenant's copy is not another's + * @param columns the undeclared columns, empty when the table has none + */ + public void recordOrphanColumns(String table, Set columns) { + if (columns.isEmpty()) { + orphanColumns.remove(table); + } else { + orphanColumns.put(table, Set.copyOf(columns)); + } + } + + /** + * The undeclared columns kept across every table, as the last reconciliation of each left them. + * + * @return the orphan column count + */ + public int getOrphanColumns() { + return orphanColumns.values() + .stream() + .mapToInt(Set::size) + .sum(); + } + /** * Records what the last AOT compiled-module discovery found (#7533). It runs on the application * ready event, independently of the synchronization passes, so the listeners are notified with the @@ -185,6 +217,7 @@ public void reset() { since = Instant.now(); artefacts = ArtefactCensus.NONE; compiledModules = null; + orphanColumns.clear(); listeners.clear(); } diff --git a/components/core/core-base/src/test/java/org/eclipse/dirigible/components/base/readiness/PlatformReadinessTest.java b/components/core/core-base/src/test/java/org/eclipse/dirigible/components/base/readiness/PlatformReadinessTest.java index 6aecfa032ab..d830a6c4982 100644 --- a/components/core/core-base/src/test/java/org/eclipse/dirigible/components/base/readiness/PlatformReadinessTest.java +++ b/components/core/core-base/src/test/java/org/eclipse/dirigible/components/base/readiness/PlatformReadinessTest.java @@ -16,6 +16,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; import org.eclipse.dirigible.components.base.readiness.PlatformReadiness.State; import org.junit.jupiter.api.BeforeEach; @@ -90,4 +91,17 @@ void theCompiledModulesCensusNotifiesTheListeners() { assertEquals(List.of(State.INITIALIZING), notified, "a consumer waiting for a clean boot must re-evaluate"); } + + @Test + void orphanColumnsAreCountedPerTableAndAReconciliationReplacesItsTablesRecord() { + readiness.recordOrphanColumns("DB.TENANT_A.INVOICE", Set.of("OLD_NOTE", "LEGACY_CODE")); + readiness.recordOrphanColumns("DB.TENANT_B.INVOICE", Set.of("OLD_NOTE")); + assertEquals(3, readiness.getOrphanColumns(), "one tenant's copy of a table does not overwrite another's"); + + readiness.recordOrphanColumns("DB.TENANT_A.INVOICE", Set.of()); + assertEquals(1, readiness.getOrphanColumns(), "a table reconciled without orphans stops counting"); + + readiness.reset(); + assertEquals(0, readiness.getOrphanColumns()); + } } diff --git a/components/core/core-healthcheck/src/main/java/org/eclipse/dirigible/components/base/healthcheck/indicator/ArtefactsHealthIndicator.java b/components/core/core-healthcheck/src/main/java/org/eclipse/dirigible/components/base/healthcheck/indicator/ArtefactsHealthIndicator.java index 74b21ab458f..4ff3220827c 100644 --- a/components/core/core-healthcheck/src/main/java/org/eclipse/dirigible/components/base/healthcheck/indicator/ArtefactsHealthIndicator.java +++ b/components/core/core-healthcheck/src/main/java/org/eclipse/dirigible/components/base/healthcheck/indicator/ArtefactsHealthIndicator.java @@ -18,9 +18,11 @@ /** * The {@code artefacts} health component (#7533): the lifecycle census of every registered artefact * as the last synchronization pass left it - how many there are, how many are failed, and of which - * type. A failed artefact is a quality signal, not an outage, so it never turns the component DOWN; - * a gate reads {@code failed}, and {@code DIRIGIBLE_READINESS_REQUIRE_CLEAN_BOOT} is the opt-in - * that makes it withhold readiness. UNKNOWN until the first pass has depleted. + * type, plus how many database columns are kept although their table's definition no longer + * declares them ({@code orphanColumns}, #7635). A failed artefact is a quality signal, not an + * outage, so it never turns the component DOWN; a gate reads {@code failed}, and + * {@code DIRIGIBLE_READINESS_REQUIRE_CLEAN_BOOT} is the opt-in that makes it withhold readiness. + * UNKNOWN until the first pass has depleted. */ @Component class ArtefactsHealthIndicator implements HealthIndicator { @@ -35,6 +37,7 @@ public Health health() { .withDetail("total", census.total()) .withDetail("failed", census.failed()) .withDetail("failedByType", census.failedByType()) + .withDetail("orphanColumns", readiness.getOrphanColumns()) .build(); } } diff --git a/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/domain/Table.java b/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/domain/Table.java index b4bf7dbba00..a8ff3ed2f5e 100644 --- a/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/domain/Table.java +++ b/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/domain/Table.java @@ -10,12 +10,14 @@ package org.eclipse.dirigible.components.data.structures.domain; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import java.util.Set; import jakarta.annotation.Nullable; import jakarta.persistence.CascadeType; import jakarta.persistence.Column; +import jakarta.persistence.Convert; import jakarta.persistence.Entity; import jakarta.persistence.FetchType; import jakarta.persistence.GeneratedValue; @@ -27,6 +29,7 @@ import jakarta.persistence.OneToOne; import org.eclipse.dirigible.components.base.artefact.Artefact; +import org.eclipse.dirigible.components.base.converters.ArrayOfStringsToCsvConverter; import org.hibernate.annotations.OnDelete; import org.hibernate.annotations.OnDeleteAction; @@ -76,6 +79,16 @@ public class Table extends Artefact { @Expose private TableConstraints constraints; + /** + * The columns this table no longer has (#7635): a live column the definition does not declare is + * kept, with its data, unless it is named here - the explicit contract step of a schema change. + */ + @Column(name = "TABLE_DROPPED_COLUMNS", columnDefinition = "VARCHAR", nullable = true, length = 2000) + @Nullable + @Convert(converter = ArrayOfStringsToCsvConverter.class) + @Expose + private String[] dropped; + /** The schema reference. */ @ManyToOne(fetch = FetchType.EAGER, optional = true) @JoinColumn(name = "SCHEMA_ID", nullable = true) @@ -283,12 +296,30 @@ public void setSchemaReference(Schema schemaReference) { * * @return the string */ + /** + * Gets the columns this table no longer has. + * + * @return the dropped column names, or null when none are declared + */ + public String[] getDropped() { + return dropped; + } + + /** + * Sets the columns this table no longer has. + * + * @param dropped the dropped column names + */ + public void setDropped(String[] dropped) { + this.dropped = dropped; + } + @Override public String toString() { return "Table [id=" + id + ", schemaName=" + schema + ", columns=" + columns + ", indexes=" + indexes + ", constraints=" - + constraints + ", location=" + location + ", name=" + name + ", type=" + type + ", description=" + description + ", key=" - + key + ", dependencies=" + dependencies + ", createdBy=" + createdBy + ", createdAt=" + createdAt + ", updatedBy=" - + updatedBy + ", updatedAt=" + updatedAt + "]"; + + constraints + ", dropped=" + Arrays.toString(dropped) + ", location=" + location + ", name=" + name + ", type=" + type + + ", description=" + description + ", key=" + key + ", dependencies=" + dependencies + ", createdBy=" + createdBy + + ", createdAt=" + createdAt + ", updatedBy=" + updatedBy + ", updatedAt=" + updatedAt + "]"; } /** diff --git a/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/domain/TableColumn.java b/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/domain/TableColumn.java index 3c2da8188ad..843182d8698 100644 --- a/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/domain/TableColumn.java +++ b/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/domain/TableColumn.java @@ -78,6 +78,15 @@ public class TableColumn { @Expose @SerializedName(value = "autoincrement", alternate = {"identity"}) private boolean autoincrement; + + /** + * The name this column had before (#7635): a live column of that name is renamed to this one, with + * its data, instead of this one being added beside it and that one left undeclared. + */ + @Column(name = "COLUMN_RENAMED_FROM", columnDefinition = "VARCHAR", nullable = true, length = 255) + @Expose + private String renamedFrom; + /** The table. */ @ManyToOne(fetch = FetchType.EAGER, optional = false) @JoinColumn(name = "TABLE_ID", nullable = false) @@ -366,11 +375,30 @@ public void setTable(Table table) { * * @return the string */ + /** + * Gets the name this column had before. + * + * @return the former name, or null when the column was not renamed + */ + public String getRenamedFrom() { + return renamedFrom; + } + + /** + * Sets the name this column had before. + * + * @param renamedFrom the former name + */ + public void setRenamedFrom(String renamedFrom) { + this.renamedFrom = renamedFrom; + } + @Override public String toString() { return "TableColumn [id=" + id + ", name=" + name + ", type=" + type + ", length=" + length + ", nullable=" + nullable + ", primaryKey=" + primaryKey + ", defaultValue=" + defaultValue + ", precision=" + precision + ", scale=" + scale - + ", unique=" + unique + ", autoincrement=" + autoincrement + ", table=" + (null == table ? null : table.getName()) + "]"; + + ", unique=" + unique + ", autoincrement=" + autoincrement + ", renamedFrom=" + renamedFrom + ", table=" + + (null == table ? null : table.getName()) + "]"; } } diff --git a/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/SchemasSynchronizer.java b/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/SchemasSynchronizer.java index d0a9729d0dd..631dbcbf5b1 100644 --- a/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/SchemasSynchronizer.java +++ b/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/SchemasSynchronizer.java @@ -309,6 +309,35 @@ private static void setTableAttributes(String location, Schema result, JsonObjec format("Error in parsing columns of table [{0}] in schema [{1}]", table.getName(), location)); } setUniqueConstraints(location, structure, table); + table.setDropped(droppedColumns(structure)); + } + + /** + * The table's {@code dropped} list (#7635) - the columns a re-publish removes from the live table. + * Written as a JSON array or as one comma-separated string. + * + * @param structure the table structure + * @return the dropped column names, or null when the structure declares none + */ + private static String[] droppedColumns(JsonObject structure) { + JsonElement element = structure.get("dropped"); + if (element == null || element.isJsonNull()) { + return null; + } + List names = new ArrayList<>(); + if (element.isJsonArray()) { + for (JsonElement name : element.getAsJsonArray()) { + names.add(name.getAsString()); + } + } else { + names.addAll(List.of(element.getAsString() + .split(","))); + } + String[] dropped = names.stream() + .map(String::trim) + .filter(name -> !name.isEmpty()) + .toArray(String[]::new); + return dropped.length == 0 ? null : dropped; } /** @@ -440,6 +469,9 @@ private static void setColumnAttributes(JsonObject column, TableColumn columnMod String precisionValue = getJsonElementValue(column, "precision", null); columnModel.setPrecision(precisionValue); + + String renamedFromValue = getJsonElementValue(column, "renamedFrom", null); + columnModel.setRenamedFrom(renamedFromValue == null || renamedFromValue.isBlank() ? null : renamedFromValue.trim()); } private static String getJsonElementValue(JsonObject jsonObject, String memberName, String defaultValue) { diff --git a/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/table/TableAlterProcessor.java b/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/table/TableAlterProcessor.java index 6d47eb7aa37..38581779e1e 100644 --- a/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/table/TableAlterProcessor.java +++ b/components/data/data-structures/src/main/java/org/eclipse/dirigible/components/data/structures/synchronizer/table/TableAlterProcessor.java @@ -22,12 +22,17 @@ import java.util.Set; import java.util.LinkedHashMap; import java.util.Map; +import java.util.TreeSet; + +import org.eclipse.dirigible.commons.config.DirigibleConfig; +import org.eclipse.dirigible.components.base.readiness.PlatformReadiness; import org.eclipse.dirigible.components.data.structures.domain.Table; import org.eclipse.dirigible.components.data.structures.domain.TableColumn; import org.eclipse.dirigible.components.database.DatabaseNameNormalizer; import org.eclipse.dirigible.database.sql.DataType; import org.eclipse.dirigible.database.sql.DataTypeUtils; +import org.eclipse.dirigible.database.sql.ISqlDialect; import org.eclipse.dirigible.database.sql.ISqlKeywords; import org.eclipse.dirigible.database.sql.SqlException; import org.eclipse.dirigible.database.sql.SqlFactory; @@ -60,22 +65,33 @@ public static void execute(Connection connection, Table tableModel) throws SQLEx logger.info("Processing Alter Table: " + tableName); Map columnDefinitions = new HashMap<>(); + // the live columns' names as the database spells them, and the ones it holds NOT NULL - both + // keyed by the canonical (upper-case) name every comparison below uses + Map liveNames = new HashMap<>(); + Set notNullColumns = new HashSet<>(); DatabaseMetaData dmd = connection.getMetaData(); String schema = connection.getSchema(); ResultSet rsColumns = dmd.getColumns(null, schema, DatabaseNameNormalizer.normalizeTableName(tableName), null); while (rsColumns.next()) { int columnType = rsColumns.getInt(5); - String columnName = rsColumns.getString(4) - .toUpperCase(); + String liveName = rsColumns.getString(4); + String columnName = liveName.toUpperCase(); try { String typeName = DataTypeUtils.getDatabaseTypeName(columnType); - columnDefinitions.put(DatabaseNameNormalizer.normalizeColumnName(columnName), typeName); + String canonicalName = DatabaseNameNormalizer.normalizeColumnName(columnName); + columnDefinitions.put(canonicalName, typeName); + liveNames.put(canonicalName, liveName); + if (rsColumns.getInt(11) == DatabaseMetaData.columnNoNulls) { + notNullColumns.add(canonicalName); + } } catch (SqlException ex) { String errorMessage = "Missing type for column [" + columnName + "] and type [" + columnType + "]"; throw new SqlException(errorMessage, ex); } } + renameColumns(connection, tableModel, columnDefinitions, liveNames, notNullColumns); + List modelColumnNames = new ArrayList<>(); // ADD iteration @@ -117,6 +133,15 @@ public static void execute(Connection connection, Table tableModel) throws SQLEx String nameOriginalCanonical = name.toUpperCase(); if (!columnDefinitions.containsKey(nameOriginalCanonical)) { + // a rename this database cannot do in place: the new column is added and the values are + // copied over, while the old column stays (undeclared, so kept) - the NOT NULL the model + // declares cannot be added to a column that starts out empty + String copyFrom = renamedLiveColumn(columnModel, liveNames); + if (copyFrom != null && !isNullable) { + logger.warn("Column [{}] of table [{}] is renamed from [{}] by adding it - it is created nullable," + + " this database cannot rename a column in place", name, tableName, copyFrom); + isNullable = true; + } AlterTableBuilder alterTableBuilder = SqlFactory.getNative(connection) .alter() @@ -135,6 +160,9 @@ public static void execute(Connection connection, Table tableModel) throws SQLEx } executeAlterBuilder(connection, alterTableBuilder); + if (copyFrom != null) { + copyColumn(connection, tableName, copyFrom, name); + } } else { String typeFromMetadata = columnDefinitions.get(nameOriginalCanonical); @@ -158,28 +186,211 @@ public static void execute(Connection connection, Table tableModel) throws SQLEx } } - // DROP iteration - for (String columnName : columnDefinitions.keySet()) { - if (!modelColumnNames.contains(columnName.toUpperCase())) { + reconcileUndeclaredColumns(connection, tableName, tableModel, modelColumnNames, liveNames, notNullColumns); + reconcileUniqueConstraints(connection, tableName, tableModel); + } + + /** + * Renames in place every live column the model declares a column {@code renamedFrom} (#7635), so + * the values move with the name instead of the old column being left behind and the new one added + * empty. A rename runs only while the old name is live and the new one is not, so a second run + * issues nothing. On a database without an in-place rename the column is left to the ADD pass, + * which copies the values over (see {@link #renamedLiveColumn}). The live-column maps are updated + * to what the database holds afterwards. + * + * @param connection the connection + * @param tableModel the model + * @param columnDefinitions the live column types, by canonical name + * @param liveNames the live column names as the database spells them, by canonical name + * @param notNullColumns the canonical names of the live NOT NULL columns + * @throws SQLException when the database refuses the rename - the data must not be split across two + * columns, so the table fails rather than falling through to the ADD + */ + private static void renameColumns(Connection connection, Table tableModel, Map columnDefinitions, + Map liveNames, Set notNullColumns) throws SQLException { + ISqlDialect dialect = SqlFactory.deriveDialect(connection); + String table = DatabaseNameNormalizer.normalizeTableName(tableModel.getName()); + for (TableColumn columnModel : tableModel.getColumns()) { + String from = renamedLiveColumn(columnModel, liveNames); + if (from == null) { + continue; + } + String to = DatabaseNameNormalizer.normalizeColumnName(columnModel.getName()); + String sql = dialect.renameColumn(table, from, to); + if (sql == null) { + continue; + } + logger.info("Renaming column [{}] of table [{}] to [{}], declared by [{}]", from, table, to, tableModel.getLocation()); + executeStatement(connection, sql); + String fromKey = canonical(from); + String toKey = canonical(to); + columnDefinitions.put(toKey, columnDefinitions.remove(fromKey)); + liveNames.remove(fromKey); + liveNames.put(toKey, to); + if (notNullColumns.remove(fromKey)) { + notNullColumns.add(toKey); + } + } + } + + /** + * The live column a model column is renamed from - only while the rename is still to be done: the + * old name is live and the new one is not. + * + * @param columnModel the model column + * @param liveNames the live column names as the database spells them, by canonical name + * @return the live name of the old column, or null when there is nothing to rename + */ + private static String renamedLiveColumn(TableColumn columnModel, Map liveNames) { + String renamedFrom = columnModel.getRenamedFrom(); + if (renamedFrom == null || renamedFrom.isBlank()) { + return null; + } + String toKey = canonical(DatabaseNameNormalizer.normalizeColumnName(columnModel.getName())); + String fromKey = canonical(DatabaseNameNormalizer.normalizeColumnName(renamedFrom.trim())); + if (fromKey.equals(toKey) || liveNames.containsKey(toKey)) { + return null; + } + return liveNames.get(fromKey); + } + + /** + * Copies the values of a renamed column into the column that replaces it, on a database that cannot + * rename in place. + * + * @param connection the connection + * @param quotedTableName the table name as the builders want it (quoted) + * @param from the live name of the old column + * @param to the new column + * @throws SQLException when the copy fails + */ + private static void copyColumn(Connection connection, String quotedTableName, String from, String to) throws SQLException { + String escape = String.valueOf(SqlFactory.deriveDialect(connection) + .getEscapeSymbol()); + String sql = SqlFactory.getNative(connection) + .update() + .table(quotedTableName) + .set("\"" + to + "\"", escape + from + escape) + .build(); + logger.info("Copying the values of column [{}] of table [{}] into its successor [{}]", from, quotedTableName, to); + executeStatement(connection, sql); + } + + /** + * Settles every live column the model does not declare (#7635). A column is dropped only when the + * table lists it as {@code dropped}, or when the operator opted into + * {@link DirigibleConfig#DATABASE_DROP_UNDECLARED_COLUMNS}; any other is KEPT with its data - a + * renamed or removed field must never cost the rows their values, in any tenant - logged at WARN + * with the artefact that owns the table, and counted in the {@code artefacts} health component. A + * kept column the database holds NOT NULL is relaxed to nullable, because the application no longer + * writes it and every insert would be refused otherwise. + * + * @param connection the connection + * @param quotedTableName the table name as the builders want it (quoted) + * @param tableModel the model + * @param modelColumnNames the canonical names of the declared columns + * @param liveNames the live column names as the database spells them, by canonical name + * @param notNullColumns the canonical names of the live NOT NULL columns + * @throws SQLException when a declared drop fails + */ + private static void reconcileUndeclaredColumns(Connection connection, String quotedTableName, Table tableModel, + List modelColumnNames, Map liveNames, Set notNullColumns) throws SQLException { + String table = DatabaseNameNormalizer.normalizeTableName(quotedTableName); + Set declaredDrops = new HashSet<>(); + if (tableModel.getDropped() != null) { + for (String dropped : tableModel.getDropped()) { + String droppedKey = canonical(DatabaseNameNormalizer.normalizeColumnName(dropped.trim())); + if (modelColumnNames.contains(droppedKey)) { + logger.warn("Column [{}] of table [{}] is both declared and listed as dropped by [{}] - it is kept", dropped, table, + tableModel.getLocation()); + } else { + declaredDrops.add(droppedKey); + } + } + } + boolean dropUndeclared = DirigibleConfig.DATABASE_DROP_UNDECLARED_COLUMNS.getBooleanValue(); + Set orphans = new TreeSet<>(); + for (Map.Entry live : liveNames.entrySet()) { + String key = live.getKey(); + String column = live.getValue(); + if (modelColumnNames.contains(key)) { + continue; + } + if (declaredDrops.contains(key) || dropUndeclared) { + logger.warn("Dropping column [{}] of table [{}] with its data - {}", column, table, + declaredDrops.contains(key) ? "listed as dropped by [" + tableModel.getLocation() + "]" + : DirigibleConfig.DATABASE_DROP_UNDECLARED_COLUMNS.getKey() + " is on"); AlterTableBuilder alterTableBuilder = SqlFactory.getNative(connection) .alter() - .table(tableName); + .table(quotedTableName); alterTableBuilder.drop() - .column("\"" + columnName + "\"", DataType.BOOLEAN); + .column("\"" + column + "\"", DataType.BOOLEAN); executeAlterBuilder(connection, alterTableBuilder); + continue; + } + orphans.add(column); + logger.warn("Column [{}] of table [{}] is not declared by [{}] and is kept with its data - list it as dropped to remove it", + column, table, tableModel.getLocation()); + if (notNullColumns.contains(key)) { + relaxNotNull(connection, table, column); } } - reconcileUniqueConstraints(connection, tableName, tableModel); + PlatformReadiness.getInstance() + .recordOrphanColumns(connection.getCatalog() + "." + connection.getSchema() + "." + table, orphans); + } + + /** + * Lets a kept undeclared column hold NULL, fail-soft: a database that cannot express it, or refuses + * it (the column is part of the primary key), leaves the column as it is with the consequence + * logged, without failing the table. + * + * @param connection the connection + * @param table the (unquoted) table + * @param column the live column name + */ + private static void relaxNotNull(Connection connection, String table, String column) { + String sql = SqlFactory.deriveDialect(connection) + .dropNotNull(table, column); + if (sql == null) { + logger.warn("Column [{}] of table [{}] is NOT NULL and this database cannot relax it here - an insert that does not set it" + + " is refused; declare the column again or list it as dropped", column, table); + return; + } + logger.info("Column [{}] of table [{}] is no longer written, so it is made nullable", column, table); + try { + executeStatement(connection, sql); + } catch (SQLException e) { + logger.error("Column [{}] of table [{}] could not be made nullable - an insert that does not set it is refused", column, table, + e); + } + } + + private static String canonical(String columnName) { + return columnName.toUpperCase(); + } + + /** + * Executes one statement of the alter path. + * + * @param connection the connection + * @param sql the statement + * @throws SQLException when the database refuses it + */ + private static void executeStatement(Connection connection, String sql) throws SQLException { + logger.info(sql); + try (PreparedStatement statement = connection.prepareStatement(sql)) { + statement.executeUpdate(); + } } /** * Brings the table's UNIQUE constraints in line with the model - the half of schema evolution the * column pass above never covered (#7019). A key the model declares (a {@code unique} column or a * composite {@code uniqueIndexes} entry) that the database lacks is ADDED; a UNIQUE the database - * enforces that the model no longer declares is DROPPED - the same policy the column pass applies - * to undeclared columns. Keys are compared as column SETS, so a differently named but equal key is - * left alone and a second run issues nothing. PRIMARY KEY and FOREIGN KEY constraints are never - * touched. + * enforces that the model no longer declares is DROPPED - a constraint holds no data, so unlike an + * undeclared column (#7635) it costs nothing to remove. Keys are compared as column SETS, so a + * differently named but equal key is left alone and a second run issues nothing. PRIMARY KEY and + * FOREIGN KEY constraints are never touched. * *

* Fails soft: a dialect without a catalog of unique constraints skips the step (no change from diff --git a/components/data/data-structures/src/test/java/org/eclipse/dirigible/components/data/structures/synchronizer/table/TableAlterProcessorTest.java b/components/data/data-structures/src/test/java/org/eclipse/dirigible/components/data/structures/synchronizer/table/TableAlterProcessorTest.java index eec01ea979c..a7091464bd4 100644 --- a/components/data/data-structures/src/test/java/org/eclipse/dirigible/components/data/structures/synchronizer/table/TableAlterProcessorTest.java +++ b/components/data/data-structures/src/test/java/org/eclipse/dirigible/components/data/structures/synchronizer/table/TableAlterProcessorTest.java @@ -11,6 +11,7 @@ import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -27,8 +28,12 @@ import java.sql.Statement; import java.sql.Types; +import org.eclipse.dirigible.commons.config.Configuration; +import org.eclipse.dirigible.commons.config.DirigibleConfig; +import org.eclipse.dirigible.components.base.readiness.PlatformReadiness; import org.eclipse.dirigible.components.data.structures.domain.Table; import org.eclipse.dirigible.components.data.structures.domain.TableColumn; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; /** @@ -36,6 +41,89 @@ */ class TableAlterProcessorTest { + @AfterEach + void resetState() { + Configuration.remove(DirigibleConfig.DATABASE_DROP_UNDECLARED_COLUMNS.getKey()); + PlatformReadiness.getInstance() + .reset(); + } + + /** + * #7635: a column the definition no longer declares - a removed or renamed field - is kept with its + * data, counted as an orphan, and relaxed to nullable so the inserts that no longer set it pass. + */ + @Test + void anUndeclaredColumnIsKeptWithItsData() throws SQLException { + try (Connection connection = connect("alter_undeclared_kept")) { + createTable(connection, "\"A\" VARCHAR(20), \"B\" VARCHAR(20) NOT NULL"); + insert(connection, "INSERT INTO \"T_NOTES\" VALUES (1, 'a', 'b')"); + Table tableModel = new Table("T_NOTES"); + new TableColumn("ID", "INTEGER", null, tableModel); + new TableColumn("A", "VARCHAR", "20", tableModel); + + TableAlterProcessor.execute(connection, tableModel); + + assertEquals("b", value(connection, "B"), "the undeclared column keeps its data"); + assertEquals(1, PlatformReadiness.getInstance() + .getOrphanColumns()); + assertDoesNotThrow(() -> insert(connection, "INSERT INTO \"T_NOTES\" (\"ID\", \"A\") VALUES (2, 'a')"), + "the kept column no longer refuses an insert that does not set it"); + } + } + + @Test + void aColumnListedAsDroppedIsDropped() throws SQLException { + try (Connection connection = connect("alter_undeclared_dropped")) { + createTable(connection, "\"A\" VARCHAR(20), \"B\" VARCHAR(20)"); + insert(connection, "INSERT INTO \"T_NOTES\" VALUES (1, 'a', 'b')"); + Table tableModel = new Table("T_NOTES"); + new TableColumn("ID", "INTEGER", null, tableModel); + new TableColumn("A", "VARCHAR", "20", tableModel); + tableModel.setDropped(new String[] {"B"}); + + TableAlterProcessor.execute(connection, tableModel); + + assertFalse(hasColumn(connection, "B")); + assertEquals(0, PlatformReadiness.getInstance() + .getOrphanColumns()); + } + } + + @Test + void theOperatorOptInDropsEveryUndeclaredColumn() throws SQLException { + try (Connection connection = connect("alter_undeclared_opt_in")) { + createTable(connection, "\"A\" VARCHAR(20), \"B\" VARCHAR(20)"); + Configuration.set(DirigibleConfig.DATABASE_DROP_UNDECLARED_COLUMNS.getKey(), "true"); + Table tableModel = new Table("T_NOTES"); + new TableColumn("ID", "INTEGER", null, tableModel); + new TableColumn("A", "VARCHAR", "20", tableModel); + + TableAlterProcessor.execute(connection, tableModel); + + assertFalse(hasColumn(connection, "B")); + } + } + + @Test + void aRenamedColumnIsRenamedInPlaceWithItsData() throws SQLException { + try (Connection connection = connect("alter_renamed")) { + createTable(connection, "\"INVOICE_DATE\" VARCHAR(20) NOT NULL"); + insert(connection, "INSERT INTO \"T_NOTES\" VALUES (1, '2026-10-05')"); + Table tableModel = new Table("T_NOTES"); + new TableColumn("ID", "INTEGER", null, tableModel); + new TableColumn("ISSUE_DATE", "VARCHAR", "20", false, false, null, null, null, false, false, tableModel).setRenamedFrom( + "INVOICE_DATE"); + + TableAlterProcessor.execute(connection, tableModel); + assertDoesNotThrow(() -> TableAlterProcessor.execute(connection, tableModel), "a second run has nothing to rename"); + + assertEquals("2026-10-05", value(connection, "ISSUE_DATE")); + assertFalse(hasColumn(connection, "INVOICE_DATE")); + assertEquals(0, PlatformReadiness.getInstance() + .getOrphanColumns()); + } + } + /** * A table whose columns the database reports in another - but equivalent - representation than the * definition declares must not fail the table: a {@code CLOB} column comes back as @@ -185,6 +273,27 @@ private static void createTable(Connection connection, String columns) throws SQ } } + private static void insert(Connection connection, String sql) throws SQLException { + try (Statement statement = connection.createStatement()) { + statement.execute(sql); + } + } + + private static String value(Connection connection, String column) throws SQLException { + try (Statement statement = connection.createStatement(); + ResultSet rs = statement.executeQuery("SELECT \"" + column + "\" FROM \"T_NOTES\" WHERE \"ID\" = 1")) { + assertTrue(rs.next(), "Missing row 1"); + return rs.getString(1); + } + } + + private static boolean hasColumn(Connection connection, String column) throws SQLException { + try (ResultSet columns = connection.getMetaData() + .getColumns(null, connection.getSchema(), "T_NOTES", column)) { + return columns.next(); + } + } + private static int columnType(Connection connection, String column) throws SQLException { try (ResultSet columns = connection.getMetaData() .getColumns(null, connection.getSchema(), "T_NOTES", column)) { diff --git a/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/generator/edm/EdmIntentGenerator.java b/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/generator/edm/EdmIntentGenerator.java index 551c2227088..94cce02160f 100644 --- a/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/generator/edm/EdmIntentGenerator.java +++ b/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/generator/edm/EdmIntentGenerator.java @@ -460,6 +460,17 @@ else if (!extension && !dependent && !setting && !compositionParents.containsVal if (entity.isHistorized()) { entityMap.put("history", "true"); } + // dropped (#7635): the columns of former fields and relations a publish removes, data + // included. A field simply deleted from the intent keeps its column and its values; this + // list is the explicit contract step. Comma-separated, so it survives the scalar-only .edm. + if (!entity.getDropped() + .isEmpty()) { + entityMap.put("dataDropped", entity.getDropped() + .stream() + .map(dropped -> IntentNaming.upperSnake(name) + "_" + + IntentNaming.upperSnake(dropped.trim())) + .collect(Collectors.joining(","))); + } // A file-child: mark it so the generated controller emits the download (and, when editable, // upload/delete) verbs and the Harmonia master/document view renders it as a Files panel // (a composition detail already, so the master-detail wiring is unchanged). A Snapshot is @@ -1373,6 +1384,13 @@ private static Map propertyMap(String entityName, FieldIntent fi p.put("description", field.getDescription() == null ? "" : field.getDescription()); p.put("tooltip", ""); p.put("dataName", column); + // The column this one was (#7635): the schema template carries it as the column's renamedFrom, + // and the publish renames the live column in place, values included. Emitted only when + // authored, so a model that does not rename anything generates byte-identically. + if (notBlank(field.getRenamedFrom())) { + p.put("dataRenamedFrom", IntentNaming.upperSnake(entityName) + "_" + IntentNaming.upperSnake(field.getRenamedFrom() + .trim())); + } p.put("dataType", dataType); p.put("dataNullable", field.isRequired() || field.isPrimaryKey() ? "false" : "true"); // Read-only in generated forms (rendered in the read-only details block, not an editable input): diff --git a/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/model/EntityIntent.java b/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/model/EntityIntent.java index 1262e7ecb0f..96e5a581325 100644 --- a/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/model/EntityIntent.java +++ b/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/model/EntityIntent.java @@ -129,6 +129,13 @@ public class EntityIntent { * {@code major} properties in control order, as before. */ private List list = new ArrayList<>(); + /** + * Optional names of fields and to-one relations this entity no longer has, whose columns a publish + * removes from the live table, data included (#7635) - the explicit contract step of a schema + * change. A field that is simply deleted from the intent keeps its column and its values, so a + * mistaken removal or a rename costs nothing; listing it here is how the data is let go. + */ + private List dropped = new ArrayList<>(); /** * Optional UI view type for this entity. {@code calendar} / {@code range} render its records as * events on a Harmonia calendar (see {@link #calendar}) - an ADDITIONAL page: the entity keeps the @@ -370,6 +377,14 @@ public void setSlots(SlotsIntent slots) { this.slots = slots; } + public List getDropped() { + return dropped; + } + + public void setDropped(List dropped) { + this.dropped = dropped == null ? new ArrayList<>() : dropped; + } + public List getOrder() { return order; } diff --git a/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/model/FieldIntent.java b/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/model/FieldIntent.java index 7e4187bb009..60c8e2a8631 100644 --- a/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/model/FieldIntent.java +++ b/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/model/FieldIntent.java @@ -166,6 +166,13 @@ public class FieldIntent { * behaviour. */ private Boolean translatable; + /** + * Optional name this field had before (#7635). A publish renames the live column in place, so the + * values follow the new name instead of staying behind in the old column while the new one starts + * empty. Harmless to keep once the rename is done: it acts only while the old column exists and the + * new one does not. + */ + private String renamedFrom; /** * Optional form-control width as a 12-column grid span (3/4/6/12: 3 = quarter, 4 = third, 6 = half, * 12 = full). Emitted as the property's {@code widgetSize}; the Harmonia form maps it to @@ -416,6 +423,14 @@ public boolean isMajor() { return major == null || major; } + public String getRenamedFrom() { + return renamedFrom; + } + + public void setRenamedFrom(String renamedFrom) { + this.renamedFrom = renamedFrom; + } + public Boolean getTranslatable() { return translatable; } diff --git a/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/parser/IntentParser.java b/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/parser/IntentParser.java index b1619ff2295..5c0ef911e31 100644 --- a/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/parser/IntentParser.java +++ b/components/engine/engine-intent/src/main/java/org/eclipse/dirigible/components/intent/parser/IntentParser.java @@ -4239,6 +4239,7 @@ private static Set validateEntities(IntentModel model, Set usesA if (!entityNames.add(name)) { issues.add("duplicate entity [" + name + "]"); } + validateSchemaEvolution(entity, issues); Set fieldNames = new HashSet<>(); int idCount = 0; for (FieldIntent field : entity.getFields()) { @@ -6998,6 +6999,71 @@ private static void validateRuleMatchIsNotTranslated(String subject, EntityInten * @param field the field carrying the marker * @param issues the collected issues, appended to */ + /** + * The expand/contract declarations of an entity's table (#7635): {@code dropped:} names the former + * fields and relations whose columns a publish removes, a field's {@code renamedFrom:} the former + * name whose column it takes over. Both are compared by the COLUMN the name maps to, because that + * is what would collide in the table: a dropped name or a rename source that is still a field or a + * relation would drop or move a column the entity still declares, and two fields renamed from one + * name would both claim its data. + * + * @param entity the entity + * @param issues the issue list to append to + */ + private static void validateSchemaEvolution(EntityIntent entity, List issues) { + String subject = "entity [" + entity.getName() + "]"; + Map declared = new HashMap<>(); + for (FieldIntent field : entity.getFields()) { + if (!isBlank(field.getName())) { + declared.put(IntentNaming.upperSnake(field.getName()), field.getName()); + } + } + for (RelationIntent relation : entity.getRelations()) { + if (!isBlank(relation.getName())) { + declared.put(IntentNaming.upperSnake(relation.getName()), relation.getName()); + } + } + Set dropped = new HashSet<>(); + for (String name : entity.getDropped()) { + if (isBlank(name)) { + issues.add(subject + " lists an empty name under `dropped`"); + continue; + } + String column = IntentNaming.upperSnake(name.trim()); + if (declared.containsKey(column)) { + issues.add(subject + " lists [" + name + "] under `dropped` but still declares [" + declared.get(column) + + "] - a dropped column is one the entity no longer has"); + } else if (!dropped.add(column)) { + issues.add(subject + " lists [" + name + "] under `dropped` twice"); + } + } + Map renameSources = new HashMap<>(); + for (FieldIntent field : entity.getFields()) { + String renamedFrom = field.getRenamedFrom(); + if (renamedFrom == null || isBlank(field.getName())) { + continue; + } + String fieldSubject = subject + " field [" + field.getName() + "]"; + if (isBlank(renamedFrom)) { + issues.add(fieldSubject + " declares an empty `renamedFrom`"); + continue; + } + String column = IntentNaming.upperSnake(renamedFrom.trim()); + if (column.equals(IntentNaming.upperSnake(field.getName()))) { + issues.add(fieldSubject + " is `renamedFrom` its own name"); + } else if (declared.containsKey(column)) { + issues.add(fieldSubject + " is `renamedFrom` [" + renamedFrom + "], which the entity still declares as [" + + declared.get(column) + "] - a rename takes over a column no field owns any more"); + } else if (dropped.contains(column)) { + issues.add(fieldSubject + " is `renamedFrom` [" + renamedFrom + "], which is also listed under `dropped`" + + " - the rename would keep the data the drop removes"); + } else if (renameSources.putIfAbsent(column, field.getName()) != null) { + issues.add(fieldSubject + " is `renamedFrom` [" + renamedFrom + "], as is field [" + renameSources.get(column) + + "] - only one field can take over its column"); + } + } + } + private static void validateTranslatable(EntityIntent entity, String subject, FieldIntent field, List issues) { if (field.isTranslatable()) { return; diff --git a/components/engine/engine-intent/src/main/resources/intent-assistant-guide.md b/components/engine/engine-intent/src/main/resources/intent-assistant-guide.md index da39c04adcf..cd8d6781df0 100644 --- a/components/engine/engine-intent/src/main/resources/intent-assistant-guide.md +++ b/components/engine/engine-intent/src/main/resources/intent-assistant-guide.md @@ -1082,6 +1082,28 @@ Generation REFUSES a rule `match:` column and an arrival lookup `by:` field that this marker; and it refuses the marker itself where it cannot mean anything (a non-multilingual entity, a non-string field). +**Renaming or removing a field of a live application (`renamedFrom:` / `dropped:`).** A publish never +drops a column on its own: a field deleted from the intent keeps its column and its values in every +tenant (the column is logged as undeclared and counted), so a mistaken removal costs nothing. Two +declarations make the change explicit. When you RENAME a field, keep its data with `renamedFrom: +` on the field - the publish renames the column in place; without it the old values stay +behind in the old column and the new field starts empty. When the user really wants a field's data +gone, list the former name under the entity's `dropped:` - that publish removes the column, values +included. Both name fields or to-one relations by their authored names, and both are refused when the +name is still declared on the entity: + +```yaml +entities: + - name: Invoice + dropped: [legacyCode] # this column and its data are removed + fields: + - { name: id, type: integer, primaryKey: true, generated: true } + - { name: issueDate, type: date, renamedFrom: invoiceDate } # the values move with the name +``` + +Propose `renamedFrom:` whenever you rename a field of an entity that may already hold rows; propose +`dropped:` only when the user asks for the data to go, never as part of a rename. + **Custom imports (`imports:` on an entity):** a multi-line string of Java `import ...;` lines injected verbatim into that entity's generated repository, so a calculated-field action (or any custom class) can be referenced from the calculated fields by simple name. Pair it with `calculatedActionOnCreate`: diff --git a/components/engine/engine-intent/src/test/java/org/eclipse/dirigible/components/intent/generator/edm/EdmIntentGeneratorTest.java b/components/engine/engine-intent/src/test/java/org/eclipse/dirigible/components/intent/generator/edm/EdmIntentGeneratorTest.java index b2ddf05d717..aaa3611eda3 100644 --- a/components/engine/engine-intent/src/test/java/org/eclipse/dirigible/components/intent/generator/edm/EdmIntentGeneratorTest.java +++ b/components/engine/engine-intent/src/test/java/org/eclipse/dirigible/components/intent/generator/edm/EdmIntentGeneratorTest.java @@ -2416,6 +2416,43 @@ void historyEntityCarriesTheModelAttribute() { assertNull(entityByName(entities, "Note").get("history"), "an entity that did not ask for a history must not carry the attribute"); } + @Test + void aRenamedFieldAndTheDroppedNamesCarryTheirFormerColumns() { + String yaml = """ + name: billing + entities: + - name: Invoice + dropped: [oldNote, legacyCode] + fields: + - { name: id, type: integer, primaryKey: true, generated: true } + - { name: issueDate, type: date, renamedFrom: invoiceDate } + """; + Map invoice = + entityByName(entities(EdmIntentGenerator.buildModelJsonForTest(IntentParser.parse(yaml), "billing")), "Invoice"); + + // #7635: the former COLUMN names, derived exactly as the current ones are, because the schema + // template hands them to the table alter path, which matches live columns by name + assertEquals("INVOICE_INVOICE_DATE", propertyByName(invoice, "IssueDate").get("dataRenamedFrom")); + assertEquals("INVOICE_OLD_NOTE,INVOICE_LEGACY_CODE", invoice.get("dataDropped")); + } + + @Test + void anEntityWithoutSchemaChangesCarriesNeitherAttribute() { + String yaml = """ + name: billing + entities: + - name: Invoice + fields: + - { name: id, type: integer, primaryKey: true, generated: true } + - { name: issueDate, type: date } + """; + Map invoice = + entityByName(entities(EdmIntentGenerator.buildModelJsonForTest(IntentParser.parse(yaml), "billing")), "Invoice"); + + assertNull(invoice.get("dataDropped"), "a model that drops nothing must generate byte-identically"); + assertNull(propertyByName(invoice, "IssueDate").get("dataRenamedFrom")); + } + @Test void aKeyFieldOfAMultilingualEntityCarriesTheNonTranslatableMarker() { String yaml = """ diff --git a/components/engine/engine-intent/src/test/java/org/eclipse/dirigible/components/intent/parser/SchemaEvolutionValidationTest.java b/components/engine/engine-intent/src/test/java/org/eclipse/dirigible/components/intent/parser/SchemaEvolutionValidationTest.java new file mode 100644 index 00000000000..fae8e8e9fc1 --- /dev/null +++ b/components/engine/engine-intent/src/test/java/org/eclipse/dirigible/components/intent/parser/SchemaEvolutionValidationTest.java @@ -0,0 +1,94 @@ +/* + * Copyright (c) 2010-2026 Eclipse Dirigible contributors + * + * All rights reserved. This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v2.0 which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v20.html + * + * SPDX-FileCopyrightText: Eclipse Dirigible contributors SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.dirigible.components.intent.parser; + +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.junit.jupiter.api.Test; + +/** + * The expand/contract declarations of an entity's table (#7635) - {@code dropped:} and a field's + * {@code renamedFrom:} - are refused when they would drop or move a column the entity still + * declares. + */ +class SchemaEvolutionValidationTest { + + @Test + void aRenameAndADropOfFormerNamesAreAccepted() { + assertDoesNotThrow(() -> IntentParser.parse(invoice("dropped: [oldNote]", "renamedFrom: invoiceDate"))); + } + + @Test + void aDroppedNameTheEntityStillDeclaresIsRefused() { + assertIssue(invoice("dropped: [total]", null), "lists [total] under `dropped` but still declares [total]"); + } + + @Test + void aDroppedRelationIsComparedByItsColumnToo() { + assertIssue(invoice("dropped: [Customer]", null), "lists [Customer] under `dropped` but still declares [customer]"); + } + + @Test + void aFieldRenamedFromItsOwnNameIsRefused() { + assertIssue(invoice(null, "renamedFrom: issueDate"), "field [issueDate] is `renamedFrom` its own name"); + } + + @Test + void aRenameFromADeclaredFieldIsRefused() { + assertIssue(invoice(null, "renamedFrom: total"), "which the entity still declares as [total]"); + } + + @Test + void aRenameFromADroppedNameIsRefused() { + assertIssue(invoice("dropped: [invoiceDate]", "renamedFrom: invoiceDate"), "which is also listed under `dropped`"); + } + + @Test + void twoFieldsRenamedFromOneNameAreRefused() { + String yaml = """ + name: billing + entities: + - name: Invoice + fields: + - { name: id, type: integer, primaryKey: true, generated: true } + - { name: issueDate, type: date, renamedFrom: invoiceDate } + - { name: dueDate, type: date, renamedFrom: invoiceDate } + """; + assertIssue(yaml, "only one field can take over its column"); + } + + private static String invoice(String dropped, String renamedFrom) { + return """ + name: billing + entities: + - name: Customer + fields: + - { name: id, type: integer, primaryKey: true, generated: true } + - name: Invoice + %s + fields: + - { name: id, type: integer, primaryKey: true, generated: true } + - { name: issueDate, type: date%s } + - { name: total, type: decimal } + relations: + - { name: customer, kind: manyToOne, to: Customer } + """.formatted(dropped == null ? "" : dropped, renamedFrom == null ? "" : ", " + renamedFrom); + } + + private static void assertIssue(String yaml, String fragment) { + IntentValidationException ex = assertThrows(IntentValidationException.class, () -> IntentParser.parse(yaml)); + assertTrue(ex.getIssues() + .stream() + .anyMatch(issue -> issue.contains(fragment)), + "expected an issue containing [" + fragment + "], got " + ex.getIssues()); + } +} diff --git a/components/template/template-application-schema/src/main/resources/META-INF/dirigible/template-application-schema/data/application.schema.template b/components/template/template-application-schema/src/main/resources/META-INF/dirigible/template-application-schema/data/application.schema.template index 3d6359f234b..cf75053d571 100644 --- a/components/template/template-application-schema/src/main/resources/META-INF/dirigible/template-application-schema/data/application.schema.template +++ b/components/template/template-application-schema/src/main/resources/META-INF/dirigible/template-application-schema/data/application.schema.template @@ -124,6 +124,11 @@ { "name": "${tablePrefix}${model.dataName}", "type": "TABLE", +## The columns of former fields a publish removes, data included (#7635). A column the model simply +## stops declaring is kept with its values; this list is the explicit contract step. +#if($model.dataDropped) + "dropped": [#foreach ($droppedColumn in $model.dataDropped.split(","))"${droppedColumn}"#if($foreach.hasNext), #end#end], +#end "columns": [ #foreach ($property in $model.properties) { @@ -156,6 +161,10 @@ #end #if($property.dataUnique) "unique": true, +#end +## The column this one was (#7635): the publish renames it in place, values included. +#if($property.dataRenamedFrom) + "renamedFrom": "${property.dataRenamedFrom}", #end "name": "${property.dataName}" #if($foreach.hasNext) diff --git a/modules/commons/commons-config/src/main/java/org/eclipse/dirigible/commons/config/DirigibleConfig.java b/modules/commons/commons-config/src/main/java/org/eclipse/dirigible/commons/config/DirigibleConfig.java index 7be7e6e75ae..966efe8ad27 100644 --- a/modules/commons/commons-config/src/main/java/org/eclipse/dirigible/commons/config/DirigibleConfig.java +++ b/modules/commons/commons-config/src/main/java/org/eclipse/dirigible/commons/config/DirigibleConfig.java @@ -138,6 +138,13 @@ public enum DirigibleConfig { /** The system data source name. */ SYSTEM_DATA_SOURCE_NAME("DIRIGIBLE_DATABASE_DATASOURCE_NAME_SYSTEM", "SystemDB"), + /** + * Drop a live column the published table definition no longer declares (#7635). Off by default: an + * undeclared column is kept, with its data, and only a table's own {@code dropped} list removes + * one. Meant for development instances, where a regenerated model is the only owner of the data. + */ + DATABASE_DROP_UNDECLARED_COLUMNS("DIRIGIBLE_DATABASE_DROP_UNDECLARED_COLUMNS", Boolean.FALSE.toString()), + /** The synchronizer frequency. */ SYNCHRONIZER_FREQUENCY("DIRIGIBLE_SYNCHRONIZER_FREQUENCY", "10"), diff --git a/modules/database/database-sql-hana/src/main/java/org/eclipse/dirigible/database/sql/dialects/hana/HanaSqlDialect.java b/modules/database/database-sql-hana/src/main/java/org/eclipse/dirigible/database/sql/dialects/hana/HanaSqlDialect.java index f2343367c66..168f1bc835f 100644 --- a/modules/database/database-sql-hana/src/main/java/org/eclipse/dirigible/database/sql/dialects/hana/HanaSqlDialect.java +++ b/modules/database/database-sql-hana/src/main/java/org/eclipse/dirigible/database/sql/dialects/hana/HanaSqlDialect.java @@ -393,5 +393,32 @@ public String getFuzzySearchIndex() { return "FUZZY SEARCH INDEX ON"; } -} + /** + * HANA names the column through its table: {@code RENAME COLUMN + * + + * . TO }. + * + * @param table the table + * @param from the current column name + * @param to the new column name + * @return the statement + */ + @Override + public String renameColumn(String table, String from, String to) { + return "RENAME COLUMN " + quoteIdentifier(table) + "." + quoteIdentifier(from) + " TO " + quoteIdentifier(to); + } + /** + * {@code NULL} is relaxed only by restating the column's whole type here ({@code ALTER (...)}), + * which the caller does not have. + * + * @param table the table + * @param column the column + * @return {@code null} + */ + @Override + public String dropNotNull(String table, String column) { + return null; + } +} diff --git a/modules/database/database-sql-hana/src/test/java/org/eclipse/dirigible/database/sql/dialects/hana/ColumnChangeStatementTest.java b/modules/database/database-sql-hana/src/test/java/org/eclipse/dirigible/database/sql/dialects/hana/ColumnChangeStatementTest.java new file mode 100644 index 00000000000..d12aa1420c0 --- /dev/null +++ b/modules/database/database-sql-hana/src/test/java/org/eclipse/dirigible/database/sql/dialects/hana/ColumnChangeStatementTest.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2010-2026 Eclipse Dirigible contributors + * + * All rights reserved. This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v2.0 which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v20.html + * + * SPDX-FileCopyrightText: Eclipse Dirigible contributors SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.dirigible.database.sql.dialects.hana; + +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +/** + * The column rename of the table alter path (#7635) on HANA. + */ +public class ColumnChangeStatementTest { + + @Test + public void renameColumnNamesTheColumnThroughItsTable() { + assertEquals("RENAME COLUMN \"T_INVOICE\".\"INVOICE_DATE\" TO \"ISSUE_DATE\"", + new HanaSqlDialect().renameColumn("T_INVOICE", "INVOICE_DATE", "ISSUE_DATE")); + } +} diff --git a/modules/database/database-sql-mariadb/src/main/java/org/eclipse/dirigible/database/sql/dialects/mariadb/MariaDBSqlDialect.java b/modules/database/database-sql-mariadb/src/main/java/org/eclipse/dirigible/database/sql/dialects/mariadb/MariaDBSqlDialect.java index ea7e8da8ab4..d25cdebe858 100644 --- a/modules/database/database-sql-mariadb/src/main/java/org/eclipse/dirigible/database/sql/dialects/mariadb/MariaDBSqlDialect.java +++ b/modules/database/database-sql-mariadb/src/main/java/org/eclipse/dirigible/database/sql/dialects/mariadb/MariaDBSqlDialect.java @@ -174,4 +174,16 @@ public MariaDBInsertBuilder insert() { return new MariaDBInsertBuilder(this); } + /** + * {@code NULL} is relaxed only by restating the column's whole type here ({@code MODIFY COLUMN}), + * which the caller does not have. + * + * @param table the table + * @param column the column + * @return {@code null} + */ + @Override + public String dropNotNull(String table, String column) { + return null; + } } diff --git a/modules/database/database-sql-mongodb/src/main/java/org/eclipse/dirigible/database/sql/dialects/mongodb/MongoDBSqlDialect.java b/modules/database/database-sql-mongodb/src/main/java/org/eclipse/dirigible/database/sql/dialects/mongodb/MongoDBSqlDialect.java index 2c252632433..a0f3fdf926b 100644 --- a/modules/database/database-sql-mongodb/src/main/java/org/eclipse/dirigible/database/sql/dialects/mongodb/MongoDBSqlDialect.java +++ b/modules/database/database-sql-mongodb/src/main/java/org/eclipse/dirigible/database/sql/dialects/mongodb/MongoDBSqlDialect.java @@ -191,4 +191,29 @@ public void importData(Connection connection, String table, InputStream input) t public Map> uniqueConstraints(Connection connection, String table) { return null; } + + /** + * A collection has no columns to rename. + * + * @param table the table + * @param from the current column name + * @param to the new column name + * @return {@code null} + */ + @Override + public String renameColumn(String table, String from, String to) { + return null; + } + + /** + * A collection has no column constraints. + * + * @param table the table + * @param column the column + * @return {@code null} + */ + @Override + public String dropNotNull(String table, String column) { + return null; + } } diff --git a/modules/database/database-sql-mssql/src/main/java/org/eclipse/dirigible/database/sql/dialects/mssql/MSSQLSqlDialect.java b/modules/database/database-sql-mssql/src/main/java/org/eclipse/dirigible/database/sql/dialects/mssql/MSSQLSqlDialect.java index df48a2525fc..44aec8557b0 100644 --- a/modules/database/database-sql-mssql/src/main/java/org/eclipse/dirigible/database/sql/dialects/mssql/MSSQLSqlDialect.java +++ b/modules/database/database-sql-mssql/src/main/java/org/eclipse/dirigible/database/sql/dialects/mssql/MSSQLSqlDialect.java @@ -251,4 +251,30 @@ public boolean existsTable(Connection connection, String table) throws SQLExcept public MSSQLSelectBuilder select() { return new MSSQLSelectBuilder(this); } + + /** + * SQL Server renames through the {@code sp_rename} procedure, not a statement of the alter path. + * + * @param table the table + * @param from the current column name + * @param to the new column name + * @return {@code null} + */ + @Override + public String renameColumn(String table, String from, String to) { + return null; + } + + /** + * {@code NULL} is relaxed only by restating the column's whole type here ({@code ALTER COLUMN}), + * which the caller does not have. + * + * @param table the table + * @param column the column + * @return {@code null} + */ + @Override + public String dropNotNull(String table, String column) { + return null; + } } diff --git a/modules/database/database-sql-mysql/src/main/java/org/eclipse/dirigible/database/sql/dialects/mysql/MySQLSqlDialect.java b/modules/database/database-sql-mysql/src/main/java/org/eclipse/dirigible/database/sql/dialects/mysql/MySQLSqlDialect.java index 7a3f40d617f..238db554386 100644 --- a/modules/database/database-sql-mysql/src/main/java/org/eclipse/dirigible/database/sql/dialects/mysql/MySQLSqlDialect.java +++ b/modules/database/database-sql-mysql/src/main/java/org/eclipse/dirigible/database/sql/dialects/mysql/MySQLSqlDialect.java @@ -174,4 +174,16 @@ public MySQLInsertBuilder insert() { return new MySQLInsertBuilder(this); } + /** + * {@code NULL} is relaxed only by restating the column's whole type here ({@code MODIFY COLUMN}), + * which the caller does not have. + * + * @param table the table + * @param column the column + * @return {@code null} + */ + @Override + public String dropNotNull(String table, String column) { + return null; + } } diff --git a/modules/database/database-sql-mysql/src/test/java/org/eclipse/dirigible/database/sql/dialects/mysql/ColumnChangeStatementTest.java b/modules/database/database-sql-mysql/src/test/java/org/eclipse/dirigible/database/sql/dialects/mysql/ColumnChangeStatementTest.java new file mode 100644 index 00000000000..457ec6edee9 --- /dev/null +++ b/modules/database/database-sql-mysql/src/test/java/org/eclipse/dirigible/database/sql/dialects/mysql/ColumnChangeStatementTest.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2010-2026 Eclipse Dirigible contributors + * + * All rights reserved. This program and the accompanying materials are made available under the + * terms of the Eclipse Public License v2.0 which accompanies this distribution, and is available at + * http://www.eclipse.org/legal/epl-v20.html + * + * SPDX-FileCopyrightText: Eclipse Dirigible contributors SPDX-License-Identifier: EPL-2.0 + */ +package org.eclipse.dirigible.database.sql.dialects.mysql; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +import org.junit.Test; + +/** + * The column statements of the table alter path (#7635) on MySQL. + */ +public class ColumnChangeStatementTest { + + private final MySQLSqlDialect dialect = new MySQLSqlDialect(); + + @Test + public void renameColumnQuotesWithBackticks() { + assertEquals("ALTER TABLE `T_INVOICE` RENAME COLUMN `INVOICE_DATE` TO `ISSUE_DATE`", + dialect.renameColumn("T_INVOICE", "INVOICE_DATE", "ISSUE_DATE")); + } + + @Test + public void dropNotNullNeedsTheWholeColumnType() { + assertNull(dialect.dropNotNull("T_INVOICE", "NOTE")); + } +} diff --git a/modules/database/database-sql/src/main/java/org/eclipse/dirigible/database/sql/ISqlDialect.java b/modules/database/database-sql/src/main/java/org/eclipse/dirigible/database/sql/ISqlDialect.java index 6f38b96af1e..a14c629cfe5 100644 --- a/modules/database/database-sql/src/main/java/org/eclipse/dirigible/database/sql/ISqlDialect.java +++ b/modules/database/database-sql/src/main/java/org/eclipse/dirigible/database/sql/ISqlDialect.java @@ -154,6 +154,30 @@ public interface ISqlDialect