From ae04d1e9c3a779f20a8e463ef88f3f038cd812da Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 22 Oct 2026 16:34:47 +0800 Subject: [PATCH 1/5] [refactor](fe) Unify external catalog metadata cache ownership ### What problem does this PR solve? Issue Number: None Problem Summary: FE external catalog names, lookup objects, and schemas had separate owners. Explicit refresh could retain a dropped table in its membership list, and independent lifecycle handling allowed stale loads or discarded database instances to outlive their cache registrations. Share one catalog invalidation owner while preserving per-database capacity/TTL. Explicit refresh invalidates the target and parent membership together; write-triggered refresh remains metadata-only, including replay. Retire database-instance entries without blocking remote loads, let schema runtimes borrow rather than close the shared owner, and keep cold replay safe after cache reconfiguration. Lookup handles retain count-based admission while names and schema payloads use configured byte budgets. ### Release note Explicit external table and database refresh now invalidates cached lookup objects and parent membership consistently. Ordinary write-triggered metadata refresh keeps cached table names. ### Check List (For Author) - Test: Unit Test (290 related tests passed); Regression test (Hive metadata refresh against an isolated FE and real HMS); FE build and Checkstyle passed. No multi-FE cluster, actual Trino, or BE data execution validation. - Behavior changed: Yes (explicit refresh also invalidates lookup and parent membership; ordinary write-triggered refresh remains metadata-only) - Does this need documentation: No (existing refresh commands and retention policy; cache-definition API documents byte-accounting boundaries) --- .../connector/cache/CatalogMetaCache.java | 24 +- .../connector/cache/MetaCacheDefinition.java | 22 + .../doris/connector/cache/ScopePath.java | 54 ++- .../cache/ScopedMetaCacheRegistry.java | 8 +- .../connector/cache/CatalogMetaCacheTest.java | 124 +++++ .../apache/doris/catalog/RefreshManager.java | 41 +- .../doris/datasource/ExternalCatalog.java | 115 ++++- .../doris/datasource/ExternalDatabase.java | 75 ++- .../datasource/ExternalMetaCacheMgr.java | 21 +- .../datasource/log/ExternalObjectLog.java | 10 + .../metacache/CatalogMetaCacheRuntime.java | 23 +- .../metacache/ExternalCatalogMetaCache.java | 15 +- .../metacache/FeMetaCacheEntry.java | 167 ++++++- .../metacache/MetaCacheEntryInvalidation.java | 3 +- .../plugin/PluginDrivenExternalTable.java | 4 +- .../datasource/FeCatalogCacheOwnerTest.java | 456 ++++++++++++++++++ .../metacache/FeMetaCacheEntryTest.java | 27 ++ .../plugin/PluginDrivenExternalTableTest.java | 8 + .../hive/test_hive_catalog_owner_refresh.out | 27 ++ .../test_hive_catalog_owner_refresh.groovy | 70 +++ 20 files changed, 1215 insertions(+), 79 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java create mode 100644 regression-test/data/external_table_p0/hive/test_hive_catalog_owner_refresh.out create mode 100644 regression-test/suites/external_table_p0/hive/test_hive_catalog_owner_refresh.groovy diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java index d84c389b04b186..9a58659d47e2c8 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java @@ -97,8 +97,9 @@ public MetaCache create(MetaCacheDefinition definition) { } MetaCacheBudgetManager.EntryBudget entryBudget = null; try { - boolean weightLimited = budgetManager.hasLimit( - catalogMaxWeight, nonNullDefinition.cacheSpec().getMaxWeight()); + boolean weightLimited = nonNullDefinition.budgetPolicy() + == MetaCacheDefinition.BudgetPolicy.WEIGHTED_PAYLOAD + && budgetManager.hasLimit(catalogMaxWeight, nonNullDefinition.cacheSpec().getMaxWeight()); if (weightLimited && nonNullDefinition.sizeEstimator() == null) { throw new IllegalArgumentException("Weighted metadata cache requires a size estimator: " + nonNullDefinition.name()); @@ -159,6 +160,25 @@ public void invalidateTable(String database, String table) { registry.invalidate(ScopePath.table(database, table)); } + /** Invalidates an object and its parent membership view at one logical publication barrier. */ + public void invalidateTableAndNames(String database, String table) { + registry.invalidate(java.util.Arrays.asList( + ScopePath.table(database, table), ScopePath.tableCollection(database))); + } + + public void invalidateDatabaseAndNames(String database) { + registry.invalidate(java.util.Arrays.asList( + ScopePath.database(database), ScopePath.databaseCollection())); + } + + public void invalidateScope(ScopePath scope) { + registry.invalidate(scope); + } + + public boolean isClosed() { + return closed.get(); + } + public void invalidatePartition(String database, String table, Object partition) { registry.invalidate(ScopePath.partition(database, table, partition)); } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java index a461b8879e44a6..7c2e4ef12c6877 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java @@ -31,6 +31,11 @@ * automatically covers it without a connector-maintained list of sibling caches. */ public final class MetaCacheDefinition { + public enum BudgetPolicy { + WEIGHTED_PAYLOAD, + COUNT_ONLY + } + private final String name; private final String budgetGroup; private final CacheSpec cacheSpec; @@ -41,6 +46,7 @@ public final class MetaCacheDefinition { private final Duration refreshAfterWrite; private final Executor refreshExecutor; private final MetaCacheSizeEstimator sizeEstimator; + private final BudgetPolicy budgetPolicy; private MetaCacheDefinition(Builder builder) { name = requireName(builder.name); @@ -53,6 +59,7 @@ private MetaCacheDefinition(Builder builder) { refreshAfterWrite = builder.refreshAfterWrite; refreshExecutor = builder.refreshExecutor; sizeEstimator = builder.sizeEstimator; + budgetPolicy = builder.budgetPolicy; if (refreshAfterWrite != null && loader == null) { throw new IllegalArgumentException("refresh-after-write requires a default loader"); } @@ -105,6 +112,10 @@ MetaCacheSizeEstimator sizeEstimator() { return sizeEstimator; } + BudgetPolicy budgetPolicy() { + return budgetPolicy; + } + private static String requireName(String name) { String nonNullName = Objects.requireNonNull(name, "name can not be null"); if (nonNullName.isEmpty()) { @@ -124,6 +135,7 @@ public static final class Builder { private Executor refreshExecutor; private MetaCacheSizeEstimator sizeEstimator; private String budgetGroup; + private BudgetPolicy budgetPolicy = BudgetPolicy.WEIGHTED_PAYLOAD; private Builder(String name, CacheSpec cacheSpec, Function scopeResolver) { this.name = name; @@ -165,6 +177,16 @@ public Builder sizeEstimator(MetaCacheSizeEstimator estimator) { return this; } + /** + * Mutable lookup handles may use count/TTL retention instead of payload-byte accounting. Their owner + * back-references are not an owned payload graph. Such entries still share invalidation and lifecycle + * with this catalog, but are explicitly excluded from global/catalog payload-byte usage statistics. + */ + public Builder budgetPolicy(BudgetPolicy policy) { + this.budgetPolicy = Objects.requireNonNull(policy, "budgetPolicy can not be null"); + return this; + } + /** Shares one configured weight limit across physical caches in the same logical entry group. */ public Builder budgetGroup(String budgetGroup) { this.budgetGroup = requireName(budgetGroup); diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java index 98d9a7abdfed3f..570b592470c7c0 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java @@ -30,7 +30,13 @@ */ public final class ScopePath { private enum SyntheticPartition { - COLLECTION + COLLECTION, + METADATA + } + + private enum NameCollection { + DATABASES, + TABLES } public enum Level { @@ -46,12 +52,18 @@ public enum Level { private final String database; private final String table; private final Object partition; + private final NameCollection nameCollection; private ScopePath(Level level, String database, String table, Object partition) { + this(level, database, table, partition, null); + } + + private ScopePath(Level level, String database, String table, Object partition, NameCollection nameCollection) { this.level = level; this.database = database; this.table = table; this.partition = partition; + this.nameCollection = nameCollection; } public static ScopePath catalog() { @@ -88,6 +100,30 @@ public static ScopePath partitionCollection(String database, String table) { return partition(database, table, SyntheticPartition.COLLECTION); } + /** The catalog's database-name snapshot, distinct from every user database. */ + public static ScopePath databaseCollection() { + return new ScopePath(Level.DATABASE, null, null, null, NameCollection.DATABASES); + } + + /** A database's table-name snapshot, distinct from every user table. */ + public static ScopePath tableCollection(String database) { + return new ScopePath(Level.TABLE, Objects.requireNonNull(database, "database"), + null, null, NameCollection.TABLES); + } + + /** FE schema/content entries; invalidating them preserves the FE table object and name snapshot. */ + public static ScopePath tableMetadata(String database, String table) { + return partition(database, table, SyntheticPartition.METADATA); + } + + Object databaseKey() { + return nameCollection == NameCollection.DATABASES ? NameCollection.DATABASES : database; + } + + Object tableKey() { + return nameCollection == NameCollection.TABLES ? NameCollection.TABLES : table; + } + public Level level() { return level; } @@ -112,13 +148,13 @@ public boolean contains(ScopePath other) { if (level == Level.CATALOG) { return true; } - if (!database.equals(other.database)) { + if (!Objects.equals(databaseKey(), other.databaseKey())) { return false; } if (level == Level.DATABASE) { return true; } - if (!table.equals(other.table)) { + if (!Objects.equals(tableKey(), other.tableKey())) { return false; } return level == Level.TABLE || partition.equals(other.partition); @@ -134,8 +170,8 @@ public boolean equals(Object obj) { } ScopePath other = (ScopePath) obj; return level == other.level - && Objects.equals(database, other.database) - && Objects.equals(table, other.table) + && Objects.equals(databaseKey(), other.databaseKey()) + && Objects.equals(tableKey(), other.tableKey()) && Objects.equals(partition, other.partition); } @@ -143,13 +179,17 @@ public boolean equals(Object obj) { public int hashCode() { int pathHashCode = 1; pathHashCode = 31 * pathHashCode + Objects.hashCode(level); - pathHashCode = 31 * pathHashCode + Objects.hashCode(database); - pathHashCode = 31 * pathHashCode + Objects.hashCode(table); + pathHashCode = 31 * pathHashCode + Objects.hashCode(databaseKey()); + pathHashCode = 31 * pathHashCode + Objects.hashCode(tableKey()); return 31 * pathHashCode + Objects.hashCode(partition); } @Override public String toString() { + if (nameCollection != null) { + return nameCollection == NameCollection.DATABASES + ? "databaseCollection" : "tableCollection[" + database + "]"; + } switch (level) { case CATALOG: return "catalog"; diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java index 918b2829a7927b..9fa587ed936306 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java @@ -334,7 +334,7 @@ private ScopeSnapshot resolveOrCreateSnapshot(ScopePath path) { return ScopeSnapshot.capture(path, catalogNode, null, null, null); } ScopeState catalogState = catalogNode.current.get(); - ScopeNode dbNode = child(catalogNode, catalogState, path.database(), ScopePath.Level.DATABASE); + ScopeNode dbNode = child(catalogNode, catalogState, path.databaseKey(), ScopePath.Level.DATABASE); if (dbNode == null) { return null; } @@ -342,7 +342,7 @@ private ScopeSnapshot resolveOrCreateSnapshot(ScopePath path) { return ScopeSnapshot.capture(path, catalogNode, dbNode, null, null); } ScopeState dbState = dbNode.current.get(); - ScopeNode tableNode = child(dbNode, dbState, path.table(), ScopePath.Level.TABLE); + ScopeNode tableNode = child(dbNode, dbState, path.tableKey(), ScopePath.Level.TABLE); if (tableNode == null) { return null; } @@ -365,7 +365,7 @@ private List resolveExisting(ScopePath path) { if (path.level() == ScopePath.Level.CATALOG) { return nodes; } - ScopeNode dbNode = catalogNode.current.get().children.get(path.database()); + ScopeNode dbNode = catalogNode.current.get().children.get(path.databaseKey()); if (dbNode == null) { return nodes; } @@ -373,7 +373,7 @@ private List resolveExisting(ScopePath path) { if (path.level() == ScopePath.Level.DATABASE) { return nodes; } - ScopeNode tableNode = dbNode.current.get().children.get(path.table()); + ScopeNode tableNode = dbNode.current.get().children.get(path.tableKey()); if (tableNode == null) { return nodes; } diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java index 3bb6ad66e80d68..ae5363e7f15c00 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java @@ -24,8 +24,14 @@ import java.time.Duration; import java.util.ArrayDeque; import java.util.Arrays; +import java.util.Collections; +import java.util.OptionalLong; import java.util.Queue; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; @@ -41,6 +47,37 @@ void unmanagedOwnershipMustBeSelectedExplicitly() throws Exception { } } + @Test + void countOnlyHandlesShareInvalidationWhilePayloadsRemainWeightBounded() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(1024L)); + try (CatalogMetaCache owner = new CatalogMetaCache(manager, 1L, "test", Collections.emptyMap())) { + AtomicInteger loads = new AtomicInteger(); + MetaCache handles = owner.create(MetaCacheDefinition + .builder("lookup", SPEC, key -> ScopePath.table("db", key)) + .budgetPolicy(MetaCacheDefinition.BudgetPolicy.COUNT_ONLY) + .loader(key -> { + loads.incrementAndGet(); + return () -> { }; + }).build()); + Runnable first = handles.get("a"); + Assertions.assertSame(first, handles.get("a")); + Assertions.assertEquals(1, loads.get()); + Assertions.assertFalse(handles.isWeightBounded()); + MetaCache payload = owner.create(MetaCacheDefinition + .builder("payload", SPEC, key -> ScopePath.table("db", key)) + .sizeEstimator(MetaCacheSizeEstimators.reflective()).build()); + Assertions.assertTrue(payload.isWeightBounded()); + payload.put("a", "x".repeat(2048)); + Assertions.assertNull(payload.getIfPresent("a")); + payload.put("a", "small"); + Assertions.assertTrue(manager.getGlobalUsedWeight() > 0L); + owner.invalidateTable("db", "a"); + Assertions.assertNull(handles.getIfPresent("a")); + Assertions.assertNull(payload.getIfPresent("a")); + } + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + @Test void tableInvalidationAutomaticallyCoversEveryPhysicalCache() { try (CatalogMetaCache catalog = CatalogMetaCache.unmanaged()) { @@ -104,6 +141,93 @@ void partitionInvalidationCoversCollectionAndSelectedPartitionsOnly() { } } + @Test + void fullTableInvalidationAlsoCoversMembershipWithoutClearingSiblings() { + try (CatalogMetaCache catalog = CatalogMetaCache.unmanaged()) { + MetaCache names = catalog.create(MetaCacheDefinition + .builder("table-names", SPEC, ScopePath::tableCollection).build()); + MetaCache objects = catalog.create(tableDefinition("objects")); + MetaCache schema = catalog.create(MetaCacheDefinition + .builder("schema", SPEC, + key -> ScopePath.tableMetadata(key.database, key.table)).build()); + TableKey first = new TableKey("db", "a"); + TableKey sibling = new TableKey("db", "b"); + names.put("db", "a,b"); + names.put("other", "unrelated"); + objects.put(first, "object-a"); + objects.put(sibling, "object-b"); + schema.put(first, "schema-a"); + schema.put(sibling, "schema-b"); + + catalog.invalidateScope(ScopePath.tableMetadata("db", "a")); + Assertions.assertEquals("a,b", names.getIfPresent("db")); + Assertions.assertEquals("object-a", objects.getIfPresent(first)); + Assertions.assertNull(schema.getIfPresent(first)); + schema.put(first, "schema-a-new"); + + catalog.invalidateTableAndNames("db", "a"); + Assertions.assertNull(names.getIfPresent("db")); + Assertions.assertNull(objects.getIfPresent(first)); + Assertions.assertNull(schema.getIfPresent(first)); + Assertions.assertEquals("object-b", objects.getIfPresent(sibling)); + Assertions.assertEquals("schema-b", schema.getIfPresent(sibling)); + Assertions.assertEquals("unrelated", names.getIfPresent("other")); + } + } + + @Test + void absentChildStillInvalidatesMembershipAndCollectionsCannotAliasUserNames() { + try (CatalogMetaCache catalog = CatalogMetaCache.unmanaged()) { + MetaCache names = catalog.create(MetaCacheDefinition + .builder("names", SPEC, ScopePath::tableCollection).build()); + names.put("db", "cold-table"); + catalog.invalidateTableAndNames("db", "cold-table"); + Assertions.assertNull(names.getIfPresent("db")); + Assertions.assertNotEquals(ScopePath.tableCollection("db"), ScopePath.table("db", "TABLES")); + Assertions.assertNotEquals(ScopePath.databaseCollection(), ScopePath.database("DATABASES")); + Assertions.assertTrue(ScopePath.database("db").contains(ScopePath.tableCollection("db"))); + Assertions.assertFalse(ScopePath.tableCollection("db").contains(ScopePath.table("db", "TABLES"))); + } + } + + @Test + void lateAsyncRefreshCannotReplaceRecreatedTableAfterFullInvalidation() throws Exception { + ExecutorService refreshes = Executors.newSingleThreadExecutor(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger loads = new AtomicInteger(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + MetaCache schema = owner.create(MetaCacheDefinition + .builder("schema", SPEC, key -> ScopePath.tableMetadata("db", key)) + .loader(key -> { + int load = loads.incrementAndGet(); + if (load == 2) { + started.countDown(); + try { + Assertions.assertTrue(release.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + return "old-schema"; + }) + .refreshAfterWrite(Duration.ofNanos(1), refreshes).build()); + Assertions.assertEquals("old-schema", schema.get("a")); + Assertions.assertEquals("old-schema", schema.get("a")); + Assertions.assertTrue(started.await(10, TimeUnit.SECONDS)); + owner.invalidateTableAndNames("db", "a"); + schema.put("a", "recreated-schema"); + release.countDown(); + refreshes.submit(() -> { }).get(10, TimeUnit.SECONDS); + Assertions.assertEquals("recreated-schema", schema.getIfPresent("a")); + } finally { + release.countDown(); + refreshes.shutdownNow(); + Assertions.assertTrue(refreshes.awaitTermination(10, TimeUnit.SECONDS)); + } + } + @Test void defaultAndContextualLoadersUseDefinitionScope() { AtomicInteger loads = new AtomicInteger(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index 8e5586b344f79c..420f509d9d5dae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -115,7 +115,11 @@ public void replayRefreshDb(ExternalObjectLog log) { try { invalidateAllConnectorCachesIfPresent(catalog); } finally { - cacheMgr.invalidateDb(log.getCatalogId(), dbId, localDbName); + try { + catalog.invalidateDatabaseLookup(localDbName); + } finally { + cacheMgr.invalidateDb(log.getCatalogId(), dbId, localDbName); + } } LOG.info("database object cache is cold when replaying refresh database; invalidated caches by " + "local name {}: {}", @@ -133,7 +137,11 @@ private void refreshDbInternal(ExternalDatabase db) { } } finally { try { - db.resetMetaToUninitialized(); + try { + db.resetMetaToUninitialized(); + } finally { + ((ExternalCatalog) db.getCatalog()).invalidateDatabaseLookup(db.getFullName()); + } } finally { Env.getCurrentEnv().getExtMetaCacheMgr() .invalidateDbRowCountCache(db.getCatalog().getId(), db.getId()); @@ -170,8 +178,8 @@ public void handleRefreshTable(String catalogName, String dbName, String tableNa return; } long updateTime = System.currentTimeMillis(); - refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, updateTime); - ExternalObjectLog log = ExternalObjectLog.createForRefreshTable( + refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, updateTime, true); + ExternalObjectLog log = ExternalObjectLog.createForExplicitRefreshTable( catalog.getId(), db.getFullName(), table.getName(), updateTime); env.getEditLog().logRefreshExternalTable(log); } @@ -225,14 +233,20 @@ public void replayRefreshTable(ExternalObjectLog log) { ((PluginDrivenExternalCatalog) catalog).getConnector().invalidateDb(db.get().getRemoteName()); } } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTable( - log.getCatalogId(), dbId, db.get().getFullName(), tableId, localTableName); + try { + if (log.isInvalidateTableLookup()) { + db.get().invalidateTableLookup(localTableName); + } + } finally { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTable( + log.getCatalogId(), dbId, db.get().getFullName(), tableId, localTableName); + } } LOG.info("table object cache is cold when replaying refresh table; invalidated caches by local name {}: {}", localTableName, log.debugForRefreshTable()); return; } - refreshTableInternal(db.get(), table.get(), log.getLastUpdateTime()); + refreshTableInternal(db.get(), table.get(), log.getLastUpdateTime(), log.isInvalidateTableLookup()); } private void replayRenameTable(ExternalObjectLog log, ExternalCatalog catalog, @@ -301,6 +315,11 @@ public void refreshExternalTableFromEvent(String catalogName, String localDbName } public void refreshTableInternal(ExternalDatabase db, ExternalTable table, long updateTime) { + refreshTableInternal(db, table, updateTime, false); + } + + private void refreshTableInternal(ExternalDatabase db, ExternalTable table, long updateTime, + boolean invalidateTableLookup) { table.unsetObjectCreated(); if (updateTime > 0) { table.setUpdateTime(updateTime); @@ -312,7 +331,13 @@ public void refreshTableInternal(ExternalDatabase db, ExternalTable table, long .invalidateTable(db.getRemoteName(), table.getRemoteName()); } } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTable(table); + try { + if (invalidateTableLookup) { + db.invalidateTableLookup(table.getName()); + } + } finally { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTable(table); + } } LOG.info("refresh table {}, id {} from db {} in catalog {}, update time: {}", table.getName(), table.getId(), db.getFullName(), db.getCatalog().getName(), updateTime); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 3934483b97bf48..a902483558856a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -40,7 +40,10 @@ import org.apache.doris.common.util.ExceptionDiagnosticUtils; import org.apache.doris.common.util.Util; import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCacheBudgetManager; +import org.apache.doris.connector.cache.ScopePath; +import org.apache.doris.datasource.doris.RemoteDorisExternalCatalog; import org.apache.doris.datasource.doris.RemoteDorisExternalDatabase; import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase; @@ -82,6 +85,7 @@ import java.util.Set; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.stream.Collectors; @@ -169,6 +173,12 @@ public abstract class ExternalCatalog private boolean objectCreated = false; protected TransactionManager transactionManager; + private transient CatalogMetaCache feMetaCacheOwner; + private transient CatalogMetaCache namingCacheOwner; + private transient boolean feCacheRetired; + private transient AtomicLong cacheEntrySequence = + new AtomicLong(); + protected FeMetaCacheEntry databaseNames; protected FeMetaCacheEntry> databases; protected transient IdNameIndex dbIdNameIndex = new IdNameIndex("external database"); @@ -343,7 +353,8 @@ public final synchronized void makeSureInitialized() { } try { initLocalObjects(); - if (!initialized) { + if (!initialized || feMetaCacheOwner == null || feMetaCacheOwner.isClosed() + || namingCacheOwner != feMetaCacheOwner) { buildMetaCache(); setLastUpdateTime(System.currentTimeMillis()); initialized = true; @@ -399,8 +410,69 @@ public boolean isInitialized() { return this.initialized; } - private void buildMetaCache() { - if (databaseNames != null && databases != null) { + /** One FE invalidation/lifecycle owner; connector caches retain their remote-name ownership. */ + public synchronized CatalogMetaCache getFeMetaCacheOwner() { + Preconditions.checkState(!feCacheRetired, "External catalog cache owner has been retired: %s", name); + if (feMetaCacheOwner == null || feMetaCacheOwner.isClosed()) { + boolean replacingClosedOwner = feMetaCacheOwner != null; + Map properties = Maps.newHashMap(getProperties()); + overlayMetaCacheConfig(properties); + String engine = this instanceof RemoteDorisExternalCatalog + ? "doris" : "default"; + feMetaCacheOwner = CatalogMetaCache.managed(id, engine, properties); + dbIdNameIndex.clear(); + if (replacingClosedOwner) { + initialized = false; + } + } + return feMetaCacheOwner; + } + + /** Cache retirement must also work before the first schema entry has been prepared. */ + public void closeFeMetaCacheOwner() { + CatalogMetaCache owner; + FeMetaCacheEntry names; + FeMetaCacheEntry> objects; + synchronized (this) { + owner = feMetaCacheOwner; + names = databaseNames; + objects = databases; + dbIdNameIndex.clear(); + initialized = false; + } + if (names != null) { + names.close(); + } + if (objects != null) { + objects.close(); + } + if (owner != null) { + owner.close(); + } + } + + /** A full database refresh also fences the catalog's database-name snapshot. */ + public void invalidateDatabaseLookup(String localDbName) { + if (databases != null) { + databases.invalidateKeyAndRun(localDbName, () -> dbIdNameIndex.removeName(localDbName)); + } else { + dbIdNameIndex.removeName(localDbName); + } + // Replay must not resurrect a retired owner or initialize a remote catalog after ALTER. + CatalogMetaCache owner = feMetaCacheOwner; + if (owner != null && !owner.isClosed()) { + owner.invalidateDatabaseAndNames(localDbName); + } + } + + /** Instance-qualified registration permits old database cleanup without touching its replacement. */ + public String nextFeCacheEntryName(String prefix) { + return prefix + "." + cacheEntrySequence.getAndIncrement(); + } + + private synchronized void buildMetaCache() { + CatalogMetaCache owner = getFeMetaCacheOwner(); + if (namingCacheOwner == owner && databaseNames != null && databases != null) { return; } if (LOG.isDebugEnabled()) { @@ -411,26 +483,29 @@ private void buildMetaCache() { Config.external_cache_expire_time_seconds_after_access, 1); // Build one immutable names snapshot so list and lower-case index share the same cache version. - databaseNames = new FeMetaCacheEntry<>( - name + ".database_names", + databaseNames = FeMetaCacheEntry.shared(owner, + ignored -> ScopePath.databaseCollection(), ScopePath.databaseCollection(), + "database_names", ignored -> NameCacheValue.of(getFilteredDatabaseNames()), namesSpec, Env.getCurrentEnv().getExtMetaCacheMgr().commonRefreshExecutor(), true, - FeMetaCacheEntry.singleKeyStripeCount()); + FeMetaCacheEntry.singleKeyStripeCount(), null); CacheSpec objectSpec = CacheSpec.of( true, Config.external_cache_expire_time_seconds_after_access, Math.max(Config.max_meta_object_cache_num, 1)); // Object entries keep the sync removal listener semantics and therefore do not enable auto refresh. - databases = FeMetaCacheEntry.withSyncRemovalListener( - name + ".databases", + databases = FeMetaCacheEntry.sharedLookup(owner, + ScopePath::database, ScopePath.catalog(), + "databases", localDbName -> buildDbForInit(null, localDbName, Util.genIdByName(name, localDbName), logType, true), objectSpec, - Env.getCurrentEnv().getExtMetaCacheMgr().commonRefreshExecutor(), + Env.getCurrentEnv().getExtMetaCacheMgr().commonRefreshExecutor(), false, FeMetaCacheEntry.defaultObjectStripeCount(), - (key, value, cause) -> value.resetMetaToUninitialized()); + (key, value, cause) -> value.releaseCachedEntries()); + namingCacheOwner = owner; } /** @@ -835,7 +910,11 @@ private ExternalDatabase getDbNullableWithoutCache(Stri String localDbName = dbNamePair.get().value(); // checkExists=false: the pair came from the live listing, so re-listing (getDbNames) is both // unnecessary and would recurse back through this bypass — build the db object directly. - return buildDbForInit(remoteDbName, localDbName, Util.genIdByName(name, localDbName), logType, false); + ExternalDatabase database = + buildDbForInit(remoteDbName, localDbName, Util.genIdByName(name, localDbName), logType, false); + // A per-session object has no catalog-cache lifecycle to retire its child registrations. + database.releaseCachedEntries(); + return database; } /** Matches a live listing's LOCAL db name against a requested name, honoring the db-name case modes. */ @@ -959,9 +1038,16 @@ public void setLastUpdateTime(long lastUpdateTime) { @Override public final void onClose() { + synchronized (this) { + feCacheRetired = true; + } Runnable accessControllerCleanup = detachAccessController(); try { - closeResourcesQuietly("registered catalog"); + try { + closeResourcesQuietly("registered catalog"); + } finally { + closeFeMetaCacheOwner(); + } } finally { accessControllerCleanup.run(); } @@ -1188,6 +1274,7 @@ public void gsonPostProcess() throws IOException { tableAutoAnalyzePolicy = Maps.newHashMap(); } this.dbIdNameIndex = new IdNameIndex("external database"); + this.cacheEntrySequence = new AtomicLong(); } public void addDatabaseForTest(ExternalDatabase db) { @@ -1593,6 +1680,10 @@ private void invalidateDatabaseNamesOnly() { } private void invalidateAllDatabaseCache() { + if (feMetaCacheOwner != null && feMetaCacheOwner.isClosed()) { + dbIdNameIndex.clear(); + return; + } if (databaseNames != null) { databaseNames.invalidateAll(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 3d24d0c3b78ce9..bccbd4cd27b3f0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -32,6 +32,8 @@ import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.Util; import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase; import org.apache.doris.datasource.metacache.FeMetaCacheEntry; @@ -93,6 +95,7 @@ public abstract class ExternalDatabase private transient IdNameIndex tableIdNameIndex = new IdNameIndex("external table"); private volatile boolean isInitializing = false; + private transient volatile boolean cacheEntriesReleased; /** * Create external database. @@ -124,12 +127,37 @@ public void resetMetaToUninitialized() { } synchronized (this) { this.initialized = false; - invalidateAllTableCache(); + if (!cacheEntriesReleased) { + invalidateAllTableCache(); + } } Env.getCurrentEnv().getExtMetaCacheMgr() .invalidateDbMetadataCache(extCatalog.getId(), getFullName()); } + /** Eviction retires only this database instance's entries, never the current shared database scope. */ + public synchronized void releaseCachedEntries() { + cacheEntriesReleased = true; + initialized = false; + if (tableNames != null) { + tableNames.close(); + } + if (tables != null) { + tables.close(); + } + tableIdNameIndex.clear(); + } + + /** Full refresh also invalidates the parent membership snapshot at the shared publication barrier. */ + public void invalidateTableLookup(String localTableName) { + if (tables != null) { + tables.invalidateKeyAndRun(localTableName, () -> tableIdNameIndex.removeName(localTableName)); + } else { + tableIdNameIndex.removeName(localTableName); + } + extCatalog.getFeMetaCacheOwner().invalidateTableAndNames(name, localTableName); + } + public boolean isInitialized() { return initialized; } @@ -137,14 +165,20 @@ public boolean isInitialized() { public final void makeSureInitialized() { // Must call this method before any operation on the database to avoid deadlock of synchronized block extCatalog.makeSureInitialized(); + CatalogMetaCache owner = extCatalog.getFeMetaCacheOwner(); synchronized (this) { + if (cacheEntriesReleased) { + // A database returned without cache admission must remain usable by the current caller. + // Its children are read through the existing transient path, never registered again. + return; + } if (isInitializing) { return; } isInitializing = true; try { if (!initialized) { - buildMetaCache(); + buildMetaCache(owner); setLastUpdateTime(System.currentTimeMillis()); initialized = true; } @@ -159,6 +193,10 @@ public final void makeSureInitialized() { } private void buildMetaCache() { + buildMetaCache(extCatalog.getFeMetaCacheOwner()); + } + + private void buildMetaCache(CatalogMetaCache owner) { if (tableNames != null && tables != null) { return; } @@ -170,13 +208,17 @@ private void buildMetaCache() { Config.external_cache_expire_time_seconds_after_access, 1); // Build one immutable names snapshot so list and lower-case index share the same cache version. - tableNames = new FeMetaCacheEntry<>( - name + ".table_names", + // Keep each database's existing capacity/TTL policy while sharing the catalog's invalidation tree. + // The database object's removal/discard hook closes these instance-qualified registrations. + String registrationName = extCatalog.nextFeCacheEntryName(name); + tableNames = FeMetaCacheEntry.shared(owner, + ignored -> ScopePath.tableCollection(name), ScopePath.tableCollection(name), + registrationName + ".table_names", ignored -> NameCacheValue.of(listTableNames()), namesSpec, Env.getCurrentEnv().getExtMetaCacheMgr().commonRefreshExecutor(), true, - FeMetaCacheEntry.singleKeyStripeCount()); + FeMetaCacheEntry.singleKeyStripeCount(), null); CacheSpec objectSpec = CacheSpec.of( true, @@ -184,8 +226,9 @@ private void buildMetaCache() { Math.max(Config.max_meta_object_cache_num, 1)); // Keep table object entries on the normal manual-miss/refresh path because they do not need // the database-level reset callback used by catalog object caches. - tables = new FeMetaCacheEntry<>( - name + ".tables", + tables = FeMetaCacheEntry.sharedLookup(owner, + localTableName -> ScopePath.table(name, localTableName), ScopePath.database(name), + registrationName + ".tables", localTableName -> buildTableForInit( null, localTableName, @@ -196,7 +239,7 @@ private void buildMetaCache() { objectSpec, Env.getCurrentEnv().getExtMetaCacheMgr().commonRefreshExecutor(), false, - FeMetaCacheEntry.defaultObjectStripeCount()); + FeMetaCacheEntry.defaultObjectStripeCount(), null); } private List> listTableNames() { @@ -472,7 +515,8 @@ public DatabaseProperty getDbProperties() { public boolean isTableExist(String tableName) { SessionContext sessionContext = SessionContext.current(); String remoteTblName = tableName; - if (extCatalog.shouldBypassTableNameCache(sessionContext) && requiresRemoteTableNameResolution()) { + if ((cacheEntriesReleased || extCatalog.shouldBypassTableNameCache(sessionContext)) + && requiresRemoteTableNameResolution()) { Optional> matched = findTableNamePairWithoutCache(sessionContext, tableName); remoteTblName = matched.map(Pair::key).orElse(null); } else if (this.isTableNamesCaseInsensitive()) { @@ -525,7 +569,7 @@ public List getTablesOnIdOrderOrThrowException(List tableIdList) throws public Set getTableNamesWithLock() { makeSureInitialized(); SessionContext sessionContext = SessionContext.current(); - if (extCatalog.shouldBypassTableNameCache(sessionContext)) { + if (cacheEntriesReleased || extCatalog.shouldBypassTableNameCache(sessionContext)) { return Sets.newHashSet(listLocalTableNamesWithoutCache(sessionContext)); } return Sets.newHashSet(listLocalTableNamesFromCache()); @@ -535,7 +579,7 @@ public Set getTableNamesWithLock() { public T getTableNullable(String tableName) { makeSureInitialized(); SessionContext sessionContext = SessionContext.current(); - if (extCatalog.shouldBypassTableNameCache(sessionContext)) { + if (cacheEntriesReleased || extCatalog.shouldBypassTableNameCache(sessionContext)) { return getTableNullableWithoutCache(sessionContext, tableName); } String finalName = getLocalTableName(tableName, false); @@ -756,6 +800,9 @@ private void invalidateAllTableCache() { @Override public T getTableNullable(long tableId) { makeSureInitialized(); + if (cacheEntriesReleased) { + return null; + } String tableName = tableIdNameIndex.getName(tableId); if (tableName == null) { return null; @@ -789,7 +836,9 @@ public void unregisterTable(String tableName) { setLastUpdateTime(System.currentTimeMillis()); localTableName = resolveTableNameForInvalidation(tableName); // Always clean local names/object/ID state, even when the object entry is cold or already evicted. - invalidateTableCache(localTableName); + if (!cacheEntriesReleased) { + invalidateTableCache(localTableName); + } } finally { Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTable( extCatalog.getId(), id, getFullName(), @@ -947,7 +996,7 @@ public String getCachedTableNameByIdForTest(long tableId) { } public void resetMetaCacheNames() { - if (tableNames != null) { + if (!cacheEntriesReleased && tableNames != null) { tableNames.invalidateAll(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index f6f4045e570285..17f28a80209442 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -219,6 +219,10 @@ public void invalidateCatalogByEngine(long catalogId, String engine) { } public void removeCatalog(long catalogId) { + CatalogIf catalog = getCatalog(catalogId); + if (catalog instanceof ExternalCatalog) { + ((ExternalCatalog) catalog).closeFeMetaCacheOwner(); + } try { try { routeCatalogEngines(catalogId, cache -> safeInvalidate( @@ -350,18 +354,19 @@ public void invalidatePartitions(long catalogId, public List getCatalogCacheStats(long catalogId) { List stats = new ArrayList<>(); - allCacheTypes().forEach(externalMetaCache -> externalMetaCache.stats(catalogId) - .forEach((entryName, entryStats) -> stats.add( - new CatalogMetaCacheStats(externalMetaCache.engine(), entryName, entryStats)))); + Map> byEngine = Maps.newHashMap(); + allCacheTypes().forEach(externalMetaCache -> byEngine + .computeIfAbsent(externalMetaCache.engine(), ignored -> Maps.newHashMap()) + .putAll(externalMetaCache.stats(catalogId))); for (CatalogMetaCache cache : MetaCacheGovernance.catalogCaches(catalogId)) { - if (ENGINE_DEFAULT.equals(cache.engine()) || ENGINE_DORIS.equals(cache.engine())) { - continue; - } + Map entries = + byEngine.computeIfAbsent(cache.engine(), ignored -> Maps.newHashMap()); for (Map.Entry> entry : cache.entries().entrySet()) { - stats.add(new CatalogMetaCacheStats( - cache.engine(), entry.getKey(), MetaCacheEntryStats.from(entry.getValue()))); + entries.put(entry.getKey(), MetaCacheEntryStats.from(entry.getValue())); } } + byEngine.forEach((engine, entries) -> entries.forEach((entryName, entryStats) -> + stats.add(new CatalogMetaCacheStats(engine, entryName, entryStats)))); stats.sort(Comparator.comparing(CatalogMetaCacheStats::getEngineName) .thenComparing(CatalogMetaCacheStats::getEntryName)); return stats; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/log/ExternalObjectLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/log/ExternalObjectLog.java index 30464457e60a80..308b3cd5184100 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/log/ExternalObjectLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/log/ExternalObjectLog.java @@ -61,6 +61,9 @@ public class ExternalObjectLog implements Writable { @SerializedName(value = "newPartitionNames") private List newPartitionNames; + @SerializedName(value = "invalidateTableLookup") + private boolean invalidateTableLookup; + @SerializedName(value = "lastUpdateTime") private long lastUpdateTime; @@ -85,6 +88,13 @@ public static ExternalObjectLog createForRefreshTable(long catalogId, String dbN return externalObjectLog; } + public static ExternalObjectLog createForExplicitRefreshTable(long catalogId, String dbName, String tblName, + long updateTime) { + ExternalObjectLog log = createForRefreshTable(catalogId, dbName, tblName, updateTime); + log.setInvalidateTableLookup(true); + return log; + } + public static ExternalObjectLog createForRefreshPartitions(long catalogId, String dbName, String tblName, List modifiedPartNames, List newPartNames, long updateTime) { ExternalObjectLog externalObjectLog = new ExternalObjectLog(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java index 9f9dbfe674086a..97c853b5ea13d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java @@ -31,10 +31,20 @@ */ final class CatalogMetaCacheRuntime { private final CatalogMetaCache owner; + private final boolean ownsOwner; private final Map> entries = new ConcurrentHashMap<>(); CatalogMetaCacheRuntime(long catalogId, String engine, Map catalogProperties) { - owner = CatalogMetaCache.managed(catalogId, engine, catalogProperties); + this(CatalogMetaCache.managed(catalogId, engine, catalogProperties), true); + } + + CatalogMetaCacheRuntime(CatalogMetaCache owner) { + this(owner, false); + } + + private CatalogMetaCacheRuntime(CatalogMetaCache owner, boolean ownsOwner) { + this.owner = Objects.requireNonNull(owner, "owner"); + this.ownsOwner = ownsOwner; } CatalogMetaCache owner() { @@ -51,7 +61,7 @@ void put(String entryName, MetaCache entry) { Map stats() { Map result = Maps.newHashMap(); - entries.forEach((name, entry) -> result.put(name, stats(entry))); + owner.entries().forEach((name, entry) -> result.put(name, stats(entry))); return result; } @@ -60,7 +70,14 @@ void invalidateAll() { } void close() { - owner.close(); + // An entry group borrows the catalog's FE owner. Reconfiguring schema entries must not close + // sibling naming/object adapters; only ExternalCatalog retires that shared owner. + if (ownsOwner) { + owner.close(); + } else { + entries.values().forEach(owner::remove); + } + entries.clear(); } private MetaCacheEntryStats stats(MetaCache entry) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java index b7f9809e264934..c762de4f750b16 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java @@ -23,6 +23,7 @@ import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; import org.apache.doris.connector.cache.MetaCacheSizeEstimators; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; @@ -88,8 +89,9 @@ public Collection aliases() { public void initCatalog(long catalogId, Map catalogProperties) { Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( catalogProperties, catalogPropertyCompatibilityMap()); - catalogEntries.computeIfAbsent(catalogId, - id -> buildCatalogRuntime(id, safeCatalogProperties)); + catalogEntries.compute(catalogId, (id, current) -> + current != null && !current.owner().isClosed() + ? current : buildCatalogRuntime(id, safeCatalogProperties)); } @Override @@ -156,13 +158,13 @@ public void invalidateDb(long catalogId, String dbName) { public void invalidateTable(long catalogId, String dbName, String tableName) { CatalogMetaCacheRuntime group = catalogEntries.get(catalogId); if (group != null) { - group.owner().invalidateTable(dbName, tableName); + group.owner().invalidateScope(ScopePath.tableMetadata(dbName, tableName)); } } @Override public void invalidatePartitions(long catalogId, String dbName, String tableName, List partitions) { - // FE schema entries are table-scoped: any partition metadata change can affect the table view. + // Partition metadata changes can also affect the table's schema/content view. invalidateTable(catalogId, dbName, tableName); } @@ -274,7 +276,10 @@ private void validateRegisteredMetaCacheEntryDef(MetaCacheEntryDef } private CatalogMetaCacheRuntime buildCatalogRuntime(long catalogId, Map catalogProperties) { - CatalogMetaCacheRuntime group = new CatalogMetaCacheRuntime(catalogId, engine, catalogProperties); + CatalogIf catalog = getCatalog(catalogId); + CatalogMetaCacheRuntime group = catalog instanceof ExternalCatalog + ? new CatalogMetaCacheRuntime(((ExternalCatalog) catalog).getFeMetaCacheOwner()) + : new CatalogMetaCacheRuntime(catalogId, engine, catalogProperties); metaCacheEntryDefs.values() .forEach(entryDef -> group.put(entryDef.getName(), newMetaCacheEntry(group, entryDef, catalogProperties))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java index 1cb7f0dcf9c444..001b6c12bf3c1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java @@ -26,6 +26,7 @@ import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; +import com.github.benmanes.caffeine.cache.RemovalCause; import com.github.benmanes.caffeine.cache.RemovalListener; import java.time.Duration; @@ -42,6 +43,7 @@ import java.util.function.BiPredicate; import java.util.function.Function; import java.util.function.Predicate; +import java.util.function.Supplier; import javax.annotation.Nullable; /** @@ -84,8 +86,13 @@ private ActionToken(K key, ActionState state) { private final boolean autoRefresh; private final int stripeCount; private final AtomicReferenceArray> stripeStates; - private final CatalogMetaCache owner = CatalogMetaCache.unmanaged(); + private final CatalogMetaCache owner; + private final ScopePath invalidationScope; private final MetaCache data; + private final BiConsumer discardedValueAction; + private final Object lifecycleLock = new Object(); + private volatile boolean retired; + private int activeUses; public FeMetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor) { this(name, loader, cacheSpec, refreshExecutor, true, false, defaultObjectStripeCount(), null); @@ -128,6 +135,39 @@ public static FeMetaCacheEntry withSyncRemovalListener(String name, FeMetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, int stripeCount, @Nullable RemovalListener removalListener) { + this(CatalogMetaCache.unmanaged(), ignored -> ScopePath.catalog(), ScopePath.catalog(), + name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, + stripeCount, removalListener, false, MetaCacheDefinition.BudgetPolicy.WEIGHTED_PAYLOAD); + } + + public static FeMetaCacheEntry shared(CatalogMetaCache owner, + Function scopeResolver, ScopePath invalidationScope, + String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, + boolean autoRefresh, int stripeCount, @Nullable RemovalListener removalListener) { + return new FeMetaCacheEntry<>(owner, scopeResolver, invalidationScope, name, loader, cacheSpec, + refreshExecutor, autoRefresh, false, stripeCount, removalListener, true, + MetaCacheDefinition.BudgetPolicy.WEIGHTED_PAYLOAD); + } + + /** Lookup handles retain their existing count/TTL policy; schema payloads use the owner's byte budget. */ + public static FeMetaCacheEntry sharedLookup(CatalogMetaCache owner, + Function scopeResolver, ScopePath invalidationScope, + String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, + boolean autoRefresh, int stripeCount, @Nullable RemovalListener removalListener) { + return new FeMetaCacheEntry<>(owner, scopeResolver, invalidationScope, name, loader, cacheSpec, + refreshExecutor, autoRefresh, false, stripeCount, removalListener, true, + MetaCacheDefinition.BudgetPolicy.COUNT_ONLY); + } + + private FeMetaCacheEntry(CatalogMetaCache owner, Function scopeResolver, + ScopePath invalidationScope, String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, int stripeCount, + @Nullable RemovalListener removalListener, boolean retireDiscardedValues, + MetaCacheDefinition.BudgetPolicy budgetPolicy) { + this.owner = Objects.requireNonNull(owner, "owner"); + discardedValueAction = retireDiscardedValues && removalListener != null + ? (key, value) -> removalListener.onRemoval(key, value, RemovalCause.EXPLICIT) : null; + this.invalidationScope = Objects.requireNonNull(invalidationScope, "invalidationScope"); this.name = Objects.requireNonNull(name, "name can not be null"); this.loader = loader; this.cacheSpec = Objects.requireNonNull(cacheSpec, "cacheSpec can not be null"); @@ -156,14 +196,20 @@ public static FeMetaCacheEntry withSyncRemovalListener(String name, effectiveEnabled = CacheSpec.isCacheEnabled( cacheSpec.isEnable(), cacheSpec.getTtlSecond(), cacheSpec.getCapacity()); MetaCacheDefinition.Builder builder = MetaCacheDefinition.builder( - name, cacheSpec, ignored -> ScopePath.catalog()) - .sizeEstimator(MetaCacheSizeEstimators.reflective()); + name, cacheSpec, scopeResolver) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) + .budgetPolicy(budgetPolicy); if (loader != null) { builder.loader(key -> loadAndPause(key, loader)); } if (removalListener != null) { builder.removalListener((key, value, reason) -> removalListener.onRemoval(key, value, toCaffeineRemovalCause(reason))); + // An uncached database is still returned to its caller. Retire its cache registrations too, + // so the caller uses the transient lookup path rather than leaking a new set of child entries. + if (discardedValueAction != null) { + builder.discardListener(discardedValueAction); + } } if (autoRefresh && Config.external_cache_refresh_time_minutes > 0) { builder.refreshAfterWrite( @@ -181,12 +227,13 @@ public V get(K key) { throw new UnsupportedOperationException(String.format( "Entry '%s' requires a contextual miss loader.", name)); } - return data.get(key); + return useCache(() -> data.get(key), () -> loadWithoutCaching(key, loader)); } public V get(K key, Function missLoader) { Function nonNullLoader = Objects.requireNonNull(missLoader, "missLoader can not be null"); - return data.get(key, loadKey -> loadAndPause(loadKey, nonNullLoader)); + return useCache(() -> data.get(key, loadKey -> loadAndPause(loadKey, nonNullLoader)), + () -> loadWithoutCaching(key, nonNullLoader)); } public V getAndRunIfCurrent(K key, BiConsumer currentValueAction) { @@ -198,6 +245,12 @@ public V getAndRunIfCurrent(K key, BiPredicate actionRequired, BiPredicate required = Objects.requireNonNull(actionRequired, "actionRequired can not be null"); BiConsumer action = Objects.requireNonNull(currentValueAction, "currentValueAction can not be null"); Function loadFunction = Objects.requireNonNull(loader, "loader can not be null"); + return useCache(() -> getAndRunIfCurrentCached(key, required, action, loadFunction), + () -> loadWithoutCaching(key, loadFunction)); + } + + private V getAndRunIfCurrentCached(K key, BiPredicate required, + BiConsumer action, Function loadFunction) { V cached = data.getIfPresent(key); if (cached != null && !required.test(key, cached)) { return cached; @@ -255,11 +308,15 @@ public V getAndRunIfCurrent(K key, BiPredicate actionRequired, } public V getIfPresent(K key) { - return data.getIfPresent(key); + return useCache(() -> data.getIfPresent(key), () -> null); } @Nullable public V findIfPresent(Predicate keyPredicate) { + return useCache(() -> findIfPresentCached(keyPredicate), () -> null); + } + + private V findIfPresentCached(Predicate keyPredicate) { Objects.requireNonNull(keyPredicate, "keyPredicate can not be null"); List result = new ArrayList<>(1); data.forEach((key, value) -> { @@ -271,6 +328,10 @@ public V findIfPresent(Predicate keyPredicate) { } public void put(K key, V value) { + useCache(() -> putCached(key, value)); + } + + private void putCached(K key, V value) { Objects.requireNonNull(value, "value can not be null"); StripeState stripe = stripeState(key); synchronized (stripe) { @@ -286,6 +347,10 @@ public V compute(K key, BiFunction remappingFunction) { } public V computeAndRun(K key, BiFunction remappingFunction, Runnable afterMutation) { + return useCache(() -> computeAndRunCached(key, remappingFunction, afterMutation), () -> null); + } + + private V computeAndRunCached(K key, BiFunction remappingFunction, Runnable afterMutation) { BiFunction remapper = Objects.requireNonNull(remappingFunction, "remappingFunction can not be null"); Runnable action = Objects.requireNonNull(afterMutation, "afterMutation can not be null"); StripeState stripe = stripeState(key); @@ -304,6 +369,10 @@ public V computeAndRun(K key, BiFunction remappingFunction, Runnable af } public V computeAfterValidation(K key, BiFunction remappingFunction, Runnable validationAction) { + return useCache(() -> computeAfterValidationCached(key, remappingFunction, validationAction), () -> null); + } + + private V computeAfterValidationCached(K key, BiFunction remappingFunction, Runnable validationAction) { BiFunction remapper = Objects.requireNonNull(remappingFunction, "remappingFunction can not be null"); Runnable validation = Objects.requireNonNull(validationAction, "validationAction can not be null"); StripeState stripe = stripeState(key); @@ -328,6 +397,10 @@ public V computeAfterValidation(K key, BiFunction remappingFunction, Ru * entry. If it throws, the value is not updated, but completed external side effects are not rolled back. */ public V computeWithCommitAction(K key, BiFunction remappingFunction, Runnable commitAction) { + return useCache(() -> computeWithCommitActionCached(key, remappingFunction, commitAction), () -> null); + } + + private V computeWithCommitActionCached(K key, BiFunction remappingFunction, Runnable commitAction) { BiFunction remapper = Objects.requireNonNull(remappingFunction, "remappingFunction can not be null"); Runnable action = Objects.requireNonNull(commitAction, "commitAction can not be null"); StripeState stripe = stripeState(key); @@ -351,15 +424,25 @@ public void invalidateKey(K key) { public void invalidateKeyAndRun(K key, Runnable afterInvalidation) { Runnable action = Objects.requireNonNull(afterInvalidation, "afterInvalidation can not be null"); - StripeState stripe = stripeState(key); - synchronized (stripe) { - bumpAction(stripe, key); - data.invalidateKey(key); + useCache(() -> { + StripeState stripe = stripeState(key); + synchronized (stripe) { + bumpAction(stripe, key); + data.invalidateKey(key); + action.run(); + } + return null; + }, () -> { action.run(); - } + return null; + }); } public void invalidateAll() { + useCache(this::invalidateAllCached); + } + + private void invalidateAllCached() { for (int i = 0; i < stripeCount; i++) { StripeState stripe = stripeStates.get(i); if (stripe != null) { @@ -370,11 +453,59 @@ public void invalidateAll() { } } } - owner.invalidateCatalog(); + owner.invalidateScope(invalidationScope); + } + + /** + * Stops accepting cached operations without waiting for a remote load. Accepted operations pin the physical + * entry until they finish; later reads use the loader without publishing cache values or ID side effects. + * Cleanup removes this exact registration, never the current database scope or a replacement instance. + */ + public void close() { + boolean remove; + synchronized (lifecycleLock) { + retired = true; + remove = activeUses == 0; + } + if (remove) { + owner.remove(data); + } } public void forEach(BiConsumer consumer) { - data.forEach(consumer); + useCache(() -> data.forEach(consumer)); + } + + private void useCache(Runnable operation) { + useCache(() -> { + operation.run(); + return null; + }, () -> null); + } + + private R useCache(Supplier operation, Supplier uncached) { + boolean accepted; + synchronized (lifecycleLock) { + accepted = !retired; + if (accepted) { + activeUses++; + } + } + if (!accepted) { + return uncached.get(); + } + try { + return operation.get(); + } finally { + boolean remove; + synchronized (lifecycleLock) { + activeUses--; + remove = retired && activeUses == 0; + } + if (remove) { + owner.remove(data); + } + } } public MetaCacheEntryStats stats() { @@ -441,6 +572,14 @@ private V loadAndPause(K key, Function loadFunction) { return loaded; } + private V loadWithoutCaching(K key, Function loadFunction) { + V loaded = loadFunction.apply(key); + if (loaded != null && discardedValueAction != null) { + discardedValueAction.accept(key, loaded); + } + return loaded; + } + private static com.github.benmanes.caffeine.cache.RemovalCause toCaffeineRemovalCause( MetaCacheRemovalReason reason) { return com.github.benmanes.caffeine.cache.RemovalCause.valueOf(reason.name()); @@ -470,7 +609,7 @@ private ActionToken beginAction(StripeState stripe, K key) { } private boolean isCurrent(StripeState stripe, ActionToken token) { - return stripe.activeActions != null + return !retired && stripe.activeActions != null && stripe.activeActions.get(token.key) == token.state && token.state.generation == token.generation; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryInvalidation.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryInvalidation.java index 7d68fbd51ff4ee..6bdc70c0b0ddfc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryInvalidation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryInvalidation.java @@ -75,8 +75,9 @@ public static MetaCacheEntryInvalidation forTableIdentity( Function dbNameExtractor, Function tableNameExtractor) { Objects.requireNonNull(dbNameExtractor, "dbNameExtractor"); Objects.requireNonNull(tableNameExtractor, "tableNameExtractor"); + // Metadata is a child of the identity scope: ordinary writes must not evict FE lookup objects. return new MetaCacheEntryInvalidation<>( - key -> ScopePath.table(dbNameExtractor.apply(key), tableNameExtractor.apply(key)), + key -> ScopePath.tableMetadata(dbNameExtractor.apply(key), tableNameExtractor.apply(key)), dbName -> key -> dbNameExtractor.apply(key).equals(dbName), (dbName, tableName) -> key -> dbNameExtractor.apply(key).equals(dbName) && tableNameExtractor.apply(key).equals(tableName), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java index 6cbea40af7505b..f424fde948aff0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java @@ -43,6 +43,7 @@ import org.apache.doris.connector.spi.write.ConnectorRowChangeStyle; import org.apache.doris.connector.spi.write.ConnectorRowLevelDmlRequest; import org.apache.doris.connector.spi.write.ConnectorWritePlanProvider; +import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; @@ -667,8 +668,7 @@ public Optional initSchema() { } Optional handleOpt = resolveConnectorTableHandle(session, metadata); if (!handleOpt.isPresent()) { - LOG.warn("Table handle not found for plugin-driven table: {}.{}", dbName, tableName); - return Optional.empty(); + throw new CacheException("Table %s.%s does not exist", null, dbName, tableName); } ConnectorTableSchema tableSchema = metadata.getTableSchema(session, handleOpt.get()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java new file mode 100644 index 00000000000000..3fde0c8a41baf1 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java @@ -0,0 +1,456 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.RefreshManager; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.common.util.Util; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCacheGovernance; +import org.apache.doris.datasource.log.ExternalObjectLog; +import org.apache.doris.datasource.metacache.FeMetaCacheEntry; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.datasource.test.TestExternalDatabase; +import org.apache.doris.datasource.test.TestExternalTable; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class FeCatalogCacheOwnerTest extends TestWithFeService { + private static final AtomicLong IDS = new AtomicLong(72000); + private CountingCatalog catalog; + private CountingDatabase database; + private Gate namesGate; + private Gate objectsGate; + private Gate schemaGate; + private ExecutorService workers; + + @Override + protected void runBeforeAll() throws Exception { + FeConstants.runningUnitTest = true; + createDefaultCtx().setThreadLocalInfo(); + } + + @BeforeEach + public void prepareCatalog() { + namesGate = new Gate(); + objectsGate = new Gate(); + schemaGate = new Gate(); + Provider.TABLES.clear(); + putTable("a", "old_column"); + putTable("b", "sibling_column"); + catalog = new CountingCatalog(IDS.incrementAndGet()); + CatalogMgr manager = Env.getCurrentEnv().getCatalogMgr(); + manager.getIdToCatalog().put(catalog.getId(), catalog); + Map> byName = Deencapsulation.getField(manager, "nameToCatalog"); + byName.put(catalog.getName(), catalog); + catalog.setInitializedForTest(true); + database = new CountingDatabase(catalog); + database.setInitializedForTest(true); + catalog.addDatabaseForTest(database); + workers = Executors.newFixedThreadPool(2); + } + + @AfterEach + public void releaseCatalog() throws Exception { + namesGate.release.countDown(); + objectsGate.release.countDown(); + schemaGate.release.countDown(); + workers.shutdownNow(); + Assertions.assertTrue(workers.awaitTermination(10, TimeUnit.SECONDS)); + catalog.closeFeMetaCacheOwner(); + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalog(catalog.getId()); + CatalogMgr manager = Env.getCurrentEnv().getCatalogMgr(); + manager.getIdToCatalog().remove(catalog.getId()); + Map> byName = Deencapsulation.getField(manager, "nameToCatalog"); + byName.remove(catalog.getName()); + } + + @Test + public void fullRefreshSharesOwnerAndPreservesSibling() throws Exception { + TestExternalTable a = database.getTableNullable("a"); + TestExternalTable b = database.getTableNullable("b"); + a.getFullSchema(); + List siblingSchema = b.getFullSchema(); + Assertions.assertEquals(1, MetaCacheGovernance.catalogCaches(catalog.getId()).size()); + Assertions.assertEquals(1, catalog.nameLoads.get()); + Provider.TABLES.remove("a"); + + new RefreshManager().handleRefreshTable(catalog.getName(), "db", "a", false); + + Assertions.assertEquals(Collections.singleton("b"), database.getTableNamesWithLock()); + Assertions.assertEquals(2, catalog.nameLoads.get()); + Assertions.assertNull(database.getTableNullable("a")); + Assertions.assertNull(database.getCachedTableNameByIdForTest(a.getId())); + Assertions.assertSame(b, database.getTableNullable("b")); + Assertions.assertSame(siblingSchema, b.getFullSchema()); + } + + @Test + public void committedWritesAndTheirReplayDoNotReloadNames() { + TestExternalTable a = database.getTableNullable("a"); + database.getTableNamesWithLock(); + RefreshManager manager = new RefreshManager(); + for (int i = 0; i < 3; i++) { + manager.refreshTableAfterExternalMutation(a); + manager.replayRefreshTable(ExternalObjectLog.createForRefreshTable( + catalog.getId(), "db", "a", i + 1L)); + Assertions.assertSame(a, database.getTableNullable("a")); + Assertions.assertEquals(Set.of("a", "b"), database.getTableNamesWithLock()); + } + Assertions.assertEquals(1, catalog.nameLoads.get()); + } + + @Test + public void explicitReplayInvalidatesNamesWhenTableObjectIsCold() { + database.getTableNamesWithLock(); + Provider.TABLES.remove("a"); + refreshExplicitly(); + Assertions.assertEquals(Collections.singleton("b"), database.getTableNamesWithLock()); + Assertions.assertEquals(2, catalog.nameLoads.get()); + } + + @Test + public void oldNameLoadCannotRefillAfterExplicitRefresh() throws Exception { + database.getTableNullable("a"); + database.resetMetaCacheNames(); + namesGate.arm(); + Future> old = workers.submit(database::getTableNamesWithLock); + namesGate.awaitStarted(); + Provider.TABLES.remove("a"); + refreshExplicitly(); + namesGate.release.countDown(); + old.get(10, TimeUnit.SECONDS); + Assertions.assertEquals(Collections.singleton("b"), database.getTableNamesWithLock()); + } + + @Test + public void oldObjectLoadCannotReplaceSameNameRecreation() throws Exception { + database.getTableNamesWithLock(); + objectsGate.arm(); + Future old = workers.submit(() -> database.getTableNullable("a")); + objectsGate.awaitStarted(); + Provider.TABLES.remove("a"); + refreshExplicitly(); + putTable("a", "new_column"); + TestExternalTable recreated = database.getTableNullable("a"); + Assertions.assertEquals("new_column", recreated.getFullSchema().get(0).getName()); + objectsGate.release.countDown(); + Assertions.assertNotSame(recreated, old.get(10, TimeUnit.SECONDS)); + Assertions.assertSame(recreated, database.getTableNullable("a")); + Assertions.assertEquals("a", database.getCachedTableNameByIdForTest(recreated.getId())); + } + + @Test + public void oldSchemaLoadCannotReplaceSameNameRecreation() throws Exception { + TestExternalTable oldTable = database.getTableNullable("a"); + schemaGate.arm(); + Future> old = workers.submit(() -> { + return oldTable.getFullSchema(); + }); + schemaGate.awaitStarted(); + Provider.TABLES.remove("a"); + refreshExplicitly(); + putTable("a", "new_column"); + TestExternalTable recreated = database.getTableNullable("a"); + List newSchema = recreated.getFullSchema(); + Assertions.assertEquals("new_column", newSchema.get(0).getName()); + schemaGate.release.countDown(); + Assertions.assertEquals("old_column", old.get(10, TimeUnit.SECONDS).get(0).getName()); + Assertions.assertSame(newSchema, recreated.getFullSchema()); + } + + @Test + public void oldDatabaseCleanupDoesNotInvalidateReplacementOrAccumulateEntries() { + database.getTableNullable("a").getFullSchema(); + for (int i = 0; i < 5; i++) { + CountingDatabase previous = database; + database = new CountingDatabase(catalog); + database.setInitializedForTest(true); + catalog.addDatabaseForTest(database); + TestExternalTable replacement = database.getTableNullable("a"); + List schema = replacement.getFullSchema(); + previous.releaseCachedEntries(); + Assertions.assertSame(replacement, database.getTableNullable("a")); + Assertions.assertSame(schema, replacement.getFullSchema()); + Assertions.assertEquals(5, catalog.getFeMetaCacheOwner().entries().size()); + Assertions.assertNotNull(previous.getTableNullable("a")); + Assertions.assertEquals(5, catalog.getFeMetaCacheOwner().entries().size()); + } + } + + @Test + public void catalogCloseReleasesNamingOwnerBeforeAnySchemaWasLoaded() { + database.getTableNamesWithLock(); + CatalogMetaCache owner = catalog.getFeMetaCacheOwner(); + catalog.onClose(); + Assertions.assertTrue(owner.isClosed()); + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalog.getId()).isEmpty()); + Assertions.assertThrows(IllegalStateException.class, catalog::getFeMetaCacheOwner); + } + + @Test + public void ownerRetirementDuringOldSchemaLoadCannotRefillReplacementRuntime() throws Exception { + TestExternalTable oldTable = database.getTableNullable("a"); + schemaGate.arm(); + Future> old = workers.submit(() -> { + return oldTable.getFullSchema(); + }); + schemaGate.awaitStarted(); + CatalogMetaCache oldOwner = catalog.getFeMetaCacheOwner(); + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalog(catalog.getId()); + putTable("a", "new_owner_column"); + ExternalDatabase replacement = catalog.getDbNullable("db"); + List schema = replacement.getTableNullable("a").getFullSchema(); + Assertions.assertTrue(oldOwner.isClosed()); + Assertions.assertNotSame(oldOwner, catalog.getFeMetaCacheOwner()); + schemaGate.release.countDown(); + Assertions.assertEquals("old_column", old.get(10, TimeUnit.SECONDS).get(0).getName()); + Assertions.assertSame(schema, replacement.getTableNullable("a").getFullSchema()); + Assertions.assertEquals("new_owner_column", schema.get(0).getName()); + Assertions.assertEquals(1, MetaCacheGovernance.catalogCaches(catalog.getId()).size()); + } + + @Test + public void perSessionDatabaseBypassDoesNotRegisterSharedChildEntries() { + catalog.bypassDatabaseNames = true; + int entries = catalog.getFeMetaCacheOwner().entries().size(); + for (int i = 0; i < 20; i++) { + ExternalDatabase transientDatabase = catalog.getDbNullable("db"); + Assertions.assertEquals(Set.of("a", "b"), transientDatabase.getTableNamesWithLock()); + Assertions.assertNotNull(transientDatabase.getTableNullable("a")); + } + Assertions.assertEquals(entries, catalog.getFeMetaCacheOwner().entries().size()); + } + + @Test + public void weightedPayloadBudgetPreservesLookupCachingAndIdResolution() { + catalog.modifyCatalogProps(Collections.singletonMap("meta.cache.max-weight", "1MB")); + ExternalDatabase current = catalog.getDbNullable("db"); + ExternalTable table = current.getTableNullable("a"); + for (int i = 0; i < 20; i++) { + Assertions.assertSame(current, catalog.getDbNullable("db")); + Assertions.assertEquals(Set.of("a", "b"), current.getTableNamesWithLock()); + Assertions.assertSame(table, current.getTableNullable("a")); + Assertions.assertEquals("old_column", table.getFullSchema().get(0).getName()); + } + Assertions.assertEquals(1, catalog.nameLoads.get()); + Assertions.assertSame(current, catalog.getDbNullable(current.getId())); + Assertions.assertSame(table, current.getTableNullable(table.getId())); + Assertions.assertEquals(5, catalog.getFeMetaCacheOwner().entries().size()); + Assertions.assertTrue(Env.getCurrentEnv().getExtMetaCacheMgr().getCatalogCacheStats(catalog.getId()).stream() + .filter(stat -> "schema".equals(stat.getEntryName())) + .anyMatch(stat -> stat.getEntryStats().getEstimatedWeight() > 0)); + } + + @Test + public void coldDatabaseRefreshReplayAfterOwnerRetirementDoesNotReinitializeCatalog() { + database.getTableNamesWithLock(); + catalog.modifyCatalogProps(Collections.singletonMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, "600")); + Assertions.assertFalse(catalog.isInitialized()); + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalog.getId()).isEmpty()); + Assertions.assertDoesNotThrow(() -> new RefreshManager().replayRefreshDb( + ExternalObjectLog.createForRefreshDb(catalog.getId(), "db"))); + Assertions.assertFalse(catalog.isInitialized()); + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalog.getId()).isEmpty()); + } + + @Test + public void schemaEntryReconfigurationDoesNotRetireTheBorrowedCatalogOwner() { + database.getTableNullable("a").getFullSchema(); + CatalogMetaCache owner = catalog.getFeMetaCacheOwner(); + catalog.modifyCatalogProps(Collections.singletonMap("meta.cache.default.schema.ttl-second", "600")); + Assertions.assertFalse(owner.isClosed()); + Assertions.assertFalse(catalog.isInitialized()); + Assertions.assertDoesNotThrow(() -> new RefreshManager().replayRefreshDb( + ExternalObjectLog.createForRefreshDb(catalog.getId(), "db"))); + Assertions.assertFalse(catalog.isInitialized()); + Assertions.assertSame(owner, catalog.getFeMetaCacheOwner()); + Assertions.assertEquals("old_column", catalog.getDbNullable("db").getTableNullable("a") + .getFullSchema().get(0).getName()); + Assertions.assertSame(owner, catalog.getFeMetaCacheOwner()); + } + + @Test + public void namingOnlyOwnerIsVisibleInStatsWithoutSchemaInitialization() { + database.getTableNamesWithLock(); + Assertions.assertEquals(4, Env.getCurrentEnv().getExtMetaCacheMgr() + .getCatalogCacheStats(catalog.getId()).size()); + database.getTableNullable("a").getFullSchema(); + Assertions.assertEquals(5, Env.getCurrentEnv().getExtMetaCacheMgr() + .getCatalogCacheStats(catalog.getId()).size()); + } + + @Test + public void retiredDatabaseEntriesReadThroughWithoutPublishingIds() { + database.getTableNamesWithLock(); + FeMetaCacheEntry entries = Deencapsulation.getField(database, "tables"); + database.releaseCachedEntries(); + AtomicInteger actions = new AtomicInteger(); + TestExternalTable table = entries.getAndRunIfCurrent("a", (key, value) -> actions.incrementAndGet()); + Assertions.assertNotNull(table); + Assertions.assertEquals(0, actions.get()); + Assertions.assertNull(entries.getIfPresent("a")); + Assertions.assertDoesNotThrow(() -> database.invalidateTableLookup("a")); + Assertions.assertNull(database.getCachedTableNameByIdForTest(table.getId())); + } + + @Test + public void databaseRetirementDoesNotCloseAnAcceptedObjectLoad() throws Exception { + database.getTableNamesWithLock(); + CountingDatabase previous = database; + objectsGate.arm(); + Future old = workers.submit(() -> previous.getTableNullable("a")); + objectsGate.awaitStarted(); + // Catalog eviction/replacement must not wait for the in-flight remote table lookup. + previous.releaseCachedEntries(); + database = new CountingDatabase(catalog); + database.setInitializedForTest(true); + catalog.addDatabaseForTest(database); + TestExternalTable replacement = database.getTableNullable("a"); + objectsGate.release.countDown(); + Assertions.assertNotSame(replacement, old.get(10, TimeUnit.SECONDS)); + Assertions.assertSame(replacement, database.getTableNullable("a")); + Assertions.assertNull(previous.getCachedTableNameByIdForTest(replacement.getId())); + Assertions.assertEquals(4, catalog.getFeMetaCacheOwner().entries().size()); + } + + @Test + public void refreshLogRetainsScopeAndOldRecordsDefaultToMetadataOnly() { + ExternalObjectLog old = GsonUtils.GSON.fromJson( + "{\"catalogId\":1,\"dbName\":\"db\",\"tableName\":\"a\"}", ExternalObjectLog.class); + Assertions.assertFalse(old.isInvalidateTableLookup()); + ExternalObjectLog full = ExternalObjectLog.createForExplicitRefreshTable(1L, "db", "a", 2L); + Assertions.assertTrue(GsonUtils.GSON.fromJson( + GsonUtils.GSON.toJson(full), ExternalObjectLog.class).isInvalidateTableLookup()); + } + + private void refreshExplicitly() { + new RefreshManager().replayRefreshTable(ExternalObjectLog.createForExplicitRefreshTable( + catalog.getId(), "db", "a", System.currentTimeMillis())); + } + + private static void putTable(String name, String column) { + Provider.TABLES.put(name, Collections.singletonList(new Column(column, PrimitiveType.INT))); + } + + public static class Provider implements TestExternalCatalog.TestCatalogProvider { + static final Map> TABLES = new ConcurrentHashMap<>(); + + @Override + public Map>> getMetadata() { + return Collections.singletonMap("db", TABLES); + } + } + + private class CountingCatalog extends TestExternalCatalog { + private final AtomicInteger nameLoads = new AtomicInteger(); + private boolean bypassDatabaseNames; + + CountingCatalog(long id) { + super(id, "owner_test_" + id, "", + new HashMap<>(Collections.singletonMap("catalog_provider.class", Provider.class.getName())), ""); + } + + @Override + protected boolean shouldBypassDbNameCache(SessionContext context) { + return bypassDatabaseNames; + } + + @Override + protected List listTableNamesFromRemote(SessionContext context, String dbName) { + nameLoads.incrementAndGet(); + List snapshot = new ArrayList<>(super.listTableNamesFromRemote(context, dbName)); + namesGate.pause(); + return snapshot; + } + } + + private class CountingDatabase extends TestExternalDatabase { + CountingDatabase(CountingCatalog owner) { + super(owner, Util.genIdByName(owner.getName(), "db"), "db", "db"); + } + + @Override + public TestExternalTable buildTableInternal(String remoteName, String localName, long id, + ExternalCatalog owner, ExternalDatabase db) { + TestExternalTable table = new TestExternalTable( + id, localName, remoteName, (TestExternalCatalog) owner, this) { + @Override + public Optional initSchema() { + List snapshot = + new ArrayList<>(((TestExternalCatalog) catalog).mockedSchema("db", getName())); + schemaGate.pause(); + return Optional.of(new SchemaCacheValue(snapshot)); + } + }; + objectsGate.pause(); + return table; + } + } + + private static class Gate { + private final AtomicInteger armed = new AtomicInteger(); + private final CountDownLatch started = new CountDownLatch(1); + private final CountDownLatch release = new CountDownLatch(1); + + void arm() { + armed.set(1); + } + + void awaitStarted() throws InterruptedException { + Assertions.assertTrue(started.await(10, TimeUnit.SECONDS)); + } + + void pause() { + if (armed.compareAndSet(1, 0)) { + started.countDown(); + try { + Assertions.assertTrue(release.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java index 22bbc80ad788f0..cb422cc548dd18 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java @@ -18,6 +18,8 @@ package org.apache.doris.datasource.metacache; import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.ScopePath; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -64,6 +66,31 @@ public void testLoadMutationAndStatsUseSharedRuntime() { } } + @Test + public void testRetiredReadThroughValuesAreDiscardedWithoutPublishingActions() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + AtomicInteger actions = new AtomicInteger(); + FeMetaCacheEntry entry = FeMetaCacheEntry.sharedLookup(owner, + key -> ScopePath.database(key), ScopePath.catalog(), "objects", + key -> new AtomicBoolean(true), ENABLED, executor, false, 8, + (key, value, cause) -> value.set(false)); + entry.close(); + AtomicBoolean transientValue = entry.getAndRunIfCurrent("db", (key, value) -> actions.incrementAndGet()); + Assertions.assertFalse(transientValue.get()); + Assertions.assertEquals(0, actions.get()); + Assertions.assertNull(entry.getIfPresent("db")); + Assertions.assertNull(entry.computeWithCommitAction("db", (key, value) -> new AtomicBoolean(true), + actions::incrementAndGet)); + Assertions.assertEquals(0, actions.get()); + entry.invalidateKeyAndRun("db", actions::incrementAndGet); + Assertions.assertEquals(1, actions.get()); + Assertions.assertTrue(owner.entries().isEmpty()); + } finally { + executor.shutdownNow(); + } + } + @Test public void testRejectedWeightedMutationReturnsWithoutRetryingForever() throws Exception { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java index f21ed9f8006890..4605f1f2784b66 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java @@ -37,6 +37,7 @@ import org.apache.doris.connector.spi.pushdown.ConnectorExpression; import org.apache.doris.connector.spi.write.ConnectorRowChangeStyle; import org.apache.doris.connector.spi.write.ConnectorWritePlanProvider; +import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.mvcc.MvccSnapshot; @@ -984,6 +985,13 @@ public void initSchemaBuildsViewSchemaFromViewDefinitionColumns() { "a view has no partition columns"); } + @Test + public void missingTableHandlePreservesNotFoundInsteadOfGenericSchemaFailure() { + PluginDrivenExternalTable table = initSchemaViewTable(null, false); + CacheException error = Assertions.assertThrows(CacheException.class, table::initSchema); + Assertions.assertEquals("Table db1.v1 does not exist", error.getMessage()); + } + @Test public void initSchemaUsesTableHandlePathForNonView() { // WHY: the isView() branch must NOT hijack ordinary tables — a non-view must still resolve its schema diff --git a/regression-test/data/external_table_p0/hive/test_hive_catalog_owner_refresh.out b/regression-test/data/external_table_p0/hive/test_hive_catalog_owner_refresh.out new file mode 100644 index 00000000000000..0f94eda759ea1d --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_hive_catalog_owner_refresh.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !names_before -- +owner_sibling +owner_target + +-- !schema_before -- +before_col int Yes true \N + +-- !sibling_before -- +sibling_col int Yes true \N + +-- !names_after_drop -- +owner_sibling + +-- !sibling_after -- +sibling_col int Yes true \N + +-- !names_after_database_refresh -- +owner_sibling +owner_target + +-- !schema_after_database_refresh -- +after_col text Yes true \N + +-- !schema_after_recreation -- +recreated_col bigint Yes true \N + diff --git a/regression-test/suites/external_table_p0/hive/test_hive_catalog_owner_refresh.groovy b/regression-test/suites/external_table_p0/hive/test_hive_catalog_owner_refresh.groovy new file mode 100644 index 00000000000000..7a9ae6e65742b3 --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_hive_catalog_owner_refresh.groovy @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_catalog_owner_refresh", "p0,external") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Hive test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String hmsPort = context.config.otherConfigs.get("hive3HmsPort") + String hdfsPort = context.config.otherConfigs.get("hive3HdfsPort") + for (String catalog : ["hive_owner_source", "hive_owner_target"]) { + sql "DROP CATALOG IF EXISTS ${catalog}" + sql """CREATE CATALOG ${catalog} PROPERTIES ( + 'type' = 'hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfsPort}' + )""" + } + + // Two independent catalogs model a change made through another engine's cache domain. + sql "DROP DATABASE IF EXISTS hive_owner_source.fe_cache_owner_refresh_db FORCE" + sql "CREATE DATABASE hive_owner_source.fe_cache_owner_refresh_db" + sql "CREATE TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_target (before_col INT)" + sql "CREATE TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_sibling (sibling_col INT)" + order_qt_names_before "SHOW TABLES FROM hive_owner_target.fe_cache_owner_refresh_db" + qt_schema_before "DESC hive_owner_target.fe_cache_owner_refresh_db.owner_target" + qt_sibling_before "DESC hive_owner_target.fe_cache_owner_refresh_db.owner_sibling" + + sql "DROP TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_target" + sql "REFRESH TABLE hive_owner_target.fe_cache_owner_refresh_db.owner_target" + // Listing first proves refresh itself invalidated membership, without a failing SELECT repairing it. + order_qt_names_after_drop "SHOW TABLES FROM hive_owner_target.fe_cache_owner_refresh_db" + test { + sql "SELECT * FROM hive_owner_target.fe_cache_owner_refresh_db.owner_target" + exception "does not exist" + } + test { + sql "EXPLAIN SELECT * FROM hive_owner_target.fe_cache_owner_refresh_db.owner_target" + exception "does not exist" + } + qt_sibling_after "DESC hive_owner_target.fe_cache_owner_refresh_db.owner_sibling" + + sql "CREATE TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_target (after_col STRING)" + sql "REFRESH DATABASE hive_owner_target.fe_cache_owner_refresh_db" + order_qt_names_after_database_refresh "SHOW TABLES FROM hive_owner_target.fe_cache_owner_refresh_db" + qt_schema_after_database_refresh "DESC hive_owner_target.fe_cache_owner_refresh_db.owner_target" + + sql "DROP TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_target" + sql "CREATE TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_target (recreated_col BIGINT)" + sql "REFRESH TABLE hive_owner_target.fe_cache_owner_refresh_db.owner_target" + qt_schema_after_recreation "DESC hive_owner_target.fe_cache_owner_refresh_db.owner_target" +} From 328deaa948135f3b4a2798e366e87dae566835d1 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 22 Oct 2026 19:26:10 +0800 Subject: [PATCH 2/5] [refactor](fe) Share catalog cache ownership with connector domains ### What problem does this PR solve? Issue Number: None Problem Summary: External catalog metadata is cached in several dependent layers: database/table membership, FE object and ID lookup, schemas, row counts, and connector metadata such as partitions, files, and snapshots. Previously these layers had separate owners and callers manually invalidated both sides. A refresh could leave a stale membership or object lookup behind; a load started before invalidation could republish an obsolete object or identity, particularly after a drop and same-name recreation. Keeping the old FE database object after eviction also allowed its cleanup to affect a replacement. Use one `CatalogMetaCache` root per external catalog. FE registrations and instance-owned connector domains share its resource generations and existing budgets. Embedded connectors use child domains. Retiring an instance removes only its exact registrations; semantic resource invalidation is independent of instance lifetime. Local names, remote names, and numeric derived-state identities use distinct coordinate spaces. `CatalogMetadataChanges` is the common boundary for refresh, successful writes/DDL, metastore events, and replay. It projects affected local and remote resources and invalidates them at one publication barrier. Connectors describe cross-resource dependencies through a pure SPI projection instead of imperative sibling invalidation. DDL journal records preserve remote identities for replay; old records fall back to broader remote-only invalidation without initializing remote metadata. Content changes preserve table-name snapshots and FE lookups, while explicit table/database refresh also invalidates the appropriate membership snapshot. ID mappings survive ordinary object eviction but participate in semantic invalidation. Publication is guarded by the same resource generation, and accepted operations pin their auxiliary index until completion. Row-count futures belong to catalog roots while a shared retention-policy index preserves the existing process-wide entry limit. Cache callbacks retain the owning plugin classloader context. Automatic row-count refresh keeps the last successful value while reloading and on failure. ADBC membership caches register under the shared collection scopes. This also addresses catalog rename/replay reactivation, disabled database caching with case-insensitive table lookup, and tests that continued to use retired database instances. The connector plugin API major version changes from 19 to 20 with updated surface coverage. ### Release note Fix stale external-catalog table membership and metadata after refresh, including drop and same-name recreation. Ordinary data writes keep cached table-name listings. ### Check List (For Author) - Test: Unit Test / Regression test - Selected connector modules and upstream dependencies packaged with their own tests, including SPI, Hive, Hudi, Iceberg, Paimon, Fluss, ADBC, and MaxCompute. - Current cache framework suite: 172 tests passed; connector SPI own suite: 82 tests passed. - Related FE coverage: 320 unique tests. A broad 317-test run had one obsolete dynamic-proxy error; after fixing its null collection return, the affected class and cache-entry lifecycle tests passed all 24 tests, including the added retirement concurrency case. The independent Astra review found two additional gaps; after reproducing both on the old implementation, the fixes passed 40 targeted tests (16 row-count and 24 ADBC tests), including successful/failed automatic refresh and root-level membership invalidation. - Full FE build and Checkstyle passed using `./build.sh --fe -j4` with JDK 17. - `test_hive_catalog_owner_refresh` passed against the newly packaged isolated FE and real HMS: 1 suite, zero failed/fatal/skipped, comparing the checked-in expected output. - No multi-FE cluster, actual Trino service, or BE data-execution validation. - Behavior changed: Yes. Explicit refresh invalidates dependent membership and metadata together; content-only changes preserve names and object lookup. Connector plugin API major version is 20. - Does this need documentation: No. Internal ownership and SPI integration are documented in `fe/fe-connector/README.md`. --- fe/fe-connector/README.md | 31 +- .../doris/connector/adbc/AdbcConnector.java | 2 +- .../connector/adbc/AdbcMetadataCache.java | 11 +- .../connector/adbc/AdbcMetadataCacheTest.java | 40 ++ .../connector/cache/CacheCallbackContext.java | 50 ++ .../connector/cache/CatalogMetaCache.java | 167 ++++- .../doris/connector/cache/MetaCache.java | 32 +- .../connector/cache/MetaCacheDefinition.java | 28 + .../doris/connector/cache/ScopePath.java | 73 +- .../connector/cache/ScopedMetaCache.java | 60 +- .../cache/ScopedMetaCacheRegistry.java | 290 ++------ .../connector/cache/StripedPhaseGate.java | 26 +- .../cache/CatalogCacheDomainTest.java | 192 ++++++ .../cache/MetaCacheWeightGovernanceTest.java | 5 +- .../connector/cache/StripedPhaseGateTest.java | 32 + .../doris/connector/fluss/FlussConnector.java | 75 ++- .../fluss/FlussConnectorLakeSiblingTest.java | 20 +- .../doris/connector/hive/HiveConnector.java | 55 +- .../hive/HiveConnectorSiblingTest.java | 62 +- .../doris/connector/hudi/HudiConnector.java | 2 +- .../connector/iceberg/IcebergConnector.java | 2 +- .../connector/iceberg/IcebergTableCache.java | 2 +- .../maxcompute/MaxComputeDorisConnector.java | 2 +- .../connector/paimon/PaimonConnector.java | 2 +- fe/fe-connector/fe-connector-spi/pom.xml | 5 + .../apache/doris/connector/spi/Connector.java | 33 +- .../doris/connector/spi/ConnectorContext.java | 11 + .../spi/ForwardingConnectorContext.java | 7 + .../spi/ConnectorPluginSurfaceTest.java | 2 +- .../resources/connector-plugin-surface.txt | 3 + fe/fe-connector/pom.xml | 2 +- .../java/org/apache/doris/catalog/Env.java | 22 +- .../apache/doris/catalog/RefreshManager.java | 174 +---- .../connector/ConnectorInstanceContext.java | 77 +++ .../connector/ConnectorPluginManager.java | 18 +- .../connector/DefaultConnectorContext.java | 52 +- .../apache/doris/datasource/CatalogMgr.java | 39 +- .../doris/datasource/ExternalCatalog.java | 23 +- .../doris/datasource/ExternalDatabase.java | 21 +- .../datasource/ExternalMetaCacheMgr.java | 15 +- .../datasource/ExternalRowCountCache.java | 221 ++++-- .../datasource/MetastoreEventSyncDriver.java | 44 +- .../datasource/log/ExternalObjectLog.java | 16 + .../metacache/CatalogMetadataChanges.java | 215 ++++++ .../metacache/ExternalCatalogMetaCache.java | 4 +- .../metacache/FeMetaCacheEntry.java | 40 +- .../datasource/metacache/IdNameIndex.java | 236 ++++--- .../metacache/MetaCacheEntryInvalidation.java | 3 +- .../plugin/PluginDrivenExternalCatalog.java | 212 ++---- .../apache/doris/persist/CreateDbInfo.java | 13 + .../apache/doris/persist/CreateTableInfo.java | 13 + .../org/apache/doris/persist/DropDbInfo.java | 13 + .../org/apache/doris/persist/DropInfo.java | 13 + .../org/apache/doris/persist/EditLog.java | 8 +- .../doris/persist/TruncateTableInfo.java | 13 + .../apache/doris/catalog/RefreshDbTest.java | 28 +- .../RefreshManagerRenameReplayTest.java | 37 +- .../doris/catalog/RefreshManagerTest.java | 632 ++++-------------- .../doris/catalog/RefreshTableTest.java | 24 +- .../connector/ConnectorCacheDomainTest.java | 169 +++++ .../DefaultConnectorContextSiblingTest.java | 5 +- .../datasource/ExternalRowCountCacheTest.java | 114 +++- .../datasource/FeCatalogCacheOwnerTest.java | 57 +- ...tastoreEventSyncDriverClassLoaderTest.java | 8 +- ...etastoreEventSyncDriverMappedNameTest.java | 17 +- .../MetastoreEventSyncDriverRenameTest.java | 48 +- .../metacache/CatalogMetadataChangesTest.java | 172 +++++ .../metacache/FeMetaCacheEntryTest.java | 34 + .../datasource/metacache/IdNameIndexTest.java | 27 + .../PluginDrivenExternalCatalogCacheTest.java | 71 +- ...nDrivenExternalCatalogConcurrencyTest.java | 2 + ...inDrivenExternalCatalogDdlRoutingTest.java | 206 +++--- 72 files changed, 2798 insertions(+), 1682 deletions(-) create mode 100644 fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheCallbackContext.java create mode 100644 fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogCacheDomainTest.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorInstanceContext.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/connector/ConnectorCacheDomainTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CatalogMetadataChangesTest.java diff --git a/fe/fe-connector/README.md b/fe/fe-connector/README.md index b455e1c0bfcda9..5dd6e4ae4a8c27 100644 --- a/fe/fe-connector/README.md +++ b/fe/fe-connector/README.md @@ -31,6 +31,35 @@ Three design rules shape everything in this directory: reached only through the SPI, and optional behavior is opt-in via capabilities (see "Reading the API"). +## Catalog cache ownership + +`ExternalCatalog` owns one `CatalogMetaCache` root. Each connector instance acquires a domain through +`ConnectorContext.createCacheDomain`; embedded siblings acquire child domains. FE entries and connector +entries share the root's registry and catalog/global budgets. Domains qualify physical registration names, +wrap plugin callbacks with the registering classloader, and retire only their own registrations. Closing an +old instance never invalidates a replacement instance's resource scope. + +Resource dependencies are independent of domain lifetime: + +``` +Catalog root + LOCAL -> database -> table -> schema/content + REMOTE -> database -> table -> partition + ID -> database id -> table id -> derived row count +``` + +Each cache value declares one most-specific `ScopePath`. Membership snapshots have typed collection scopes. +`CatalogMetadataChanges` projects a semantic refresh, mutation, DDL, event or replay into these coordinates +and detaches all affected scopes at one publication barrier. Content changes preserve lookup objects and +name lists; explicit lookup refresh also invalidates the target identity and parent membership. Newly +written journals persist local/remote identities and the operation's lookup flag. Old records widen only +the remote scope when a cache-only lookup cannot recover the remote identity. + +Gateways with different physical identities implement the pure `Connector.cacheDependencies` projection; +it must not perform I/O or construct siblings. Partition names are translated by `partitionCacheScopes`. +The host applies the resulting scopes; gateways do not maintain a second list of invalidation callbacks. +Immutable path-addressed caches can stay at catalog scope and survive table refreshes. + ## Module Map Roles only — one line each. For anything deeper, read the module's javadoc @@ -61,7 +90,7 @@ endpoint properties) | Module | Role | |---|---| -| `fe-connector-cache` | Self-contained caching framework used by several connectors. No fe-core dependency; it is bundled into each consuming plugin, so shared third-party libraries stay at the consumers' lowest common version (see the version notes in consumer poms). | +| `fe-connector-cache` | Shared metadata-cache framework with no fe-core dependency. The parent-first `org.apache.doris.connector.*` loader policy gives FE and plugins one runtime identity; plugin packaging does not create private cache owners. ConnectorContext supplies instance domains under the catalog root. | | `fe-connector-hms-hive-shade` | Slim, relocated HMS metastore-client closure for connectors that speak HMS thrift. The pom comments say exactly what relocates where and why. | | `fe-connector-paimon-hive-shade` | Paimon-private relocated HMS-thrift closure; same idea, different owner. | diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java index 97ba7288d673cf..d51c3140e243da 100644 --- a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java @@ -67,7 +67,7 @@ public AdbcConnector(Map properties, ConnectorContext context) { // The raw map, because the cache knobs are the shared framework's keys rather than this // connector's: CacheSpec owns their names, and mirroring them as fields here would give the // validator and the reader two things to drift apart. - this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "adbc", props.getRaw()); + this.metaCache = context.createCacheDomain("adbc", props.getRaw()); try { this.metadataCache = new AdbcMetadataCache(metaCache, props.getRaw()); } catch (RuntimeException | Error e) { diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java index 27a7a3b2a4fc80..90bdbfcc8f6c90 100644 --- a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java @@ -105,13 +105,14 @@ public AdbcMetadataCache(Map properties) { this.owner = Objects.requireNonNull(owner, "owner can not be null"); CacheSpec spec = cacheSpec(properties); this.namespaces = owner.create(MetaCacheDefinition - .>builder("adbc-namespaces", spec, ignored -> ScopePath.catalog()) + .>builder("adbc-namespaces", spec, + ignored -> ScopePath.databaseCollection()) .budgetGroup(ENTRY) .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); this.tableNames = owner.create(MetaCacheDefinition .>builder("adbc-table-names", spec, - key -> ScopePath.database(key.dorisDbName)) + key -> ScopePath.tableCollection(key.dorisDbName)) .budgetGroup(ENTRY) .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); @@ -174,11 +175,7 @@ Schema tableSchema(AdbcTableHandle handle, Supplier loader) { * new name in, and the one the user tried would appear to do nothing. */ void invalidateTable(String dbName, String tableName) { - owner.invalidateTable(dbName, tableName); - // A table refresh must also forget the parent name listing, so a remotely-created table can be found. - // This is an ancestor materialization, not a sibling-cache dependency, and therefore cannot be selected - // by the table's descendant scope. - tableNames.invalidateKey(TableNameKey.forDatabase(dbName)); + owner.invalidateTableAndNames(dbName, tableName); } /** {@code REFRESH DATABASE}: that database's table listing and every schema in it. */ diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcMetadataCacheTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcMetadataCacheTest.java index 5bc224778584ed..58537f96615e01 100644 --- a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcMetadataCacheTest.java +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcMetadataCacheTest.java @@ -20,6 +20,7 @@ import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCacheGovernance; +import org.apache.doris.connector.cache.ScopePath; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; @@ -41,6 +42,45 @@ */ class AdbcMetadataCacheTest { + @Test + void rootResourceInvalidationIncludesRememberedMemberships() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + CatalogMetaCache domain = root.openDomain("adbc", ScopePath.Coordinates.REMOTE, + getClass().getClassLoader()); + AdbcMetadataCache cache = new AdbcMetadataCache(domain, Map.of()); + Counting> namespaces = new Counting<>(List.of(MAIN, OTHER)); + Counting> main = new Counting<>(List.of("t1")); + Counting> other = new Counting<>(List.of("t9")); + Counting schema = new Counting<>(SCHEMA); + cache.namespaces(namespaces); + cache.tableNames(MAIN, main); + cache.tableNames(OTHER, other); + cache.tableSchema(MAIN_T1, schema); + + root.invalidateScopes(List.of( + ScopePath.table("main", "t1").in(ScopePath.Coordinates.REMOTE), + ScopePath.tableCollection("main").in(ScopePath.Coordinates.REMOTE))); + cache.namespaces(namespaces); + cache.tableNames(MAIN, main); + cache.tableNames(OTHER, other); + cache.tableSchema(MAIN_T1, schema); + Assertions.assertEquals(1, namespaces.calls); + Assertions.assertEquals(2, main.calls); + Assertions.assertEquals(1, other.calls); + Assertions.assertEquals(2, schema.calls); + + root.invalidateScopes(List.of( + ScopePath.database("main").in(ScopePath.Coordinates.REMOTE), + ScopePath.databaseCollection().in(ScopePath.Coordinates.REMOTE))); + cache.namespaces(namespaces); + cache.tableNames(MAIN, main); + cache.tableNames(OTHER, other); + Assertions.assertEquals(2, namespaces.calls); + Assertions.assertEquals(3, main.calls); + Assertions.assertEquals(1, other.calls); + } + } + @Test void managedCachesAccountAllPayloadsAndReleaseReservations() { long baseline = MetaCacheGovernance.globalEstimatedWeight(); diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheCallbackContext.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheCallbackContext.java new file mode 100644 index 00000000000000..aa5b0c8ac5e475 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheCallbackContext.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.util.function.Supplier; + +/** Restores the registering plugin's loader around callbacks, including asynchronous cleanup. */ +public final class CacheCallbackContext { + private final ClassLoader loader; + + public CacheCallbackContext(ClassLoader loader) { + this.loader = loader; + } + + public T call(Supplier callback) { + Thread thread = Thread.currentThread(); + ClassLoader previous = thread.getContextClassLoader(); + if (previous == loader) { + return callback.get(); + } + try { + thread.setContextClassLoader(loader); + return callback.get(); + } finally { + thread.setContextClassLoader(previous); + } + } + + public void run(Runnable callback) { + call(() -> { + callback.run(); + return null; + }); + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java index 9a58659d47e2c8..641ca7e535cab0 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java @@ -26,6 +26,7 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; /** * Catalog-local owner of all framework metadata caches. @@ -40,6 +41,14 @@ public final class CatalogMetaCache implements AutoCloseable { private final String engine; private final OptionalLong catalogMaxWeight; private final boolean managed; + private final CatalogMetaCache parent; + private final ScopePath.Coordinates coordinates; + private final CacheCallbackContext callbacks; + private final CatalogMetaCache rootOwner; + private final String registrationPrefix; + private final AtomicLong domainSequence; + private final Set domains = ConcurrentHashMap.newKeySet(); + private final Map> allEntries; private final Set names = ConcurrentHashMap.newKeySet(); private final Map> entries = new ConcurrentHashMap<>(); private final AtomicBoolean closed = new AtomicBoolean(false); @@ -71,26 +80,77 @@ public static CatalogMetaCache unmanaged() { public static CatalogMetaCache managed(long catalogId, String engine, Map catalogProperties) { + return managed(catalogId, engine, catalogProperties, Thread.currentThread().getContextClassLoader()); + } + + public static CatalogMetaCache managed(long catalogId, String engine, + Map catalogProperties, ClassLoader classLoader) { MetaCacheBudgetManager manager = MetaCacheGovernance.budgetManager(); CatalogMetaCache cache = new CatalogMetaCache(new ScopedMetaCacheRegistry(), manager, - catalogId, engine, manager.parseCatalogMaxWeight(catalogProperties), true); + catalogId, engine, manager.parseCatalogMaxWeight(catalogProperties), true, classLoader); MetaCacheGovernance.register(cache); return cache; } CatalogMetaCache(ScopedMetaCacheRegistry registry, MetaCacheBudgetManager budgetManager, long catalogId, String engine, OptionalLong catalogMaxWeight, boolean managed) { + this(registry, budgetManager, catalogId, engine, catalogMaxWeight, managed, + Thread.currentThread().getContextClassLoader()); + } + + private CatalogMetaCache(ScopedMetaCacheRegistry registry, MetaCacheBudgetManager budgetManager, + long catalogId, String engine, OptionalLong catalogMaxWeight, boolean managed, ClassLoader classLoader) { this.registry = Objects.requireNonNull(registry, "registry can not be null"); this.budgetManager = Objects.requireNonNull(budgetManager, "budgetManager can not be null"); this.catalogId = catalogId; this.engine = Objects.requireNonNull(engine, "engine can not be null"); this.catalogMaxWeight = Objects.requireNonNull(catalogMaxWeight, "catalogMaxWeight can not be null"); this.managed = managed; + this.parent = null; + this.coordinates = null; + this.callbacks = new CacheCallbackContext(classLoader); + this.rootOwner = this; + this.registrationPrefix = ""; + this.domainSequence = new AtomicLong(); + this.allEntries = new ConcurrentHashMap<>(); + } + + private CatalogMetaCache(CatalogMetaCache parent, String engine, + ScopePath.Coordinates coordinates, ClassLoader classLoader) { + this.coordinates = coordinates; + this.callbacks = new CacheCallbackContext(classLoader); + this.registry = parent.registry; + this.budgetManager = parent.budgetManager; + this.catalogId = parent.catalogId; + this.engine = Objects.requireNonNull(engine, "engine"); + this.catalogMaxWeight = parent.catalogMaxWeight; + this.managed = false; + this.parent = parent; + this.rootOwner = parent.rootOwner; + this.domainSequence = rootOwner.domainSequence; + this.registrationPrefix = engine + "@" + domainSequence.incrementAndGet() + "."; + this.allEntries = rootOwner.allEntries; + } + + /** Opens one instance-owned domain. Resource invalidation is shared; close only retires this instance. */ + public CatalogMetaCache openDomain(String engine) { + return openDomain(engine, coordinates, Thread.currentThread().getContextClassLoader()); + } + + public CatalogMetaCache openDomain(String engine, ScopePath.Coordinates coordinates, ClassLoader classLoader) { + checkOpen(); + CatalogMetaCache domain = new CatalogMetaCache(this, engine, coordinates, classLoader); + domains.add(domain); + if (isClosed()) { + domain.close(); + throw new IllegalStateException("Parent metadata cache domain was retired during construction"); + } + return domain; } public MetaCache create(MetaCacheDefinition definition) { MetaCacheDefinition nonNullDefinition = - Objects.requireNonNull(definition, "definition can not be null"); + Objects.requireNonNull(definition, "definition can not be null").forDomain(coordinates, callbacks); checkOpen(); if (!names.add(nonNullDefinition.name())) { throw new IllegalArgumentException("Duplicate meta cache name: " + nonNullDefinition.name()); @@ -109,26 +169,25 @@ public MetaCache create(MetaCacheDefinition definition) { nonNullDefinition.name(), nonNullDefinition.budgetGroup(), catalogMaxWeight, nonNullDefinition.cacheSpec().getMaxWeight()); } - MetaCache created = new MetaCache<>(nonNullDefinition, - registry.createCacheWithMetaRemovalListener(nonNullDefinition.name(), + String registrationName = registrationPrefix + nonNullDefinition.name(); + MetaCache created = new MetaCache<>(nonNullDefinition, engine, registrationName, callbacks, + registry.createCacheWithMetaRemovalListener(registrationName, nonNullDefinition.cacheSpec(), nonNullDefinition.removalListener(), nonNullDefinition.discardListener(), nonNullDefinition.refreshAfterWrite(), nonNullDefinition.refreshExecutor(), entryBudget == null ? null : nonNullDefinition.sizeEstimator(), entryBudget)); entries.put(nonNullDefinition.name(), created); + allEntries.put(created.registrationName(), created); + if (isClosed()) { + remove(created); + throw new IllegalStateException("Metadata cache domain was retired during entry construction"); + } return created; } catch (RuntimeException | Error throwable) { if (entryBudget != null) { entryBudget.close(); } names.remove(nonNullDefinition.name()); - if (managed) { - try { - close(); - } catch (RuntimeException | Error closeFailure) { - throwable.addSuppressed(closeFailure); - } - } throw throwable; } } @@ -140,6 +199,7 @@ public MetaCache create(MetaCacheDefinition definition) { public void remove(MetaCache cache) { MetaCache nonNullCache = Objects.requireNonNull(cache, "cache can not be null"); if (entries.remove(nonNullCache.name(), nonNullCache)) { + allEntries.remove(nonNullCache.registrationName(), nonNullCache); try { nonNullCache.closeFromOwner(); } finally { @@ -149,42 +209,53 @@ public void remove(MetaCache cache) { } public void invalidateCatalog() { - registry.invalidate(ScopePath.catalog()); + invalidateScope(ScopePath.catalog()); } public void invalidateDatabase(String database) { - registry.invalidate(ScopePath.database(database)); + invalidateScope(ScopePath.database(database)); } public void invalidateTable(String database, String table) { - registry.invalidate(ScopePath.table(database, table)); + invalidateScope(ScopePath.table(database, table)); } /** Invalidates an object and its parent membership view at one logical publication barrier. */ public void invalidateTableAndNames(String database, String table) { - registry.invalidate(java.util.Arrays.asList( + invalidateScopes(java.util.Arrays.asList( ScopePath.table(database, table), ScopePath.tableCollection(database))); } public void invalidateDatabaseAndNames(String database) { - registry.invalidate(java.util.Arrays.asList( + invalidateScopes(java.util.Arrays.asList( ScopePath.database(database), ScopePath.databaseCollection())); } public void invalidateScope(ScopePath scope) { - registry.invalidate(scope); + registry.invalidate(project(scope)); + } + + /** One barrier covers local and remote dependencies, even when their object caches are cold. */ + public void invalidateScopes(Collection scopes) { + List projected = new ArrayList<>(scopes.size()); + scopes.forEach(scope -> projected.add(project(scope))); + registry.invalidate(projected); + } + + private ScopePath project(ScopePath scope) { + return coordinates == null ? scope : scope.in(coordinates); } public boolean isClosed() { - return closed.get(); + return closed.get() || (parent != null && parent.isClosed()); } public void invalidatePartition(String database, String table, Object partition) { - registry.invalidate(ScopePath.partition(database, table, partition)); + invalidateScope(ScopePath.partition(database, table, partition)); } public void invalidatePartitionCollection(String database, String table) { - registry.invalidate(ScopePath.partitionCollection(database, table)); + invalidateScope(ScopePath.partitionCollection(database, table)); } /** @@ -197,7 +268,7 @@ public void invalidatePartitions(String database, String table, Collection pa List paths = new ArrayList<>(partitions.size() + 1); paths.add(ScopePath.partitionCollection(database, table)); partitions.forEach(partition -> paths.add(ScopePath.partition(database, table, partition))); - registry.invalidate(paths); + invalidateScopes(paths); } public ScopedMetaCacheRegistry.ScopeMetrics metrics() { @@ -208,6 +279,11 @@ public ScopedMetaCacheRegistry.ScopeMetrics metrics() { return java.util.Collections.unmodifiableMap(entries); } + /** Catalog-wide registrations, including every live connector instance domain. */ + public Map> allEntries() { + return java.util.Collections.unmodifiableMap(allEntries); + } + public long catalogId() { return catalogId; } @@ -226,22 +302,49 @@ public boolean hasEnclosingWeightLimit() { @Override public void close() { - if (closed.compareAndSet(false, true)) { - try { - registry.close(); - } finally { - names.clear(); - entries.clear(); - if (managed) { - MetaCacheGovernance.unregister(this); - } + if (!closed.compareAndSet(false, true)) { + return; + } + Throwable failure = null; + for (CatalogMetaCache domain : new ArrayList<>(domains)) { + failure = closeAndCollect(domain::close, failure); + } + for (MetaCache entry : new ArrayList<>(entries.values())) { + failure = closeAndCollect(() -> remove(entry), failure); + } + domains.clear(); + if (parent == null) { + failure = closeAndCollect(registry::close, failure); + allEntries.clear(); + if (managed) { + MetaCacheGovernance.unregister(this); + } + } else { + parent.domains.remove(this); + } + if (failure instanceof Error) { + throw (Error) failure; + } + if (failure != null) { + throw (RuntimeException) failure; + } + } + + private static Throwable closeAndCollect(Runnable cleanup, Throwable failure) { + try { + cleanup.run(); + } catch (RuntimeException | Error error) { + if (failure == null) { + return error; } + failure.addSuppressed(error); } + return failure; } private void checkOpen() { - if (closed.get()) { - throw new IllegalStateException("Catalog meta cache is closed"); + if (isClosed()) { + throw new IllegalStateException("Catalog meta cache domain is closed"); } } } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java index 07afd00576d4bf..74ab6d496b4716 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java @@ -26,8 +26,15 @@ public final class MetaCache { private final MetaCacheDefinition definition; private final ScopedMetaCache delegate; - - MetaCache(MetaCacheDefinition definition, ScopedMetaCache delegate) { + private final String engine; + private final String registrationName; + private final CacheCallbackContext callbacks; + + MetaCache(MetaCacheDefinition definition, String engine, String registrationName, + CacheCallbackContext callbacks, ScopedMetaCache delegate) { + this.callbacks = callbacks; + this.engine = engine; + this.registrationName = registrationName; this.definition = Objects.requireNonNull(definition, "definition can not be null"); this.delegate = Objects.requireNonNull(delegate, "delegate can not be null"); } @@ -36,6 +43,14 @@ public String name() { return definition.name(); } + public String engine() { + return engine; + } + + public String registrationName() { + return registrationName; + } + public V get(K key) { Function loader = definition.loader(); if (loader == null) { @@ -46,7 +61,8 @@ public V get(K key) { public V get(K key, Function loader) { K nonNullKey = Objects.requireNonNull(key, "key can not be null"); - return delegate.get(nonNullKey, definition.scope(nonNullKey), loader); + return delegate.get(nonNullKey, definition.scope(nonNullKey), + keyToLoad -> callbacks.call(() -> loader.apply(keyToLoad))); } /** @@ -57,8 +73,8 @@ public V get(K key, Function loader) { public V getWithPublicationAction(K key, Function loader, BiConsumer> publicationCoordinator) { K nonNullKey = Objects.requireNonNull(key, "key can not be null"); - return delegate.getWithPublicationAction(nonNullKey, definition.scope(nonNullKey), loader, - publicationCoordinator); + return delegate.getWithPublicationAction(nonNullKey, definition.scope(nonNullKey), + keyToLoad -> callbacks.call(() -> loader.apply(keyToLoad)), publicationCoordinator); } public V getIfPresent(K key) { @@ -89,6 +105,10 @@ public boolean compareAndSet(K key, V expectedValue, V updatedValue, Runnable co Objects.requireNonNull(commitAction, "commitAction can not be null")); } + public boolean runIfCurrent(K key, V expected, Runnable action) { + return delegate.runIfCurrent(key, definition.scope(key), expected, action); + } + public void invalidateKey(K key) { delegate.invalidateKey(Objects.requireNonNull(key, "key can not be null")); } @@ -99,7 +119,7 @@ public void invalidateKey(K key) { * returns a currentness fence, but {@link BulkLoad#publish(Object, Object)} retains no values. */ public BulkLoad beginBulkLoad(ScopePath parentScope) { - return new BulkLoad<>(this, delegate.beginBulkLoad(parentScope)); + return new BulkLoad<>(this, delegate.beginBulkLoad(definition.project(parentScope))); } public ScopedMetaCache.CacheMetrics metrics() { diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java index 7c2e4ef12c6877..539824232fbd70 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java @@ -47,8 +47,10 @@ public enum BudgetPolicy { private final Executor refreshExecutor; private final MetaCacheSizeEstimator sizeEstimator; private final BudgetPolicy budgetPolicy; + private final ScopePath.Coordinates coordinates; private MetaCacheDefinition(Builder builder) { + coordinates = builder.coordinates; name = requireName(builder.name); budgetGroup = requireName(builder.budgetGroup == null ? builder.name : builder.budgetGroup); cacheSpec = Objects.requireNonNull(builder.cacheSpec, "cacheSpec can not be null"); @@ -116,6 +118,31 @@ BudgetPolicy budgetPolicy() { return budgetPolicy; } + MetaCacheDefinition forDomain(ScopePath.Coordinates coordinates, CacheCallbackContext callbacks) { + Builder copy = builder(name, cacheSpec, key -> callbacks.call(() -> { + ScopePath path = scope(key); + return coordinates == null ? path : path.in(coordinates); + })); + copy.budgetGroup = budgetGroup; + copy.budgetPolicy = budgetPolicy; + copy.refreshAfterWrite = refreshAfterWrite; + copy.refreshExecutor = refreshExecutor == null ? null + : command -> refreshExecutor.execute(() -> callbacks.run(command)); + copy.loader = loader == null ? null : key -> callbacks.call(() -> loader.apply(key)); + copy.removalListener = removalListener == null ? null + : (key, value, reason) -> callbacks.run(() -> removalListener.onRemoval(key, value, reason)); + copy.discardListener = discardListener == null ? null + : (key, value) -> callbacks.run(() -> discardListener.accept(key, value)); + copy.sizeEstimator = sizeEstimator == null ? null + : (key, value) -> callbacks.call(() -> sizeEstimator.estimate(key, value)); + copy.coordinates = coordinates; + return copy.build(); + } + + ScopePath project(ScopePath path) { + return coordinates == null ? path : path.in(coordinates); + } + private static String requireName(String name) { String nonNullName = Objects.requireNonNull(name, "name can not be null"); if (nonNullName.isEmpty()) { @@ -135,6 +162,7 @@ public static final class Builder { private Executor refreshExecutor; private MetaCacheSizeEstimator sizeEstimator; private String budgetGroup; + private ScopePath.Coordinates coordinates; private BudgetPolicy budgetPolicy = BudgetPolicy.WEIGHTED_PAYLOAD; private Builder(String name, CacheSpec cacheSpec, Function scopeResolver) { diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java index 570b592470c7c0..8324225215e8a2 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java @@ -43,7 +43,14 @@ public enum Level { CATALOG, DATABASE, TABLE, - PARTITION + PARTITION, + NAMESPACE + } + + public enum Coordinates { + LOCAL, + REMOTE, + ID } private static final ScopePath CATALOG = new ScopePath(Level.CATALOG, null, null, null); @@ -53,13 +60,20 @@ public enum Level { private final String table; private final Object partition; private final NameCollection nameCollection; + private final Coordinates coordinates; private ScopePath(Level level, String database, String table, Object partition) { this(level, database, table, partition, null); } private ScopePath(Level level, String database, String table, Object partition, NameCollection nameCollection) { + this(level, database, table, partition, nameCollection, null); + } + + private ScopePath(Level level, String database, String table, Object partition, + NameCollection nameCollection, Coordinates coordinates) { this.level = level; + this.coordinates = coordinates; this.database = database; this.table = table; this.partition = partition; @@ -116,6 +130,42 @@ public static ScopePath tableMetadata(String database, String table) { return partition(database, table, SyntheticPartition.METADATA); } + /** Selects a name coordinate system without embedding names or plugin instances in string prefixes. */ + public ScopePath in(Coordinates coordinates) { + Objects.requireNonNull(coordinates, "coordinates"); + return new ScopePath(level == Level.CATALOG ? Level.NAMESPACE : level, + database, table, partition, nameCollection, coordinates); + } + + int depth() { + return level == Level.NAMESPACE ? 1 : level.ordinal() + (coordinates == null ? 0 : 1); + } + + Object keyAtDepth(int depth) { + int resourceDepth = depth; + if (coordinates != null) { + if (depth == 1) { + return coordinates; + } + resourceDepth--; + } + switch (resourceDepth) { + case 1: + return databaseKey(); + case 2: + return tableKey(); + case 3: + return partition; + default: + throw new IllegalArgumentException("Invalid scope depth: " + depth); + } + } + + Level levelAtDepth(int depth) { + return coordinates != null && depth == 1 ? Level.NAMESPACE + : Level.values()[depth - (coordinates == null ? 0 : 1)]; + } + Object databaseKey() { return nameCollection == NameCollection.DATABASES ? NameCollection.DATABASES : database; } @@ -142,22 +192,18 @@ public Object partition() { public boolean contains(ScopePath other) { Objects.requireNonNull(other, "other can not be null"); - if (level.ordinal() > other.level.ordinal()) { - return false; - } if (level == Level.CATALOG) { return true; } - if (!Objects.equals(databaseKey(), other.databaseKey())) { + if (coordinates != other.coordinates || depth() > other.depth()) { return false; } - if (level == Level.DATABASE) { - return true; - } - if (!Objects.equals(tableKey(), other.tableKey())) { - return false; + for (int i = 1; i <= depth(); i++) { + if (!Objects.equals(keyAtDepth(i), other.keyAtDepth(i))) { + return false; + } } - return level == Level.TABLE || partition.equals(other.partition); + return true; } @Override @@ -169,7 +215,7 @@ public boolean equals(Object obj) { return false; } ScopePath other = (ScopePath) obj; - return level == other.level + return level == other.level && coordinates == other.coordinates && Objects.equals(databaseKey(), other.databaseKey()) && Objects.equals(tableKey(), other.tableKey()) && Objects.equals(partition, other.partition); @@ -179,6 +225,7 @@ public boolean equals(Object obj) { public int hashCode() { int pathHashCode = 1; pathHashCode = 31 * pathHashCode + Objects.hashCode(level); + pathHashCode = 31 * pathHashCode + Objects.hashCode(coordinates); pathHashCode = 31 * pathHashCode + Objects.hashCode(databaseKey()); pathHashCode = 31 * pathHashCode + Objects.hashCode(tableKey()); return 31 * pathHashCode + Objects.hashCode(partition); @@ -193,6 +240,8 @@ public String toString() { switch (level) { case CATALOG: return "catalog"; + case NAMESPACE: + return "namespace[" + coordinates + "]"; case DATABASE: return "database[" + database + "]"; case TABLE: diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java index b58ad13bd9dfa5..5fa04ea067a5f6 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java @@ -186,26 +186,32 @@ public V getWithPublicationAction(K key, ScopePath path, Function loader, BiConsumer> coordinator = Objects.requireNonNull( publicationCoordinator, "publicationCoordinator can not be null"); checkOpen(); + if (!effectiveEnabled) { recordAccess(false); - V loaded = loadAndRecord(key, loadFunction); - if (loaded != null) { - AtomicBoolean commitInvoked = new AtomicBoolean(false); - try { - coordinator.accept(loaded, beforePublication -> { - if (!commitInvoked.compareAndSet(false, true)) { - throw new IllegalStateException("Metadata cache publication callback was invoked twice"); + // Disabled retention keeps the original per-request load behavior, but auxiliary publication still + // captures its resource generation before I/O and is rejected if an invalidation races the load. + try (PublicationLease lease = acquirePublicationLease(key, path, true)) { + V loaded = loadAndRecord(key, loadFunction); + if (loaded != null) { + AtomicBoolean invoked = new AtomicBoolean(); + try { + coordinator.accept(loaded, action -> { + if (!invoked.compareAndSet(false, true)) { + throw new IllegalStateException( + "Metadata cache publication callback was invoked twice"); + } + commitLoaded(lease, key, loaded, action); + }); + if (!invoked.get()) { + throw new IllegalStateException("Metadata cache publication callback was not invoked"); } - Objects.requireNonNull(beforePublication, "beforePublication can not be null").run(); - }); - if (!commitInvoked.get()) { - throw new IllegalStateException("Metadata cache publication callback was not invoked"); + } finally { + notifyDiscarded(key, loaded); } - } finally { - notifyDiscarded(key, loaded); } + return loaded; } - return loaded; } VersionedValue presentVersioned = currentVersionedValue(key, path); @@ -349,6 +355,21 @@ public void put(K key, ScopePath path, V value) { } } + /** Runs a short side effect only while the exact retained value and its resource scope are current. */ + public boolean runIfCurrent(K key, ScopePath path, V expected, Runnable action) { + checkOpen(); + try (PublicationLease lease = acquirePublicationLease(key, path, false)) { + return guardedCommit(lease, () -> { + VersionedValue current = currentVersionedValue(key, path); + if (current == null || current.value != expected) { + return false; + } + action.run(); + return true; + }); + } + } + public boolean compareAndSet(K key, ScopePath path, V expectedValue, V updatedValue) { return compareAndSet(key, path, expectedValue, updatedValue, NO_OP); } @@ -360,8 +381,12 @@ public boolean compareAndSet( Runnable action = Objects.requireNonNull(commitAction, "commitAction can not be null"); checkOpen(); if (!effectiveEnabled) { - action.run(); - return true; + try (PublicationLease lease = acquirePublicationLease(key, path, false)) { + return guardedCommit(lease, () -> { + action.run(); + return true; + }); + } } try (PublicationLease lease = acquirePublicationLease(key, path, false)) { AtomicReference> preparedRef = new AtomicReference<>(); @@ -711,6 +736,9 @@ private boolean guardedCommit(PublicationLease lease, BooleanSupplier comm private VersionedValue prepareVersionedValue( PublicationLease lease, K key, V value) { + if (!effectiveEnabled) { + return null; + } MetaCacheBudgetManager.AdmissionReservation reservation = reserve(key, value); if (weightBounded && reservation == null) { return null; diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java index 9fa587ed936306..ba04642aa6ef6a 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java @@ -198,7 +198,7 @@ void invalidate(ScopePath path, Runnable afterStateReplacement) { InvalidatedScope invalidated = publicationGate.write(() -> { List existing = resolveExisting(path); bumpPublicationStates(existing); - if (existing.size() != path.level().ordinal() + 1) { + if (existing.size() != path.depth() + 1) { return null; } ScopeNode target = existing.get(existing.size() - 1); @@ -224,7 +224,7 @@ void invalidate(List paths, Runnable afterStateReplacement) { Objects.requireNonNull(path, "path can not be null"); List existing = resolveExisting(path); publicationNodes.addAll(existing); - if (existing.size() == path.level().ordinal() + 1) { + if (existing.size() == path.depth() + 1) { ScopeNode target = existing.get(existing.size() - 1); if (targets.add(target)) { resolvedScopes.add(existing); @@ -329,61 +329,28 @@ private ScopeState replaceState(ScopeNode node) { } private ScopeSnapshot resolveOrCreateSnapshot(ScopePath path) { - ScopeNode catalogNode = root; - if (path.level() == ScopePath.Level.CATALOG) { - return ScopeSnapshot.capture(path, catalogNode, null, null, null); - } - ScopeState catalogState = catalogNode.current.get(); - ScopeNode dbNode = child(catalogNode, catalogState, path.databaseKey(), ScopePath.Level.DATABASE); - if (dbNode == null) { - return null; - } - if (path.level() == ScopePath.Level.DATABASE) { - return ScopeSnapshot.capture(path, catalogNode, dbNode, null, null); - } - ScopeState dbState = dbNode.current.get(); - ScopeNode tableNode = child(dbNode, dbState, path.tableKey(), ScopePath.Level.TABLE); - if (tableNode == null) { - return null; - } - if (path.level() == ScopePath.Level.TABLE) { - return ScopeSnapshot.capture(path, catalogNode, dbNode, tableNode, null); - } - ScopeState tableState = tableNode.current.get(); - ScopeNode partitionNode = child( - tableNode, tableState, path.partition(), ScopePath.Level.PARTITION); - if (partitionNode == null) { - return null; + ScopeNode[] nodes = new ScopeNode[path.depth() + 1]; + nodes[0] = root; + for (int depth = 1; depth < nodes.length; depth++) { + ScopeNode parent = nodes[depth - 1]; + nodes[depth] = child(parent, parent.current.get(), path.keyAtDepth(depth), path.levelAtDepth(depth)); + if (nodes[depth] == null) { + return null; + } } - return ScopeSnapshot.capture(path, catalogNode, dbNode, tableNode, partitionNode); + return ScopeSnapshot.capture(path, nodes); } private List resolveExisting(ScopePath path) { - List nodes = new ArrayList<>(path.level().ordinal() + 1); - ScopeNode catalogNode = root; - nodes.add(catalogNode); - if (path.level() == ScopePath.Level.CATALOG) { - return nodes; - } - ScopeNode dbNode = catalogNode.current.get().children.get(path.databaseKey()); - if (dbNode == null) { - return nodes; - } - nodes.add(dbNode); - if (path.level() == ScopePath.Level.DATABASE) { - return nodes; - } - ScopeNode tableNode = dbNode.current.get().children.get(path.tableKey()); - if (tableNode == null) { - return nodes; - } - nodes.add(tableNode); - if (path.level() == ScopePath.Level.TABLE) { - return nodes; - } - ScopeNode partitionNode = tableNode.current.get().children.get(path.partition()); - if (partitionNode != null) { - nodes.add(partitionNode); + List nodes = new ArrayList<>(path.depth() + 1); + ScopeNode node = root; + nodes.add(node); + for (int depth = 1; depth <= path.depth(); depth++) { + node = node.current.get().children.get(path.keyAtDepth(depth)); + if (node == null) { + break; + } + nodes.add(node); } return nodes; } @@ -446,30 +413,16 @@ private boolean retain(ScopeSnapshot snapshot) { if (!snapshot.leafNode().tryRetain()) { return false; } - retainedActiveLoads[ScopePath.Level.CATALOG.ordinal()].increment(); - if (snapshot.databaseNode != null) { - retainedActiveLoads[ScopePath.Level.DATABASE.ordinal()].increment(); - } - if (snapshot.tableNode != null) { - retainedActiveLoads[ScopePath.Level.TABLE.ordinal()].increment(); - } - if (snapshot.partitionNode != null) { - retainedActiveLoads[ScopePath.Level.PARTITION.ordinal()].increment(); + for (ScopeNode node : snapshot.nodes) { + retainedActiveLoads[node.level.ordinal()].increment(); } return true; } private void release(ScopeSnapshot snapshot) { snapshot.leafNode().releaseRetain(); - retainedActiveLoads[ScopePath.Level.CATALOG.ordinal()].decrement(); - if (snapshot.databaseNode != null) { - retainedActiveLoads[ScopePath.Level.DATABASE.ordinal()].decrement(); - } - if (snapshot.tableNode != null) { - retainedActiveLoads[ScopePath.Level.TABLE.ordinal()].decrement(); - } - if (snapshot.partitionNode != null) { - retainedActiveLoads[ScopePath.Level.PARTITION.ordinal()].decrement(); + for (ScopeNode node : snapshot.nodes) { + retainedActiveLoads[node.level.ordinal()].decrement(); } tryPrune(snapshot); } @@ -481,14 +434,8 @@ private void tryPrune(List nodes) { } private void tryPrune(ScopeSnapshot snapshot) { - if (snapshot.partitionNode != null) { - tryPrune(snapshot.partitionNode); - } - if (snapshot.tableNode != null) { - tryPrune(snapshot.tableNode); - } - if (snapshot.databaseNode != null) { - tryPrune(snapshot.databaseNode); + for (int i = snapshot.nodes.length - 1; i > 0; i--) { + tryPrune(snapshot.nodes[i]); } } @@ -525,6 +472,8 @@ private void collectMetrics(ScopeNode node, ScopeMetricsAccumulator accumulator) accumulator.registrations += state.entries.size(); state.children.values().forEach(child -> { switch (child.level) { + case NAMESPACE: + break; case DATABASE: accumulator.databaseNodes++; break; @@ -551,7 +500,7 @@ private static LongAdder[] newCounters(int count) { void forceGenerationForTest(ScopePath path, long generation) { List existing = resolveExisting(path); - if (existing.size() != path.level().ordinal() + 1) { + if (existing.size() != path.depth() + 1) { throw new IllegalStateException("Scope does not exist: " + path); } ScopeNode node = existing.get(existing.size() - 1); @@ -568,7 +517,7 @@ void forceGenerationForTest(ScopePath path, long generation) { long generationForTest(ScopePath path) { List existing = resolveExisting(path); - if (existing.size() != path.level().ordinal() + 1) { + if (existing.size() != path.depth() + 1) { throw new IllegalStateException("Scope does not exist: " + path); } return existing.get(existing.size() - 1).current.get().generation; @@ -626,114 +575,48 @@ private InvalidatedScope(List existing, ScopeState oldState) { static final class ScopeSnapshot { private final ScopePath path; - private final ScopeNode catalogNode; - private final ScopeState catalogState; - private final ScopeNode databaseNode; - private final ScopeState databaseState; - private final ScopeNode tableNode; - private final ScopeState tableState; - private final ScopeNode partitionNode; - private final ScopeState partitionState; + private final ScopeNode[] nodes; + private final ScopeState[] states; private final int pathHashCode; - private ScopeSnapshot( - ScopePath path, - ScopeNode catalogNode, - ScopeState catalogState, - ScopeNode databaseNode, - ScopeState databaseState, - ScopeNode tableNode, - ScopeState tableState, - ScopeNode partitionNode, - ScopeState partitionState) { + private ScopeSnapshot(ScopePath path, ScopeNode[] nodes, ScopeState[] states) { this.path = path; - this.catalogNode = catalogNode; - this.catalogState = catalogState; - this.databaseNode = databaseNode; - this.databaseState = databaseState; - this.tableNode = tableNode; - this.tableState = tableState; - this.partitionNode = partitionNode; - this.partitionState = partitionState; + this.nodes = nodes; + this.states = states; this.pathHashCode = path.hashCode(); } - static ScopeSnapshot capture( - ScopePath path, - ScopeNode catalogNode, - ScopeNode databaseNode, - ScopeNode tableNode, - ScopeNode partitionNode) { - ScopeState catalogState = catalogNode.current.get(); - ScopeState databaseState = databaseNode == null ? null : databaseNode.current.get(); - ScopeState tableState = tableNode == null ? null : tableNode.current.get(); - ScopeState partitionState = partitionNode == null ? null : partitionNode.current.get(); - ScopeState leafState = partitionState != null - ? partitionState - : tableState != null ? tableState : databaseState != null ? databaseState : catalogState; - ScopeSnapshot currentSnapshot = leafState.scopeSnapshot; - if (currentSnapshot != null && currentSnapshot.matches( - catalogNode, - catalogState, - databaseNode, - databaseState, - tableNode, - tableState, - partitionNode, - partitionState)) { - return currentSnapshot; + static ScopeSnapshot capture(ScopePath path, ScopeNode[] nodes) { + ScopeState[] states = new ScopeState[nodes.length]; + for (int i = 0; i < nodes.length; i++) { + states[i] = nodes[i].current.get(); } - synchronized (leafState) { - currentSnapshot = leafState.scopeSnapshot; - if (currentSnapshot == null || !currentSnapshot.matches( - catalogNode, - catalogState, - databaseNode, - databaseState, - tableNode, - tableState, - partitionNode, - partitionState)) { - currentSnapshot = new ScopeSnapshot( - path, - catalogNode, - catalogState, - databaseNode, - databaseState, - tableNode, - tableState, - partitionNode, - partitionState); - leafState.scopeSnapshot = currentSnapshot; + ScopeState leaf = states[states.length - 1]; + ScopeSnapshot current = leaf.scopeSnapshot; + if (current != null && current.matches(nodes, states)) { + return current; + } + synchronized (leaf) { + current = leaf.scopeSnapshot; + if (current == null || !current.matches(nodes, states)) { + current = new ScopeSnapshot(path, nodes, states); + leaf.scopeSnapshot = current; } - return currentSnapshot; + return current; } } boolean isCurrent(ScopedMetaCacheRegistry registry) { - if (registry.closed.get() || registry.root != catalogNode - || catalogNode.current.get() != catalogState) { - return false; - } - if (databaseNode == null) { - return true; - } - if (catalogState.children.get(databaseNode.childKey) != databaseNode - || databaseNode.current.get() != databaseState) { - return false; - } - if (tableNode == null) { - return true; - } - if (databaseState.children.get(tableNode.childKey) != tableNode - || tableNode.current.get() != tableState) { + if (registry.closed.get() || registry.root != nodes[0]) { return false; } - if (partitionNode == null) { - return true; + for (int i = 0; i < nodes.length; i++) { + if (nodes[i].current.get() != states[i] + || (i > 0 && states[i - 1].children.get(nodes[i].childKey) != nodes[i])) { + return false; + } } - return tableState.children.get(partitionNode.childKey) == partitionNode - && partitionNode.current.get() == partitionState; + return true; } ScopePath path() { @@ -741,64 +624,31 @@ ScopePath path() { } ScopeNode leafNode() { - if (partitionNode != null) { - return partitionNode; - } - if (tableNode != null) { - return tableNode; - } - if (databaseNode != null) { - return databaseNode; - } - return catalogNode; + return nodes[nodes.length - 1]; } ScopeState leafState() { - if (partitionState != null) { - return partitionState; - } - if (tableState != null) { - return tableState; - } - if (databaseState != null) { - return databaseState; - } - return catalogState; + return states[states.length - 1]; } boolean sameGeneration(ScopeSnapshot other) { - return matches( - other.catalogNode, - other.catalogState, - other.databaseNode, - other.databaseState, - other.tableNode, - other.tableState, - other.partitionNode, - other.partitionState); + return matches(other.nodes, other.states); } int pathHashCode() { return pathHashCode; } - private boolean matches( - ScopeNode expectedCatalogNode, - ScopeState expectedCatalogState, - ScopeNode expectedDatabaseNode, - ScopeState expectedDatabaseState, - ScopeNode expectedTableNode, - ScopeState expectedTableState, - ScopeNode expectedPartitionNode, - ScopeState expectedPartitionState) { - return catalogNode == expectedCatalogNode - && catalogState == expectedCatalogState - && databaseNode == expectedDatabaseNode - && databaseState == expectedDatabaseState - && tableNode == expectedTableNode - && tableState == expectedTableState - && partitionNode == expectedPartitionNode - && partitionState == expectedPartitionState; + private boolean matches(ScopeNode[] expectedNodes, ScopeState[] expectedStates) { + if (nodes.length != expectedNodes.length) { + return false; + } + for (int i = 0; i < nodes.length; i++) { + if (nodes[i] != expectedNodes[i] || states[i] != expectedStates[i]) { + return false; + } + } + return true; } } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/StripedPhaseGate.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/StripedPhaseGate.java index b585e8f020b64f..e69d09ea34e3bd 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/StripedPhaseGate.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/StripedPhaseGate.java @@ -24,7 +24,8 @@ /** * Writer-preferred phase gate for short cache-publication critical sections. * - *

The gate is intentionally non-reentrant: an action must not enter either phase of the same gate. In particular, + *

Read phases are reentrant so one guarded publication can update a second cache in the same catalog. + * Read-to-write upgrade is forbidden. In particular, * synchronous removal callbacks must run after the read phase. Reader actions must also avoid remote I/O or other * unbounded work because a waiting writer and newly arriving readers spin until the current reader phase drains. */ @@ -37,13 +38,14 @@ final class StripedPhaseGate { private final AtomicIntegerArray activeReaders = new AtomicIntegerArray(READER_STRIPE_COUNT * READER_STRIDE); private volatile boolean writerActive; + private final ThreadLocal readDepth = new ThreadLocal<>(); boolean readBoolean(BooleanSupplier action) { int readerIndex = enterRead(); try { return action.getAsBoolean(); } finally { - activeReaders.decrementAndGet(readerIndex); + exitRead(readerIndex); } } @@ -52,7 +54,7 @@ T read(Supplier action) { try { return action.get(); } finally { - activeReaders.decrementAndGet(readerIndex); + exitRead(readerIndex); } } @@ -64,6 +66,9 @@ void write(Runnable action) { } T write(Supplier action) { + if (readDepth.get() != null) { + throw new IllegalStateException("A cache publication action cannot invalidate its own scope"); + } synchronized (writerLock) { writerActive = true; try { @@ -76,6 +81,11 @@ T write(Supplier action) { } private int enterRead() { + Integer depth = readDepth.get(); + if (depth != null) { + readDepth.set(depth + 1); + return -1; + } int readerIndex = readerIndex(); while (true) { while (writerActive) { @@ -83,12 +93,22 @@ private int enterRead() { } activeReaders.incrementAndGet(readerIndex); if (!writerActive) { + readDepth.set(1); return readerIndex; } activeReaders.decrementAndGet(readerIndex); } } + private void exitRead(int readerIndex) { + if (readerIndex < 0) { + readDepth.set(readDepth.get() - 1); + } else { + readDepth.remove(); + activeReaders.decrementAndGet(readerIndex); + } + } + private void awaitReaders() { while (hasReaders()) { Thread.onSpinWait(); diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogCacheDomainTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogCacheDomainTest.java new file mode 100644 index 00000000000000..ca47b1bb2dca15 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogCacheDomainTest.java @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +class CatalogCacheDomainTest { + private static final CacheSpec SPEC = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 100); + + private MetaCache entry(CatalogMetaCache domain) { + return domain.create(MetaCacheDefinition.builder("table", SPEC, + key -> ScopePath.table("db", key)).build()); + } + + @Test + void sameNameInstancesShareInvalidationButNotRetirement() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + CatalogMetaCache old = root.openDomain("hive"); + CatalogMetaCache replacement = root.openDomain("hive"); + MetaCache first = entry(old); + MetaCache second = entry(replacement); + first.put("a", "old"); + second.put("a", "new"); + Assertions.assertEquals(2, root.allEntries().size()); + old.close(); + Assertions.assertEquals("new", second.getIfPresent("a")); + Assertions.assertEquals(1, root.allEntries().size()); + Assertions.assertThrows(IllegalStateException.class, () -> old.openDomain("iceberg")); + root.invalidateTable("db", "a"); + Assertions.assertNull(second.getIfPresent("a")); + } + } + + @Test + void constructionRollbackDoesNotRetireTheRootOrSiblings() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + MetaCache sibling = entry(root.openDomain("iceberg")); + sibling.put("a", "live"); + CatalogMetaCache failing = root.openDomain("hive"); + entry(failing).put("a", "partial"); + Assertions.assertThrows(IllegalArgumentException.class, () -> failing.create(MetaCacheDefinition + .builder("bad", CacheSpec.of(true, -1L, -1L), + key -> ScopePath.table("db", key)).build())); + failing.close(); + Assertions.assertFalse(root.isClosed()); + Assertions.assertEquals("live", sibling.getIfPresent("a")); + Assertions.assertEquals(1, root.allEntries().size()); + } + } + + @Test + void localAndRemoteCoordinatesDoNotCollideAndCanBeFencedTogether() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + CatalogMetaCache local = root.openDomain("fe", ScopePath.Coordinates.LOCAL, getClass().getClassLoader()); + CatalogMetaCache remote = root.openDomain("hive", ScopePath.Coordinates.REMOTE, getClass().getClassLoader()); + MetaCache fe = entry(local); + MetaCache source = entry(remote); + fe.put("a", "local"); + source.put("a", "remote"); + local.invalidateTable("db", "a"); + Assertions.assertNull(fe.getIfPresent("a")); + Assertions.assertEquals("remote", source.getIfPresent("a")); + fe.put("a", "local2"); + root.invalidateScopes(Arrays.asList(ScopePath.table("db", "a").in(ScopePath.Coordinates.LOCAL), + ScopePath.table("db", "a").in(ScopePath.Coordinates.REMOTE))); + Assertions.assertNull(fe.getIfPresent("a")); + Assertions.assertNull(source.getIfPresent("a")); + } + } + + @Test + void oneRemoteInvalidationFencesAnInFlightSiblingLoad() throws Exception { + ExecutorService worker = Executors.newSingleThreadExecutor(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finish = new CountDownLatch(1); + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + CatalogMetaCache hive = root.openDomain("hive", ScopePath.Coordinates.REMOTE, getClass().getClassLoader()); + MetaCache sibling = entry(hive.openDomain("iceberg")); + Future old = worker.submit(() -> sibling.get("a", key -> { + started.countDown(); + try { + Assertions.assertTrue(finish.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return "old"; + })); + Assertions.assertTrue(started.await(10, TimeUnit.SECONDS)); + hive.invalidateTable("db", "a"); + sibling.put("a", "new"); + finish.countDown(); + Assertions.assertEquals("old", old.get(10, TimeUnit.SECONDS)); + Assertions.assertEquals("new", sibling.getIfPresent("a")); + } finally { + finish.countDown(); + worker.shutdownNow(); + } + } + + @Test + void bulkLoadsProjectTheirParentIntoTheDomainCoordinates() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + CatalogMetaCache domain = root.openDomain("hive", ScopePath.Coordinates.REMOTE, + getClass().getClassLoader()); + MetaCache cache = entry(domain); + try (MetaCache.BulkLoad load = cache.beginBulkLoad(ScopePath.database("db"))) { + Assertions.assertTrue(load.publish("a", "first")); + domain.invalidateDatabase("db"); + Assertions.assertFalse(load.publish("b", "stale")); + Assertions.assertNull(cache.getIfPresent("a")); + Assertions.assertNull(cache.getIfPresent("b")); + } + } + } + + @Test + void disabledCacheStillFencesPublicationSideEffectsBeforeLoading() throws Exception { + ExecutorService worker = Executors.newSingleThreadExecutor(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finish = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicInteger actions = new java.util.concurrent.atomic.AtomicInteger(); + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + MetaCache cache = root.create(MetaCacheDefinition.builder("disabled", + CacheSpec.of(false, -1, 10), key -> ScopePath.table("db", key)).build()); + Future load = worker.submit(() -> cache.getWithPublicationAction("a", key -> { + started.countDown(); + try { + Assertions.assertTrue(finish.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + return "old"; + }, (value, commit) -> commit.accept(actions::incrementAndGet))); + Assertions.assertTrue(started.await(10, TimeUnit.SECONDS)); + root.invalidateTable("db", "a"); + finish.countDown(); + Assertions.assertEquals("old", load.get(10, TimeUnit.SECONDS)); + Assertions.assertEquals(0, actions.get()); + Assertions.assertNull(cache.getIfPresent("a")); + } finally { + finish.countDown(); + worker.shutdownNow(); + } + } + + @Test + void pluginCallbacksRestoreTheirRegisteringClassLoader() { + ClassLoader original = Thread.currentThread().getContextClassLoader(); + ClassLoader plugin = new ClassLoader(original) { }; + AtomicReference loaded = new AtomicReference<>(); + AtomicReference removed = new AtomicReference<>(); + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + CatalogMetaCache domain = root.openDomain("hive", ScopePath.Coordinates.REMOTE, plugin); + MetaCache cache = domain.create(MetaCacheDefinition.builder("table", SPEC, + key -> ScopePath.table("db", key)) + .loader(key -> { + loaded.set(Thread.currentThread().getContextClassLoader()); + return "v"; + }).removalListener((key, value, cause) -> + removed.set(Thread.currentThread().getContextClassLoader())).build()); + cache.get("a"); + domain.close(); + Assertions.assertSame(plugin, loaded.get()); + Assertions.assertSame(plugin, removed.get()); + Assertions.assertSame(original, Thread.currentThread().getContextClassLoader()); + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java index 59acec37a200c2..804d8fd6c3c5c8 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java @@ -247,7 +247,7 @@ void budgetPressureReclaimsOneColdEntryBeforeTheNextAdmission() { } @Test - void managedOwnerClosesWhenPhysicalCacheConstructionFails() { + void failedEntryConstructionDoesNotCloseTheCatalogRoot() { long catalogId = Long.MIN_VALUE + 31L; Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalogId).isEmpty()); CatalogMetaCache owner = CatalogMetaCache.managed(catalogId, "iceberg", Collections.emptyMap()); @@ -258,6 +258,9 @@ void managedOwnerClosesWhenPhysicalCacheConstructionFails() { .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(512L)) .build())); + Assertions.assertFalse(owner.isClosed()); + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalogId).contains(owner)); + owner.close(); Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalogId).isEmpty()); } diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/StripedPhaseGateTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/StripedPhaseGateTest.java index 1e562ec85ec3a6..5cce549c3f1283 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/StripedPhaseGateTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/StripedPhaseGateTest.java @@ -147,6 +147,38 @@ public void exceptionReleasesReaderAndWriterPhases() { Assertions.assertTrue(gate.readBoolean(() -> true)); } + @Test + public void nestedPublicationCanFinishWhileAnInvalidationWriterWaits() throws Exception { + StripedPhaseGate gate = new StripedPhaseGate(); + java.lang.reflect.Field activeWriter = StripedPhaseGate.class.getDeclaredField("writerActive"); + activeWriter.setAccessible(true); + ExecutorService workers = Executors.newFixedThreadPool(2); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch nest = new CountDownLatch(1); + try { + Future publication = workers.submit(() -> gate.readBoolean(() -> { + entered.countDown(); + await(nest); + return gate.readBoolean(() -> true); + })); + await(entered); + Future invalidation = workers.submit(() -> gate.write(() -> { })); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + while (!activeWriter.getBoolean(gate) && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + Assertions.assertTrue(activeWriter.getBoolean(gate)); + nest.countDown(); + Assertions.assertTrue(publication.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + invalidation.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + Assertions.assertThrows(IllegalStateException.class, + () -> gate.read(() -> gate.write(() -> true))); + } finally { + nest.countDown(); + workers.shutdownNow(); + } + } + private static void await(CountDownLatch latch) { try { if (!latch.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) { diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java index 1d3d7f364cca84..4bd721640a35f7 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java @@ -17,6 +17,8 @@ package org.apache.doris.connector.fluss; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorCapability; import org.apache.doris.connector.spi.ConnectorContext; @@ -37,7 +39,6 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -92,12 +93,14 @@ public class FlussConnector implements Connector { // Source Fluss identity -> physical lake identity. The latter may be overridden independently for the // database and table, and targeted invalidation must follow the same path as lookup. + private final CatalogMetaCache metaCache; private final Map lakeTablePaths = new LinkedHashMap<>(); public FlussConnector(FlussCatalogProperties properties, ConnectorContext context) { this.properties = properties; this.catalogName = context.getCatalogName(); this.context = context; + this.metaCache = context.createCacheDomain("fluss", properties.getRawCatalogProperties()); } @Override @@ -229,53 +232,50 @@ synchronized void rememberLakePath(TablePath sourcePath, TablePath lakePath) { } @Override - public synchronized void invalidateTable(String dbName, String tableName) { - if (lakeSibling != null) { - // Keep the mapping until the table is resolved again (which replaces it) or the whole catalog - // is invalidated. A single engine refresh may emit more than one targeted invalidation, and a - // second one must not fall back to the source name merely because the first already ran. - TablePath lakePath = lakeTablePaths.get(TablePath.of(dbName, tableName)); - lakeSibling.invalidateTable( - lakePath == null ? dbName : lakePath.getDatabaseName(), - lakePath == null ? tableName : lakePath.getTableName()); + public synchronized List cacheDependencies(ScopePath scope) { + List dependencies = new java.util.ArrayList<>(); + if (scope.database() == null) { + return dependencies; } - } - - @Override - public synchronized void invalidateDb(String dbName) { - if (lakeSibling != null) { - Set lakeDatabases = new LinkedHashSet<>(); - // An unresolved table cannot have populated the sibling's table cache, but the sibling may - // still hold database-level metadata for the source namespace. Invalidating it as well as every - // known physical override is conservative and prevents one overridden table from accidentally - // suppressing invalidation of ordinary lake tables in the same Fluss database. - lakeDatabases.add(dbName); + if (scope.table() != null) { + TablePath lake = lakeTablePaths.get(TablePath.of(scope.database(), scope.table())); + if (lake != null) { + dependencies.add(ScopePath.table(lake.getDatabaseName(), lake.getTableName())); + } + } else { lakeTablePaths.forEach((source, lake) -> { - if (source.getDatabaseName().equals(dbName)) { - lakeDatabases.add(lake.getDatabaseName()); + if (source.getDatabaseName().equals(scope.database())) { + dependencies.add(scope.level() == ScopePath.Level.TABLE + ? ScopePath.tableCollection(lake.getDatabaseName()) + : ScopePath.database(lake.getDatabaseName())); } }); - lakeDatabases.forEach(lakeSibling::invalidateDb); } + return dependencies; } @Override - public synchronized void invalidateAll() { - lakeTablePaths.clear(); - if (lakeSibling != null) { - lakeSibling.invalidateAll(); - } + public void invalidateTable(String database, String table) { + List scopes = new java.util.ArrayList<>(cacheDependencies(ScopePath.table(database, table))); + scopes.add(ScopePath.table(database, table)); + metaCache.invalidateScopes(scopes); } @Override - public synchronized void invalidatePartition( - String dbName, String tableName, List partitionNames) { - if (lakeSibling != null) { - TablePath lakePath = lakeTablePaths.get(TablePath.of(dbName, tableName)); - lakeSibling.invalidatePartition( - lakePath == null ? dbName : lakePath.getDatabaseName(), - lakePath == null ? tableName : lakePath.getTableName(), partitionNames); - } + public void invalidateDb(String database) { + List scopes = new java.util.ArrayList<>(cacheDependencies(ScopePath.database(database))); + scopes.add(ScopePath.database(database)); + metaCache.invalidateScopes(scopes); + } + + @Override + public void invalidateAll() { + metaCache.invalidateCatalog(); + } + + @Override + public void invalidatePartition(String database, String table, List partitions) { + invalidateTable(database, table); } @Override @@ -296,6 +296,7 @@ public ConnectorTestResult testConnection(ConnectorSession session) { */ @Override public void close() throws IOException { + metaCache.close(); Connector sibling; Connection toClose; synchronized (this) { diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java index 5081030231c5f4..9675cc69af0857 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java @@ -186,8 +186,7 @@ public void invalidationsReachOnlyAnAlreadyBuiltLakeSibling() { connector.invalidatePartition("db", "tbl", Collections.singletonList("dt=20260101")); connector.invalidateAll(); - Assertions.assertEquals(Arrays.asList("invalidateTable:db.tbl", "invalidateDb:db", - "invalidatePartition:db.tbl:[dt=20260101]", "invalidateAll"), sibling.calls); + Assertions.assertTrue(sibling.calls.isEmpty(), "shared scopes replace manual forwarding"); } @Test @@ -208,16 +207,13 @@ public void targetedInvalidationsUseThePhysicalLakeIdentity() { connector.invalidateDb("db"); connector.invalidateDb("db"); - Assertions.assertEquals(Arrays.asList( - "invalidatePartition:archive.orders_lake:[dt=20260101]", - "invalidateTable:archive.orders_lake", - "invalidateTable:archive.orders_lake", - "invalidateDb:db", - "invalidateDb:archive", - "invalidateDb:cold", - "invalidateDb:db", - "invalidateDb:archive", - "invalidateDb:cold"), sibling.calls); + Assertions.assertTrue(sibling.calls.isEmpty()); + Assertions.assertEquals(Collections.singletonList( + org.apache.doris.connector.cache.ScopePath.table("archive", "orders_lake")), + connector.cacheDependencies(org.apache.doris.connector.cache.ScopePath.table("db", "orders"))); + Assertions.assertEquals(Arrays.asList(org.apache.doris.connector.cache.ScopePath.database("archive"), + org.apache.doris.connector.cache.ScopePath.database("cold")), + connector.cacheDependencies(org.apache.doris.connector.cache.ScopePath.database("db"))); } @Test diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java index 7b6c5656806bfe..de67dbfa6a2b52 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java @@ -53,15 +53,14 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; -import java.util.function.Consumer; /** * Hive connector implementation. Manages the lifecycle of @@ -151,7 +150,7 @@ public HiveConnector(Map properties, ConnectorContext context) { this.props = HiveCatalogProperties.of(properties); this.properties = props.getRaw(); this.context = context; - this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "hive", this.properties); + this.metaCache = context.createCacheDomain("hive", this.properties); this.fileListingCache = new HiveFileListingCache(metaCache, props); // Reads its own meta.cache.hive.partition_view.(enable|ttl-second|capacity) from the catalog properties // via the framework's CacheSpec (default ON / 24h / 1000). @@ -378,12 +377,12 @@ public Set getCapabilities() { * schema, partitions AND files are all mutable, so a REFRESH must re-read all of them (unlike iceberg, whose * manifests are immutable and kept across REFRESH TABLE). fe-core already routes {@code REFRESH TABLE} to * {@code connector.invalidateTable} for a plugin-driven catalog. - * Also forwarded to the already-built embedded siblings — see {@link #forEachBuiltSibling}. + * Sibling domains share the REMOTE scope tree and participate without explicit forwarding. */ @Override public void invalidateTable(String dbName, String tableName) { metaCache.invalidateTable(dbName, tableName); - forEachBuiltSibling(sibling -> sibling.invalidateTable(dbName, tableName)); + } /** @@ -391,23 +390,23 @@ public void invalidateTable(String dbName, String tableName) { * layers (metastore-metadata + {@link HiveFileListingCache#invalidateDb} * directory listings). Same no-force-build read of the client as {@link #invalidateTable(String, String)}. * fe-core routes {@code REFRESH DATABASE} to {@code connector.invalidateDb} for a plugin-driven catalog. - * Also forwarded to the already-built embedded siblings — see {@link #forEachBuiltSibling}. + * Sibling domains share the REMOTE scope tree and participate without explicit forwarding. */ @Override public void invalidateDb(String dbName) { metaCache.invalidateDatabase(dbName); - forEachBuiltSibling(sibling -> sibling.invalidateDb(dbName)); + } /** * REFRESH CATALOG hook: drop ALL of this catalog's connector-owned scan caches — every metastore-metadata * entry and every directory listing. - * Also forwarded to the already-built embedded siblings — see {@link #forEachBuiltSibling}. + * Sibling domains share the REMOTE scope tree and participate without explicit forwarding. */ @Override public void invalidateAll() { metaCache.invalidateCatalog(); - forEachBuiltSibling(Connector::invalidateAll); + } /** @@ -418,17 +417,21 @@ public void invalidateAll() { * partition-metadata cache. Deriving values from the name * (rather than looking the partition up) is exactly what stops an evicted partition-metadata entry from * leaving a stale file listing — the #65334 failure mode. Table-level column stats and the table object are - * intentionally left intact (legacy did not drop them on a partition-level refresh). Also forwarded to the - * already-built embedded siblings. + * intentionally left intact. Embedded siblings participate through the shared remote resource scopes. */ @Override public void invalidatePartition(String dbName, String tableName, List partitionNames) { - Set> partitionValues = new HashSet<>(); - for (String name : partitionNames) { - partitionValues.add(HiveWriteUtils.toPartitionValues(name)); + metaCache.invalidateScopes(partitionCacheScopes(dbName, tableName, partitionNames)); + } + + @Override + public List partitionCacheScopes(String database, String table, List partitions) { + List scopes = new ArrayList<>(); + scopes.add(ScopePath.partitionCollection(database, table)); + for (String name : partitions) { + scopes.add(ScopePath.partition(database, table, HiveWriteUtils.toPartitionValues(name))); } - metaCache.invalidatePartitions(dbName, tableName, partitionValues); - forEachBuiltSibling(sibling -> sibling.invalidatePartition(dbName, tableName, partitionNames)); + return scopes; } /** @@ -448,26 +451,6 @@ public ConnectorEventSource getEventSource() { return new HmsEventSource(getOrCreateClient(), props.getHmsEventsBatchSizePerRpc()); } - /** - * Runs one invalidation action on each ALREADY-BUILT embedded sibling (iceberg, hudi). fe-core routes - * {@code REFRESH TABLE/DATABASE/CATALOG} only to a catalog's PRIMARY connector, so the gateway must - * propagate invalidation to the sibling-owned caches — e.g. the iceberg sibling's latest-snapshot pin, - * whose ACCESS-based expiry keeps a continuously-queried stale entry alive indefinitely, making an explicit - * REFRESH the only way to unpin an iceberg-on-HMS table's staleness. Mirrors {@link #close()}: plain - * volatile reads, never force-builds (a never-built sibling has no cache to drop, and building one just to - * flush it could fail-loud spuriously on a catalog whose sibling plugin is absent). - */ - private void forEachBuiltSibling(Consumer action) { - Connector iceberg = icebergSibling; - if (iceberg != null) { - action.accept(iceberg); - } - Connector hudi = hudiSibling; - if (hudi != null) { - action.accept(hudi); - } - } - /** The connector-owned directory-listing cache, exposed for unit tests (mirrors iceberg manifestCacheForTest). */ HiveFileListingCache fileListingCacheForTest() { return fileListingCache; diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java index 05570d79db6e98..42a1d879d3a083 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java @@ -387,34 +387,40 @@ public Connector createSiblingConnector(String catalogType, Map } @Test - public void refreshHooksForwardToBothBuiltSiblings() { - // WHY: fe-core routes REFRESH TABLE/DATABASE/CATALOG only to a catalog's PRIMARY connector. If the - // gateway did not forward its invalidate hooks, the iceberg sibling's latest-snapshot pin could NEVER - // be dropped by an explicit REFRESH — and its access-based expiry keeps a continuously-queried stale - // entry alive indefinitely, so staleness would be unbounded (breaks "bounded by TTL + REFRESH"). - FakeSibling icebergSibling = new FakeSibling(); - FakeSibling hudiSibling = new FakeSibling(); - FakeConnectorContext context = new FakeConnectorContext() { - @Override - public Connector createSiblingConnector(String catalogType, Map properties) { - return "hudi".equals(catalogType) ? hudiSibling : icebergSibling; - } - }; - HiveConnector connector = new HiveConnector(HiveTestProperties.minimalMap(), context); - connector.getOrCreateIcebergSibling(); - connector.getOrCreateHudiSibling(); - - connector.invalidateTable("db1", "t1"); - connector.invalidateDb("db2"); - connector.invalidateAll(); - - for (FakeSibling sibling : new FakeSibling[] {icebergSibling, hudiSibling}) { - Assertions.assertEquals("db1.t1", sibling.lastInvalidatedTable, - "invalidateTable must forward the (db, table) pair to each built sibling"); - Assertions.assertEquals("db2", sibling.lastInvalidatedDb, - "invalidateDb must forward the db to each built sibling"); - Assertions.assertEquals(1, sibling.invalidateAllCount, - "invalidateAll must forward to each built sibling exactly once"); + public void refreshHooksInvalidateBuiltSiblingDomainsWithoutForwarding() { + try (org.apache.doris.connector.cache.CatalogMetaCache root = + org.apache.doris.connector.cache.CatalogMetaCache.unmanaged()) { + FakeSibling icebergSibling = new FakeSibling(); + FakeConnectorContext context = new FakeConnectorContext() { + @Override + public org.apache.doris.connector.cache.CatalogMetaCache createCacheDomain( + String engine, Map properties) { + return root.openDomain(engine); + } + + @Override + public Connector createSiblingConnector(String type, Map properties) { + return icebergSibling; + } + }; + HiveConnector connector = new HiveConnector(HiveTestProperties.minimalMap(), context); + connector.getOrCreateIcebergSibling(); + org.apache.doris.connector.cache.MetaCache siblingCache = root.openDomain("iceberg") + .create(org.apache.doris.connector.cache.MetaCacheDefinition.builder("table", + org.apache.doris.connector.cache.CacheSpec.of(true, -1, 10), + key -> org.apache.doris.connector.cache.ScopePath.table("db", key)).build()); + siblingCache.put("a", "old"); + siblingCache.put("b", "other"); + connector.invalidateTable("db", "a"); + Assertions.assertNull(siblingCache.getIfPresent("a")); + Assertions.assertEquals("other", siblingCache.getIfPresent("b")); + connector.invalidateDb("db"); + Assertions.assertNull(siblingCache.getIfPresent("b")); + siblingCache.put("a", "reloaded"); + connector.invalidateAll(); + Assertions.assertNull(siblingCache.getIfPresent("a")); + Assertions.assertEquals(0, icebergSibling.invalidateAllCount); + Assertions.assertNull(icebergSibling.lastInvalidatedTable); } } diff --git a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java index 9c031bea67e7f7..5b65934b93557a 100644 --- a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java +++ b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java @@ -191,7 +191,7 @@ public HudiConnector(Map properties, ConnectorContext context) { this.props = HudiCatalogProperties.of(properties); this.properties = props.getRaw(); this.context = context; - this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "hudi", this.properties); + this.metaCache = context.createCacheDomain("hudi", this.properties); } @Override diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java index afc79d395440d9..07fe4844b8d2cd 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java @@ -269,7 +269,7 @@ public IcebergConnector(Map properties, ConnectorContext context // authenticator never logs in — so without this the DDL/read hits secured HDFS as SIMPLE auth. this.context = new TcclPinningConnectorContext(context, getClass().getClassLoader(), this::pluginAuthenticator); - this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "iceberg", this.properties); + this.metaCache = context.createCacheDomain("iceberg", this.properties); this.manifestCache = new IcebergManifestCache(metaCache, this.properties); // The latest-snapshot, format and comment caches have no meta.cache.iceberg. keys of their own; they // take the table entry's enable / ttl-second / capacity (the legacy IcebergExternalMetaCache served the diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java index 5526a52587dcb8..2b4fd30f37f9d7 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java @@ -215,7 +215,7 @@ void invalidateAll() { /** Seals new borrows and retires every cache owner, including loaders that publish after invalidation. */ void close() { if (closed.compareAndSet(false, true)) { - owner.invalidateCatalog(); + owner.remove(entry); } } diff --git a/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeDorisConnector.java b/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeDorisConnector.java index 4b1095ae5af769..56acd8062097f2 100644 --- a/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeDorisConnector.java +++ b/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeDorisConnector.java @@ -78,7 +78,7 @@ public MaxComputeDorisConnector(Map properties, ConnectorContext context) { this.props = MCCatalogProperties.of(properties); this.context = context; - this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "max_compute", props.getRaw()); + this.metaCache = context.createCacheDomain("max_compute", props.getRaw()); try { this.partitionCache = new MaxComputePartitionCache(metaCache, props.getRaw(), (db, t) -> structureHelper.getPartitions(odps, db, t)); diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java index a5154b81f86049..3b33f6b8042d76 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java @@ -178,7 +178,7 @@ public PaimonConnector(Map properties, ConnectorContext context) // this a DDL/read against secured HDFS negotiates SIMPLE auth. See TcclPinningConnectorContext. this.context = new TcclPinningConnectorContext(context, getClass().getClassLoader(), this::pluginAuthenticator); - this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "paimon", properties); + this.metaCache = context.createCacheDomain("paimon", properties); this.schemaAtMemo = new PaimonSchemaAtMemo(metaCache, PaimonSchemaAtMemo.DEFAULT_MAX_SIZE); this.latestSnapshotCache = new PaimonLatestSnapshotCache( diff --git a/fe/fe-connector/fe-connector-spi/pom.xml b/fe/fe-connector/fe-connector-spi/pom.xml index d64936b49fcd92..a0a60fa57d69a3 100644 --- a/fe/fe-connector/fe-connector-spi/pom.xml +++ b/fe/fe-connector/fe-connector-spi/pom.xml @@ -55,6 +55,11 @@ under the License. + + ${project.groupId} + fe-connector-cache + ${project.version} + ${project.groupId} fe-thrift diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/Connector.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/Connector.java index f7675ba6d0a24b..407a79d424c923 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/Connector.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/Connector.java @@ -17,6 +17,7 @@ package org.apache.doris.connector.spi; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.event.ConnectorEventSource; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; import org.apache.doris.connector.spi.procedure.ConnectorProcedureOps; @@ -53,6 +54,19 @@ */ public interface Connector extends Closeable { + /** + * Additional physical resource identities used by this gateway's already-resolved tables. Pure projection: + * no loading, connector construction or invalidation. The host invalidates all returned scopes together. + */ + default List cacheDependencies(ScopePath remoteScope) { + return Collections.emptyList(); + } + + /** Translate event partition names into the identities used by this connector's caches, without I/O. */ + default List partitionCacheScopes(String database, String table, List partitions) { + return Collections.singletonList(ScopePath.table(database, table)); + } + /** * Returns the metadata interface for the given session. The engine calls this exactly once per catalog per * statement through its own single entry point and closes the result when the statement ends, so an @@ -220,29 +234,30 @@ default void close() throws IOException { /** * Invalidates any connector-side per-table cache (e.g. a latest-snapshot/version cache) so a subsequent - * read reflects the latest external state. Called by the engine on {@code REFRESH TABLE} and before applying - * structural table events (create, drop, rename, and same-name view recreation). The names are the REMOTE - * db/table names (as seen by the connector). Default no-op for connectors that cache nothing. + * read reflects the latest external state. This is a direct connector/standalone entry point; engine refresh, + * DDL and events invalidate shared resource scopes instead, including {@link #cacheDependencies(ScopePath)}. + * The names are the REMOTE db/table names. Default no-op for connectors that cache nothing. */ default void invalidateTable(String dbName, String tableName) { } - /** Invalidates all connector-side per-table caches. Default no-op. */ + /** Direct connector invalidation entry point. Engine changes use the shared root. Default no-op. */ default void invalidateAll() { } /** - * Invalidates the connector-side caches for every table in one database. Called by the engine on - * {@code REFRESH DATABASE} and before applying structural database events (create, drop, and rename). - * The name is the REMOTE db name (as seen by the connector). Default no-op for connectors that cache nothing. + * Directly invalidates the connector-side caches for every table in one database. Engine refresh, DDL and + * events use the shared root's resource scopes. The name is the REMOTE db name. + * Default no-op for connectors that cache nothing. */ default void invalidateDb(String dbName) { } /** * Invalidates the connector-side caches for specific partitions of a table so a subsequent read - * reflects the latest external state. Driven by the engine's metastore-event sync when partitions - * are added/dropped/altered. The names are the REMOTE db/table names and canonical partition names + * reflects the latest external state. Engine events project partition scopes through + * {@link #partitionCacheScopes(String, String, List)} and invalidate them at the shared root instead. + * The names are the REMOTE db/table names and canonical partition names * ({@code "col=val/.../colN=valN"}); an empty/whole-table drop is expressed by * {@link #invalidateTable(String, String)}. A connector whose partition cache cannot target a single * name may degrade to invalidating the whole table's partition caches (correctness-safe when the diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorContext.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorContext.java index e0a6245733d1b9..c5ec2632ce89ac 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorContext.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorContext.java @@ -17,6 +17,8 @@ package org.apache.doris.connector.spi; +import org.apache.doris.connector.cache.CatalogMetaCache; + import java.util.Collections; import java.util.Map; import java.util.concurrent.Callable; @@ -32,6 +34,15 @@ */ public interface ConnectorContext { + /** + * Obtains this connector instance's cache domain. Engine contexts bind it to the catalog root before + * construction; standalone/test contexts own an independent root and close it with their connector. + * Decorators must forward this service. A sibling receives a new instance context and a child domain. + */ + default CatalogMetaCache createCacheDomain(String engine, Map properties) { + return CatalogMetaCache.managed(getCatalogId(), engine, properties); + } + /** Returns the catalog name. */ String getCatalogName(); diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ForwardingConnectorContext.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ForwardingConnectorContext.java index d65d9f4c4313b3..06fbe3bd0e844f 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ForwardingConnectorContext.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ForwardingConnectorContext.java @@ -17,6 +17,8 @@ package org.apache.doris.connector.spi; +import org.apache.doris.connector.cache.CatalogMetaCache; + import java.util.Map; import java.util.Objects; import java.util.concurrent.Callable; @@ -65,6 +67,11 @@ protected final ConnectorContext delegate() { return delegate; } + @Override + public CatalogMetaCache createCacheDomain(String engine, Map properties) { + return delegate.createCacheDomain(engine, properties); + } + @Override public String getCatalogName() { return delegate.getCatalogName(); diff --git a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java index cce4173053f2f7..fc3d710b433d2c 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java +++ b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java @@ -90,7 +90,7 @@ public void connectorApiMajorTracksTheRecordedSurfaceChange() throws IOException } // Major 19 adds ConnectorColumn#isSourceNullable()/withSourceNullable(boolean) - the nullability the // source declares, kept separate from the nullability the engine reads the column with. - Assertions.assertEquals("19.0", version.getProperty("api.version")); + Assertions.assertEquals("20.0", version.getProperty("api.version")); } /** Root entry points plus provider/handle types returned to connector plugins. */ diff --git a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt index 81918e6daa1431..c925bf1fd03533 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt +++ b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt @@ -1,3 +1,4 @@ +org.apache.doris.connector.spi.Connector#cacheDependencies(org.apache.doris.connector.cache.ScopePath):java.util.List org.apache.doris.connector.spi.Connector#close():void org.apache.doris.connector.spi.Connector#defaultTestConnection():boolean org.apache.doris.connector.spi.Connector#deriveStorageProperties(java.util.Map):java.util.Map @@ -16,6 +17,7 @@ org.apache.doris.connector.spi.Connector#invalidateDb(java.lang.String):void org.apache.doris.connector.spi.Connector#invalidatePartition(java.lang.String,java.lang.String,java.util.List):void org.apache.doris.connector.spi.Connector#invalidateTable(java.lang.String,java.lang.String):void org.apache.doris.connector.spi.Connector#ownsHandle(org.apache.doris.connector.spi.handle.ConnectorTableHandle):boolean +org.apache.doris.connector.spi.Connector#partitionCacheScopes(java.lang.String,java.lang.String,java.util.List):java.util.List org.apache.doris.connector.spi.Connector#preCreateValidation(org.apache.doris.connector.spi.ConnectorValidationContext):void org.apache.doris.connector.spi.Connector#schemaCacheTtlSecondOverride():java.util.OptionalLong org.apache.doris.connector.spi.Connector#testConnection(org.apache.doris.connector.spi.ConnectorSession):org.apache.doris.connector.spi.ConnectorTestResult @@ -63,6 +65,7 @@ org.apache.doris.connector.spi.ConnectorColumn#withSpecified(boolean,boolean):or org.apache.doris.connector.spi.ConnectorColumn#withStringWriteType(org.apache.doris.connector.spi.ConnectorType):org.apache.doris.connector.spi.ConnectorColumn org.apache.doris.connector.spi.ConnectorColumn#withTimeZone():org.apache.doris.connector.spi.ConnectorColumn org.apache.doris.connector.spi.ConnectorColumn#withUniqueId(int):org.apache.doris.connector.spi.ConnectorColumn +org.apache.doris.connector.spi.ConnectorContext#createCacheDomain(java.lang.String,java.util.Map):org.apache.doris.connector.cache.CatalogMetaCache org.apache.doris.connector.spi.ConnectorContext#createSiblingConnector(java.lang.String,java.util.Map):org.apache.doris.connector.spi.Connector org.apache.doris.connector.spi.ConnectorContext#executeAuthenticated(java.util.concurrent.Callable):java.lang.Object org.apache.doris.connector.spi.ConnectorContext#getCatalogId():long diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml index 50adfde9967a05..d1d8f2fb4c445f 100644 --- a/fe/fe-connector/pom.xml +++ b/fe/fe-connector/pom.xml @@ -55,7 +55,7 @@ under the License. of the latter two means bumping this property as well (and fe-extension-spi means bumping all five families). --> - 19.0 + 20.0 diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 1b8c36a751e93c..7ff06e47f3a751 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -107,6 +107,7 @@ import org.apache.doris.datasource.ExternalMetaIdMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.MetastoreEventSyncDriver; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; import org.apache.doris.datasource.plugin.PluginDrivenSysExternalTable; import org.apache.doris.datasource.split.SplitSourceManager; @@ -3737,7 +3738,12 @@ public void replayCreateDb(CreateDbInfo dbInfo) { } else { ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(dbInfo.getCtlName()); if (externalCatalog != null) { - externalCatalog.replayCreateDb(dbInfo.getDbName()); + if (dbInfo.getExternalCacheChange() != null) { + CatalogMetadataChanges.apply( + externalCatalog, dbInfo.getExternalCacheChange()); + } else { + externalCatalog.replayCreateDb(dbInfo.getDbName()); + } } } } @@ -3759,7 +3765,12 @@ public void replayDropDb(DropDbInfo info) throws DdlException { } else { ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(info.getCtlName()); if (externalCatalog != null) { - externalCatalog.replayDropDb(info.getDbName()); + if (info.getExternalCacheChange() != null) { + CatalogMetadataChanges.apply( + externalCatalog, info.getExternalCacheChange()); + } else { + externalCatalog.replayDropDb(info.getDbName()); + } } } } @@ -5102,7 +5113,12 @@ public void replayCreateTable(org.apache.doris.persist.CreateTableInfo info) thr } else { ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(info.getCtlName()); if (externalCatalog != null) { - externalCatalog.replayCreateTable(info.getDbName(), info.getTblName()); + if (info.getExternalCacheChange() != null) { + CatalogMetadataChanges.apply( + externalCatalog, info.getExternalCacheChange()); + } else { + externalCatalog.replayCreateTable(info.getDbName(), info.getTblName()); + } } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index 420f509d9d5dae..4208b7c9d72d72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -21,15 +21,13 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.UserException; -import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; -import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.log.CatalogLog; import org.apache.doris.datasource.log.ExternalObjectLog; -import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.persist.OperationType; import com.google.common.base.Strings; @@ -39,7 +37,6 @@ import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -92,7 +89,8 @@ public void handleRefreshDb(String catalogName, String dbName) throws DdlExcepti DatabaseIf db = catalog.getDbOrDdlException(dbName); refreshDbInternal((ExternalDatabase) db); - ExternalObjectLog log = ExternalObjectLog.createForRefreshDb(catalog.getId(), db.getFullName()); + ExternalObjectLog log = ExternalObjectLog.createForRefreshDb(catalog.getId(), db.getFullName()) + .withRemoteNames(((ExternalDatabase) db).getRemoteName(), null); Env.getCurrentEnv().getEditLog().logRefreshExternalDb(log); } @@ -107,46 +105,13 @@ public void replayRefreshDb(ExternalObjectLog log) { LOG.warn("refresh database replay log has no local name: {}", log.debugForRefreshDb()); return; } - long dbId = Util.genIdByName(catalog.getName(), localDbName); - Optional> db = catalog.getDbForReplay(localDbName); - - if (!db.isPresent()) { - ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); - try { - invalidateAllConnectorCachesIfPresent(catalog); - } finally { - try { - catalog.invalidateDatabaseLookup(localDbName); - } finally { - cacheMgr.invalidateDb(log.getCatalogId(), dbId, localDbName); - } - } - LOG.info("database object cache is cold when replaying refresh database; invalidated caches by " - + "local name {}: {}", - localDbName, log.debugForRefreshDb()); - return; - } - refreshDbInternal(db.get()); + CatalogMetadataChanges.apply(catalog, log); } private void refreshDbInternal(ExternalDatabase db) { - // Connector metadata is the row-count source, so invalidate it before engine and row-count caches. - try { - if (db.getCatalog() instanceof PluginDrivenExternalCatalog) { - ((PluginDrivenExternalCatalog) db.getCatalog()).getConnector().invalidateDb(db.getRemoteName()); - } - } finally { - try { - try { - db.resetMetaToUninitialized(); - } finally { - ((ExternalCatalog) db.getCatalog()).invalidateDatabaseLookup(db.getFullName()); - } - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr() - .invalidateDbRowCountCache(db.getCatalog().getId(), db.getId()); - } - } + CatalogMetadataChanges.apply((ExternalCatalog) db.getCatalog(), + ExternalObjectLog.createForRefreshDb(db.getCatalog().getId(), db.getFullName()) + .withRemoteNames(db.getRemoteName(), null)); LOG.info("refresh database {} in catalog {}", db.getFullName(), db.getCatalog().getName()); } @@ -179,8 +144,7 @@ public void handleRefreshTable(String catalogName, String dbName, String tableNa } long updateTime = System.currentTimeMillis(); refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, updateTime, true); - ExternalObjectLog log = ExternalObjectLog.createForExplicitRefreshTable( - catalog.getId(), db.getFullName(), table.getName(), updateTime); + ExternalObjectLog log = CatalogMetadataChanges.table((ExternalTable) table, true, updateTime); env.getEditLog().logRefreshExternalTable(log); } @@ -188,8 +152,7 @@ public void handleRefreshTable(String catalogName, String dbName, String tableNa public void refreshTableAfterExternalMutation(ExternalTable table) { ExternalDatabase db = table.getDb(); long updateTime = System.currentTimeMillis(); - ExternalObjectLog log = ExternalObjectLog.createForRefreshTable( - table.getCatalog().getId(), db.getFullName(), table.getName(), updateTime); + ExternalObjectLog log = CatalogMetadataChanges.table(table, false, updateTime); Env.getCurrentEnv().getEditLog().logRefreshExternalTable(log); refreshTableInternal(db, table, updateTime); } @@ -206,90 +169,11 @@ public void replayRefreshTable(ExternalObjectLog log) { LOG.warn("refresh table replay log has no local name: {}", log.debugForRefreshTable()); return; } - long dbId = Util.genIdByName(catalog.getName(), localDbName); - long tableId = Util.genIdByName(catalog.getName(), localDbName, localTableName); - Optional> db = catalog.getDbForReplay(localDbName); - if (!Strings.isNullOrEmpty(log.getNewTableName())) { - replayRenameTable(log, catalog, db); - return; - } - if (!db.isPresent()) { - try { - invalidateAllConnectorCachesIfPresent(catalog); - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTable( - log.getCatalogId(), dbId, localDbName, tableId, localTableName); - } - LOG.info("database object cache is cold when replaying refresh table; " - + "invalidated caches by local names {}.{}: {}", - localDbName, localTableName, log.debugForRefreshTable()); - return; - } - Optional table = db.get().getTableForReplay(localTableName); - if (!table.isPresent()) { - try { - if (catalog instanceof PluginDrivenExternalCatalog) { - // A cold table object cannot provide the mapped remote table name. - ((PluginDrivenExternalCatalog) catalog).getConnector().invalidateDb(db.get().getRemoteName()); - } - } finally { - try { - if (log.isInvalidateTableLookup()) { - db.get().invalidateTableLookup(localTableName); - } - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTable( - log.getCatalogId(), dbId, db.get().getFullName(), tableId, localTableName); - } - } - LOG.info("table object cache is cold when replaying refresh table; invalidated caches by local name {}: {}", - localTableName, log.debugForRefreshTable()); - return; - } - refreshTableInternal(db.get(), table.get(), log.getLastUpdateTime(), log.isInvalidateTableLookup()); - } - - private void replayRenameTable(ExternalObjectLog log, ExternalCatalog catalog, - Optional> db) { - ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); - String localDbName = log.getDbName(); - String localTableName = log.getTableName(); - long dbId = Util.genIdByName(catalog.getName(), localDbName); - long sourceTableId = Util.genIdByName(catalog.getName(), localDbName, localTableName); - long destinationTableId = Util.genIdByName(catalog.getName(), localDbName, log.getNewTableName()); - try { - if (!db.isPresent()) { - invalidateAllConnectorCachesIfPresent(catalog); - LOG.info("database object cache is cold when replaying rename table; invalidated connector caches for " - + "{}: {}", - localDbName, log.debugForRefreshTable()); - } else { - ExternalDatabase database = db.get(); - try { - if (catalog instanceof PluginDrivenExternalCatalog) { - // Persisted names are local identities, so database scope is the replay-safe connector key. - ((PluginDrivenExternalCatalog) catalog).getConnector() - .invalidateDb(database.getRemoteName()); - } - } finally { - database.invalidateTableRename(localTableName, log.getNewTableName()); - } - } - } finally { - try { - Env.getCurrentEnv().getConstraintManager().renameTable( - new TableNameInfo(catalog.getName(), localDbName, localTableName), - new TableNameInfo(catalog.getName(), localDbName, log.getNewTableName())); - } finally { - cacheMgr.invalidateTableRename(log.getCatalogId(), dbId, localDbName, - sourceTableId, localTableName, destinationTableId, log.getNewTableName()); - } - } - } - - private void invalidateAllConnectorCachesIfPresent(ExternalCatalog catalog) { - if (catalog instanceof PluginDrivenExternalCatalog) { - ((PluginDrivenExternalCatalog) catalog).invalidateAllConnectorCachesIfPresent(); + CatalogMetadataChanges.apply(catalog, log); + if (log.getNewTableName() != null) { + Env.getCurrentEnv().getConstraintManager().renameTable( + new TableNameInfo(catalog.getName(), localDbName, localTableName), + new TableNameInfo(catalog.getName(), localDbName, log.getNewTableName())); } } @@ -324,21 +208,8 @@ private void refreshTableInternal(ExternalDatabase db, ExternalTable table, long if (updateTime > 0) { table.setUpdateTime(updateTime); } - // Connector metadata is the row-count source, so invalidate it before engine and row-count caches. - try { - if (table.getCatalog() instanceof PluginDrivenExternalCatalog) { - ((PluginDrivenExternalCatalog) table.getCatalog()).getConnector() - .invalidateTable(db.getRemoteName(), table.getRemoteName()); - } - } finally { - try { - if (invalidateTableLookup) { - db.invalidateTableLookup(table.getName()); - } - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTable(table); - } - } + CatalogMetadataChanges.apply((ExternalCatalog) table.getCatalog(), + CatalogMetadataChanges.table(table, invalidateTableLookup, updateTime)); LOG.info("refresh table {}, id {} from db {} in catalog {}, update time: {}", table.getName(), table.getId(), db.getFullName(), db.getCatalog().getName(), updateTime); } @@ -364,15 +235,10 @@ public void refreshPartitionsFromEvent(String catalogName, String localDbName, S } ExternalTable externalTable = (ExternalTable) table; - try { - if (externalTable.getCatalog() instanceof PluginDrivenExternalCatalog) { - ((PluginDrivenExternalCatalog) externalTable.getCatalog()).getConnector().invalidatePartition( - ((ExternalDatabase) db).getRemoteName(), externalTable.getRemoteName(), partitionNames); - } - externalTable.setUpdateTime(updateTime); - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableRowCountCache(externalTable); - } + CatalogMetadataChanges.apply((ExternalCatalog) externalTable.getCatalog(), + ExternalObjectLog.createForRefreshPartitions(catalog.getId(), localDbName, localTableName, + partitionNames, null, updateTime).withRemoteNames( + ((ExternalDatabase) db).getRemoteName(), externalTable.getRemoteName())); } public void addToRefreshMap(long catalogId, Integer[] sec) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorInstanceContext.java b/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorInstanceContext.java new file mode 100644 index 00000000000000..f249ae24d6bad3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorInstanceContext.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector; + +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.ScopePath; +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ForwardingConnectorContext; + +import java.util.Map; + +/** One construction/instance boundary, shared by its decorators but never by another connector instance. */ +final class ConnectorInstanceContext extends ForwardingConnectorContext { + private final String provider; + private final Map properties; + private final ClassLoader loader; + private CatalogMetaCache domain; + + ConnectorInstanceContext(ConnectorContext delegate, String provider, Map properties, + ClassLoader loader) { + super(delegate); + this.provider = provider; + this.properties = properties; + this.loader = loader; + } + + @Override + public synchronized CatalogMetaCache createCacheDomain(String engine, Map properties) { + if (domain == null) { + domain = delegate() instanceof ConnectorInstanceContext + ? ((ConnectorInstanceContext) delegate()).childDomain(engine, loader) + : delegate().createCacheDomain(engine, properties); + } + if (domain.isClosed()) { + throw new IllegalStateException("Connector cache domain has been retired"); + } + return domain; + } + + private synchronized CatalogMetaCache childDomain(String engine, ClassLoader loader) { + return createCacheDomain(provider, properties).openDomain(engine, ScopePath.Coordinates.REMOTE, loader); + } + + @Override + public Connector createSiblingConnector(String type, Map properties) { + synchronized (this) { + createCacheDomain(provider, this.properties); + } + return ConnectorFactory.createConnector(type, properties, this); + } + + void rollback() { + CatalogMetaCache captured; + synchronized (this) { + captured = domain; + } + if (captured != null) { + captured.close(); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorPluginManager.java b/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorPluginManager.java index 4eddb0563460d3..dd704162486406 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorPluginManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorPluginManager.java @@ -414,7 +414,23 @@ private Connector createConnector(String catalogType, Map proper } LOG.info("Creating connector via provider '{}' for catalogType='{}'", provider.getType(), catalogType); - return provider.create(properties, withConnectorConfig(context, provider)); + ConnectorInstanceContext instance = new ConnectorInstanceContext(context, provider.getType(), + properties, provider.getClass().getClassLoader()); + Thread thread = Thread.currentThread(); + ClassLoader previous = thread.getContextClassLoader(); + try { + thread.setContextClassLoader(provider.getClass().getClassLoader()); + return provider.create(properties, withConnectorConfig(instance, provider)); + } catch (RuntimeException | Error failure) { + try { + instance.rollback(); + } catch (RuntimeException | Error cleanup) { + failure.addSuppressed(cleanup); + } + throw failure; + } finally { + thread.setContextClassLoader(previous); + } } } LOG.debug("No ConnectorProvider supports catalogType='{}' (standaloneOnly={}). Registered: {}", diff --git a/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java b/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java index f519f6de28b368..1e993ac022a5d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java @@ -25,6 +25,8 @@ import org.apache.doris.common.EnvUtils; import org.apache.doris.common.Version; import org.apache.doris.common.util.LocationPath; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorBrokerAddress; import org.apache.doris.connector.spi.ConnectorContext; @@ -110,6 +112,8 @@ public class DefaultConnectorContext implements ConnectorContext, ConnectorStora private final Object fsLock = new Object(); private volatile FileSystem catalogFileSystem; private volatile boolean closed; + private Supplier cacheRootSupplier; + private CatalogMetaCache standaloneCacheRoot; private final ConnectorHttpSecurityHook httpSecurityHook = new ConnectorHttpSecurityHook() { @Override @@ -166,6 +170,36 @@ public DefaultConnectorContext(String catalogName, long catalogId, this.environment = buildEnvironment(); } + /** Runtime catalogs inject their root; validation contexts own their standalone connector root. */ + public synchronized void bindCacheRoot(Supplier supplier) { + this.cacheRootSupplier = Objects.requireNonNull(supplier, "cacheRootSupplier"); + } + + @Override + public CatalogMetaCache createCacheDomain(String engine, Map properties) { + Supplier supplier; + synchronized (this) { + if (closed) { + throw new IllegalStateException("Connector context has been closed"); + } + supplier = cacheRootSupplier; + if (supplier == null) { + if (standaloneCacheRoot == null) { + standaloneCacheRoot = CatalogMetaCache.managed(catalogId, engine, properties); + } + return standaloneCacheRoot; + } + } + // Catalog lookup may take the catalog monitor; never hold this context's retirement monitor here. + CatalogMetaCache domain = supplier.get().openDomain(engine, ScopePath.Coordinates.REMOTE, + Thread.currentThread().getContextClassLoader()); + if (closed) { + domain.close(); + throw new IllegalStateException("Connector context was retired during cache domain construction"); + } + return domain; + } + @Override public String getCatalogName() { return catalogName; @@ -395,17 +429,27 @@ FileSystem buildCatalogFileSystem(Map storageProp */ @Override public void close() throws IOException { - FileSystem fs; - synchronized (fsLock) { + CatalogMetaCache standalone; + synchronized (this) { if (closed) { return; } closed = true; + standalone = standaloneCacheRoot; + } + FileSystem fs; + synchronized (fsLock) { fs = catalogFileSystem; catalogFileSystem = null; } - if (fs != null) { - fs.close(); + try { + if (fs != null) { + fs.close(); + } + } finally { + if (standalone != null) { + standalone.close(); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 00300420aa1514..8e6651875f1336 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -38,7 +38,8 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.log.CatalogLog; -import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; +import org.apache.doris.datasource.log.ExternalObjectLog; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; import org.apache.doris.persist.OperationType; @@ -111,12 +112,14 @@ private void initInternalCatalog() { } private void addCatalog(CatalogIf catalog) { - nameToCatalog.put(catalog.getName(), catalog); - idToCatalog.put(catalog.getId(), catalog); String catalogName = catalog.getName(); if (!catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { - ((ExternalCatalog) catalog).resetToUninitialized(false); + ExternalCatalog external = (ExternalCatalog) catalog; + external.resetToUninitialized(false); + external.onRegistered(); } + nameToCatalog.put(catalogName, catalog); + idToCatalog.put(catalog.getId(), catalog); } private RemovedCatalog removeCatalog(long catalogId) { @@ -892,16 +895,10 @@ public void addExternalPartitionsFromEvent(String catalogName, if (table == null) { return; } - try { - if (catalog instanceof PluginDrivenExternalCatalog) { - ((PluginDrivenExternalCatalog) catalog).getConnector().invalidatePartition( - ((ExternalDatabase) db).getRemoteName(), ((ExternalTable) table).getRemoteName(), - partitionNames); - ((ExternalTable) table).setUpdateTime(updateTime); - } - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableRowCountCache((ExternalTable) table); - } + CatalogMetadataChanges.apply((ExternalCatalog) catalog, + ExternalObjectLog.createForRefreshPartitions(catalog.getId(), localDbName, localTableName, + partitionNames, null, updateTime).withRemoteNames( + ((ExternalDatabase) db).getRemoteName(), ((ExternalTable) table).getRemoteName())); } public void dropExternalPartitionsFromEvent(String catalogName, @@ -924,16 +921,10 @@ public void dropExternalPartitionsFromEvent(String catalogName, return; } - try { - if (catalog instanceof PluginDrivenExternalCatalog) { - ((PluginDrivenExternalCatalog) catalog).getConnector().invalidatePartition( - ((ExternalDatabase) db).getRemoteName(), ((ExternalTable) table).getRemoteName(), - partitionNames); - ((ExternalTable) table).setUpdateTime(updateTime); - } - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableRowCountCache((ExternalTable) table); - } + CatalogMetadataChanges.apply((ExternalCatalog) catalog, + ExternalObjectLog.createForRefreshPartitions(catalog.getId(), localDbName, localTableName, + partitionNames, null, updateTime).withRemoteNames( + ((ExternalDatabase) db).getRemoteName(), ((ExternalTable) table).getRemoteName())); } public void registerCatalogRefreshListener(Env env) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index a902483558856a..ea4f6b9907acfe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -173,7 +173,7 @@ public abstract class ExternalCatalog private boolean objectCreated = false; protected TransactionManager transactionManager; - private transient CatalogMetaCache feMetaCacheOwner; + private transient volatile CatalogMetaCache feMetaCacheOwner; private transient CatalogMetaCache namingCacheOwner; private transient boolean feCacheRetired; private transient AtomicLong cacheEntrySequence = @@ -410,7 +410,7 @@ public boolean isInitialized() { return this.initialized; } - /** One FE invalidation/lifecycle owner; connector caches retain their remote-name ownership. */ + /** Catalog root shared by FE entries and connector instance domains. */ public synchronized CatalogMetaCache getFeMetaCacheOwner() { Preconditions.checkState(!feCacheRetired, "External catalog cache owner has been retired: %s", name); if (feMetaCacheOwner == null || feMetaCacheOwner.isClosed()) { @@ -419,7 +419,7 @@ public synchronized CatalogMetaCache getFeMetaCacheOwner() { overlayMetaCacheConfig(properties); String engine = this instanceof RemoteDorisExternalCatalog ? "doris" : "default"; - feMetaCacheOwner = CatalogMetaCache.managed(id, engine, properties); + feMetaCacheOwner = CatalogMetaCache.managed(id, engine, properties, ExternalCatalog.class.getClassLoader()); dbIdNameIndex.clear(); if (replacingClosedOwner) { initialized = false; @@ -428,6 +428,11 @@ public synchronized CatalogMetaCache getFeMetaCacheOwner() { return feMetaCacheOwner; } + public CatalogMetaCache getMetaCacheOwnerIfPresent() { + CatalogMetaCache owner = feMetaCacheOwner; + return owner == null || owner.isClosed() ? null : owner; + } + /** Cache retirement must also work before the first schema entry has been prepared. */ public void closeFeMetaCacheOwner() { CatalogMetaCache owner; @@ -461,7 +466,9 @@ public void invalidateDatabaseLookup(String localDbName) { // Replay must not resurrect a retired owner or initialize a remote catalog after ALTER. CatalogMetaCache owner = feMetaCacheOwner; if (owner != null && !owner.isClosed()) { - owner.invalidateDatabaseAndNames(localDbName); + owner.invalidateScopes(java.util.Arrays.asList( + ScopePath.database(localDbName).in(ScopePath.Coordinates.LOCAL), + ScopePath.databaseCollection().in(ScopePath.Coordinates.LOCAL))); } } @@ -478,6 +485,7 @@ private synchronized void buildMetaCache() { if (LOG.isDebugEnabled()) { LOG.debug("buildMetaCache for catalog: {}:{}", this.name, this.id, new Exception()); } + dbIdNameIndex.bind(owner, "database_ids", ScopePath::database); CacheSpec namesSpec = CacheSpec.of( true, Config.external_cache_expire_time_seconds_after_access, @@ -1036,6 +1044,11 @@ public void setLastUpdateTime(long lastUpdateTime) { this.lastUpdateTime = lastUpdateTime; } + /** Only registry publication may reactivate an instance retired by rename cleanup. */ + synchronized void onRegistered() { + feCacheRetired = false; + } + @Override public final void onClose() { synchronized (this) { @@ -1588,7 +1601,7 @@ private List listLocalDatabaseNamesFromCache() { return namesValue.localNames(); } - private String resolveDatabaseNameForInvalidation(String dbName) { + public String resolveDatabaseNameForInvalidation(String dbName) { int mode = getLowerCaseDatabaseNames(); if (mode == 1) { return dbName.toLowerCase(Locale.ROOT); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index bccbd4cd27b3f0..951be1d72225b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -145,7 +145,15 @@ public synchronized void releaseCachedEntries() { if (tables != null) { tables.close(); } - tableIdNameIndex.clear(); + // The table adapter pins its auxiliary identity index until accepted table operations finish. + if (tables == null) { + CatalogMetaCache owner = extCatalog.getMetaCacheOwnerIfPresent(); + if (owner != null) { + tableIdNameIndex.close(owner); + } else { + tableIdNameIndex.clear(); + } + } } /** Full refresh also invalidates the parent membership snapshot at the shared publication barrier. */ @@ -155,7 +163,9 @@ public void invalidateTableLookup(String localTableName) { } else { tableIdNameIndex.removeName(localTableName); } - extCatalog.getFeMetaCacheOwner().invalidateTableAndNames(name, localTableName); + extCatalog.getFeMetaCacheOwner().invalidateScopes(java.util.Arrays.asList( + ScopePath.table(name, localTableName).in(ScopePath.Coordinates.LOCAL), + ScopePath.tableCollection(name).in(ScopePath.Coordinates.LOCAL))); } public boolean isInitialized() { @@ -203,6 +213,8 @@ private void buildMetaCache(CatalogMetaCache owner) { if (LOG.isDebugEnabled()) { LOG.debug("buildMetaCache for database: {}:{}", this.name, this.id, new Exception()); } + tableIdNameIndex.bind(owner, extCatalog.nextFeCacheEntryName("table_ids"), + tableName -> ScopePath.table(name, tableName)); CacheSpec namesSpec = CacheSpec.of( true, Config.external_cache_expire_time_seconds_after_access, @@ -240,6 +252,7 @@ private void buildMetaCache(CatalogMetaCache owner) { Env.getCurrentEnv().getExtMetaCacheMgr().commonRefreshExecutor(), false, FeMetaCacheEntry.defaultObjectStripeCount(), null); + tables.onRetired(() -> tableIdNameIndex.close(owner)); } private List> listTableNames() { @@ -713,11 +726,11 @@ private boolean matchesLocalTableName(String localTableName, String requestedTab return localTableName.equals(requestedTableName); } - private String resolveTableNameForInvalidation(String tableName) { + public String resolveTableNameForInvalidation(String tableName) { if (isStoredTableNamesLowerCase()) { return tableName.toLowerCase(Locale.ROOT); } - if (!isTableNamesCaseInsensitive()) { + if (cacheEntriesReleased || !isTableNamesCaseInsensitive() || !isInitialized()) { return tableName; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 17f28a80209442..14874a6b49a35b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -198,6 +198,13 @@ public void prepareCatalogByEngine(long catalogId, String engine) { } public void invalidateCatalog(long catalogId) { + CatalogIf catalog = getCatalog(catalogId); + if (catalog instanceof ExternalCatalog) { + CatalogMetaCache owner = ((ExternalCatalog) catalog).getMetaCacheOwnerIfPresent(); + if (owner != null) { + owner.invalidateCatalog(); + } + } try { try { routeCatalogEngines(catalogId, cache -> safeInvalidate( @@ -359,10 +366,10 @@ public List getCatalogCacheStats(long catalogId) { .computeIfAbsent(externalMetaCache.engine(), ignored -> Maps.newHashMap()) .putAll(externalMetaCache.stats(catalogId))); for (CatalogMetaCache cache : MetaCacheGovernance.catalogCaches(catalogId)) { - Map entries = - byEngine.computeIfAbsent(cache.engine(), ignored -> Maps.newHashMap()); - for (Map.Entry> entry : cache.entries().entrySet()) { - entries.put(entry.getKey(), MetaCacheEntryStats.from(entry.getValue())); + for (MetaCache entry : cache.allEntries().values()) { + Map entries = + byEngine.computeIfAbsent(entry.engine(), ignored -> Maps.newHashMap()); + entries.put(entry.registrationName(), MetaCacheEntryStats.from(entry)); } } byEngine.forEach((engine, entries) -> entries.forEach((entryName, entryStats) -> diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java index 85d3f2be12cb49..fb19659b48d3c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java @@ -17,42 +17,155 @@ package org.apache.doris.datasource; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; -import org.apache.doris.common.CacheFactory; import org.apache.doris.common.Config; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheRemovalReason; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.cache.BasicAsyncCacheLoader; import org.apache.doris.statistics.util.StatisticsUtil; -import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import lombok.Getter; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.time.Duration; +import java.util.Map; import java.util.Optional; -import java.util.OptionalLong; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.LongFunction; public class ExternalRowCountCache { private static final Logger LOG = LogManager.getLogger(ExternalRowCountCache.class); - private final AsyncLoadingCache> rowCountCache; - // Serialize future publication with explicit invalidation. The read lock ends as soon as Caffeine has - // published the future, before this class waits for the row-count result. - private final ReentrantReadWriteLock publicationLock = new ReentrantReadWriteLock(); + private final ExecutorService executor; + private final LongFunction rootResolver; + private final RowCountCacheLoader loader = new RowCountCacheLoader(); + private final Map entries = new ConcurrentHashMap<>(); + // A process-wide retention policy, not a second metadata store. Values/futures belong exclusively to + // catalog roots. Keep the existing global count bound while invalidation uses root resource scopes. + private final Cache retention = Caffeine.newBuilder() + .maximumSize(Config.max_external_table_row_count_cache_num) + .removalListener((RowCountKey key, Retention token, + com.github.benmanes.caffeine.cache.RemovalCause cause) -> retire(token)) + .build(); public ExternalRowCountCache(ExecutorService executor) { - // 1. set expireAfterWrite to 1 day, avoid too many entries - // 2. set refreshAfterWrite to 10min(default), so that the cache will be refreshed after 10min - CacheFactory rowCountCacheFactory = new CacheFactory( - OptionalLong.of(Config.external_cache_expire_time_seconds_after_access), - OptionalLong.of(Config.external_cache_refresh_time_minutes * 60), - Config.max_external_table_row_count_cache_num, - false, - null); - rowCountCache = rowCountCacheFactory.buildAsyncCache(new RowCountCacheLoader(), executor); + this(executor, id -> { + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(id); + return catalog instanceof ExternalCatalog ? ((ExternalCatalog) catalog).getFeMetaCacheOwner() : null; + }); + } + + ExternalRowCountCache(ExecutorService executor, LongFunction rootResolver) { + this.executor = executor; + this.rootResolver = rootResolver; + } + + private Entry entry(long catalogId) { + CatalogMetaCache root = rootResolver.apply(catalogId); + if (root == null) { + throw new IllegalStateException("No external catalog for row count: " + catalogId); + } + return entries.compute(catalogId, (id, old) -> old != null && old.owner == root + ? old : new Entry(root)); + } + + private void retire(Retention token) { + if (token.entry.owner.isClosed() || retention.getIfPresent(token.key) == token) { + return; + } + try { + while (true) { + RowCountValue current = token.entry.cache.getIfPresent(token.key); + if (current == null || current.retention != token + || token.entry.cache.compareAndSet(token.key, current, null)) { + return; + } + } + } catch (IllegalStateException failure) { + // Catalog close may retire the physical entry between the owner check and the exact removal. + if (!token.entry.owner.isClosed()) { + throw failure; + } + } + } + + private final class Entry { + private final CatalogMetaCache owner; + private final MetaCache cache; + + private Entry(CatalogMetaCache owner) { + this.owner = owner; + MetaCacheDefinition.Builder definition = + MetaCacheDefinition.builder("row_count", + CacheSpec.of(true, Config.external_cache_expire_time_seconds_after_access, + Config.max_external_table_row_count_cache_num), + key -> ScopePath.table(Long.toString(key.dbId), Long.toString(key.tableId)) + .in(ScopePath.Coordinates.ID)) + .budgetPolicy(MetaCacheDefinition.BudgetPolicy.COUNT_ONLY) + .loader(key -> load(key, false)) + .removalListener((key, value, reason) -> { + // Automatic refresh retains the same policy token across future replacement. + if (reason != MetaCacheRemovalReason.REPLACED) { + retention.asMap().remove(key, value.retention); + } + }); + if (Config.external_cache_refresh_time_minutes > 0) { + definition.refreshAfterWrite(Duration.ofMinutes(Config.external_cache_refresh_time_minutes), executor); + } + cache = owner.create(definition.build()); + } + + private RowCountValue load(RowCountKey key, boolean fillMetaCache) { + RowCountValue current = cache.getIfPresent(key); + Retention token = current == null ? new Retention(this, key) : current.retention; + if (current != null) { + // The framework runs refreshes on the background executor. Keep the previous successful + // future visible until this load succeeds; submitting and waiting on the same pool can deadlock. + // Like the original async-cache loader, automatic refresh does not fill metadata caches. + Optional refreshed = loader.doLoad(key); + return refreshed == null ? null + : new RowCountValue(token, CompletableFuture.completedFuture(refreshed)); + } + CompletableFuture> future = fillMetaCache + ? CompletableFuture.supplyAsync(() -> loadRowCount(key, true), executor) + : loader.asyncLoad(key, executor); + return new RowCountValue(token, future); + } + + private void touch(RowCountKey key, RowCountValue value) { + cache.runIfCurrent(key, value, () -> retention.put(key, value.retention)); + } + } + + private static final class Retention { + private final Entry entry; + private final RowCountKey key; + + private Retention(Entry entry, RowCountKey key) { + this.entry = entry; + this.key = key; + } + } + + private static final class RowCountValue { + private final Retention retention; + private final CompletableFuture> future; + + private RowCountValue(Retention retention, CompletableFuture> future) { + this.retention = retention; + this.future = future; + } } @Getter @@ -126,23 +239,17 @@ static Optional loadRowCount(RowCountKey rowCountKey, boolean fillMetaCach public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean fillMetaCache) { RowCountKey key = new RowCountKey(catalogId, dbId, tableId); try { - CompletableFuture> f; - publicationLock.readLock().lock(); - try { - f = fillMetaCache - ? rowCountCache.get(key, (rowCountKey, executor) -> CompletableFuture.supplyAsync( - () -> loadRowCount(rowCountKey, true), executor)) - : rowCountCache.get(key); - } finally { - publicationLock.readLock().unlock(); - } + Entry entry = entry(catalogId); + RowCountValue value = entry.cache.get(key, rowCountKey -> entry.load(rowCountKey, fillMetaCache)); + entry.touch(key, value); + CompletableFuture> f = value.future; // Get row count synchronously by default. if (ConnectContext.get() == null || ConnectContext.get().getSessionVariable().fetchHiveRowCountSync) { - return f.get().orElse(TableIf.UNKNOWN_ROW_COUNT); + return result(entry, key, value); } else { if (f.isDone()) { - return f.get().orElse(TableIf.UNKNOWN_ROW_COUNT); + return result(entry, key, value); } LOG.info("Row count for table {}.{}.{} is still processing.", catalogId, dbId, tableId); } @@ -160,11 +267,16 @@ public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean f public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) { RowCountKey key = new RowCountKey(catalogId, dbId, tableId); try { - CompletableFuture> f = rowCountCache.getIfPresent(key); - if (f == null) { + Entry entry = entries.get(catalogId); + if (entry == null || entry.owner.isClosed()) { + return TableIf.UNKNOWN_ROW_COUNT; + } + RowCountValue value = entry.cache.getIfPresent(key); + if (value == null) { return -1; - } else if (f.isDone()) { - return f.get().orElse(-1L); + } else if (value.future.isDone()) { + entry.touch(key, value); + return result(entry, key, value); } } catch (Exception e) { LOG.warn("Unexpected exception while returning row count if present", e); @@ -172,33 +284,42 @@ public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) return -1; } - // Catalog/db invalidation is O(N): row-count keys are numeric ids, and Caffeine - // does not support prefix invalidation by catalog or database id. - void invalidateCatalog(long catalogId) { - publicationLock.writeLock().lock(); + private long result(Entry entry, RowCountKey key, RowCountValue value) throws Exception { try { - rowCountCache.asMap().keySet().removeIf(key -> key.catalogId == catalogId); + Optional loaded = value.future.get(); + return loaded == null ? TableIf.UNKNOWN_ROW_COUNT : loaded.orElse(TableIf.UNKNOWN_ROW_COUNT); } finally { - publicationLock.writeLock().unlock(); + if (value.future.isDone() && (value.future.isCompletedExceptionally() + || value.future.getNow(null) == null)) { + entry.cache.compareAndSet(key, value, null); + } + } + } + + // These compatibility entry points share the same resource generations as the unified change plan. + void invalidateCatalog(long catalogId) { + Entry entry = entries.get(catalogId); + if (entry != null) { + if (entry.owner.isClosed()) { + entries.remove(catalogId, entry); + } else { + entry.owner.invalidateScope(ScopePath.catalog().in(ScopePath.Coordinates.ID)); + } } } void invalidateDb(long catalogId, long dbId) { - publicationLock.writeLock().lock(); - try { - rowCountCache.asMap().keySet().removeIf(key -> key.catalogId == catalogId && key.dbId == dbId); - } finally { - publicationLock.writeLock().unlock(); + Entry entry = entries.get(catalogId); + if (entry != null && !entry.owner.isClosed()) { + entry.owner.invalidateScope(ScopePath.database(Long.toString(dbId)).in(ScopePath.Coordinates.ID)); } } void invalidateTable(long catalogId, long dbId, long tableId) { - publicationLock.writeLock().lock(); - try { - rowCountCache.synchronous().invalidate(new RowCountKey(catalogId, dbId, tableId)); - } finally { - publicationLock.writeLock().unlock(); + Entry entry = entries.get(catalogId); + if (entry != null && !entry.owner.isClosed()) { + entry.owner.invalidateScope(ScopePath.table(Long.toString(dbId), Long.toString(tableId)) + .in(ScopePath.Coordinates.ID)); } } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/MetastoreEventSyncDriver.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/MetastoreEventSyncDriver.java index 53dc1c9142debb..d7005f55e14d6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/MetastoreEventSyncDriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/MetastoreEventSyncDriver.java @@ -30,7 +30,9 @@ import org.apache.doris.connector.spi.event.EventPollResult; import org.apache.doris.connector.spi.event.MetastoreChangeDescriptor; import org.apache.doris.datasource.log.CatalogLog; +import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.datasource.log.MetaIdMappingsLog; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.MasterOpExecutor; @@ -234,7 +236,7 @@ private void applyDescriptors(PluginDrivenExternalCatalog catalog, Connector con List descriptors) { for (MetastoreChangeDescriptor descriptor : descriptors) { try { - applyOne(catalog, connector, descriptor); + applyOne(catalog, descriptor); } catch (Exception e) { throw new RuntimeException( "Failed to apply metastore change " + descriptor + " on catalog " @@ -245,15 +247,14 @@ private void applyDescriptors(PluginDrivenExternalCatalog catalog, Connector con // Applies one neutral descriptor via the engine's own (connector-agnostic) mutators — the same ones the // legacy event.process() bodies called, now generalized to work on a flipped catalog. - private void applyOne(PluginDrivenExternalCatalog catalog, Connector connector, - MetastoreChangeDescriptor descriptor) throws Exception { + private void applyOne(PluginDrivenExternalCatalog catalog, MetastoreChangeDescriptor descriptor) throws Exception { EventIdentity before = EventIdentity.from( catalog, descriptor.getDbName(), descriptor.getTableName()); EventIdentity after = descriptor.getDbNameAfter() == null ? null : EventIdentity.from( catalog, descriptor.getDbNameAfter(), descriptor.getTableNameAfter()); String catalogName = catalog.getName(); CatalogMgr catalogMgr = Env.getCurrentEnv().getCatalogMgr(); - invalidateStructuralEventCaches(connector, descriptor.getOp(), before, after); + invalidateStructuralEventCaches(catalog, descriptor.getOp(), before, after); switch (descriptor.getOp()) { case REGISTER_DATABASE: catalogMgr.registerExternalDatabaseFromEvent( @@ -310,39 +311,36 @@ private void applyOne(PluginDrivenExternalCatalog catalog, Connector connector, } } - /** - * Invalidates connector-owned caches for structural events before publishing the corresponding FE mutation. - * Connector cache keys use remote identities; FE mutators below use the local identities from the same values. - */ - private void invalidateStructuralEventCaches(Connector connector, + /** Structural events invalidate both coordinates before the incremental FE snapshot mutation. */ + private void invalidateStructuralEventCaches(PluginDrivenExternalCatalog catalog, MetastoreChangeDescriptor.Op op, EventIdentity before, EventIdentity after) { switch (op) { case REGISTER_DATABASE: case UNREGISTER_DATABASE: - connector.invalidateDb(before.remoteDbName); - break; - case RENAME_DATABASE: - connector.invalidateDb(before.remoteDbName); - if (!Objects.equals(before.remoteDbName, after.remoteDbName)) { - connector.invalidateDb(after.remoteDbName); - } - break; case REGISTER_TABLE: case UNREGISTER_TABLE: - connector.invalidateTable(before.remoteDbName, before.remoteTableName); + CatalogMetadataChanges.apply(catalog, structuralChange(catalog, before)); break; + case RENAME_DATABASE: case RENAME_TABLE: - connector.invalidateTable(before.remoteDbName, before.remoteTableName); - if (!Objects.equals(before.remoteDbName, after.remoteDbName) - || !Objects.equals(before.remoteTableName, after.remoteTableName)) { - connector.invalidateTable(after.remoteDbName, after.remoteTableName); - } + ExternalObjectLog source = structuralChange(catalog, before); + ExternalObjectLog destination = structuralChange(catalog, after); + CatalogMetadataChanges.apply(catalog, source.equals(destination) + ? java.util.Collections.singletonList(source) : java.util.Arrays.asList(source, destination)); break; default: break; } } + private ExternalObjectLog structuralChange(PluginDrivenExternalCatalog catalog, EventIdentity identity) { + ExternalObjectLog change = identity.localTableName == null + ? ExternalObjectLog.createForRefreshDb(catalog.getId(), identity.localDbName) + : ExternalObjectLog.createForExplicitRefreshTable( + catalog.getId(), identity.localDbName, identity.localTableName, 0); + return change.withRemoteNames(identity.remoteDbName, identity.remoteTableName); + } + /** One remote connector identity and its canonical FE-local counterpart. */ private static final class EventIdentity { private final String remoteDbName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/log/ExternalObjectLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/log/ExternalObjectLog.java index 308b3cd5184100..881df2ac1070ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/log/ExternalObjectLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/log/ExternalObjectLog.java @@ -52,6 +52,15 @@ public class ExternalObjectLog implements Writable { @SerializedName(value = "ntn") private String newTableName; // for rename table op + @SerializedName(value = "remoteDbName") + private String remoteDbName; + + @SerializedName(value = "remoteTableName") + private String remoteTableName; + + @SerializedName(value = "remoteNewTableName") + private String remoteNewTableName; + @SerializedName(value = "invalidCache") private boolean invalidCache; @@ -117,6 +126,13 @@ public static ExternalObjectLog createForRenameTable(long catalogId, String dbNa return externalObjectLog; } + /** Persist both coordinates; replay must not need a warm object cache to recover remote identities. */ + public ExternalObjectLog withRemoteNames(String database, String table) { + setRemoteDbName(database); + setRemoteTableName(table); + return this; + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java new file mode 100644 index 00000000000000..127ca9ec71fd87 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.util.Util; +import org.apache.doris.connector.cache.CacheCallbackContext; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.ScopePath; +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.log.ExternalObjectLog; +import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; + +import com.google.common.base.Strings; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * The engine's semantic change boundary. FE names, remote connector identities and numeric derived state are + * projected once and detached at one catalog publication barrier. Projection never loads remote metadata. + * Connector domains own registrations/lifetime; resource scopes own invalidation, independently of domains. + */ +public final class CatalogMetadataChanges { + private CatalogMetadataChanges() { + } + + public static ExternalObjectLog table(ExternalTable table, boolean lookup, long updateTime) { + ExternalObjectLog change = lookup + ? ExternalObjectLog.createForExplicitRefreshTable(table.getCatalog().getId(), + table.getDb().getFullName(), table.getName(), updateTime) + : ExternalObjectLog.createForRefreshTable(table.getCatalog().getId(), + table.getDb().getFullName(), table.getName(), updateTime); + change.setDbId(table.getDb().getId()); + change.setTableId(table.getId()); + return change.withRemoteNames(table.getRemoteDbName(), table.getRemoteName()); + } + + public static void apply(ExternalCatalog catalog, ExternalObjectLog change) { + apply(catalog, java.util.Collections.singletonList(change)); + } + + /** A rename event can span databases; all old/new coordinates share one publication barrier. */ + public static void apply(ExternalCatalog catalog, List changes) { + List scopes = new ArrayList<>(); + RuntimeException failure = null; + try { + for (ExternalObjectLog change : changes) { + try { + plan(catalog, change, scopes); + } catch (RuntimeException error) { + if (failure == null) { + failure = error; + } else { + failure.addSuppressed(error); + } + } + } + } finally { + CatalogMetaCache owner = catalog.getMetaCacheOwnerIfPresent(); + if (owner != null) { + owner.invalidateScopes(scopes); + } + if (Env.getCurrentEnv().getSortedPartitionsCacheManager() != null) { + for (ExternalObjectLog change : changes) { + invalidateSortedPartitions(catalog, change); + } + } + } + if (failure != null) { + throw failure; + } + } + + private static void invalidateSortedPartitions(ExternalCatalog catalog, ExternalObjectLog change) { + if (change.getTableName() == null) { + Env.getCurrentEnv().getSortedPartitionsCacheManager().invalidateAll(); + } else { + Env.getCurrentEnv().getSortedPartitionsCacheManager().invalidateTable( + catalog.getName(), change.getDbName(), change.getTableName()); + if (change.getNewTableName() != null) { + Env.getCurrentEnv().getSortedPartitionsCacheManager().invalidateTable( + catalog.getName(), change.getDbName(), change.getNewTableName()); + } + } + } + + private static void plan(ExternalCatalog catalog, ExternalObjectLog change, List scopes) { + Optional> db = catalog.getDbForReplay(change.getDbName()); + Optional table = Strings.isNullOrEmpty(change.getTableName()) + ? Optional.empty() : db.flatMap(database -> database.getTableForReplay(change.getTableName())); + long dbId = change.getDbId() != 0 ? change.getDbId() + : Util.genIdByName(catalog.getName(), change.getDbName()); + long tableId = change.getTableId() != 0 ? change.getTableId() + : change.getTableName() == null ? 0L + : Util.genIdByName(catalog.getName(), change.getDbName(), change.getTableName()); + String remoteDb = change.getRemoteDbName() != null ? change.getRemoteDbName() + : db.map(ExternalDatabase::getRemoteName).orElse(null); + String remoteTable = change.getRemoteTableName() != null ? change.getRemoteTableName() + : table.map(ExternalTable::getRemoteName).orElse(null); + Connector connector = catalog instanceof PluginDrivenExternalCatalog + ? ((PluginDrivenExternalCatalog) catalog).getConnectorIfPresent() : null; + Set remote = new LinkedHashSet<>(); + boolean database = Strings.isNullOrEmpty(change.getTableName()); + boolean partitions = change.getPartitionNames() != null; + String canonicalDb = catalog.resolveDatabaseNameForInvalidation(change.getDbName()); + if (!canonicalDb.equals(change.getDbName())) { + scopes.add(local(ScopePath.database(canonicalDb))); + scopes.add(id(ScopePath.database(Long.toString(Util.genIdByName(catalog.getName(), canonicalDb))))); + } + if (database) { + scopes.add(local(ScopePath.database(change.getDbName()))); + scopes.add(local(ScopePath.databaseCollection())); + scopes.add(id(ScopePath.database(Long.toString(dbId)))); + remote.add(remoteDb == null ? ScopePath.catalog() : ScopePath.database(remoteDb)); + remote.add(ScopePath.databaseCollection()); + } else { + table.ifPresent(value -> { + if (!partitions) { + value.unsetObjectCreated(); + } + if (change.getLastUpdateTime() > 0) { + value.setUpdateTime(change.getLastUpdateTime()); + } + }); + scopes.add(id(ScopePath.table(Long.toString(dbId), Long.toString(tableId)))); + if (!partitions) { + if (db.isPresent() && change.isInvalidateTableLookup()) { + String canonical = db.get().resolveTableNameForInvalidation(change.getTableName()); + scopes.add(local(ScopePath.table(db.get().getFullName(), canonical))); + scopes.add(id(ScopePath.table(Long.toString(dbId), Long.toString(Util.genIdByName( + catalog.getName(), db.get().getFullName(), canonical))))); + } + scopes.add(local(change.isInvalidateTableLookup() + ? ScopePath.table(change.getDbName(), change.getTableName()) + : ScopePath.tableMetadata(change.getDbName(), change.getTableName()))); + if (change.isInvalidateTableLookup() || change.getNewTableName() != null) { + scopes.add(local(ScopePath.tableCollection(change.getDbName()))); + } + } + if (remoteDb == null || remoteTable == null) { + // Old logs carry only local identities. Widen only the remote coordinate, without I/O. + remote.add(remoteDb == null ? ScopePath.catalog() : ScopePath.database(remoteDb)); + } else { + remote.add(ScopePath.table(remoteDb, remoteTable)); + } + if (remoteDb != null && (change.isInvalidateTableLookup() || change.getNewTableName() != null)) { + remote.add(ScopePath.tableCollection(remoteDb)); + } + if (change.getNewTableName() != null) { + scopes.add(local(ScopePath.table(change.getDbName(), change.getTableName()))); + scopes.add(local(ScopePath.table(change.getDbName(), change.getNewTableName()))); + scopes.add(id(ScopePath.table(Long.toString(dbId), Long.toString(Util.genIdByName( + catalog.getName(), change.getDbName(), change.getNewTableName()))))); + remote.add(remoteDb != null && change.getRemoteNewTableName() != null + ? ScopePath.table(remoteDb, change.getRemoteNewTableName()) + : remoteDb == null ? ScopePath.catalog() : ScopePath.database(remoteDb)); + } + } + try { + if (connector != null) { + new CacheCallbackContext(connector.getClass().getClassLoader()).run(() -> { + if (partitions && remoteDb != null && remoteTable != null) { + remote.clear(); + remote.addAll(connector.partitionCacheScopes( + remoteDb, remoteTable, change.getPartitionNames())); + } + List dependencies = new ArrayList<>(); + for (ScopePath scope : remote) { + if (scope.database() != null) { + dependencies.addAll(connector.cacheDependencies(scope)); + } + } + remote.addAll(dependencies); + }); + } + } catch (RuntimeException | Error failure) { + // A gateway failed to project its dependencies. Fence every remote cache rather than retain a + // dependency whose identity is unknown, then report the failure to the caller. + remote.add(ScopePath.catalog()); + throw failure; + } finally { + remote.forEach(scope -> scopes.add(scope.in(ScopePath.Coordinates.REMOTE))); + } + } + + private static ScopePath local(ScopePath scope) { + return scope.in(ScopePath.Coordinates.LOCAL); + } + + private static ScopePath id(ScopePath scope) { + return scope.in(ScopePath.Coordinates.ID); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java index c762de4f750b16..c9f945ab59dc87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java @@ -150,7 +150,7 @@ public void invalidateCatalogEntries(long catalogId) { public void invalidateDb(long catalogId, String dbName) { CatalogMetaCacheRuntime group = catalogEntries.get(catalogId); if (group != null) { - group.owner().invalidateDatabase(dbName); + group.owner().invalidateScope(ScopePath.database(dbName).in(ScopePath.Coordinates.LOCAL)); } } @@ -158,7 +158,7 @@ public void invalidateDb(long catalogId, String dbName) { public void invalidateTable(long catalogId, String dbName, String tableName) { CatalogMetaCacheRuntime group = catalogEntries.get(catalogId); if (group != null) { - group.owner().invalidateScope(ScopePath.tableMetadata(dbName, tableName)); + group.owner().invalidateScope(ScopePath.tableMetadata(dbName, tableName).in(ScopePath.Coordinates.LOCAL)); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java index 001b6c12bf3c1b..94e5b4147325c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java @@ -91,6 +91,8 @@ private ActionToken(K key, ActionState state) { private final MetaCache data; private final BiConsumer discardedValueAction; private final Object lifecycleLock = new Object(); + private Runnable retirementCleanup; + private final AtomicBoolean registrationReleased = new AtomicBoolean(); private volatile boolean retired; private int activeUses; @@ -144,7 +146,8 @@ public static FeMetaCacheEntry shared(CatalogMetaCache owner, Function scopeResolver, ScopePath invalidationScope, String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, boolean autoRefresh, int stripeCount, @Nullable RemovalListener removalListener) { - return new FeMetaCacheEntry<>(owner, scopeResolver, invalidationScope, name, loader, cacheSpec, + return new FeMetaCacheEntry<>(owner, scopeResolver, invalidationScope.in(ScopePath.Coordinates.LOCAL), + name, loader, cacheSpec, refreshExecutor, autoRefresh, false, stripeCount, removalListener, true, MetaCacheDefinition.BudgetPolicy.WEIGHTED_PAYLOAD); } @@ -154,7 +157,8 @@ public static FeMetaCacheEntry sharedLookup(CatalogMetaCache owner, Function scopeResolver, ScopePath invalidationScope, String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, boolean autoRefresh, int stripeCount, @Nullable RemovalListener removalListener) { - return new FeMetaCacheEntry<>(owner, scopeResolver, invalidationScope, name, loader, cacheSpec, + return new FeMetaCacheEntry<>(owner, scopeResolver, invalidationScope.in(ScopePath.Coordinates.LOCAL), + name, loader, cacheSpec, refreshExecutor, autoRefresh, false, stripeCount, removalListener, true, MetaCacheDefinition.BudgetPolicy.COUNT_ONLY); } @@ -168,6 +172,8 @@ private FeMetaCacheEntry(CatalogMetaCache owner, Function scopeRes discardedValueAction = retireDiscardedValues && removalListener != null ? (key, value) -> removalListener.onRemoval(key, value, RemovalCause.EXPLICIT) : null; this.invalidationScope = Objects.requireNonNull(invalidationScope, "invalidationScope"); + Function entryScope = retireDiscardedValues + ? key -> scopeResolver.apply(key).in(ScopePath.Coordinates.LOCAL) : scopeResolver; this.name = Objects.requireNonNull(name, "name can not be null"); this.loader = loader; this.cacheSpec = Objects.requireNonNull(cacheSpec, "cacheSpec can not be null"); @@ -196,7 +202,7 @@ private FeMetaCacheEntry(CatalogMetaCache owner, Function scopeRes effectiveEnabled = CacheSpec.isCacheEnabled( cacheSpec.isEnable(), cacheSpec.getTtlSecond(), cacheSpec.getCapacity()); MetaCacheDefinition.Builder builder = MetaCacheDefinition.builder( - name, cacheSpec, scopeResolver) + name, cacheSpec, entryScope) .sizeEstimator(MetaCacheSizeEstimators.reflective()) .budgetPolicy(budgetPolicy); if (loader != null) { @@ -292,7 +298,7 @@ private V getAndRunIfCurrentCached(K key, BiPredicate required, && (!effectiveEnabled || data.getIfPresent(key) == value); if (current && required.test(key, value) && isCurrent(stripe, token)) { try { - action.accept(key, value); + data.runIfCurrent(key, value, () -> action.accept(key, value)); } catch (RuntimeException | Error throwable) { data.invalidateKey(key); throw throwable; @@ -456,6 +462,28 @@ private void invalidateAllCached() { owner.invalidateScope(invalidationScope); } + /** Auxiliary state has the same accepted-operation lifetime as this exact physical registration. */ + public void onRetired(Runnable cleanup) { + synchronized (lifecycleLock) { + if (retired || retirementCleanup != null) { + throw new IllegalStateException("Retirement cleanup must be bound once before retirement"); + } + retirementCleanup = Objects.requireNonNull(cleanup, "cleanup"); + } + } + + private void releaseRegistration() { + if (registrationReleased.compareAndSet(false, true)) { + try { + owner.remove(data); + } finally { + if (retirementCleanup != null) { + retirementCleanup.run(); + } + } + } + } + /** * Stops accepting cached operations without waiting for a remote load. Accepted operations pin the physical * entry until they finish; later reads use the loader without publishing cache values or ID side effects. @@ -468,7 +496,7 @@ public void close() { remove = activeUses == 0; } if (remove) { - owner.remove(data); + releaseRegistration(); } } @@ -503,7 +531,7 @@ private R useCache(Supplier operation, Supplier uncached) { remove = retired && activeUses == 0; } if (remove) { - owner.remove(data); + releaseRegistration(); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/IdNameIndex.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/IdNameIndex.java index fcc075b80fe799..0426685bf1ce87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/IdNameIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/IdNameIndex.java @@ -17,136 +17,210 @@ package org.apache.doris.datasource.metacache; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; import org.jetbrains.annotations.Nullable; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.function.Function; /** - * Runtime-only one-to-one index between metadata IDs and canonical local names. - * - *

Reads by ID are lock-free. All mutations are serialized so the two maps have a stable one-to-one relationship - * after each mutation completes. The reverse map is intentionally kept internal: production callers use it only - * indirectly for uniqueness validation and constant-time removal by name. + * Runtime-only identity index. Identities survive object TTL/eviction, but share their semantic resource scope. + * Physical cleanup removes the exact identity record, so delayed cleanup cannot erase a same-name recreation. */ public final class IdNameIndex { private final String scope; - private final ConcurrentMap idToName = Maps.newConcurrentMap(); - private final ConcurrentMap nameToId = Maps.newConcurrentMap(); + private final ConcurrentMap byId = new ConcurrentHashMap<>(); + private final ConcurrentMap byName = new ConcurrentHashMap<>(); + private volatile Binding binding; public IdNameIndex(String scope) { - this.scope = Objects.requireNonNull(scope, "scope can not be null"); + this.scope = Objects.requireNonNull(scope, "scope"); + } + + /** Bind once per owning catalog/database instance; standalone indexes retain the same validation contract. */ + public synchronized void bind(CatalogMetaCache owner, String entryName, Function resolver) { + if (binding != null && binding.owner == owner) { + return; + } + if (binding != null) { + close(binding.owner); + } + MetaCache cache = owner.create(MetaCacheDefinition.builder(entryName, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, Long.MAX_VALUE), + name -> resolver.apply(name).in(ScopePath.Coordinates.LOCAL)) + .budgetPolicy(MetaCacheDefinition.BudgetPolicy.COUNT_ONLY) + .removalListener((name, identity, reason) -> removeExact(identity)) + .build()); + binding = new Binding(owner, cache); } @Nullable public String getName(long id) { - return idToName.get(id); + Identity identity = byId.get(id); + return isCurrent(identity) ? identity.name : null; + } + + private boolean isCurrent(Identity identity) { + if (identity == null || byId.get(identity.id) != identity || byName.get(identity.name) != identity) { + return false; + } + Binding current = binding; + if (current == null) { + return true; + } + try { + return !current.owner.isClosed() && current.cache.getIfPresent(identity.name) == identity; + } catch (IllegalStateException failure) { + // The catalog root may close the physical registration concurrently with this index read. + if (binding != current || current.owner.isClosed()) { + return false; + } + throw failure; + } } - /** - * Validate one identity without publishing it. - * - *

Callers use this before mutating names/object caches so a pre-existing identity conflict fails without - * leaving those caches partially updated. The final {@link #put(long, String)} must still run inside the - * FeMetaCacheEntry publication action to keep the ID side effect ordered with object invalidation. - */ public void checkCanPut(long id, String name) { - Objects.requireNonNull(name, "name can not be null"); - Long existingId = nameToId.get(name); - String existingName = idToName.get(id); - - // Keep the common exact-match and cold-index paths lock-free. A later put remains the authoritative - // validation inside the caller's FeMetaCacheEntry publication window. - if ((existingId == null && existingName == null) - || (Long.valueOf(id).equals(existingId) && name.equals(existingName))) { + Identity named = byName.get(name); + Identity identified = byId.get(id); + if (named == identified && (named == null || (named.id == id && named.name.equals(name)))) { return; } - - // Re-read suspicious snapshots under the mutation lock. This both confirms real conflicts and avoids - // rejecting a reader that observed the two maps between the writes of one synchronized mutation. synchronized (this) { checkCanPutLocked(id, name); } } private void checkCanPutLocked(long id, String name) { - Long existingId = nameToId.get(name); - String existingName = idToName.get(id); - - Preconditions.checkState(existingId == null || existingId == id, + Objects.requireNonNull(name, "name"); + Identity named = byName.get(name); + Identity identified = byId.get(id); + if (!isCurrent(named)) { + removeExact(named); + named = null; + } + if (!isCurrent(identified)) { + removeExact(identified); + identified = null; + } + Preconditions.checkState(named == null || named.id == id, "%s name '%s' is already mapped to id %s, can not map to id %s", - scope, name, existingId, id); - Preconditions.checkState(existingName == null || existingName.equals(name), + scope, name, named == null ? null : named.id, id); + Preconditions.checkState(identified == null || identified.name.equals(name), "%s id %s is already mapped to name '%s', can not map to name '%s'", - scope, id, existingName, name); - Preconditions.checkState((existingId == null) == (existingName == null), - "%s index is inconsistent for id %s and name '%s': existing id %s, existing name '%s'", - scope, id, name, existingId, existingName); + scope, id, identified == null ? null : identified.name, name); + Preconditions.checkState(named == identified, "%s index is inconsistent for %s:%s", scope, id, name); + } + + /** Called inside the object cache's guarded publication action. */ + public void put(long id, String name) { + checkCanPut(id, name); + Binding captured = binding; + MetaCache cache = captured == null ? null : captured.cache; + Identity identity = new Identity(id, name); + // Enter cache publication before taking the index monitor. Other guarded object publications may + // hold the catalog read phase while waiting for this monitor; the reverse order would deadlock. + if (cache != null) { + while (true) { + Identity current = cache.getIfPresent(name); + if (current != null) { + Preconditions.checkState(current.id == id, "%s identity conflict for %s", scope, name); + identity = current; + break; + } + if (cache.compareAndSet(name, null, identity)) { + break; + } + } + } + synchronized (this) { + if (binding != captured || (cache != null && cache.getIfPresent(name) != identity)) { + return; + } + checkCanPutLocked(id, name); + byId.put(id, identity); + byName.put(name, identity); + } } - /** - * Publish one identity. Re-publishing the same pair is idempotent; every other conflict is a correctness error. - */ - public synchronized void put(long id, String name) { - Objects.requireNonNull(name, "name can not be null"); - checkCanPutLocked(id, name); - idToName.put(id, name); - nameToId.put(name, id); + private synchronized void removeExact(Identity identity) { + if (identity != null) { + byId.remove(identity.id, identity); + byName.remove(identity.name, identity); + } } - /** - * Remove the identity associated with a canonical local name in constant time. - */ public synchronized void removeName(String name) { - Objects.requireNonNull(name, "name can not be null"); - Long id = nameToId.get(name); - if (id == null) { - return; + Objects.requireNonNull(name, "name"); + Identity identity = byName.get(name); + Binding current = binding; + if (current != null && !current.owner.isClosed()) { + current.cache.invalidateKey(name); } - String indexedName = idToName.get(id); - Preconditions.checkState(name.equals(indexedName), - "%s index is inconsistent for name '%s' and id %s: indexed name '%s'", - scope, name, id, indexedName); - - nameToId.remove(name); - boolean removed = idToName.remove(id, name); - Preconditions.checkState(removed, - "%s index failed to remove id %s and name '%s'", scope, id, name); + removeExact(identity); } - /** - * Find a retained canonical name for the mode-2 cache-only invalidation fallback. - * - *

This deliberately preserves the existing O(N) fallback behavior. Normal lookup and exact-name cleanup do - * not use this method. - */ @Nullable public String findNameIgnoreCase(String requestedName) { - Objects.requireNonNull(requestedName, "requestedName can not be null"); - return idToName.values().stream() - .filter(name -> name.equalsIgnoreCase(requestedName)) - .findFirst() - .orElse(null); + return byId.values().stream().filter(this::isCurrent).map(identity -> identity.name) + .filter(name -> name.equalsIgnoreCase(requestedName)).findFirst().orElse(null); } public synchronized void clear() { - idToName.clear(); - nameToId.clear(); + Binding current = binding; + if (current != null && !current.owner.isClosed()) { + byName.keySet().forEach(current.cache::invalidateKey); + } + byId.clear(); + byName.clear(); + } + + /** Retires this instance's registration without invalidating a replacement resource. */ + public synchronized void close(CatalogMetaCache owner) { + Binding current = binding; + binding = null; + if (current != null) { + owner.remove(current.cache); + } + byId.clear(); + byName.clear(); } @VisibleForTesting - public synchronized boolean containsMappingForTest(long id, String name) { - return name.equals(idToName.get(id)) && Long.valueOf(id).equals(nameToId.get(name)); + public boolean containsMappingForTest(long id, String name) { + return name.equals(getName(id)); } @VisibleForTesting public synchronized int sizeForTest() { - Preconditions.checkState(idToName.size() == nameToId.size(), - "%s index size mismatch: idToName=%s, nameToId=%s", - scope, idToName.size(), nameToId.size()); - return idToName.size(); + return (int) byId.values().stream().filter(this::isCurrent).count(); + } + + private static final class Binding { + private final CatalogMetaCache owner; + private final MetaCache cache; + + private Binding(CatalogMetaCache owner, MetaCache cache) { + this.owner = owner; + this.cache = cache; + } + } + + private static final class Identity { + private final long id; + private final String name; + + private Identity(long id, String name) { + this.id = id; + this.name = name; + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryInvalidation.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryInvalidation.java index 6bdc70c0b0ddfc..d2e69593281d84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryInvalidation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryInvalidation.java @@ -77,7 +77,8 @@ public static MetaCacheEntryInvalidation forTableIdentity( Objects.requireNonNull(tableNameExtractor, "tableNameExtractor"); // Metadata is a child of the identity scope: ordinary writes must not evict FE lookup objects. return new MetaCacheEntryInvalidation<>( - key -> ScopePath.tableMetadata(dbNameExtractor.apply(key), tableNameExtractor.apply(key)), + key -> ScopePath.tableMetadata(dbNameExtractor.apply(key), tableNameExtractor.apply(key)) + .in(ScopePath.Coordinates.LOCAL), dbName -> key -> dbNameExtractor.apply(key).equals(dbName), (dbName, tableName) -> key -> dbNameExtractor.apply(key).equals(dbName) && tableNameExtractor.apply(key).equals(tableName), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java index 60c2a5422e4334..92616b7312aecc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java @@ -61,7 +61,6 @@ import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalFunctionRules; -import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SessionContext; import org.apache.doris.datasource.connector.converter.ConnectorBranchTagConverter; @@ -69,6 +68,7 @@ import org.apache.doris.datasource.connector.converter.ConnectorPartitionFieldConverter; import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.datasource.log.InitCatalogLog; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.foundation.property.StoragePropertiesException; import org.apache.doris.foundation.security.JdbcDriverUrlSecurity; import org.apache.doris.fs.FileSystemFactory; @@ -215,6 +215,7 @@ protected Connector createConnectorFromProperties() { DefaultConnectorContext context = new DefaultConnectorContext(name, id, this::getExecutionAuthenticator, () -> catalogProperty.getStorageAdaptersMap(), catalogProperty::getEffectiveRawStorageProperties); + context.bindCacheRoot(this::getFeMetaCacheOwner); this.connectorContext = context; // The standalone entry point, same as CatalogFactory uses: this is the second door onto a catalog (the // lazy build after image deserialization), and both doors must agree on what may become a catalog. @@ -441,33 +442,9 @@ public void notifyPropertiesUpdated(Map updatedProps) { super.notifyPropertiesUpdated(updatedProps); } - /** - * Invalidates connector-owned metadata before FE caches because connector metadata feeds row-count loading. - * The connector field is read directly so refreshing an uninitialized catalog does not initialize it. - */ - @Override - public void onRefreshCache(boolean invalidCache) { - if (!invalidCache) { - super.onRefreshCache(false); - return; - } - try { - invalidateAllConnectorCachesIfPresent(); - } finally { - super.onRefreshCache(true); - } - } - - /** - * Invalidates connector-owned caches without initializing or rebuilding the connector. - * - *

Replay also uses this when no database object is cached and remote cache keys cannot be recovered. - */ - public void invalidateAllConnectorCachesIfPresent() { - Connector localConnector = connector; - if (localConnector != null) { - localConnector.invalidateAll(); - } + /** Cache coordination is cache-only; replay must never initialize a connector. */ + public Connector getConnectorIfPresent() { + return connector; } @Override @@ -646,7 +623,7 @@ public void overlayMetaCacheConfig(Map metaCacheProperties) { /** * Routes CREATE TABLE through the connector SPI. Returning {@code true} for an existing table preserves * the caller's CTAS short-circuit contract. A newly created table persists canonical local names; every - * successful path invalidates connector, metadata, and row-count caches in that order. + * successful path applies the same catalog cache change to connector metadata and derived FE state. */ @Override public boolean createTable(CreateTableInfo createTableInfo) throws UserException { @@ -702,7 +679,10 @@ public boolean createTable(CreateTableInfo createTableInfo) throws UserException } org.apache.doris.persist.CreateTableInfo persistInfo = new org.apache.doris.persist.CreateTableInfo( - getName(), db.getFullName(), localTableName); + getName(), db.getFullName(), localTableName).withExternalCacheChange( + ExternalObjectLog.createForExplicitRefreshTable(getId(), db.getFullName(), + localTableName, 0L).withRemoteNames( + db.getRemoteName(), createTableInfo.getTableName())); Env.getCurrentEnv().getEditLog().logCreateTable(persistInfo); invalidateCreatedTableCaches(db, createTableInfo.getTableName(), localTableName); LOG.info("finished to create table {}.{}.{}", getName(), @@ -751,7 +731,9 @@ public void createDb(String dbName, boolean ifNotExists, Map pro } throw new DdlException(e.getMessage(), e); } - Env.getCurrentEnv().getEditLog().logCreateDb(new CreateDbInfo(getName(), localDbName, null)); + Env.getCurrentEnv().getEditLog().logCreateDb(new CreateDbInfo(getName(), localDbName, null) + .withExternalCacheChange(ExternalObjectLog.createForRefreshDb(getId(), localDbName) + .withRemoteNames(dbName, null))); invalidateCreatedDatabaseCaches(dbName, localDbName); LOG.info("finished to create database {}.{}", getName(), dbName); } @@ -788,12 +770,11 @@ public void dropDb(String dbName, boolean ifExists, boolean force) throws DdlExc } catch (DorisConnectorException e) { throw new DdlException(e.getMessage(), e); } - Env.getCurrentEnv().getEditLog().logDropDb(new DropDbInfo(getName(), db.getFullName())); - try { - connector.invalidateDb(db.getRemoteName()); - } finally { - unregisterDatabase(db.getFullName()); - } + Env.getCurrentEnv().getEditLog().logDropDb(new DropDbInfo(getName(), db.getFullName()) + .withExternalCacheChange(ExternalObjectLog.createForRefreshDb(getId(), db.getFullName()) + .withRemoteNames(db.getRemoteName(), null))); + CatalogMetadataChanges.apply(this, ExternalObjectLog.createForRefreshDb(getId(), db.getFullName()) + .withRemoteNames(db.getRemoteName(), null)); LOG.info("finished to drop database {}.{}", getName(), dbName); } @@ -946,7 +927,8 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa } long updateTime = System.currentTimeMillis(); Env.getCurrentEnv().getEditLog().logTruncateTable( - new TruncateTableInfo(getName(), db.getFullName(), dorisTable.getName(), partitions, updateTime)); + new TruncateTableInfo(getName(), db.getFullName(), dorisTable.getName(), partitions, updateTime) + .withExternalCacheChange(CatalogMetadataChanges.table(dorisTable, false, updateTime))); Env.getCurrentEnv().getRefreshManager().refreshTableInternal(db, dorisTable, updateTime); LOG.info("finished to truncate table {}.{}.{}", getName(), dbName, tableName); } @@ -958,95 +940,29 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa */ @Override public void replayTruncateTable(TruncateTableInfo info) { - Optional> db = getDbForReplay(info.getDb()); - Optional table = db.flatMap(database -> database.getTableForReplay(info.getTable())); - if (table.isPresent()) { - Env.getCurrentEnv().getRefreshManager() - .refreshTableInternal(db.get(), table.get(), info.getUpdateTime()); - return; - } - ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); - try { - Connector replayConnector = connector; - if (isInitialized() && replayConnector != null) { - if (db.isPresent()) { - replayConnector.invalidateDb(db.get().getRemoteName()); - } else { - replayConnector.invalidateAll(); - } - } - } finally { - long dbId = Util.genIdByName(getName(), info.getDb()); - cacheMgr.invalidateTable(getId(), dbId, info.getDb(), - Util.genIdByName(getName(), info.getDb(), info.getTable()), info.getTable()); - } + CatalogMetadataChanges.apply(this, info.getExternalCacheChange() != null ? info.getExternalCacheChange() + : ExternalObjectLog.createForRefreshTable( + getId(), info.getDb(), info.getTable(), info.getUpdateTime())); } @Override - public void replayDropTable(String dbName, String tblName) { - try { - invalidateTableConnectorCacheForReplay(dbName, tblName); - } finally { - super.replayDropTable(dbName, tblName); - } + public void replayDropTable(String dbName, String tableName) { + applyTableLookupChange(dbName, tableName, null, null); } @Override public void replayDropDb(String dbName) { - try { - invalidateDatabaseConnectorCacheForReplay(dbName); - } finally { - super.replayDropDb(dbName); - } + CatalogMetadataChanges.apply(this, ExternalObjectLog.createForRefreshDb(getId(), dbName)); } @Override - public void replayCreateTable(String dbName, String tblName) { - try { - invalidateDatabaseConnectorCacheForReplay(dbName); - } finally { - super.replayCreateTable(dbName, tblName); - } + public void replayCreateTable(String dbName, String tableName) { + applyTableLookupChange(dbName, tableName, null, null); } @Override public void replayCreateDb(String dbName) { - try { - invalidateDatabaseConnectorCacheForReplay(dbName); - } finally { - super.replayCreateDb(dbName); - } - } - - // Replay never initializes a cold catalog; missing local-to-remote identity widens the live connector scope. - private void invalidateDatabaseConnectorCacheForReplay(String localDbName) { - Connector replayConnector = connector; - if (isInitialized() && replayConnector != null) { - Optional> db = getDbForReplay(localDbName); - if (db.isPresent()) { - replayConnector.invalidateDb(db.get().getRemoteName()); - } else { - replayConnector.invalidateAll(); - } - } - } - - private void invalidateTableConnectorCacheForReplay(String localDbName, String localTableName) { - Connector replayConnector = connector; - if (!isInitialized() || replayConnector == null) { - return; - } - Optional> db = getDbForReplay(localDbName); - if (!db.isPresent()) { - replayConnector.invalidateAll(); - return; - } - Optional table = db.get().getTableForReplay(localTableName); - if (table.isPresent()) { - replayConnector.invalidateTable(db.get().getRemoteName(), table.get().getRemoteName()); - } else { - replayConnector.invalidateDb(db.get().getRemoteName()); - } + CatalogMetadataChanges.apply(this, ExternalObjectLog.createForRefreshDb(getId(), dbName)); } /** @@ -1450,8 +1366,7 @@ private static ConnectorColumnPath toConnectorPath(ColumnPath columnPath) { /** Persists replay identity, then refreshes the cached table when present or the resolved transient table. */ protected void afterExternalDdl(ExternalTable externalTable, long updateTime) { Env.getCurrentEnv().getEditLog().logRefreshExternalTable( - ExternalObjectLog.createForRefreshTable( - getId(), externalTable.getDbName(), externalTable.getName(), updateTime)); + CatalogMetadataChanges.table(externalTable, false, updateTime)); ExternalTable refreshTarget = getDbForReplay(externalTable.getDbName()) .flatMap(db -> db.getTableForReplay(externalTable.getName())) .orElse(externalTable); @@ -1460,62 +1375,41 @@ protected void afterExternalDdl(ExternalTable externalTable, long updateTime) { } private void invalidateCreatedDatabaseCaches(String remoteDbName, String localDbName) { - try { - connector.invalidateDb(remoteDbName); - } finally { - invalidateCreatedDatabase(localDbName); - } + CatalogMetadataChanges.apply(this, ExternalObjectLog.createForRefreshDb(getId(), localDbName) + .withRemoteNames(remoteDbName, null)); } private void invalidateCreatedTableCaches(ExternalDatabase db, String remoteTableName, String localTableName) { - try { - connector.invalidateTable(db.getRemoteName(), remoteTableName); - } finally { - invalidateCreatedTable(db.getFullName(), localTableName); - } + applyTableLookupChange(db.getFullName(), localTableName, db.getRemoteName(), remoteTableName); + } + + private void applyTableLookupChange(String database, String table, String remoteDb, String remoteTable) { + CatalogMetadataChanges.apply(this, ExternalObjectLog.createForExplicitRefreshTable( + getId(), database, table, 0L).withRemoteNames(remoteDb, remoteTable)); } private void finishDropTable(ExternalDatabase db, ExternalTable table) { Env.getCurrentEnv().getEditLog().logDropTable( - new DropInfo(getName(), db.getFullName(), table.getName())); - try { - connector.invalidateTable(table.getRemoteDbName(), table.getRemoteName()); - } finally { - invalidateDroppedTable(db.getFullName(), table.getName()); - } + new DropInfo(getName(), db.getFullName(), table.getName()) + .withExternalCacheChange(CatalogMetadataChanges.table(table, true, 0L))); + CatalogMetadataChanges.apply(this, CatalogMetadataChanges.table(table, true, 0L)); } protected void afterExternalRename(ExternalDatabase db, ExternalTable table, String remoteNewTableName, String localNewTableName) { - String dbName = db.getFullName(); - String oldTableName = table.getName(); - // The remote rename has committed; persist replay identity before fallible local bookkeeping. - Env.getCurrentEnv().getEditLog().logRefreshExternalTable( - ExternalObjectLog.createForRenameTable( - getId(), dbName, oldTableName, localNewTableName)); - try { - try { - connector.invalidateTable(table.getRemoteDbName(), table.getRemoteName()); - } finally { - connector.invalidateTable(table.getRemoteDbName(), remoteNewTableName); - } - } finally { - try { - getDbForReplay(dbName).ifPresent( - cachedDb -> cachedDb.invalidateTableRename(oldTableName, localNewTableName)); - } finally { - try { - Env.getCurrentEnv().getConstraintManager().renameTable( - new TableNameInfo(getName(), dbName, oldTableName), - new TableNameInfo(getName(), dbName, localNewTableName)); - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableRename( - getId(), db.getId(), dbName, table.getId(), oldTableName, - Util.genIdByName(getName(), dbName, localNewTableName), localNewTableName); - } - } - } + ExternalObjectLog change = ExternalObjectLog.createForRenameTable( + getId(), db.getFullName(), table.getName(), localNewTableName) + .withRemoteNames(table.getRemoteDbName(), table.getRemoteName()); + change.setRemoteNewTableName(remoteNewTableName); + change.setDbId(db.getId()); + change.setTableId(table.getId()); + // Persist the committed remote identities before fallible local bookkeeping. + Env.getCurrentEnv().getEditLog().logRefreshExternalTable(change); + CatalogMetadataChanges.apply(this, change); + Env.getCurrentEnv().getConstraintManager().renameTable( + new TableNameInfo(getName(), db.getFullName(), table.getName()), + new TableNameInfo(getName(), db.getFullName(), localNewTableName)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java index 522ae02142915a..5749e0438df555 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Database; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.persist.gson.GsonUtils; import com.google.gson.annotations.SerializedName; @@ -29,6 +30,18 @@ import java.io.IOException; public class CreateDbInfo implements Writable { + @SerializedName("externalCacheChange") + private ExternalObjectLog externalCacheChange; + + public ExternalObjectLog getExternalCacheChange() { + return externalCacheChange; + } + + public CreateDbInfo withExternalCacheChange(ExternalObjectLog change) { + this.externalCacheChange = change; + return this; + } + @SerializedName("ctl") private String ctlName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java index 31a6bb0bc83e10..7b5faa62b521a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java @@ -21,6 +21,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.persist.gson.GsonUtils; import com.google.common.base.Strings; @@ -34,6 +35,18 @@ import java.util.Objects; public class CreateTableInfo implements Writable { + @SerializedName("externalCacheChange") + private ExternalObjectLog externalCacheChange; + + public ExternalObjectLog getExternalCacheChange() { + return externalCacheChange; + } + + public CreateTableInfo withExternalCacheChange(ExternalObjectLog change) { + this.externalCacheChange = change; + return this; + } + public static final Logger LOG = LoggerFactory.getLogger(CreateTableInfo.class); @SerializedName(value = "ctl") diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java index c9d2009521ad24..de338649921ac3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java @@ -19,6 +19,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.persist.gson.GsonUtils; import com.google.gson.annotations.SerializedName; @@ -28,6 +29,18 @@ import java.io.IOException; public class DropDbInfo implements Writable { + @SerializedName("externalCacheChange") + private ExternalObjectLog externalCacheChange; + + public ExternalObjectLog getExternalCacheChange() { + return externalCacheChange; + } + + public DropDbInfo withExternalCacheChange(ExternalObjectLog change) { + this.externalCacheChange = change; + return this; + } + @SerializedName(value = "ctl") private String ctlName; @SerializedName(value = "dbName") diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java index c2082ecc22610d..a4d6e44259e779 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java @@ -20,6 +20,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.persist.gson.GsonUtils; import com.google.common.base.Strings; @@ -30,6 +31,18 @@ import java.io.IOException; public class DropInfo implements Writable { + @SerializedName("externalCacheChange") + private ExternalObjectLog externalCacheChange; + + public ExternalObjectLog getExternalCacheChange() { + return externalCacheChange; + } + + public DropInfo withExternalCacheChange(ExternalObjectLog change) { + this.externalCacheChange = change; + return this; + } + @SerializedName(value = "ctl") private String ctl; @SerializedName(value = "db") diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 4530be5af9ae34..5cd90207998182 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -67,6 +67,7 @@ import org.apache.doris.datasource.log.InitCatalogLog; import org.apache.doris.datasource.log.InitDatabaseLog; import org.apache.doris.datasource.log.MetaIdMappingsLog; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.dictionary.Dictionary; import org.apache.doris.ha.MasterInfo; import org.apache.doris.indexpolicy.DropIndexPolicyLog; @@ -415,7 +416,12 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { ExternalCatalog ctl = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr() .getCatalog(info.getCtl()); if (ctl != null) { - ctl.replayDropTable(info.getDb(), info.getTableName()); + if (info.getExternalCacheChange() != null) { + CatalogMetadataChanges.apply( + ctl, info.getExternalCacheChange()); + } else { + ctl.replayDropTable(info.getDb(), info.getTableName()); + } } } break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java index 7edffceaf0be2c..2513a1ff21f569 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.persist.gson.GsonUtils; import com.google.common.collect.Lists; @@ -33,6 +34,18 @@ import java.util.Map; public class TruncateTableInfo implements Writable { + @SerializedName("externalCacheChange") + private ExternalObjectLog externalCacheChange; + + public ExternalObjectLog getExternalCacheChange() { + return externalCacheChange; + } + + public TruncateTableInfo withExternalCacheChange(ExternalObjectLog change) { + this.externalCacheChange = change; + return this; + } + @SerializedName(value = "ctl") private String ctl; @SerializedName(value = "dbId") diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java index 8699ff566201fc..3cbf693b77c8b4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java @@ -22,7 +22,6 @@ import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.CatalogIf; -import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.datasource.test.TestExternalDatabase; import org.apache.doris.datasource.test.TestExternalTable; @@ -97,21 +96,22 @@ public void testRefreshDatabase() throws Exception { RefreshDatabaseCommand refreshDatabaseCommand = new RefreshDatabaseCommand("test1", "db1", null); refreshDatabaseCommand.run(connectContext, null); - long l3 = db1.getLastUpdateTime(); - Assertions.assertTrue(l3 == l2); - // the table will be recreated after refresh. - // so we need to get table again - table = db1.getTable("tbl11").get(); - Assertions.assertFalse(table.isObjectCreated()); - test1.getDbNullable("db1").getTables(); - Assertions.assertFalse(table.isObjectCreated()); + TestExternalDatabase replacement = (TestExternalDatabase) test1.getDbNullable("db1"); + Assertions.assertNotSame(db1, replacement); + Assertions.assertFalse(replacement.isInitialized()); + TestExternalTable replacementTable = replacement.getTable("tbl11").get(); + Assertions.assertNotSame(table, replacementTable); + Assertions.assertFalse(replacementTable.isObjectCreated()); + Assertions.assertTrue(replacement.getLastUpdateTime() > 0); + Assertions.assertEquals(l2, db1.getLastUpdateTime()); refreshDatabaseCommand.run(connectContext, null); - Assertions.assertFalse(((ExternalDatabase) test1.getDbNullable("db1")).isInitialized()); - table.makeSureInitialized(); - long l4 = db1.getLastUpdateTime(); - Assertions.assertTrue(l4 > l3); - Assertions.assertTrue(((ExternalDatabase) test1.getDbNullable("db1")).isInitialized()); + TestExternalDatabase current = (TestExternalDatabase) test1.getDbNullable("db1"); + Assertions.assertNotSame(replacement, current); + Assertions.assertFalse(current.isInitialized()); + current.getTable("tbl11").get().makeSureInitialized(); + Assertions.assertTrue(current.isInitialized()); + Assertions.assertTrue(current.getLastUpdateTime() > 0); refreshDatabaseCommand.run(connectContext, null); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerRenameReplayTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerRenameReplayTest.java index 16084561377fac..c5028794961da5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerRenameReplayTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerRenameReplayTest.java @@ -19,6 +19,11 @@ import org.apache.doris.catalog.constraint.ConstraintManager; import org.apache.doris.common.util.Util; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.ExternalDatabase; @@ -28,9 +33,9 @@ import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.InOrder; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -40,6 +45,7 @@ public class RefreshManagerRenameReplayTest { private MockedStatic mockedEnv; + private final CatalogMetaCache root = CatalogMetaCache.unmanaged(); private CatalogMgr catalogMgr; private ExternalMetaCacheMgr metaCacheMgr; private ConstraintManager constraintManager; @@ -69,7 +75,9 @@ public void setUp() { db = mockDb(); Mockito.doReturn(catalog).when(catalogMgr).getCatalog(7L); Mockito.when(catalog.getName()).thenReturn("c"); - Mockito.when(catalog.getConnector()).thenReturn(connector); + Mockito.when(catalog.getConnectorIfPresent()).thenReturn(connector); + Mockito.when(catalog.getMetaCacheOwnerIfPresent()).thenReturn(root); + Mockito.when(catalog.resolveDatabaseNameForInvalidation("db1")).thenReturn("db1"); Mockito.doReturn(Optional.of(db)).when(catalog).getDbForReplay("db1"); Mockito.when(db.getRemoteName()).thenReturn("DB1"); dbId = Util.genIdByName("c", "db1"); @@ -79,6 +87,7 @@ public void setUp() { @AfterEach public void tearDown() { + root.close(); if (mockedEnv != null) { mockedEnv.close(); } @@ -93,24 +102,26 @@ private static ExternalDatabase mockDb() { public void testRenameReplayInvalidatesConnectorDatabaseOnFollower() { replayRename(); - InOrder order = Mockito.inOrder(connector, db, metaCacheMgr); - order.verify(connector).invalidateDb("DB1"); - order.verify(db).invalidateTableRename("t1", "t2"); - order.verify(metaCacheMgr).invalidateTableRename( - 7L, dbId, "db1", sourceTableId, "t1", destinationTableId, "t2"); + Mockito.verify(connector).cacheDependencies(ScopePath.database("DB1")); + Mockito.verify(connector, Mockito.never()).invalidateDb(Mockito.anyString()); + Mockito.verify(constraintManager).renameTable(Mockito.any(), Mockito.any()); } @Test public void testRenameReplayWithoutDatabaseObjectInvalidatesBothIdentitiesWithoutConnectorInit() { Mockito.doReturn(Optional.empty()).when(catalog).getDbForReplay("db1"); - + MetaCache old = root.create(MetaCacheDefinition.builder("old", + CacheSpec.of(true, -1, 10), key -> ScopePath.table("db1", "t1") + .in(ScopePath.Coordinates.LOCAL)).build()); + MetaCache destination = root.create(MetaCacheDefinition.builder("new", + CacheSpec.of(true, -1, 10), key -> ScopePath.table("db1", "t2") + .in(ScopePath.Coordinates.LOCAL)).build()); + old.put("key", "old"); + destination.put("key", "previous incarnation"); replayRename(); - - Mockito.verify(metaCacheMgr).invalidateTableRename( - 7L, dbId, "db1", sourceTableId, "t1", destinationTableId, "t2"); - Mockito.verify(catalog).invalidateAllConnectorCachesIfPresent(); + Assertions.assertNull(old.getIfPresent("key")); + Assertions.assertNull(destination.getIfPresent("key")); Mockito.verify(catalog, Mockito.never()).getConnector(); - Mockito.verifyNoInteractions(connector); } private void replayRename() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java index 44059931620e82..6e8285b0ee27cf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java @@ -17,574 +17,170 @@ package org.apache.doris.catalog; -import org.apache.doris.catalog.constraint.ConstraintManager; -import org.apache.doris.catalog.info.TableNameInfo; -import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.util.Util; import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; -import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.CatalogMgr; -import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; -import org.apache.doris.datasource.ExternalMetaCacheMgr; -import org.apache.doris.datasource.ExternalRowCountCache; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.log.ExternalObjectLog; -import org.apache.doris.datasource.metacache.ExternalMetaCache; -import org.apache.doris.datasource.metacache.FeMetaCacheEntry; -import org.apache.doris.datasource.metacache.MetaCacheEntryStats; import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; -import org.apache.doris.datasource.test.TestExternalCatalog; -import org.apache.doris.datasource.test.TestExternalDatabase; -import org.apache.doris.datasource.test.TestExternalTable; import org.apache.doris.persist.EditLog; -import com.google.common.util.concurrent.MoreExecutors; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.mockito.InOrder; -import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.mockito.Mockito; -import java.util.Collections; -import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; +/** Refresh entry points use one resource change for FE, connector and derived cache state. */ public class RefreshManagerTest { - private static final long CATALOG_ID = 10L; - private static final String CATALOG_NAME = "test_catalog"; - private static final String DATABASE_NAME = "db1"; - private static final String TABLE_NAME = "tbl1"; - private static final String NEW_TABLE_NAME = "tbl2"; - private static final long DATABASE_ID = Util.genIdByName(CATALOG_NAME, DATABASE_NAME); - private static final long TABLE_ID = Util.genIdByName(CATALOG_NAME, DATABASE_NAME, TABLE_NAME); - - private MockedStatic mockedEnv; - private TestExternalCatalog catalog; - private CountingDatabase database; - private RecordingExternalMetaCache engineCache; - private ExternalMetaCacheMgr metaCacheMgr; - private RecordingConstraintManager constraintManager; + private final CatalogMetaCache root = CatalogMetaCache.unmanaged(); + private MockedStatic env; + private CatalogMgr catalogs; + private PluginDrivenExternalCatalog catalog; + private ExternalDatabase db; + private ExternalTable table; + private Connector connector; private EditLog editLog; - private AtomicInteger databaseObjectLoadCalls; - private TestingCatalogMgr testingCatalogMgr; @BeforeEach - public void setUp() { - Map properties = Collections.singletonMap( - "catalog_provider.class", EmptyCatalogProvider.class.getName()); - catalog = new TestExternalCatalog(CATALOG_ID, CATALOG_NAME, "", properties, ""); - catalog.setInitializedForTest(true); - - database = new CountingDatabase(catalog, DATABASE_ID, DATABASE_NAME, DATABASE_NAME); - database.setInitializedForTest(true); - catalog.addDatabaseForTest(database); - - engineCache = new RecordingExternalMetaCache(); - databaseObjectLoadCalls = new AtomicInteger(); - metaCacheMgr = Mockito.spy(new ExternalMetaCacheMgr(true)); - Map cacheTypes = Deencapsulation.getField(metaCacheMgr, "cacheTypes"); - cacheTypes.clear(); - cacheTypes.put(engineCache.engine(), engineCache); - constraintManager = new RecordingConstraintManager(); - testingCatalogMgr = new TestingCatalogMgr(catalog); - TestingEnv testingEnv = new TestingEnv(testingCatalogMgr, metaCacheMgr, constraintManager); + public void setup() { + Env current = Mockito.mock(Env.class); + env = Mockito.mockStatic(Env.class); + env.when(Env::getCurrentEnv).thenReturn(current); + catalogs = Mockito.mock(CatalogMgr.class); editLog = Mockito.mock(EditLog.class); - testingEnv.setEditLog(editLog); - mockedEnv = Mockito.mockStatic(Env.class); - mockedEnv.when(Env::getCurrentEnv).thenReturn(testingEnv); + Mockito.when(current.getCatalogMgr()).thenReturn(catalogs); + Mockito.when(current.getEditLog()).thenReturn(editLog); + catalog = Mockito.mock(PluginDrivenExternalCatalog.class); + db = Mockito.mock(ExternalDatabase.class); + table = Mockito.mock(ExternalTable.class); + connector = Mockito.mock(Connector.class); + Mockito.doReturn(catalog).when(catalogs).getCatalog(10L); + Mockito.when(catalog.getId()).thenReturn(10L); + Mockito.when(catalog.getName()).thenReturn("c"); + Mockito.when(catalog.getMetaCacheOwnerIfPresent()).thenReturn(root); + Mockito.when(catalog.getConnectorIfPresent()).thenReturn(connector); + Mockito.when(catalog.resolveDatabaseNameForInvalidation("db")).thenReturn("db"); + Mockito.doReturn(Optional.of(db)).when(catalog).getDbForReplay("db"); + Mockito.when(db.getCatalog()).thenReturn(catalog); + Mockito.when(db.getFullName()).thenReturn("db"); + Mockito.when(db.getRemoteName()).thenReturn("DB"); + Mockito.when(db.getId()).thenReturn(Util.genIdByName("c", "db")); + Mockito.when(db.resolveTableNameForInvalidation(Mockito.anyString())).thenAnswer(inv -> inv.getArgument(0)); + Mockito.doReturn(Optional.of(table)).when(db).getTableForReplay("t"); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.getDb()).thenReturn(db); + Mockito.when(table.getId()).thenReturn(Util.genIdByName("c", "db", "t")); + Mockito.when(table.getName()).thenReturn("t"); + Mockito.when(table.getRemoteDbName()).thenReturn("DB"); + Mockito.when(table.getRemoteName()).thenReturn("T"); } @AfterEach - public void tearDown() { - if (mockedEnv != null) { - mockedEnv.close(); - mockedEnv = null; - } + public void cleanup() { + root.close(); + env.close(); } - @Test - public void testReplayRefreshDbReturnsWhenCatalogIsMissing() { - testingCatalogMgr.setCatalog(null); - Mockito.clearInvocations(metaCacheMgr); - - new RefreshManager().replayRefreshDb( - ExternalObjectLog.createForRefreshDb(CATALOG_ID, DATABASE_NAME)); - - Mockito.verifyNoInteractions(metaCacheMgr); + private MetaCache cache(String name, ScopePath scope) { + MetaCache cache = root.create(MetaCacheDefinition.builder(name, + CacheSpec.of(true, -1, 10), key -> scope).build()); + cache.put("key", "value"); + return cache; } @Test - public void testReplayHotRefreshDbInvalidatesMetadataThenRowCountOnce() { - ExternalObjectLog log = ExternalObjectLog.createForRefreshDb(CATALOG_ID, DATABASE_NAME); - - new RefreshManager().replayRefreshDb(log); - - InOrder order = Mockito.inOrder(metaCacheMgr); - order.verify(metaCacheMgr).invalidateDbMetadataCache(CATALOG_ID, DATABASE_NAME); - order.verify(metaCacheMgr).invalidateDbRowCountCache(CATALOG_ID, DATABASE_ID); - Mockito.verify(metaCacheMgr, Mockito.never()).invalidateDb( - Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + public void replayMissingCatalogDoesNothing() { + Mockito.doReturn(null).when(catalogs).getCatalog(10L); + new RefreshManager().replayRefreshDb(ExternalObjectLog.createForRefreshDb(10, "db")); + Mockito.verifyNoInteractions(catalog, connector); } @Test - public void testReplayHotRefreshDbKeepsRowCountBarrierWhenMetadataInvalidationFails() { - engineCache.failDbInvalidation = true; - ExternalObjectLog log = ExternalObjectLog.createForRefreshDb(CATALOG_ID, DATABASE_NAME); - - Assertions.assertThrows(IllegalStateException.class, () -> new RefreshManager().replayRefreshDb(log)); - - InOrder order = Mockito.inOrder(metaCacheMgr); - order.verify(metaCacheMgr).invalidateDbMetadataCache(CATALOG_ID, DATABASE_NAME); - order.verify(metaCacheMgr).invalidateDbRowCountCache(CATALOG_ID, DATABASE_ID); + public void writeRefreshPreservesLookupAndMembershipWhileUpdatingContent() { + MetaCache names = cache("names", + ScopePath.tableCollection("db").in(ScopePath.Coordinates.LOCAL)); + MetaCache object = cache("object", + ScopePath.table("db", "t").in(ScopePath.Coordinates.LOCAL)); + MetaCache source = cache("source", + ScopePath.table("DB", "T").in(ScopePath.Coordinates.REMOTE)); + new RefreshManager().refreshTableInternal(db, table, 123L); + Assertions.assertEquals("value", names.getIfPresent("key")); + Assertions.assertEquals("value", object.getIfPresent("key")); + Assertions.assertNull(source.getIfPresent("key")); + Mockito.verify(table, Mockito.atLeastOnce()).unsetObjectCreated(); + Mockito.verify(table, Mockito.atLeastOnce()).setUpdateTime(123L); + Mockito.verify(connector, Mockito.never()).invalidateTable(Mockito.anyString(), Mockito.anyString()); } @Test - public void testReplayNameBasedRefreshInvalidatesColdTableByLogName() { - seedAndEvictTableObject(); - ExternalObjectLog log = ExternalObjectLog.createForRefreshTable( - CATALOG_ID, DATABASE_NAME, TABLE_NAME, 123L); - - new RefreshManager().replayRefreshTable(log); - - assertColdTableInvalidatedByName(); - Mockito.verify(metaCacheMgr).invalidateTable( - CATALOG_ID, DATABASE_ID, DATABASE_NAME, TABLE_ID, TABLE_NAME); - } - - @Test - public void testRefreshTableRemovesRowCountLoadedDuringConnectorInvalidation() throws Exception { - PluginCatalogFixture fixture = usePluginCatalog(); - ExternalTable table = Mockito.mock(ExternalTable.class); - Mockito.when(table.getCatalog()).thenReturn(fixture.catalog); - Mockito.when(table.getDb()).thenReturn(fixture.database); - Mockito.when(table.getId()).thenReturn(TABLE_ID); - Mockito.when(table.getDbName()).thenReturn(DATABASE_NAME); - Mockito.when(table.getName()).thenReturn(TABLE_NAME); - Mockito.when(table.getRemoteName()).thenReturn(TABLE_NAME); - AtomicLong sourceRowCount = new AtomicLong(100L); - Mockito.when(table.fetchRowCountWithMetaCache(false)).thenAnswer(inv -> sourceRowCount.get()); - - CountDownLatch connectorInvalidationStarted = new CountDownLatch(1); - CountDownLatch finishConnectorInvalidation = new CountDownLatch(1); - Mockito.doAnswer(inv -> { - connectorInvalidationStarted.countDown(); - Assertions.assertTrue(finishConnectorInvalidation.await(3L, TimeUnit.SECONDS)); - sourceRowCount.set(200L); - return null; - }).when(fixture.connector).invalidateTable(DATABASE_NAME, TABLE_NAME); - - ExecutorService executor = Executors.newSingleThreadExecutor(); - try (MockedConstruction mocked = - Mockito.mockConstruction(ExternalRowCountCache.RowCountCacheLoader.class, - (loader, context) -> Mockito.when(loader.asyncLoad(Mockito.any(), Mockito.any())) - .thenAnswer(inv -> CompletableFuture.completedFuture( - Optional.of(sourceRowCount.get()))))) { - ExternalRowCountCache rowCountCache = - new ExternalRowCountCache(MoreExecutors.newDirectExecutorService()); - Deencapsulation.setField(metaCacheMgr, "rowCountCache", rowCountCache); - - Future loadDuringConnectorInvalidation = executor.submit(() -> { - Assertions.assertTrue(connectorInvalidationStarted.await(3L, TimeUnit.SECONDS)); - try { - return rowCountCache.getCachedRowCount(CATALOG_ID, DATABASE_ID, TABLE_ID, false); - } finally { - finishConnectorInvalidation.countDown(); - } - }); - new RefreshManager().refreshTableInternal(fixture.database, table, 123L); - - Assertions.assertEquals(100L, loadDuringConnectorInvalidation.get(3L, TimeUnit.SECONDS).longValue()); - Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, - rowCountCache.getCachedRowCountIfPresent(CATALOG_ID, DATABASE_ID, TABLE_ID)); - Assertions.assertEquals(200L, - rowCountCache.getCachedRowCount(CATALOG_ID, DATABASE_ID, TABLE_ID, false)); - Mockito.verify(metaCacheMgr).invalidateTable( - CATALOG_ID, DATABASE_ID, DATABASE_NAME, TABLE_ID, TABLE_NAME); - } finally { - finishConnectorInvalidation.countDown(); - executor.shutdownNow(); - } - } - - @Test - public void testRefreshTableKeepsLocalBarrierWhenConnectorInvalidationFails() { - PluginCatalogFixture fixture = usePluginCatalog(); - ExternalTable table = Mockito.mock(ExternalTable.class); - Mockito.when(table.getCatalog()).thenReturn(fixture.catalog); - Mockito.when(table.getDb()).thenReturn(fixture.database); - Mockito.when(table.getId()).thenReturn(TABLE_ID); - Mockito.when(table.getDbName()).thenReturn(DATABASE_NAME); - Mockito.when(table.getName()).thenReturn(TABLE_NAME); - Mockito.when(table.getRemoteName()).thenReturn(TABLE_NAME); - Mockito.doThrow(new IllegalStateException("connector invalidation failed")) - .when(fixture.connector).invalidateTable(DATABASE_NAME, TABLE_NAME); - - Assertions.assertThrows(IllegalStateException.class, - () -> new RefreshManager().refreshTableInternal(fixture.database, table, 123L)); - - Mockito.verify(metaCacheMgr).invalidateTable(table); + public void coldExplicitReplayUsesPersistedCoordinatesWithoutInitialization() { + Mockito.doReturn(Optional.empty()).when(catalog).getDbForReplay("db"); + MetaCache names = cache("names", + ScopePath.tableCollection("db").in(ScopePath.Coordinates.LOCAL)); + MetaCache remote = cache("remote", + ScopePath.table("DB", "T").in(ScopePath.Coordinates.REMOTE)); + new RefreshManager().replayRefreshTable(ExternalObjectLog.createForExplicitRefreshTable(10, "db", "t", 0) + .withRemoteNames("DB", "T")); + Assertions.assertNull(names.getIfPresent("key")); + Assertions.assertNull(remote.getIfPresent("key")); + Mockito.verify(catalog, Mockito.never()).getConnector(); + Mockito.verify(catalog, Mockito.never()).getFeMetaCacheOwner(); } @Test - public void testRefreshTableAfterExternalMutationLogsBeforeFallibleInvalidation() { - ExternalCatalog externalCatalog = Mockito.mock(ExternalCatalog.class); - ExternalDatabase database = Mockito.mock(ExternalDatabase.class); - ExternalTable table = Mockito.mock(ExternalTable.class); - Mockito.when(externalCatalog.getId()).thenReturn(CATALOG_ID); - Mockito.when(database.getFullName()).thenReturn(DATABASE_NAME); - Mockito.when(table.getCatalog()).thenReturn(externalCatalog); - Mockito.when(table.getDb()).thenReturn(database); - Mockito.when(table.getName()).thenReturn(TABLE_NAME); - RefreshManager refreshManager = Mockito.spy(new RefreshManager()); - IllegalStateException refreshFailure = new IllegalStateException("cache invalidation failed"); - Mockito.doThrow(refreshFailure).when(refreshManager) - .refreshTableInternal(Mockito.eq(database), Mockito.eq(table), Mockito.anyLong()); - - IllegalStateException thrown = Assertions.assertThrows(IllegalStateException.class, - () -> refreshManager.refreshTableAfterExternalMutation(table)); - Assertions.assertSame(refreshFailure, thrown); - - InOrder order = Mockito.inOrder(editLog, refreshManager); - ArgumentCaptor logCaptor = ArgumentCaptor.forClass(ExternalObjectLog.class); - order.verify(editLog).logRefreshExternalTable(logCaptor.capture()); - ExternalObjectLog log = logCaptor.getValue(); - Assertions.assertEquals(CATALOG_ID, log.getCatalogId()); - Assertions.assertEquals(DATABASE_NAME, log.getDbName()); - Assertions.assertEquals(TABLE_NAME, log.getTableName()); - order.verify(refreshManager).refreshTableInternal(database, table, log.getLastUpdateTime()); + public void mutationJournalPrecedesFallibleInvalidationAndCarriesBothCoordinates() { + RefreshManager manager = Mockito.spy(new RefreshManager()); + IllegalStateException failure = new IllegalStateException("invalidation"); + Mockito.doThrow(failure).when(manager).refreshTableInternal(Mockito.eq(db), Mockito.eq(table), + Mockito.anyLong()); + Assertions.assertSame(failure, Assertions.assertThrows(IllegalStateException.class, + () -> manager.refreshTableAfterExternalMutation(table))); + InOrder order = Mockito.inOrder(editLog, manager); + ArgumentCaptor change = ArgumentCaptor.forClass(ExternalObjectLog.class); + order.verify(editLog).logRefreshExternalTable(change.capture()); + order.verify(manager).refreshTableInternal(db, table, change.getValue().getLastUpdateTime()); + Assertions.assertEquals("db", change.getValue().getDbName()); + Assertions.assertEquals("DB", change.getValue().getRemoteDbName()); + Assertions.assertEquals("T", change.getValue().getRemoteTableName()); + Assertions.assertFalse(change.getValue().isInvalidateTableLookup()); } @Test - public void testReplayRefreshDbKeepsLocalBarrierWhenConnectorInvalidationFails() { - PluginCatalogFixture fixture = usePluginCatalog(); - Mockito.doThrow(new IllegalStateException("connector invalidation failed")) - .when(fixture.connector).invalidateDb(DATABASE_NAME); - + public void projectionFailureStillInvalidatesKnownLocalAndNumericDependencies() { + MetaCache schema = cache("schema", + ScopePath.tableMetadata("db", "t").in(ScopePath.Coordinates.LOCAL)); + MetaCache rows = cache("rows", ScopePath.table( + Long.toString(db.getId()), Long.toString(table.getId())).in(ScopePath.Coordinates.ID)); + Mockito.when(connector.cacheDependencies(Mockito.any())) + .thenThrow(new IllegalStateException("projection")); Assertions.assertThrows(IllegalStateException.class, - () -> new RefreshManager().replayRefreshDb( - ExternalObjectLog.createForRefreshDb(CATALOG_ID, DATABASE_NAME))); - - Assertions.assertFalse(fixture.database.isInitialized()); - Mockito.verify(metaCacheMgr).invalidateDbMetadataCache(CATALOG_ID, DATABASE_NAME); - Mockito.verify(metaCacheMgr).invalidateDbRowCountCache(CATALOG_ID, DATABASE_ID); + () -> new RefreshManager().refreshTableInternal(db, table, 0)); + Assertions.assertNull(schema.getIfPresent("key")); + Assertions.assertNull(rows.getIfPresent("key")); } @Test - public void testReplayRenameEvictsPreexistingSourceAndDestinationRowCounts() { - long sourceTableId = Util.genIdByName(catalog.getName(), DATABASE_NAME, TABLE_NAME); - long destinationTableId = Util.genIdByName(catalog.getName(), DATABASE_NAME, NEW_TABLE_NAME); - try (MockedConstruction mocked = - Mockito.mockConstruction(ExternalRowCountCache.RowCountCacheLoader.class, - (loader, context) -> Mockito.when(loader.asyncLoad(Mockito.any(), Mockito.any())) - .thenAnswer(inv -> { - ExternalRowCountCache.RowCountKey key = inv.getArgument(0); - long rowCount = key.getTableId() == sourceTableId ? 100L : 200L; - return CompletableFuture.completedFuture(Optional.of(rowCount)); - }))) { - ExternalRowCountCache rowCountCache = - new ExternalRowCountCache(MoreExecutors.newDirectExecutorService()); - Deencapsulation.setField(metaCacheMgr, "rowCountCache", rowCountCache); - Assertions.assertEquals(100L, - rowCountCache.getCachedRowCount(CATALOG_ID, DATABASE_ID, sourceTableId, false)); - Assertions.assertEquals(200L, - rowCountCache.getCachedRowCount(CATALOG_ID, DATABASE_ID, destinationTableId, false)); - - database.addTableForTest( - new TestExternalTable(sourceTableId, TABLE_NAME, TABLE_NAME, catalog, database)); - database.addTableForTest( - new TestExternalTable(destinationTableId, NEW_TABLE_NAME, NEW_TABLE_NAME, catalog, database)); - Assertions.assertNotNull(database.getCachedTableForTest(NEW_TABLE_NAME)); - new RefreshManager().replayRefreshTable(ExternalObjectLog.createForRenameTable( - CATALOG_ID, DATABASE_NAME, TABLE_NAME, NEW_TABLE_NAME)); - - Assertions.assertNull(database.getCachedTableForTest(TABLE_NAME)); - Assertions.assertNull(database.getCachedTableForTest(NEW_TABLE_NAME)); - Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, - rowCountCache.getCachedRowCountIfPresent(CATALOG_ID, DATABASE_ID, sourceTableId)); - Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, - rowCountCache.getCachedRowCountIfPresent(CATALOG_ID, DATABASE_ID, destinationTableId)); - } - } - - @Test - public void testReplayNameBasedRenameMigratesColdLocalState() { - seedAndEvictTableObject(); - ExternalObjectLog log = ExternalObjectLog.createForRenameTable( - CATALOG_ID, DATABASE_NAME, TABLE_NAME, NEW_TABLE_NAME); - - new RefreshManager().replayRefreshTable(log); - - assertColdRenameMigrated(); - } - - private void seedAndEvictTableObject() { - TestExternalTable table = new TestExternalTable(TABLE_ID, TABLE_NAME, TABLE_NAME, catalog, database); - database.addTableForTest(table); - database.evictTableObjectForTest(TABLE_NAME); - Assertions.assertFalse(database.getTableForReplay(TABLE_NAME).isPresent()); - } - - private void disableDatabaseObjectCacheWithTtlZero() { - disableDatabaseObjectCacheWithTtlZero(catalog, database); - } - - private void disableDatabaseObjectCacheWithTtlZero( - ExternalCatalog targetCatalog, ExternalDatabase targetDatabase) { - FeMetaCacheEntry> disabledDatabases = new FeMetaCacheEntry<>( - "ttl_zero_databases", - ignored -> { - databaseObjectLoadCalls.incrementAndGet(); - return targetDatabase; - }, - CacheSpec.of(true, 0L, 10L), - Env.getCurrentEnv().getExtMetaCacheMgr().commonRefreshExecutor(), - false); - Deencapsulation.setField(targetCatalog, "databases", disabledDatabases); - Assertions.assertFalse(targetCatalog.getDbForReplay(DATABASE_NAME).isPresent()); - } - - private Connector usePluginCatalogWithDisabledDatabaseObjectCache() { - PluginCatalogFixture fixture = usePluginCatalog(); - disableDatabaseObjectCacheWithTtlZero(fixture.catalog, fixture.database); - return fixture.connector; - } - - private PluginCatalogFixture usePluginCatalog() { - Connector connector = Mockito.mock(Connector.class); - Map properties = Collections.singletonMap("type", "test"); - PluginDrivenExternalCatalog pluginCatalog = new PluginDrivenExternalCatalog( - CATALOG_ID, CATALOG_NAME, null, properties, "", connector); - pluginCatalog.setInitializedForTest(true); - PluginDatabase pluginDatabase = - new PluginDatabase(pluginCatalog, DATABASE_ID, DATABASE_NAME, DATABASE_NAME); - pluginDatabase.setInitializedForTest(true); - pluginCatalog.addDatabaseForTest(pluginDatabase); - testingCatalogMgr.setCatalog(pluginCatalog); - return new PluginCatalogFixture(connector, pluginCatalog, pluginDatabase); - } - - private void assertColdTableInvalidatedByName() { - Assertions.assertEquals(1, engineCache.invalidateTableCalls.get()); - Assertions.assertEquals(CATALOG_ID, engineCache.lastCatalogId); - Assertions.assertEquals(DATABASE_NAME, engineCache.lastDatabaseName); - Assertions.assertEquals(TABLE_NAME, engineCache.lastTableName); - Assertions.assertFalse(database.getTableForReplay(TABLE_NAME).isPresent()); - Assertions.assertEquals(0, database.buildTableCalls.get()); - } - - private void assertColdRenameMigrated() { - Assertions.assertEquals(CATALOG_ID, engineCache.lastCatalogId); - Assertions.assertEquals(DATABASE_NAME, engineCache.lastDatabaseName); - Assertions.assertEquals(NEW_TABLE_NAME, engineCache.lastTableName); - Assertions.assertNull(database.getCachedTableNameByIdForTest(TABLE_ID)); - Assertions.assertFalse(database.getTableForReplay(TABLE_NAME).isPresent()); - Assertions.assertNull(database.getCachedTableNamesForTest()); - Assertions.assertEquals(0, database.buildTableCalls.get()); - Assertions.assertEquals(1, constraintManager.renameTableCalls.get()); - Assertions.assertEquals("test_catalog", constraintManager.oldTableName.getCtl()); - Assertions.assertEquals(DATABASE_NAME, constraintManager.oldTableName.getDb()); - Assertions.assertEquals(TABLE_NAME, constraintManager.oldTableName.getTbl()); - Assertions.assertEquals("test_catalog", constraintManager.newTableName.getCtl()); - Assertions.assertEquals(DATABASE_NAME, constraintManager.newTableName.getDb()); - Assertions.assertEquals(NEW_TABLE_NAME, constraintManager.newTableName.getTbl()); - } - - public static class EmptyCatalogProvider implements TestExternalCatalog.TestCatalogProvider { - @Override - public Map>> getMetadata() { - return Collections.emptyMap(); - } - } - - private static class CountingDatabase extends TestExternalDatabase { - private final AtomicInteger buildTableCalls = new AtomicInteger(); - - CountingDatabase(ExternalCatalog catalog, long id, String name, String remoteName) { - super(catalog, id, name, remoteName); - } - - @Override - public TestExternalTable buildTableInternal(String remoteTableName, String localTableName, long tableId, - org.apache.doris.datasource.ExternalCatalog externalCatalog, ExternalDatabase db) { - buildTableCalls.incrementAndGet(); - return super.buildTableInternal(remoteTableName, localTableName, tableId, externalCatalog, db); - } - } - - private static class PluginCatalogFixture { - private final Connector connector; - private final PluginDrivenExternalCatalog catalog; - private final PluginDatabase database; - - PluginCatalogFixture( - Connector connector, PluginDrivenExternalCatalog catalog, PluginDatabase database) { - this.connector = connector; - this.catalog = catalog; - this.database = database; - } - } - - private static class PluginDatabase extends ExternalDatabase { - private final AtomicInteger buildTableCalls = new AtomicInteger(); - - PluginDatabase(ExternalCatalog catalog, long id, String name, String remoteName) { - super(catalog, id, name, remoteName); - } - - @Override - public ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tableId, - ExternalCatalog catalog, ExternalDatabase db) { - buildTableCalls.incrementAndGet(); - return new ExternalTable(tableId, localTableName, remoteTableName, catalog, db, - TableIf.TableType.PLUGIN_EXTERNAL_TABLE); - } - } - - private static class TestingCatalogMgr extends CatalogMgr { - private CatalogIf> catalog; - - TestingCatalogMgr(CatalogIf> catalog) { - this.catalog = catalog; - } - - void setCatalog(CatalogIf> catalog) { - this.catalog = catalog; - } - - @Override - public CatalogIf> getCatalog(long id) { - return catalog != null && id == catalog.getId() ? catalog : null; - } - } - - private static class TestingEnv extends Env { - private final CatalogMgr catalogMgr; - private final ExternalMetaCacheMgr metaCacheMgr; - private final ConstraintManager constraintManager; - - TestingEnv(CatalogMgr catalogMgr, ExternalMetaCacheMgr metaCacheMgr, - ConstraintManager constraintManager) { - super(true); - this.catalogMgr = catalogMgr; - this.metaCacheMgr = metaCacheMgr; - this.constraintManager = constraintManager; - } - - @Override - public CatalogMgr getCatalogMgr() { - return catalogMgr; - } - - @Override - public ExternalMetaCacheMgr getExtMetaCacheMgr() { - return metaCacheMgr; - } - - @Override - public ConstraintManager getConstraintManager() { - return constraintManager; - } - } - - private static class RecordingConstraintManager extends ConstraintManager { - private final AtomicInteger renameTableCalls = new AtomicInteger(); - private TableNameInfo oldTableName; - private TableNameInfo newTableName; - - @Override - public void renameTable(TableNameInfo oldTableInfo, TableNameInfo newTableInfo) { - oldTableName = oldTableInfo; - newTableName = newTableInfo; - renameTableCalls.incrementAndGet(); - } - } - - private static class RecordingExternalMetaCache implements ExternalMetaCache { - private final AtomicInteger invalidateDbCalls = new AtomicInteger(); - private final AtomicInteger invalidateTableCalls = new AtomicInteger(); - private boolean failDbInvalidation; - private long lastCatalogId; - private String lastDatabaseName; - private String lastTableName; - - @Override - public String engine() { - return "default"; - } - - @Override - public List aliases() { - return Collections.emptyList(); - } - - @Override - public void initCatalog(long catalogId, Map catalogProperties) { - } - - @Override - public MetaCache entry( - long catalogId, String entryName, Class keyType, Class valueType) { - throw new UnsupportedOperationException(); - } - - @Override - public void checkCatalogInitialized(long catalogId) { - } - - @Override - public boolean isCatalogInitialized(long catalogId) { - return true; - } - - @Override - public void invalidateCatalog(long catalogId) { - } - - @Override - public void invalidateDb(long catalogId, String dbName) { - lastCatalogId = catalogId; - lastDatabaseName = dbName; - invalidateDbCalls.incrementAndGet(); - if (failDbInvalidation) { - throw new IllegalStateException("database metadata invalidation failed"); - } - } - - @Override - public void invalidateTable(long catalogId, String dbName, String tableName) { - lastCatalogId = catalogId; - lastDatabaseName = dbName; - lastTableName = tableName; - invalidateTableCalls.incrementAndGet(); - } - - @Override - public void invalidatePartitions(long catalogId, String dbName, String tableName, List partitions) { - } - - @Override - public Map stats(long catalogId) { - return Collections.emptyMap(); - } - - @Override - public void close() { - } + public void databaseReplayInvalidatesBothCoordinatesAndParentMembership() { + MetaCache names = cache("names", + ScopePath.databaseCollection().in(ScopePath.Coordinates.LOCAL)); + MetaCache source = cache("source", + ScopePath.table("DB", "T").in(ScopePath.Coordinates.REMOTE)); + new RefreshManager().replayRefreshDb(ExternalObjectLog.createForRefreshDb(10, "db") + .withRemoteNames("DB", null)); + Assertions.assertNull(names.getIfPresent("key")); + Assertions.assertNull(source.getIfPresent("key")); + Mockito.verify(catalog, Mockito.never()).getConnector(); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java index 36edaacdf0197c..0f3b073cda6c8d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java @@ -95,22 +95,16 @@ public void testRefreshTable() throws Exception { long l2 = table.getUpdateTime(); Assertions.assertTrue(l2 >= l1); TableNameInfo tableNameInfo = new TableNameInfo("test1", "db1", "tbl11"); - try { - Env.getCurrentEnv().getRefreshManager() - .handleRefreshTable(tableNameInfo.getCtl(), tableNameInfo.getDb(), tableNameInfo.getTbl(), false); - } catch (Exception e) { - // Do nothing - } + Env.getCurrentEnv().getRefreshManager().handleRefreshTable( + tableNameInfo.getCtl(), tableNameInfo.getDb(), tableNameInfo.getTbl(), false); Assertions.assertFalse(table.isObjectCreated()); - long l3 = table.getUpdateTime(); - Assertions.assertTrue(l3 >= l2); - table.getFullSchema(); - // only table.getFullSchema() can change table.lastUpdateTime - long l4 = table.getUpdateTime(); - Assertions.assertTrue(l4 > l3); - // updateTime is equal to schema update time as default - long l5 = table.getUpdateTime(); - Assertions.assertTrue(l5 >= l4); + long retiredUpdateTime = table.getUpdateTime(); + TestExternalTable replacement = (TestExternalTable) test1.getDbNullable("db1").getTable("tbl11").get(); + Assertions.assertNotSame(table, replacement); + Assertions.assertFalse(replacement.isObjectCreated()); + replacement.getFullSchema(); + Assertions.assertTrue(replacement.getUpdateTime() > 0); + Assertions.assertEquals(retiredUpdateTime, table.getUpdateTime()); // external info schema db ExternalInfoSchemaDatabase infoDb = (ExternalInfoSchemaDatabase) test1.getDbNullable(InfoSchemaDb.DATABASE_NAME); diff --git a/fe/fe-core/src/test/java/org/apache/doris/connector/ConnectorCacheDomainTest.java b/fe/fe-core/src/test/java/org/apache/doris/connector/ConnectorCacheDomainTest.java new file mode 100644 index 00000000000000..4171f483cf5756 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/connector/ConnectorCacheDomainTest.java @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector; + +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorProvider; +import org.apache.doris.connector.spi.ConnectorSession; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +class ConnectorCacheDomainTest { + @AfterEach + void reset() { + ConnectorFactory.clearPluginManager(); + } + + private void providers() { + ConnectorPluginManager manager = new ConnectorPluginManager(); + for (String type : new String[] {"primary", "sibling"}) { + manager.registerProvider(new ConnectorProvider() { + @Override + public String getType() { + return type; + } + + @Override + public Connector create(Map properties, ConnectorContext context) { + CachedConnector connector = new CachedConnector(context, type, properties); + if (properties.containsKey("fail")) { + throw new IllegalArgumentException("partial construction"); + } + return connector; + } + }); + } + ConnectorFactory.initPluginManager(manager); + } + + @Test + void factoryInstancesShareRootAndResourceInvalidationButOwnTheirLifetimes() throws Exception { + providers(); + try (CatalogMetaCache root = CatalogMetaCache.unmanaged(); + DefaultConnectorContext context = new DefaultConnectorContext("c", 7L)) { + context.bindCacheRoot(() -> root); + CachedConnector first = (CachedConnector) ConnectorFactory.createConnector( + "primary", Collections.emptyMap(), context); + CachedConnector child = (CachedConnector) first.context.createSiblingConnector( + "sibling", Collections.emptyMap()); + CachedConnector replacement = (CachedConnector) ConnectorFactory.createConnector( + "primary", Collections.emptyMap(), context); + child.cache.put("t", "child"); + replacement.cache.put("t", "new"); + first.close(); + Assertions.assertTrue(child.domain.isClosed()); + Assertions.assertEquals("new", replacement.cache.getIfPresent("t")); + Assertions.assertFalse(root.isClosed()); + Assertions.assertEquals(1, root.allEntries().size()); + Assertions.assertThrows(IllegalStateException.class, () -> first.context.createSiblingConnector( + "sibling", Collections.emptyMap())); + root.invalidateScope(ScopePath.table("db", "t").in(ScopePath.Coordinates.REMOTE)); + Assertions.assertNull(replacement.cache.getIfPresent("t")); + } + } + + @Test + void failedFactoryConstructionRollsBackOnlyItsInstanceDomain() throws Exception { + providers(); + try (CatalogMetaCache root = CatalogMetaCache.unmanaged(); + DefaultConnectorContext context = new DefaultConnectorContext("c", 7L)) { + context.bindCacheRoot(() -> root); + CachedConnector live = (CachedConnector) ConnectorFactory.createConnector( + "primary", Collections.emptyMap(), context); + live.cache.put("t", "live"); + Assertions.assertThrows(IllegalArgumentException.class, () -> ConnectorFactory.createConnector( + "primary", Collections.singletonMap("fail", "true"), context)); + Assertions.assertEquals("live", live.cache.getIfPresent("t")); + Assertions.assertEquals(1, root.allEntries().size()); + Assertions.assertFalse(root.isClosed()); + } + } + + @Test + void contextRetirementDoesNotWaitForTheCatalogMonitorAndRejectsLateDomains() throws Exception { + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch finish = new CountDownLatch(1); + ExecutorService workers = Executors.newFixedThreadPool(2); + try (CatalogMetaCache root = CatalogMetaCache.unmanaged(); + DefaultConnectorContext context = new DefaultConnectorContext("c", 7L)) { + context.bindCacheRoot(() -> { + entered.countDown(); + try { + Assertions.assertTrue(finish.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + return root; + }); + Future construction = workers.submit(() -> Assertions.assertThrows(IllegalStateException.class, + () -> context.createCacheDomain("primary", Collections.emptyMap()))); + Assertions.assertTrue(entered.await(10, TimeUnit.SECONDS)); + workers.submit(() -> { + context.close(); + return null; + }).get(10, TimeUnit.SECONDS); + finish.countDown(); + construction.get(10, TimeUnit.SECONDS); + Assertions.assertFalse(root.isClosed()); + Assertions.assertTrue(root.allEntries().isEmpty()); + } finally { + finish.countDown(); + workers.shutdownNow(); + } + } + + private static final class CachedConnector implements Connector { + private final ConnectorContext context; + private final CatalogMetaCache domain; + private final MetaCache cache; + + private CachedConnector(ConnectorContext context, String engine, Map properties) { + this.context = context; + domain = context.createCacheDomain(engine, properties); + cache = domain.create(MetaCacheDefinition.builder("table", CacheSpec.of(true, -1, 10), + key -> ScopePath.table("db", key)).build()); + } + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session) { + return null; + } + + @Override + public void close() { + domain.close(); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextSiblingTest.java b/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextSiblingTest.java index ac1ac5f2cdffd2..07d6bb5ea7c3bd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextSiblingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextSiblingTest.java @@ -67,8 +67,9 @@ void createSiblingConnector_buildsViaFactory_passesPropsAndThisContext() { "the sibling must be exactly the connector the matching provider produced"); Assertions.assertSame(props, provider.lastProperties, "the caller-synthesized properties must be forwarded to the sibling provider unchanged"); - Assertions.assertSame(ctx, provider.lastContext, - "the gateway's own context (this) must be passed to the sibling so it shares id/auth/storage"); + Assertions.assertEquals(ctx.getCatalogId(), provider.lastContext.getCatalogId()); + Assertions.assertEquals(ctx.getCatalogName(), provider.lastContext.getCatalogName()); + Assertions.assertInstanceOf(ConnectorInstanceContext.class, provider.lastContext); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java index c73ac018f444f3..288f0f8545a7d9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java @@ -20,10 +20,12 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Config; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.util.Util; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.datasource.metacache.ExternalMetaCache; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.datasource.test.TestExternalDatabase; @@ -46,14 +48,103 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; public class ExternalRowCountCacheTest { + private ExternalRowCountCache standalone(ExecutorService executor) { + Map owners = new java.util.concurrent.ConcurrentHashMap<>(); + return new ExternalRowCountCache(executor, id -> owners.computeIfAbsent(id, + ignored -> CatalogMetaCache.unmanaged())); + } + + @Test + public void testAutomaticRefreshKeepsLastSuccessfulCountUntilCompletion() throws Exception { + verifyAutomaticRefresh(false); + } + + @Test + public void testFailedAutomaticRefreshKeepsLastSuccessfulCount() throws Exception { + verifyAutomaticRefresh(true); + } + + private void verifyAutomaticRefresh(boolean failRefresh) throws Exception { + CountDownLatch refreshStarted = new CountDownLatch(1); + ExecutorService executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>()) { + @Override + protected void afterExecute(Runnable task, Throwable failure) { + // Also make an incorrectly published pending future observable before the assertion. + refreshStarted.countDown(); + } + }; + CompletableFuture> refresh = new CompletableFuture<>(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged(); + MockedConstruction mocked = + Mockito.mockConstruction(ExternalRowCountCache.RowCountCacheLoader.class, + (loader, context) -> { + Mockito.when(loader.asyncLoad(Mockito.any(), Mockito.any())) + .thenReturn(CompletableFuture.completedFuture(Optional.of(100L))) + .thenReturn(refresh); + Mockito.when(loader.doLoad(Mockito.any())).thenAnswer(call -> { + refreshStarted.countDown(); + return refresh.get(5L, TimeUnit.SECONDS); + }); + })) { + ExternalRowCountCache cache = new ExternalRowCountCache(executor, ignored -> owner); + Assertions.assertEquals(100L, cache.getCachedRowCount(1, 2, 3, false)); + Object delegate = Deencapsulation.getField(owner.entries().get("row_count"), "delegate"); + com.github.benmanes.caffeine.cache.Cache data = Deencapsulation.getField(delegate, "data"); + Object versioned = data.asMap().values().iterator().next(); + Deencapsulation.setField(versioned, "writeTimeNanos", System.nanoTime() + - TimeUnit.MINUTES.toNanos(Config.external_cache_refresh_time_minutes + 1L)); + Assertions.assertEquals(100L, cache.getCachedRowCount(1, 2, 3, false)); + Assertions.assertTrue(refreshStarted.await(3L, TimeUnit.SECONDS)); + // A refresh is pending, but callers must still see the last successfully loaded result. + Assertions.assertEquals(100L, cache.getCachedRowCountIfPresent(1, 2, 3)); + refresh.complete(failRefresh ? null : Optional.of(200L)); + executor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assertions.assertEquals(failRefresh ? 100L : 200L, cache.getCachedRowCountIfPresent(1, 2, 3)); + } finally { + refresh.complete(Optional.of(200L)); + executor.shutdownNow(); + } + } + + @Test + public void rowCountRetentionLimitRemainsGlobalAcrossCatalogRoots() throws Exception { + long original = Config.max_external_table_row_count_cache_num; + Config.max_external_table_row_count_cache_num = 1; + try (CatalogMetaCache first = CatalogMetaCache.unmanaged(); + CatalogMetaCache second = CatalogMetaCache.unmanaged(); + MockedStatic stats = Mockito.mockStatic(StatisticsUtil.class)) { + ExternalTable table = Mockito.mock(ExternalTable.class); + Mockito.when(table.fetchRowCountWithMetaCache(false)).thenReturn(100L); + stats.when(() -> StatisticsUtil.findTable(1, 11, 111)).thenReturn(table); + stats.when(() -> StatisticsUtil.findTable(2, 22, 222)).thenReturn(table); + ExternalRowCountCache cache = new ExternalRowCountCache(MoreExecutors.newDirectExecutorService(), + id -> id == 1 ? first : second); + Assertions.assertEquals(100L, cache.getCachedRowCount(1, 11, 111, false)); + Assertions.assertEquals(100L, cache.getCachedRowCount(2, 22, 222, false)); + com.github.benmanes.caffeine.cache.Cache retention = + Deencapsulation.getField(cache, "retention"); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (first.entries().get("row_count").size() + second.entries().get("row_count").size() > 1 + && System.nanoTime() < deadline) { + retention.cleanUp(); + Thread.sleep(10); + } + Assertions.assertTrue(first.entries().get("row_count").size() + + second.entries().get("row_count").size() <= 1); + } finally { + Config.max_external_table_row_count_cache_num = original; + } + } + @Test public void testRowCountKeyUsesTableIdAsCacheIdentity() { ExternalRowCountCache.RowCountKey key1 = new ExternalRowCountCache.RowCountKey(1, 2, 3); @@ -91,7 +182,7 @@ public void testGetCachedRowCountPassesFillMetaCacheToLoader() { mockedStatisticsUtil.when(() -> StatisticsUtil.findTable(1, 2, 3)).thenReturn(table); mockedStatisticsUtil.when(() -> StatisticsUtil.findTable(1, 2, 4)).thenReturn(table); - ExternalRowCountCache cache = new ExternalRowCountCache(MoreExecutors.newDirectExecutorService()); + ExternalRowCountCache cache = standalone(MoreExecutors.newDirectExecutorService()); Assertions.assertEquals(100L, cache.getCachedRowCount(1, 2, 3, true)); Assertions.assertEquals(200L, cache.getCachedRowCount(1, 2, 4, false)); } @@ -109,7 +200,7 @@ public void testInvalidateTableReloadsRowCount() { try (MockedStatic mockedStatisticsUtil = Mockito.mockStatic(StatisticsUtil.class)) { mockedStatisticsUtil.when(() -> StatisticsUtil.findTable(1, 2, 3)).thenReturn(table); - ExternalRowCountCache cache = new ExternalRowCountCache(MoreExecutors.newDirectExecutorService()); + ExternalRowCountCache cache = standalone(MoreExecutors.newDirectExecutorService()); Assertions.assertEquals(100L, cache.getCachedRowCount(1, 2, 3, false)); rowCount.set(200L); @@ -140,7 +231,7 @@ public void testCatalogAndDbInvalidationKeepUnrelatedRowCounts() { mockedStatisticsUtil.when(() -> StatisticsUtil.findTable(1, 12, 112)).thenReturn(secondTable); mockedStatisticsUtil.when(() -> StatisticsUtil.findTable(2, 11, 211)).thenReturn(otherCatalogTable); - ExternalRowCountCache cache = new ExternalRowCountCache(MoreExecutors.newDirectExecutorService()); + ExternalRowCountCache cache = standalone(MoreExecutors.newDirectExecutorService()); Assertions.assertEquals(100L, cache.getCachedRowCount(1, 11, 111, false)); Assertions.assertEquals(200L, cache.getCachedRowCount(1, 12, 112, false)); Assertions.assertEquals(300L, cache.getCachedRowCount(2, 11, 211, false)); @@ -162,7 +253,7 @@ public void testCatalogAndDbInvalidationKeepUnrelatedRowCounts() { } @Test - public void testTableInvalidationWaitsForFuturePublication() throws Exception { + public void testTableInvalidationFencesPublicationWithoutWaitingForExecutor() throws Exception { ExecutorService loadExecutor = Mockito.mock(ExecutorService.class); CountDownLatch loadSubmitted = new CountDownLatch(1); CountDownLatch allowPublication = new CountDownLatch(1); @@ -178,7 +269,7 @@ public void testTableInvalidationWaitsForFuturePublication() throws Exception { Mockito.withSettings().defaultAnswer(Mockito.CALLS_REAL_METHODS), (loader, context) -> Mockito.doReturn(Optional.of(100L)) .when(loader).doLoad(Mockito.any()))) { - ExternalRowCountCache cache = new ExternalRowCountCache(loadExecutor); + ExternalRowCountCache cache = standalone(loadExecutor); Future load = callers.submit(() -> cache.getCachedRowCount(1, 2, 3, false)); Assertions.assertTrue(loadSubmitted.await(3L, TimeUnit.SECONDS)); @@ -188,8 +279,7 @@ public void testTableInvalidationWaitsForFuturePublication() throws Exception { cache.invalidateTable(1, 2, 3); }); Assertions.assertTrue(invalidationStarted.await(3L, TimeUnit.SECONDS)); - Assertions.assertThrows(TimeoutException.class, - () -> invalidation.get(200L, TimeUnit.MILLISECONDS)); + invalidation.get(3L, TimeUnit.SECONDS); allowPublication.countDown(); Assertions.assertEquals(100L, load.get(3L, TimeUnit.SECONDS)); @@ -214,7 +304,7 @@ public void testTableInvalidationDiscardsPendingLoadCompletion() throws Exceptio return loadedRowCount; }))) { ExternalRowCountCache cache = - new ExternalRowCountCache(MoreExecutors.newDirectExecutorService()); + standalone(MoreExecutors.newDirectExecutorService()); Future load = callers.submit(() -> cache.getCachedRowCount(1, 2, 3, false)); Assertions.assertTrue(loadStarted.await(3L, TimeUnit.SECONDS)); @@ -337,7 +427,7 @@ public void testCatalogRefreshScansRowCountCacheOnceForMultipleHotDatabases() { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); ExternalRowCountCache rowCountCache = Mockito.spy( - new ExternalRowCountCache(MoreExecutors.newDirectExecutorService())); + standalone(MoreExecutors.newDirectExecutorService())); Deencapsulation.setField(metaCacheMgr, "rowCountCache", rowCountCache); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); @@ -364,7 +454,7 @@ public void testInvalidateDbMetadataCacheDoesNotAccessDbForRowCount() { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); ExternalRowCountCache rowCountCache = Mockito.spy( - new ExternalRowCountCache(MoreExecutors.newDirectExecutorService())); + standalone(MoreExecutors.newDirectExecutorService())); Deencapsulation.setField(metaCacheMgr, "rowCountCache", rowCountCache); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); @@ -399,7 +489,7 @@ public void testLoadWithException() throws Exception { }).when(mock).doLoad(Mockito.any()); })) { - ExternalRowCountCache cache = new ExternalRowCountCache(executor); + ExternalRowCountCache cache = standalone(executor); long cachedRowCount = cache.getCachedRowCount(1, 1, 1, false); Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cachedRowCount); for (int i = 0; i < 60; i++) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java index 3fde0c8a41baf1..de530783963056 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java @@ -21,11 +21,13 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.RefreshManager; +import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.util.Util; import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCacheGovernance; +import org.apache.doris.datasource.log.CatalogLog; import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.datasource.metacache.FeMetaCacheEntry; import org.apache.doris.datasource.test.TestExternalCatalog; @@ -105,6 +107,49 @@ public void releaseCatalog() throws Exception { byName.remove(catalog.getName()); } + @Test + public void catalogRenameAndReplayReactivateOnlyTheRegisteredInstance() throws Exception { + database.getTableNullable("a").getFullSchema(); + CatalogMetaCache before = catalog.getFeMetaCacheOwner(); + CatalogMgr manager = Env.getCurrentEnv().getCatalogMgr(); + String oldName = catalog.getName(); + String renamed = oldName + "_renamed"; + manager.alterCatalogName(oldName, renamed); + Assertions.assertNull(manager.getCatalog(oldName)); + Assertions.assertSame(catalog, manager.getCatalog(renamed)); + Assertions.assertTrue(before.isClosed()); + Assertions.assertNotNull(catalog.getDbNullable("db").getTableNullable("a").getFullSchema()); + CatalogMetaCache after = catalog.getFeMetaCacheOwner(); + Assertions.assertNotSame(before, after); + + CatalogLog replay = new CatalogLog(); + replay.setCatalogId(catalog.getId()); + replay.setNewCatalogName(renamed + "_replayed"); + manager.replayAlterCatalogName(replay); + Assertions.assertTrue(after.isClosed()); + Assertions.assertNotNull(catalog.getDbNullable("db").getTableNullable("a").getFullSchema()); + Assertions.assertNotSame(after, catalog.getFeMetaCacheOwner()); + Assertions.assertEquals(1, MetaCacheGovernance.catalogCaches(catalog.getId()).size()); + } + + @Test + public void disabledDatabaseCacheSupportsCaseInsensitiveDropAndRenameEvents() { + long originalTtl = Config.external_cache_expire_time_seconds_after_access; + try { + Config.external_cache_expire_time_seconds_after_access = 0; + catalog.modifyCatalogProps(Collections.singletonMap(ExternalCatalog.LOWER_CASE_TABLE_NAMES, "2")); + catalog.closeFeMetaCacheOwner(); + ExternalDatabase transientDb = catalog.getDbNullable("db"); + Assertions.assertNotNull(transientDb); + Assertions.assertFalse(transientDb.isInitialized()); + Assertions.assertDoesNotThrow(() -> transientDb.unregisterTable("MixedTable")); + Assertions.assertDoesNotThrow(() -> transientDb.invalidateTableRename("MixedTable", "RenamedTable")); + Assertions.assertNotSame(transientDb, catalog.getDbNullable("db")); + } finally { + Config.external_cache_expire_time_seconds_after_access = originalTtl; + } + } + @Test public void fullRefreshSharesOwnerAndPreservesSibling() throws Exception { TestExternalTable a = database.getTableNullable("a"); @@ -212,9 +257,9 @@ public void oldDatabaseCleanupDoesNotInvalidateReplacementOrAccumulateEntries() previous.releaseCachedEntries(); Assertions.assertSame(replacement, database.getTableNullable("a")); Assertions.assertSame(schema, replacement.getFullSchema()); - Assertions.assertEquals(5, catalog.getFeMetaCacheOwner().entries().size()); + Assertions.assertEquals(7, catalog.getFeMetaCacheOwner().entries().size()); Assertions.assertNotNull(previous.getTableNullable("a")); - Assertions.assertEquals(5, catalog.getFeMetaCacheOwner().entries().size()); + Assertions.assertEquals(7, catalog.getFeMetaCacheOwner().entries().size()); } } @@ -276,7 +321,7 @@ public void weightedPayloadBudgetPreservesLookupCachingAndIdResolution() { Assertions.assertEquals(1, catalog.nameLoads.get()); Assertions.assertSame(current, catalog.getDbNullable(current.getId())); Assertions.assertSame(table, current.getTableNullable(table.getId())); - Assertions.assertEquals(5, catalog.getFeMetaCacheOwner().entries().size()); + Assertions.assertEquals(7, catalog.getFeMetaCacheOwner().entries().size()); Assertions.assertTrue(Env.getCurrentEnv().getExtMetaCacheMgr().getCatalogCacheStats(catalog.getId()).stream() .filter(stat -> "schema".equals(stat.getEntryName())) .anyMatch(stat -> stat.getEntryStats().getEstimatedWeight() > 0)); @@ -313,10 +358,10 @@ public void schemaEntryReconfigurationDoesNotRetireTheBorrowedCatalogOwner() { @Test public void namingOnlyOwnerIsVisibleInStatsWithoutSchemaInitialization() { database.getTableNamesWithLock(); - Assertions.assertEquals(4, Env.getCurrentEnv().getExtMetaCacheMgr() + Assertions.assertEquals(6, Env.getCurrentEnv().getExtMetaCacheMgr() .getCatalogCacheStats(catalog.getId()).size()); database.getTableNullable("a").getFullSchema(); - Assertions.assertEquals(5, Env.getCurrentEnv().getExtMetaCacheMgr() + Assertions.assertEquals(7, Env.getCurrentEnv().getExtMetaCacheMgr() .getCatalogCacheStats(catalog.getId()).size()); } @@ -351,7 +396,7 @@ public void databaseRetirementDoesNotCloseAnAcceptedObjectLoad() throws Exceptio Assertions.assertNotSame(replacement, old.get(10, TimeUnit.SECONDS)); Assertions.assertSame(replacement, database.getTableNullable("a")); Assertions.assertNull(previous.getCachedTableNameByIdForTest(replacement.getId())); - Assertions.assertEquals(4, catalog.getFeMetaCacheOwner().entries().size()); + Assertions.assertEquals(6, catalog.getFeMetaCacheOwner().entries().size()); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverClassLoaderTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverClassLoaderTest.java index 759e984d2c14f5..a75a5037518fbb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverClassLoaderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverClassLoaderTest.java @@ -54,7 +54,7 @@ public void eventSourceAcquisitionUsesConnectorClassLoaderAndRestoresCaller() th connectorLoader, new Class[] {Connector.class}, (proxy, method, args) -> { if ("getEventSource".equals(method.getName())) { acquisitionClassLoader.set(Thread.currentThread().getContextClassLoader()); - return null; + return Collections.emptyList(); } if ("getMetadata".equals(method.getName()) || "close".equals(method.getName())) { return null; @@ -187,12 +187,12 @@ private void assertDescriptorApplicationClassLoader(boolean failInvalidation) th }); Connector connector = (Connector) Proxy.newProxyInstance( connectorLoader, new Class[] {Connector.class}, (proxy, method, args) -> { - if ("invalidateDb".equals(method.getName())) { + if ("cacheDependencies".equals(method.getName())) { invalidationClassLoader.set(Thread.currentThread().getContextClassLoader()); if (failInvalidation) { throw new IllegalStateException("expected invalidation failure"); } - return null; + return Collections.emptyList(); } if ("getMetadata".equals(method.getName()) || "close".equals(method.getName())) { return null; @@ -204,6 +204,8 @@ private void assertDescriptorApplicationClassLoader(boolean failInvalidation) th Mockito.when(catalog.getId()).thenReturn(1L); Mockito.when(catalog.getName()).thenReturn("test_catalog"); Mockito.when(catalog.canonicalLocalDatabaseNameFromRemote("db1")).thenReturn("db1"); + Mockito.when(catalog.resolveDatabaseNameForInvalidation("db1")).thenReturn("db1"); + Mockito.when(catalog.getConnectorIfPresent()).thenReturn(connector); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); Mockito.doAnswer(invocation -> { mutationClassLoader.set(Thread.currentThread().getContextClassLoader()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverMappedNameTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverMappedNameTest.java index 4d68fd102721f4..95b4065aedb7b7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverMappedNameTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverMappedNameTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.RefreshManager; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.event.MetastoreChangeDescriptor; import org.apache.doris.connector.spi.event.MetastoreChangeDescriptor.Op; @@ -47,12 +48,13 @@ public void databaseRenameKeepsRemoteConnectorKeysAndUsesLocalFeNames() throws E apply(catalog, connector, descriptor, env); InOrder inOrder = Mockito.inOrder(connector, catalogMgr); - inOrder.verify(connector).invalidateDb("OldDb"); - inOrder.verify(connector).invalidateDb("NewDb"); + inOrder.verify(connector).cacheDependencies(ScopePath.database("OldDb")); + inOrder.verify(connector).cacheDependencies(ScopePath.database("NewDb")); inOrder.verify(catalogMgr).unregisterExternalDatabaseFromEvent("local_OldDb", "test_catalog"); inOrder.verify(catalogMgr).registerExternalDatabaseFromEvent( "NewDb", "local_NewDb", "test_catalog"); - Mockito.verifyNoMoreInteractions(connector, catalogMgr); + Mockito.verifyNoMoreInteractions(catalogMgr); + Mockito.verify(connector, Mockito.never()).getMetadata(Mockito.any()); } @Test @@ -68,13 +70,14 @@ public void tableRenameKeepsRemoteConnectorKeysAndUsesBothTableIdentities() thro apply(catalog, connector, descriptor, env); InOrder inOrder = Mockito.inOrder(connector, catalogMgr); - inOrder.verify(connector).invalidateTable("OldDb", "OldTable"); - inOrder.verify(connector).invalidateTable("NewDb", "NewTable"); + inOrder.verify(connector).cacheDependencies(ScopePath.table("OldDb", "OldTable")); + inOrder.verify(connector).cacheDependencies(ScopePath.table("NewDb", "NewTable")); inOrder.verify(catalogMgr).unregisterExternalTableFromEvent( "local_OldDb", "local_OldDb_OldTable", "test_catalog"); inOrder.verify(catalogMgr).registerExternalTableFromEvent( "local_NewDb", "NewTable", "local_NewDb_NewTable", "test_catalog", 2L); - Mockito.verifyNoMoreInteractions(connector, catalogMgr); + Mockito.verifyNoMoreInteractions(catalogMgr); + Mockito.verify(connector, Mockito.never()).getMetadata(Mockito.any()); } @Test @@ -122,7 +125,7 @@ private static void apply(PluginDrivenExternalCatalog catalog, Connector connect try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); Deencapsulation.invoke( - new MetastoreEventSyncDriver(), "applyOne", catalog, connector, descriptor); + new MetastoreEventSyncDriver(), "applyOne", catalog, descriptor); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverRenameTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverRenameTest.java index 5d447f4a2a12be..4a47d4b6f7b6bb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverRenameTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverRenameTest.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.event.MetastoreChangeDescriptor; import org.apache.doris.connector.spi.event.MetastoreChangeDescriptor.Op; @@ -46,15 +47,16 @@ public void databaseRenameAlwaysCleansOldNameAndRegistersNewName() throws Except try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); Deencapsulation.invoke( - new MetastoreEventSyncDriver(), "applyOne", catalog, connector, descriptor); + new MetastoreEventSyncDriver(), "applyOne", catalog, descriptor); } InOrder inOrder = Mockito.inOrder(connector, catalogMgr); - inOrder.verify(connector).invalidateDb("OldDb"); - inOrder.verify(connector).invalidateDb("NewDb"); + inOrder.verify(connector).cacheDependencies(ScopePath.database("OldDb")); + inOrder.verify(connector).cacheDependencies(ScopePath.database("NewDb")); inOrder.verify(catalogMgr).unregisterExternalDatabaseFromEvent("OldDb", "test_catalog"); inOrder.verify(catalogMgr).registerExternalDatabaseFromEvent("NewDb", "NewDb", "test_catalog"); - Mockito.verifyNoMoreInteractions(connector, catalogMgr); + Mockito.verifyNoMoreInteractions(catalogMgr); + Mockito.verify(connector, Mockito.never()).getMetadata(Mockito.any()); } @Test @@ -70,17 +72,18 @@ public void tableRenameAlwaysCleansOldNameAndRegistersNewName() throws Exception try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); Deencapsulation.invoke( - new MetastoreEventSyncDriver(), "applyOne", catalog, connector, descriptor); + new MetastoreEventSyncDriver(), "applyOne", catalog, descriptor); } InOrder inOrder = Mockito.inOrder(connector, catalogMgr); - inOrder.verify(connector).invalidateTable("OldDb", "OldTable"); - inOrder.verify(connector).invalidateTable("NewDb", "NewTable"); + inOrder.verify(connector).cacheDependencies(ScopePath.table("OldDb", "OldTable")); + inOrder.verify(connector).cacheDependencies(ScopePath.table("NewDb", "NewTable")); inOrder.verify(catalogMgr).unregisterExternalTableFromEvent( "OldDb", "OldTable", "test_catalog"); inOrder.verify(catalogMgr).registerExternalTableFromEvent( "NewDb", "NewTable", "NewTable", "test_catalog", 2L); - Mockito.verifyNoMoreInteractions(connector, catalogMgr); + Mockito.verifyNoMoreInteractions(catalogMgr); + Mockito.verify(connector, Mockito.never()).getMetadata(Mockito.any()); } @Test @@ -96,16 +99,17 @@ public void sameNameViewRecreateInvalidatesConnectorOnceBeforeReplacement() thro try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); Deencapsulation.invoke( - new MetastoreEventSyncDriver(), "applyOne", catalog, connector, descriptor); + new MetastoreEventSyncDriver(), "applyOne", catalog, descriptor); } InOrder inOrder = Mockito.inOrder(connector, catalogMgr); - inOrder.verify(connector).invalidateTable("db1", "view1"); + inOrder.verify(connector).cacheDependencies(ScopePath.table("db1", "view1")); inOrder.verify(catalogMgr).unregisterExternalTableFromEvent( "db1", "view1", "test_catalog"); inOrder.verify(catalogMgr).registerExternalTableFromEvent( "db1", "view1", "view1", "test_catalog", 2L); - Mockito.verifyNoMoreInteractions(connector, catalogMgr); + Mockito.verifyNoMoreInteractions(catalogMgr); + Mockito.verify(connector, Mockito.never()).getMetadata(Mockito.any()); } @Test @@ -123,18 +127,19 @@ public void sameNameTableDropCreateInvalidatesBothIncarnations() throws Exceptio try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); MetastoreEventSyncDriver driver = new MetastoreEventSyncDriver(); - Deencapsulation.invoke(driver, "applyOne", catalog, connector, drop); - Deencapsulation.invoke(driver, "applyOne", catalog, connector, create); + Deencapsulation.invoke(driver, "applyOne", catalog, drop); + Deencapsulation.invoke(driver, "applyOne", catalog, create); } InOrder inOrder = Mockito.inOrder(connector, catalogMgr); - inOrder.verify(connector).invalidateTable("db1", "tbl1"); + inOrder.verify(connector).cacheDependencies(ScopePath.table("db1", "tbl1")); inOrder.verify(catalogMgr).unregisterExternalTableFromEvent( "db1", "tbl1", "test_catalog"); - inOrder.verify(connector).invalidateTable("db1", "tbl1"); + inOrder.verify(connector).cacheDependencies(ScopePath.table("db1", "tbl1")); inOrder.verify(catalogMgr).registerExternalTableFromEvent( "db1", "tbl1", "tbl1", "test_catalog", 3L); - Mockito.verifyNoMoreInteractions(connector, catalogMgr); + Mockito.verifyNoMoreInteractions(catalogMgr); + Mockito.verify(connector, Mockito.never()).getMetadata(Mockito.any()); } @Test @@ -152,16 +157,17 @@ public void sameNameDatabaseDropCreateInvalidatesBothIncarnations() throws Excep try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); MetastoreEventSyncDriver driver = new MetastoreEventSyncDriver(); - Deencapsulation.invoke(driver, "applyOne", catalog, connector, drop); - Deencapsulation.invoke(driver, "applyOne", catalog, connector, create); + Deencapsulation.invoke(driver, "applyOne", catalog, drop); + Deencapsulation.invoke(driver, "applyOne", catalog, create); } InOrder inOrder = Mockito.inOrder(connector, catalogMgr); - inOrder.verify(connector).invalidateDb("db1"); + inOrder.verify(connector).cacheDependencies(ScopePath.database("db1")); inOrder.verify(catalogMgr).unregisterExternalDatabaseFromEvent("db1", "test_catalog"); - inOrder.verify(connector).invalidateDb("db1"); + inOrder.verify(connector).cacheDependencies(ScopePath.database("db1")); inOrder.verify(catalogMgr).registerExternalDatabaseFromEvent("db1", "db1", "test_catalog"); - Mockito.verifyNoMoreInteractions(connector, catalogMgr); + Mockito.verifyNoMoreInteractions(catalogMgr); + Mockito.verify(connector, Mockito.never()).getMetadata(Mockito.any()); } private static class IdentityEventCatalog extends PluginDrivenExternalCatalog { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CatalogMetadataChangesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CatalogMetadataChangesTest.java new file mode 100644 index 00000000000000..ef15261f72f911 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CatalogMetadataChangesTest.java @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.util.Util; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.datasource.log.ExternalObjectLog; +import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; +import org.apache.doris.persist.CreateDbInfo; +import org.apache.doris.persist.CreateTableInfo; +import org.apache.doris.persist.DropDbInfo; +import org.apache.doris.persist.DropInfo; +import org.apache.doris.persist.TruncateTableInfo; +import org.apache.doris.persist.gson.GsonUtils; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.Optional; + +class CatalogMetadataChangesTest { + private final CatalogMetaCache root = CatalogMetaCache.unmanaged(); + private PluginDrivenExternalCatalog catalog; + private MockedStatic env; + + @BeforeEach + void setup() { + env = Mockito.mockStatic(Env.class); + env.when(Env::getCurrentEnv).thenReturn(Mockito.mock(Env.class)); + catalog = Mockito.mock(PluginDrivenExternalCatalog.class); + Mockito.when(catalog.getName()).thenReturn("c"); + Mockito.when(catalog.getMetaCacheOwnerIfPresent()).thenReturn(root); + Mockito.when(catalog.getDbForReplay("db")).thenReturn(Optional.empty()); + Mockito.when(catalog.resolveDatabaseNameForInvalidation("db")).thenReturn("db"); + } + + @AfterEach + void cleanup() { + root.close(); + env.close(); + } + + private MetaCache cache(String name, ScopePath scope) { + MetaCache cache = root.create(MetaCacheDefinition.builder(name, + CacheSpec.of(true, -1, 100), key -> scope).build()); + cache.put("key", "value"); + return cache; + } + + private ScopePath local(ScopePath scope) { + return scope.in(ScopePath.Coordinates.LOCAL); + } + + private ScopePath remote(ScopePath scope) { + return scope.in(ScopePath.Coordinates.REMOTE); + } + + @Test + void contentChangePreservesNamesAndLookupButInvalidatesEveryContentDependency() { + MetaCache names = cache("names", local(ScopePath.tableCollection("db"))); + MetaCache object = cache("object", local(ScopePath.table("db", "t"))); + MetaCache schema = cache("schema", local(ScopePath.tableMetadata("db", "t"))); + MetaCache hive = cache("hive", remote(ScopePath.table("DB", "T"))); + MetaCache iceberg = cache("iceberg", remote(ScopePath.table("DB", "T"))); + MetaCache other = cache("other", remote(ScopePath.table("DB", "OTHER"))); + MetaCache rows = cache("rows", ScopePath.table( + Long.toString(Util.genIdByName("c", "db")), Long.toString(Util.genIdByName("c", "db", "t"))) + .in(ScopePath.Coordinates.ID)); + CatalogMetadataChanges.apply(catalog, ExternalObjectLog.createForRefreshTable(1, "db", "t", 0) + .withRemoteNames("DB", "T")); + Assertions.assertEquals("value", names.getIfPresent("key")); + Assertions.assertEquals("value", object.getIfPresent("key")); + Assertions.assertNull(schema.getIfPresent("key")); + Assertions.assertNull(hive.getIfPresent("key")); + Assertions.assertNull(iceberg.getIfPresent("key")); + Assertions.assertNull(rows.getIfPresent("key")); + Assertions.assertEquals("value", other.getIfPresent("key")); + Mockito.verify(catalog, Mockito.never()).getConnector(); + Mockito.verify(catalog, Mockito.never()).getFeMetaCacheOwner(); + } + + @Test + void coldRenameUsesPersistedRemoteNamesAndFencesOldAndNewResourcesOnly() { + MetaCache old = cache("old", local(ScopePath.table("db", "old"))); + MetaCache replacement = cache("new", local(ScopePath.table("db", "new"))); + MetaCache remoteOld = cache("remoteOld", remote(ScopePath.table("DB", "OLD"))); + MetaCache remoteNew = cache("remoteNew", remote(ScopePath.table("DB", "NEW"))); + MetaCache sibling = cache("sibling", remote(ScopePath.table("DB", "OTHER"))); + ExternalObjectLog change = ExternalObjectLog.createForRenameTable(1, "db", "old", "new") + .withRemoteNames("DB", "OLD"); + change.setRemoteNewTableName("NEW"); + change = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(change), ExternalObjectLog.class); + CatalogMetadataChanges.apply(catalog, change); + Assertions.assertNull(old.getIfPresent("key")); + Assertions.assertNull(replacement.getIfPresent("key")); + Assertions.assertNull(remoteOld.getIfPresent("key")); + Assertions.assertNull(remoteNew.getIfPresent("key")); + Assertions.assertEquals("value", sibling.getIfPresent("key")); + } + + @Test + void oldColdLogWidensOnlyRemoteCoordinates() { + MetaCache remote = cache("remote", remote(ScopePath.table("ANY", "T"))); + MetaCache names = cache("names", local(ScopePath.tableCollection("db"))); + MetaCache sibling = cache("sibling", local(ScopePath.table("db", "other"))); + CatalogMetadataChanges.apply(catalog, ExternalObjectLog.createForRefreshTable(1, "db", "t", 0)); + Assertions.assertNull(remote.getIfPresent("key")); + Assertions.assertEquals("value", names.getIfPresent("key")); + Assertions.assertEquals("value", sibling.getIfPresent("key")); + } + + @Test + void gatewayDependencyProjectionParticipatesInTheSameChange() { + Connector connector = Mockito.mock(Connector.class); + Mockito.when(catalog.getConnectorIfPresent()).thenReturn(connector); + Mockito.when(connector.cacheDependencies(ScopePath.table("DB", "T"))) + .thenReturn(Collections.singletonList(ScopePath.table("lake", "physical"))); + MetaCache source = cache("source", remote(ScopePath.table("DB", "T"))); + MetaCache lake = cache("lake", remote(ScopePath.table("lake", "physical"))); + CatalogMetadataChanges.apply(catalog, ExternalObjectLog.createForRefreshTable(1, "db", "t", 0) + .withRemoteNames("DB", "T")); + Assertions.assertNull(source.getIfPresent("key")); + Assertions.assertNull(lake.getIfPresent("key")); + Mockito.verify(connector, Mockito.never()).invalidateTable(Mockito.anyString(), Mockito.anyString()); + } + + @Test + void ddlJournalRecordsPreserveTheSameChangeForReplay() { + ExternalObjectLog change = ExternalObjectLog.createForExplicitRefreshTable(1, "db", "t", 0) + .withRemoteNames("DB", "T"); + Object[] records = { + new CreateDbInfo("c", "db", null).withExternalCacheChange(change), + new CreateTableInfo("c", "db", "t").withExternalCacheChange(change), + new DropDbInfo("c", "db").withExternalCacheChange(change), + new DropInfo("c", "db", "t").withExternalCacheChange(change), + new TruncateTableInfo("c", "db", "t", null, 0).withExternalCacheChange(change) + }; + for (Object record : records) { + String json = GsonUtils.GSON.toJson(record); + Object restored = GsonUtils.GSON.fromJson(json, record.getClass()); + Assertions.assertEquals(GsonUtils.GSON.toJsonTree(record).getAsJsonObject().get("externalCacheChange"), + GsonUtils.GSON.toJsonTree(restored).getAsJsonObject().get("externalCacheChange")); + Assertions.assertTrue(json.contains("remoteTableName")); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java index cb422cc548dd18..b2a997ea713ae5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java @@ -91,6 +91,40 @@ public void testRetiredReadThroughValuesAreDiscardedWithoutPublishingActions() { } } + @Test + public void testRetirementPinsAuxiliaryStateUntilAcceptedActionFinishes() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + ExecutorService worker = Executors.newSingleThreadExecutor(); + CountDownLatch actionEntered = new CountDownLatch(1); + CountDownLatch finishAction = new CountDownLatch(1); + AtomicInteger cleanups = new AtomicInteger(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + FeMetaCacheEntry entry = FeMetaCacheEntry.sharedLookup(owner, + ScopePath::database, ScopePath.catalog(), "objects", + String::length, ENABLED, executor, false, 8, (key, value, cause) -> { }); + entry.onRetired(cleanups::incrementAndGet); + Assertions.assertEquals(2, entry.get("db")); + Future read = worker.submit(() -> entry.getAndRunIfCurrent("db", (key, value) -> { + actionEntered.countDown(); + await(finishAction); + Assertions.assertEquals(0, cleanups.get()); + })); + Assertions.assertTrue(actionEntered.await(3L, TimeUnit.SECONDS)); + entry.close(); + Assertions.assertEquals(0, cleanups.get()); + finishAction.countDown(); + Assertions.assertEquals(2, read.get(3L, TimeUnit.SECONDS)); + Assertions.assertEquals(1, cleanups.get()); + Assertions.assertTrue(owner.entries().isEmpty()); + entry.close(); + Assertions.assertEquals(1, cleanups.get()); + } finally { + finishAction.countDown(); + worker.shutdownNow(); + executor.shutdownNow(); + } + } + @Test public void testRejectedWeightedMutationReturnsWithoutRetryingForever() throws Exception { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/IdNameIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/IdNameIndexTest.java index 4ea6773e24db34..176f8832817333 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/IdNameIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/IdNameIndexTest.java @@ -17,6 +17,12 @@ package org.apache.doris.datasource.metacache; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -160,6 +166,27 @@ public void testConcurrentConflictingPublicationKeepsOneCompleteMapping() throws } } + @Test + public void identitySurvivesObjectEvictionButNotSemanticRefresh() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + IdNameIndex index = new IdNameIndex("table"); + ScopePath scope = ScopePath.table("db", "t").in(ScopePath.Coordinates.LOCAL); + index.bind(root, "ids", name -> ScopePath.table("db", name)); + MetaCache objects = root.create(MetaCacheDefinition.builder("objects", + CacheSpec.of(true, -1, 1), key -> scope).build()); + objects.put("t", "old"); + index.put(1, "t"); + objects.invalidateKey("t"); + Assertions.assertEquals("t", index.getName(1)); + root.invalidateScope(scope); + Assertions.assertNull(index.getName(1)); + index.put(2, "t"); + Assertions.assertEquals("t", index.getName(2)); + Assertions.assertNull(index.getName(1)); + index.close(root); + } + } + private static boolean putAfterStart( IdNameIndex index, CountDownLatch start, long id, String name) throws InterruptedException { start.await(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogCacheTest.java index 7c61d641747475..f13f4182a16488 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogCacheTest.java @@ -18,64 +18,65 @@ package org.apache.doris.datasource.plugin; import org.apache.doris.catalog.Env; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; +import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.mockito.InOrder; import org.mockito.MockedStatic; import org.mockito.Mockito; -import java.util.HashMap; -import java.util.Map; +import java.util.Collections; -/** Verifies connector-owned caches follow catalog refresh semantics. */ +/** Catalog refresh reaches connector domains through the root even before schema caches are initialized. */ public class PluginDrivenExternalCatalogCacheTest { - - private static PluginDrivenExternalCatalog catalogWith(Connector connector) { - Map props = new HashMap<>(); - props.put("type", "iceberg"); - return new PluginDrivenExternalCatalog(1L, "test_ctl", null, props, "", connector); - } - @Test public void refreshCatalogWithInvalidateDropsConnectorCaches() { - Connector connector = Mockito.mock(Connector.class); - PluginDrivenExternalCatalog catalog = catalogWith(connector); - Env env = Mockito.mock(Env.class); - ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); - Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - catalog.onRefreshCache(true); - } - // Connector state must be cleared before the engine invalidates row counts derived from it. - InOrder order = Mockito.inOrder(connector, cacheMgr); - order.verify(connector).invalidateAll(); - order.verify(cacheMgr).invalidateCatalog(1L); + refresh(true); } @Test - public void refreshCatalogWithoutInvalidateDoesNotTouchConnector() { + public void refreshCatalogWithoutInvalidatePreservesConnectorCaches() { + refresh(false); + } + + private void refresh(boolean invalidate) { Connector connector = Mockito.mock(Connector.class); - PluginDrivenExternalCatalog catalog = catalogWith(connector); - Env env = Mockito.mock(Env.class); - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - catalog.onRefreshCache(false); + PluginDrivenExternalCatalog catalog = new PluginDrivenExternalCatalog(1L, "c", null, + Collections.singletonMap("type", "iceberg"), "", connector); + try (CatalogMetaCache root = catalog.getFeMetaCacheOwner()) { + MetaCache remote = root.openDomain("iceberg", ScopePath.Coordinates.REMOTE, + getClass().getClassLoader()).create(MetaCacheDefinition.builder("table", + CacheSpec.of(true, -1, 10), key -> ScopePath.table("DB", key)).build()); + remote.put("T", "old"); + Env env = Mockito.mock(Env.class); + CatalogMgr catalogs = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogs).getCatalog(1L); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogs); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(new ExternalMetaCacheMgr(true)); + try (MockedStatic current = Mockito.mockStatic(Env.class)) { + current.when(Env::getCurrentEnv).thenReturn(env); + catalog.onRefreshCache(invalidate); + } + Assertions.assertEquals(invalidate ? null : "old", remote.getIfPresent("T")); + Mockito.verify(connector, Mockito.never()).invalidateAll(); } - Mockito.verify(connector, Mockito.never()).invalidateAll(); } @Test public void refreshCatalogWithNullConnectorIsSafe() { - // A reset catalog has no connector to invalidate. - PluginDrivenExternalCatalog catalog = catalogWith(null); + PluginDrivenExternalCatalog catalog = new PluginDrivenExternalCatalog(1L, "c", null, + Collections.singletonMap("type", "iceberg"), "", null); Env env = Mockito.mock(Env.class); Mockito.when(env.getExtMetaCacheMgr()).thenReturn(Mockito.mock(ExternalMetaCacheMgr.class)); - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); + try (MockedStatic current = Mockito.mockStatic(Env.class)) { + current.when(Env::getCurrentEnv).thenReturn(env); Assertions.assertDoesNotThrow(() -> catalog.onRefreshCache(true)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java index 18a0e80c8998aa..6bfe4cf49719c3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java @@ -72,6 +72,8 @@ public class PluginDrivenExternalCatalogConcurrencyTest { public void setUp() { Env env = Mockito.mock(Env.class); cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(cacheMgr.commonRefreshExecutor()) + .thenReturn(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService()); AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); Mockito.when(env.getAccessManager()).thenReturn(accessManager); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogDdlRoutingTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogDdlRoutingTest.java index 9605674d694a6e..29fe3023bfdc25 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogDdlRoutingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogDdlRoutingTest.java @@ -35,6 +35,8 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.util.Util; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.ddl.CreateTableInfoToConnectorRequestConverter; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorColumn; @@ -97,6 +99,7 @@ public class PluginDrivenExternalCatalogDdlRoutingTest { private ConnectorMetadata metadata; private ConnectorSession session; private TestablePluginCatalog catalog; + private CatalogMetaCache cacheRoot; @BeforeEach public void setUp() { @@ -126,10 +129,18 @@ public void setUp() { Mockito.when(mockEnv.getRefreshManager()).thenReturn(mockRefreshManager); Mockito.when(mockEnv.getConstraintManager()).thenReturn(mockConstraintManager); Mockito.when(mockEnv.getExtMetaCacheMgr()).thenReturn(mockMetaCacheMgr); + Mockito.when(mockMetaCacheMgr.commonRefreshExecutor()) + .thenReturn(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService()); + cacheRoot = Mockito.spy(CatalogMetaCache.unmanaged()); + Deencapsulation.setField(catalog, "feMetaCacheOwner", cacheRoot); + org.apache.doris.datasource.CatalogMgr catalogs = Mockito.mock(org.apache.doris.datasource.CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogs).getCatalog(1L); + Mockito.when(mockEnv.getCatalogMgr()).thenReturn(catalogs); } @AfterEach public void tearDown() { + cacheRoot.close(); if (mockedEnv != null) { mockedEnv.close(); } @@ -146,11 +157,9 @@ public void testCreateDbRoutesToConnectorAndInvalidatesCache() throws Exception Mockito.verify(metadata).createDatabase(session, "db1", props); Mockito.verify(mockEditLog).logCreateDb(Mockito.any()); - Assertions.assertEquals(1, catalog.resetMetaCacheNamesCount); - InOrder order = Mockito.inOrder(connector, mockMetaCacheMgr); - order.verify(connector).invalidateDb("db1"); - order.verify(mockMetaCacheMgr).invalidateDb( - 1L, Util.genIdByName("test-catalog", "db1"), "db1"); + verifyInvalidation(ScopePath.databaseCollection().in(ScopePath.Coordinates.LOCAL)); + verifyRemoteDatabaseInvalidation("db1"); + verifyDeterministicDatabaseInvalidation("db1"); } @Test @@ -162,9 +171,8 @@ public void testCreateDbPersistsCanonicalLocalName() throws Exception { ArgumentCaptor log = ArgumentCaptor.forClass(CreateDbInfo.class); Mockito.verify(mockEditLog).logCreateDb(log.capture()); Assertions.assertEquals("local_db", log.getValue().getDbName()); - Mockito.verify(connector).invalidateDb("REMOTE_DB"); - Mockito.verify(mockMetaCacheMgr).invalidateDb( - 1L, Util.genIdByName("test-catalog", "local_db"), "local_db"); + verifyRemoteDatabaseInvalidation("REMOTE_DB"); + verifyDeterministicDatabaseInvalidation("local_db"); } @Test @@ -175,10 +183,9 @@ public void testCreateDbIfNotExistsRefreshesCachesWhenDbExists() throws Exceptio Mockito.verify(metadata, Mockito.never()).createDatabase(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.verify(mockEditLog, Mockito.never()).logCreateDb(Mockito.any()); - Mockito.verify(connector).invalidateDb("DB1"); - Assertions.assertEquals(1, catalog.resetMetaCacheNamesCount); - Mockito.verify(mockMetaCacheMgr).invalidateDb( - 1L, Util.genIdByName("test-catalog", "db1"), "db1"); + verifyRemoteDatabaseInvalidation("DB1"); + verifyInvalidation(ScopePath.databaseCollection().in(ScopePath.Coordinates.LOCAL)); + verifyDeterministicDatabaseInvalidation("db1"); } @Test @@ -201,9 +208,8 @@ public void testCreateDbIfNotExistsSkipsWhenRemoteExists() throws Exception { Mockito.verify(metadata).databaseExists(session, "db1"); Mockito.verify(metadata, Mockito.never()).createDatabase(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.verify(mockEditLog, Mockito.never()).logCreateDb(Mockito.any()); - Assertions.assertEquals(1, catalog.resetMetaCacheNamesCount); - Mockito.verify(mockMetaCacheMgr).invalidateDb( - 1L, Util.genIdByName("test-catalog", "db1"), "db1"); + verifyInvalidation(ScopePath.databaseCollection().in(ScopePath.Coordinates.LOCAL)); + verifyDeterministicDatabaseInvalidation("db1"); } @Test @@ -217,7 +223,7 @@ public void testCreateDbIfNotExistsCreatesWhenRemoteAbsent() throws Exception { Mockito.verify(metadata).databaseExists(session, "db1"); Mockito.verify(metadata).createDatabase(session, "db1", props); Mockito.verify(mockEditLog).logCreateDb(Mockito.any()); - Assertions.assertEquals(1, catalog.resetMetaCacheNamesCount); + verifyInvalidation(ScopePath.databaseCollection().in(ScopePath.Coordinates.LOCAL)); } @Test @@ -230,9 +236,8 @@ public void testCreateDbIfNotExistsLosingTheCreateRaceReturnsWithoutEditLog() th Mockito.verify(metadata, Mockito.times(2)).databaseExists(session, "db1"); Mockito.verify(mockEditLog, Mockito.never()).logCreateDb(Mockito.any()); - Mockito.verify(connector).invalidateDb("db1"); - Mockito.verify(mockMetaCacheMgr).invalidateDb( - 1L, Util.genIdByName("test-catalog", "db1"), "db1"); + verifyRemoteDatabaseInvalidation("db1"); + verifyDeterministicDatabaseInvalidation("db1"); } @Test @@ -277,9 +282,7 @@ public void testDropDbRoutesToConnectorAndUnregisters() throws Exception { Mockito.verify(metadata).dropDatabase(session, "db1", false, false); Mockito.verify(mockEditLog).logDropDb(Mockito.any()); - Assertions.assertEquals("db1", catalog.unregisteredDb, - "dropDb must remove the db from the cache (legacy afterDropDb parity)"); - Mockito.verify(mockMetaCacheMgr).invalidateDb(1L, DATABASE_ID, "db1"); + verifyDeterministicDatabaseInvalidation("db1"); } @Test @@ -350,9 +353,8 @@ public void testDropDbResolvesRemoteNameRoutesAndUnregisters() throws Exception Mockito.verify(mockEditLog).logDropDb(dropDbInfo.capture()); Assertions.assertEquals("db1", dropDbInfo.getValue().getDbName(), "edit-log DropDbInfo must carry the LOCAL db name for follower replay"); - Assertions.assertEquals("db1", catalog.unregisteredDb, - "cache invalidation must use the LOCAL db name"); - Mockito.verify(connector).invalidateDb("REMOTE_DB1"); + verifyDeterministicDatabaseInvalidation("db1"); + verifyRemoteDatabaseInvalidation("REMOTE_DB1"); } // ==================== DROP TABLE ==================== @@ -366,6 +368,8 @@ public void testDropTableKeepsExactCleanupAfterDatabaseCacheEviction() throws Ex Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("T1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; ExternalDatabase replayDb = mockExternalDatabase(); catalog.dbForReplayResult = Optional.of(replayDb); @@ -391,9 +395,9 @@ public void testDropTableKeepsExactCleanupAfterDatabaseCacheEviction() throws Ex Assertions.assertEquals("db1", catalog.lastGetDbForReplayArg); Mockito.verify(replayDb, Mockito.never()).unregisterTable(Mockito.anyString()); Mockito.verify(db, Mockito.never()).unregisterTable(Mockito.anyString()); - Mockito.verify(mockMetaCacheMgr).invalidateTable(1L, DATABASE_ID, "db1", TABLE_ID, "t1"); + verifyDeterministicTableInvalidation("db1", "t1"); // Connector caches are keyed by the remote identity. - Mockito.verify(connector).invalidateTable("DB1", "TBL1"); + verifyRemoteTableInvalidation("DB1", "TBL1"); } @Test @@ -438,6 +442,8 @@ public void testDropTemporaryTableNeverDropsPermanentExternalTable() { Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; DdlException ex = Assertions.assertThrows(DdlException.class, @@ -455,6 +461,8 @@ public void testDropTemporaryTableIfExistsIsNoopAndKeepsPermanentExternalTable() ExternalTable table = Mockito.mock(ExternalTable.class); Mockito.when(table.getName()).thenReturn("t1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; catalog.dropTable("db1", "t1", false, false, false, true, true, false); @@ -473,6 +481,8 @@ public void testDropTableHandleAbsentAfterLocalResolveCleansLocalStateWithIfExis Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; Mockito.when(metadata.getTableHandle(session, "DB1", "TBL1")).thenReturn(Optional.empty()); @@ -480,9 +490,9 @@ public void testDropTableHandleAbsentAfterLocalResolveCleansLocalStateWithIfExis Mockito.verify(metadata).getTableHandle(session, "DB1", "TBL1"); Mockito.verify(metadata, Mockito.never()).dropTable(Mockito.any(), Mockito.any()); - Mockito.verify(connector).invalidateTable("DB1", "TBL1"); + verifyRemoteTableInvalidation("DB1", "TBL1"); Mockito.verify(mockEditLog).logDropTable(Mockito.any()); - Mockito.verify(mockMetaCacheMgr).invalidateTable(1L, DATABASE_ID, "db1", TABLE_ID, "t1"); + verifyDeterministicTableInvalidation("db1", "t1"); } @Test @@ -492,6 +502,8 @@ public void testDropTableHandleAbsentAfterLocalResolveThrowsWithoutIfExists() { Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; Mockito.when(metadata.getTableHandle(session, "DB1", "TBL1")).thenReturn(Optional.empty()); @@ -507,6 +519,8 @@ public void testDropTableWrapsConnectorException() { Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class); @@ -528,6 +542,8 @@ public void testDropTableRoutesViewToDropViewAndUnregisters() throws Exception { Mockito.when(view.getRemoteDbName()).thenReturn("DB1"); Mockito.when(view.getRemoteName()).thenReturn("V1"); Mockito.doReturn(view).when(db).getTableNullable("v1"); + Mockito.doReturn(db).when(view).getDb(); + Mockito.when(view.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; ExternalDatabase replayDb = mockExternalDatabase(); catalog.dbForReplayResult = Optional.of(replayDb); @@ -546,8 +562,8 @@ public void testDropTableRoutesViewToDropViewAndUnregisters() throws Exception { Assertions.assertEquals("v1", dropInfo.getValue().getTableName(), "edit-log DropInfo must carry the LOCAL view name for follower replay"); Assertions.assertEquals("db1", catalog.lastGetDbForReplayArg); - Mockito.verify(replayDb).unregisterTable("v1"); - Mockito.verify(connector).invalidateTable("DB1", "V1"); + verifyDeterministicTableInvalidation("db1", "v1"); + verifyRemoteTableInvalidation("DB1", "V1"); } @Test @@ -557,6 +573,8 @@ public void testDropViewWrapsConnectorException() { Mockito.when(view.getRemoteDbName()).thenReturn("DB1"); Mockito.when(view.getRemoteName()).thenReturn("V1"); Mockito.doReturn(view).when(db).getTableNullable("v1"); + Mockito.doReturn(db).when(view).getDb(); + Mockito.when(view.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; Mockito.when(metadata.viewExists(session, "DB1", "V1")).thenReturn(true); Mockito.doThrow(new DorisConnectorException("boom")).when(metadata).dropView(session, "DB1", "V1"); @@ -578,6 +596,8 @@ public void testRenameTableKeepsCanonicalExactCleanupAfterDatabaseCacheEviction( Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); Mockito.when(db.canonicalLocalTableNameFromRemote("REMOTE_T2")).thenReturn("local_t2"); catalog.dbNullableResult = db; ExternalDatabase replayDb = mockExternalDatabase(); @@ -606,11 +626,10 @@ public void testRenameTableKeepsCanonicalExactCleanupAfterDatabaseCacheEviction( Assertions.assertEquals("t1", logCap.getValue().getTableName()); Assertions.assertEquals("local_t2", logCap.getValue().getNewTableName()); // The destination has no pre-rename mapping, so its remote name is the requested new name. - Mockito.verify(connector).invalidateTable("DB1", "TBL1"); - Mockito.verify(connector).invalidateTable("DB1", "REMOTE_T2"); - Mockito.verify(mockMetaCacheMgr).invalidateTableRename( - 1L, DATABASE_ID, "db1", TABLE_ID, "t1", - Util.genIdByName("test-catalog", "db1", "local_t2"), "local_t2"); + verifyRemoteTableInvalidation("DB1", "TBL1"); + verifyRemoteTableInvalidation("DB1", "REMOTE_T2"); + verifyDeterministicTableInvalidation("db1", "t1"); + verifyDeterministicTableInvalidation("db1", "local_t2"); } @Test @@ -620,6 +639,8 @@ public void testRenameResolvesDestinationIdentityBeforeRemoteMutation() throws E Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); Mockito.when(db.canonicalLocalTableNameFromRemote("TBL2")) .thenThrow(new IllegalStateException("name mapping failed")); catalog.dbNullableResult = db; @@ -642,6 +663,8 @@ public void testRenameTableUsesStructuralSourceInvalidationAndDestinationBarrier Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; ExternalDatabase cachedDb = mockExternalDatabase(); catalog.dbForReplayResult = Optional.of(cachedDb); @@ -650,14 +673,13 @@ public void testRenameTableUsesStructuralSourceInvalidationAndDestinationBarrier catalog.renameTable("db1", "t1", "t2"); - long destinationId = Util.genIdByName("test-catalog", "db1", "t2"); InOrder order = Mockito.inOrder(mockEditLog, connector, cachedDb, mockMetaCacheMgr); order.verify(mockEditLog).logRefreshExternalTable(Mockito.any()); - order.verify(connector).invalidateTable("DB1", "TBL1"); - order.verify(connector).invalidateTable("DB1", "t2"); - order.verify(cachedDb).invalidateTableRename("t1", "t2"); - order.verify(mockMetaCacheMgr).invalidateTableRename( - 1L, DATABASE_ID, "db1", TABLE_ID, "t1", destinationId, "t2"); + verifyRemoteTableInvalidation("DB1", "TBL1"); + verifyRemoteTableInvalidation("DB1", "t2"); + verifyDeterministicTableInvalidation("db1", "t1"); + verifyDeterministicTableInvalidation("db1", "t1"); + verifyDeterministicTableInvalidation("db1", "t2"); } @Test @@ -685,6 +707,8 @@ public void testRenameTableWrapsConnectorExceptionAndSkipsBookkeeping() { Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; ExternalDatabase replayDb = mockExternalDatabase(); catalog.dbForReplayResult = Optional.of(replayDb); @@ -763,9 +787,8 @@ public void testCreateTableInvalidatesDbCacheUsingLocalNames() throws UserExcept "edit-log CreateTableInfo must carry the LOCAL db name for follower replay"); Assertions.assertEquals("t1", persist.getValue().getTblName(), "edit-log CreateTableInfo must carry the LOCAL table name for follower replay"); - Mockito.verify(mockMetaCacheMgr).invalidateTable( - 1L, DATABASE_ID, "db1", Util.genIdByName("test-catalog", "db1", "t1"), "t1"); - Mockito.verify(connector).invalidateTable("DB1", "t1"); + verifyDeterministicTableInvalidation("db1", "t1"); + verifyRemoteTableInvalidation("DB1", "t1"); } } @@ -793,9 +816,8 @@ public void testCreateTablePersistsCanonicalLocalTableName() throws UserExceptio Mockito.verify(mockEditLog).logCreateTable(log.capture()); Assertions.assertEquals("db1", log.getValue().getDbName()); Assertions.assertEquals("local_t1", log.getValue().getTblName()); - Mockito.verify(connector).invalidateTable("DB1", "REMOTE_T1"); - Mockito.verify(mockMetaCacheMgr).invalidateTable( - 1L, DATABASE_ID, "db1", Util.genIdByName("test-catalog", "db1", "local_t1"), "local_t1"); + verifyRemoteTableInvalidation("DB1", "REMOTE_T1"); + verifyDeterministicTableInvalidation("db1", "local_t1"); } } @@ -831,7 +853,7 @@ public void testCreateTableIfNotExistsEvictsPreexistingRowCount() throws Excepti Assertions.assertTrue(result); Mockito.verify(metadata, Mockito.never()).createTable(Mockito.any(), Mockito.any()); Mockito.verify(mockEditLog, Mockito.never()).logCreateTable(Mockito.any()); - Mockito.verify(connector).invalidateTable("DB1", "t1"); + verifyRemoteTableInvalidation("DB1", "t1"); Assertions.assertEquals(200L, rowCountCache.getCachedRowCount(1L, DATABASE_ID, tableId, false)); } } @@ -860,9 +882,8 @@ public void testCreateTableIfNotExistsLosingTheCreateRaceReturnsTrueWithoutEditL Assertions.assertTrue(res, "losing the create race under IF NOT EXISTS must return true"); Mockito.verify(mockEditLog, Mockito.never()).logCreateTable(Mockito.any()); - Mockito.verify(connector).invalidateTable("DB1", "t1"); - Mockito.verify(mockMetaCacheMgr).invalidateTable( - 1L, DATABASE_ID, "db1", Util.genIdByName("test-catalog", "db1", "t1"), "t1"); + verifyRemoteTableInvalidation("DB1", "t1"); + verifyDeterministicTableInvalidation("db1", "t1"); } @Test @@ -908,8 +929,7 @@ public void testCreateTableIfNotExistsExistingLocalTableReturnsTrue() throws Exc "an existing IF NOT EXISTS target must remain a no-op even if new validation rules exist"); Mockito.verify(metadata, Mockito.never()).createTable(Mockito.any(), Mockito.any()); Mockito.verify(mockEditLog, Mockito.never()).logCreateTable(Mockito.any()); - Mockito.verify(mockMetaCacheMgr).invalidateTable( - 1L, DATABASE_ID, "db1", Util.genIdByName("test-catalog", "db1", "t1"), "t1"); + verifyDeterministicTableInvalidation("db1", "t1"); } @Test @@ -997,10 +1017,13 @@ public void testCreateTableLocalConflictWithoutIfNotExistsRejects() throws Excep public void testTruncateTablePersistsCanonicalLocalNames() throws Exception { ExternalDatabase db = mockExternalDatabase(); Mockito.when(db.getId()).thenReturn(DATABASE_ID); + Mockito.when(db.resolveTableNameForInvalidation(Mockito.anyString())).thenAnswer(inv -> inv.getArgument(0)); ExternalTable table = mockAlterTable(); Mockito.when(table.getId()).thenReturn(TABLE_ID); Mockito.when(db.getFullName()).thenReturn("db1"); Mockito.doReturn(table).when(db).getTableNullable("T1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class); Mockito.when(metadata.getTableHandle(session, "DB1", "TBL1")).thenReturn(Optional.of(handle)); @@ -1025,8 +1048,10 @@ public void testReplayTruncateUsesNormalRefreshWhenTableObjectIsCached() { catalog.replayTruncateTable(info); - Mockito.verify(mockRefreshManager).refreshTableInternal(db, table, 123L); - Mockito.verifyNoInteractions(mockMetaCacheMgr); + Mockito.verify(table).unsetObjectCreated(); + Mockito.verify(table).setUpdateTime(123L); + verifyDeterministicTableInvalidation("db1", "t1"); + verifyRemoteTableInvalidation("DB1", "TBL1"); } @Test @@ -1038,11 +1063,8 @@ public void testReplayTruncateInvalidatesByCanonicalNameWhenTableObjectIsCold() catalog.replayTruncateTable(new TruncateTableInfo("test-catalog", "db1", "t1", null, 123L)); - InOrder order = Mockito.inOrder(connector, mockMetaCacheMgr); - order.verify(connector).invalidateDb("DB1"); - order.verify(mockMetaCacheMgr).invalidateTable( - 1L, Util.genIdByName("test-catalog", "db1"), "db1", - Util.genIdByName("test-catalog", "db1", "t1"), "t1"); + verifyRemoteDatabaseInvalidation("DB1"); + verifyDeterministicTableInvalidation("db1", "t1"); } @Test @@ -1051,11 +1073,8 @@ public void testReplayTruncateInvalidatesAllConnectorCachesWhenDatabaseObjectIsC catalog.replayTruncateTable(new TruncateTableInfo("test-catalog", "db1", "t1", null, 123L)); - InOrder order = Mockito.inOrder(connector, mockMetaCacheMgr); - order.verify(connector).invalidateAll(); - order.verify(mockMetaCacheMgr).invalidateTable( - 1L, Util.genIdByName("test-catalog", "db1"), "db1", - Util.genIdByName("test-catalog", "db1", "t1"), "t1"); + verifyInvalidation(ScopePath.catalog().in(ScopePath.Coordinates.REMOTE)); + verifyDeterministicTableInvalidation("db1", "t1"); } // ==================== EDIT-LOG REPLAY (follower / observer propagation) ==================== @@ -1071,8 +1090,8 @@ public void testReplayDropTableInvalidatesConnectorOnFollower() { catalog.replayDropTable("db1", "t1"); - Mockito.verify(connector).invalidateTable("DB1", "TBL1"); - Mockito.verify(replayDb).unregisterTable("t1"); + verifyRemoteTableInvalidation("DB1", "TBL1"); + verifyDeterministicTableInvalidation("db1", "t1"); } @Test @@ -1097,8 +1116,8 @@ public void testReplayDropTableInvalidatesDatabaseConnectorCacheWhenTableObjectI catalog.replayDropTable("db1", "t1"); - Mockito.verify(connector).invalidateDb("DB1"); - Mockito.verify(replayDb).unregisterTable("t1"); + verifyRemoteDatabaseInvalidation("DB1"); + verifyDeterministicTableInvalidation("db1", "t1"); } @Test @@ -1107,7 +1126,7 @@ public void testReplayDropTableInvalidatesAllConnectorCachesWhenDatabaseObjectIs catalog.replayDropTable("db1", "t1"); - Mockito.verify(connector).invalidateAll(); + verifyInvalidation(ScopePath.catalog().in(ScopePath.Coordinates.REMOTE)); verifyDeterministicTableInvalidation("db1", "t1"); } @@ -1131,8 +1150,7 @@ public void testReplayDropDbInvalidatesConnectorOnFollower() { catalog.replayDropDb("db1"); - Mockito.verify(connector).invalidateDb("DB1"); - Assertions.assertEquals("db1", catalog.unregisteredDb); + verifyRemoteDatabaseInvalidation("DB1"); verifyDeterministicDatabaseInvalidation("db1"); } @@ -1142,8 +1160,7 @@ public void testReplayDropDbInvalidatesAllWhenDatabaseObjectIsCold() { catalog.replayDropDb("db1"); - Mockito.verify(connector).invalidateAll(); - Assertions.assertEquals("db1", catalog.unregisteredDb); + verifyInvalidation(ScopePath.catalog().in(ScopePath.Coordinates.REMOTE)); verifyDeterministicDatabaseInvalidation("db1"); } @@ -1155,7 +1172,7 @@ public void testReplayCreateTableInvalidatesConnectorOnFollower() { catalog.replayCreateTable("db1", "t1"); - Mockito.verify(connector).invalidateDb("DB1"); + verifyRemoteDatabaseInvalidation("DB1"); verifyDeterministicTableInvalidation("db1", "t1"); } @@ -1165,7 +1182,7 @@ public void testReplayCreateTableInvalidatesAllWhenDatabaseObjectIsCold() { catalog.replayCreateTable("db1", "t1"); - Mockito.verify(connector).invalidateAll(); + verifyInvalidation(ScopePath.catalog().in(ScopePath.Coordinates.REMOTE)); verifyDeterministicTableInvalidation("db1", "t1"); } @@ -1173,8 +1190,8 @@ public void testReplayCreateTableInvalidatesAllWhenDatabaseObjectIsCold() { public void testReplayCreateDbInvalidatesDatabaseCaches() { catalog.replayCreateDb("db1"); - Assertions.assertEquals(1, catalog.resetMetaCacheNamesCount); - Mockito.verify(connector).invalidateAll(); + verifyInvalidation(ScopePath.databaseCollection().in(ScopePath.Coordinates.LOCAL)); + verifyInvalidation(ScopePath.catalog().in(ScopePath.Coordinates.REMOTE)); verifyDeterministicDatabaseInvalidation("db1"); } @@ -1575,6 +1592,7 @@ private ExternalTable mockAlterTable() { Mockito.when(table.getRemoteName()).thenReturn("TBL1"); ExternalDatabase db = mockExternalDatabase(); Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); return table; } @@ -1586,15 +1604,30 @@ private ConnectorTableHandle stubAlterHandle() { } /** A nullable INT Doris column (iceberg add/modify reject non-nullable adds). */ - private void verifyDeterministicDatabaseInvalidation(String dbName) { - Mockito.verify(mockMetaCacheMgr).invalidateDb( - 1L, Util.genIdByName("test-catalog", dbName), dbName); + private void verifyInvalidation(ScopePath expected) { + boolean invalidated = Mockito.mockingDetails(cacheRoot).getInvocations().stream() + .filter(invocation -> invocation.getMethod().getName().equals("invalidateScopes")) + .map(invocation -> (java.util.Collection) invocation.getArgument(0)) + .flatMap(java.util.Collection::stream) + .map(ScopePath.class::cast) + .anyMatch(scope -> scope.contains(expected)); + Assertions.assertTrue(invalidated, "Missing resource invalidation: " + expected); + } + + private void verifyRemoteTableInvalidation(String database, String table) { + verifyInvalidation(ScopePath.table(database, table).in(ScopePath.Coordinates.REMOTE)); + } + + private void verifyRemoteDatabaseInvalidation(String database) { + verifyInvalidation(ScopePath.database(database).in(ScopePath.Coordinates.REMOTE)); + } + + private void verifyDeterministicDatabaseInvalidation(String database) { + verifyInvalidation(ScopePath.database(database).in(ScopePath.Coordinates.LOCAL)); } - private void verifyDeterministicTableInvalidation(String dbName, String tableName) { - Mockito.verify(mockMetaCacheMgr).invalidateTable( - 1L, Util.genIdByName("test-catalog", dbName), dbName, - Util.genIdByName("test-catalog", dbName, tableName), tableName); + private void verifyDeterministicTableInvalidation(String database, String table) { + verifyInvalidation(ScopePath.tableMetadata(database, table).in(ScopePath.Coordinates.LOCAL)); } private static Column nullableIntColumn(String name) { @@ -1608,6 +1641,7 @@ private ExternalDatabase mockExternalDatabase() { Mockito.when(db.getFullName()).thenReturn("db1"); Mockito.when(db.getRemoteName()).thenReturn("DB1"); Mockito.when(db.getId()).thenReturn(DATABASE_ID); + Mockito.when(db.resolveTableNameForInvalidation(Mockito.anyString())).thenAnswer(inv -> inv.getArgument(0)); Mockito.when(db.isInitialized()).thenReturn(true); Mockito.when(db.canonicalLocalTableNameFromRemote(Mockito.anyString())) .thenAnswer(inv -> inv.getArgument(0)); From 450da63290057dac1995f90561fdaa97e152e64d Mon Sep 17 00:00:00 2001 From: 924060929 Date: Fri, 23 Oct 2026 01:30:17 +0800 Subject: [PATCH 3/5] [test](fe) Align external cache tests with unified ownership ### What problem does this PR solve? Issue Number: None Problem Summary: Full CI exposed four FE test assumptions left behind by the unified metadata cache lifecycle: missing connector handles now fail explicitly, initialization constructs an empty shared cache owner, and the deadlock fixture needs real catalog properties. Update the assertions and fixture while retaining the per-session isolation and lock-order checks. Database DROP now invalidates membership immediately, so the nested Iceberg namespace regression must expect live inferred parents rather than a stale pre-refresh list. Regenerate its expected output with the regression runner. ### Release note None ### Check List (For Author) - Test: Unit Test (36 related FE tests passed); Checkstyle (zero violations); Regression test (complete iceberg_and_internal_nested_namespace suite generated and then compared successfully against the packaged FE and real REST/MinIO). The temporary native ARM runtime used a cached 4.1 BE with be_exec_version=13; no new BE build or full CI success is claimed. - Behavior changed: No. Tests follow the already implemented cache semantics. - Does this need documentation: No --- .../doris/datasource/ExternalCatalogDeadlockTest.java | 2 ++ .../datasource/ExternalDatabaseSessionContextTest.java | 4 +++- .../apache/doris/datasource/PluginDrivenSysTableTest.java | 6 +++--- .../plugin/PluginDrivenExternalTableEngineTest.java | 8 ++++---- .../iceberg/iceberg_and_internal_nested_namespace.out | 2 +- .../iceberg/iceberg_and_internal_nested_namespace.groovy | 8 ++++---- 6 files changed, 17 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogDeadlockTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogDeadlockTest.java index c9a46b4c1dd772..0c46f865e6da52 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogDeadlockTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogDeadlockTest.java @@ -74,6 +74,7 @@ public void testResetToUninitializedShouldNotDeadlockWithCacheLoader() throws Ex Assertions.assertTrue(refreshReady.await(DEADLOCK_DETECTION_TIMEOUT_SECONDS, TimeUnit.SECONDS)); assertNoDeadlock(queryThread, refreshThread, backgroundFailure); + catalog.onClose(); } private static void assertNoDeadlock(Thread queryThread, Thread refreshThread, @@ -125,6 +126,7 @@ private static class DeadlockCatalog extends ExternalCatalog { DeadlockCatalog() { super(1L, "deadlock-catalog", Type.TEST, ""); + catalogProperty = new CatalogProperty(null, java.util.Collections.emptyMap()); initialized = true; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseSessionContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseSessionContextTest.java index ece97c60979410..e3faff8c7bc386 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseSessionContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseSessionContextTest.java @@ -284,7 +284,9 @@ private static Map props(int lowerCaseTableNames) { } boolean hasSharedDatabaseCacheState(long dbId) { - return databaseNames != null || databases != null || dbIdNameIndex.getName(dbId) != null; + return (databaseNames != null && databaseNames.getIfPresent("") != null) + || (databases != null && databases.stats().getEstimatedSize() > 0) + || dbIdNameIndex.getName(dbId) != null; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/PluginDrivenSysTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/PluginDrivenSysTableTest.java index d164ccf3a97b14..1bd10f7c859ddb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/PluginDrivenSysTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/PluginDrivenSysTableTest.java @@ -256,7 +256,7 @@ protected synchronized void makeSureInitialized() { } @Test - public void testSysTableEmptyWhenBaseHandleMissing() { + public void testSysTableFailsWhenBaseHandleMissing() { ConnectorMetadata metadata = Mockito.mock(ConnectorMetadata.class); ConnectorSession session = Mockito.mock(ConnectorSession.class); Mockito.when(session.getStatementScope()).thenReturn(ConnectorStatementScope.NONE); @@ -272,8 +272,8 @@ protected synchronized void makeSureInitialized() { } }; - Assertions.assertFalse(sysTable.initSchema().isPresent(), - "no base handle -> no sys handle -> empty schema (no spurious getSysTableHandle)"); + CacheException failure = Assertions.assertThrows(CacheException.class, sysTable::initSchema); + Assertions.assertTrue(failure.getMessage().contains("REMOTE_DB.REMOTE_TBL$snapshots does not exist")); Mockito.verify(metadata, Mockito.never()) .getSysTableHandle(Mockito.any(), Mockito.any(), Mockito.anyString()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableEngineTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableEngineTest.java index 67f872d27250c7..745ebb49c7e1f1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableEngineTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableEngineTest.java @@ -31,6 +31,7 @@ import org.apache.doris.connector.spi.ConnectorTableSchema; import org.apache.doris.connector.spi.ConnectorType; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.SessionContext; @@ -206,13 +207,12 @@ public void testTableTypeIsAlwaysPluginExternalTable() { } @Test - public void testInitSchemaReturnsEmptyWhenTableHandleMissing() { + public void testInitSchemaFailsWhenTableHandleMissing() { Connector connector = createMockConnector(false, false); PluginDrivenExternalTable table = createTableWithCatalogType("jdbc", connector); - // Return an empty schema result when the connector cannot resolve the table handle. - Assertions.assertFalse(table.initSchema().isPresent(), - "Missing connector table handles should produce an empty schema result"); + CacheException failure = Assertions.assertThrows(CacheException.class, table::initSchema); + Assertions.assertTrue(failure.getMessage().contains("test_db.test_table does not exist")); } @Test diff --git a/regression-test/data/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.out b/regression-test/data/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.out index b4b3deb56ec3c3..016e287e04717b 100644 --- a/regression-test/data/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.out +++ b/regression-test/data/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.out @@ -40,9 +40,9 @@ nested_tbl2 nested_tbl3 -- !sql15 -- +ns1.ns2 -- !sql17 -- -ns1.ns2 -- !sql18 -- diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.groovy index abf060b85cec7c..02fe9eb464536d 100644 --- a/regression-test/suites/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.groovy +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.groovy @@ -208,14 +208,14 @@ suite("iceberg_and_internal_nested_namespace", "p0,external") { sql """refresh table `ns1.ns2`.nested_tbl2xxx""" exception """Table nested_tbl2xxx does not exist in db ns1.ns2""" } - // drop ns1.ns2 first, we can still see it after refresh, because ns1.ns2.ns3 still exists + // DROP invalidates database membership immediately. The remaining child keeps its parent visible. sql """drop database `ns1.ns2` force""" - qt_sql15 """show databases like "ns1.ns2"""" // empty + qt_sql15 """show databases like "ns1.ns2"""" // 1 sql """refresh catalog ${catalog_name}""" qtWithRefreshRetry("sql16", """show databases like "ns1.ns2"""", "ns1.ns2") // 1 - // then we drop ns1.ns2.ns3, after refresh, ns1.ns2 also disappear + // Removing the last child also removes the inferred parent, without requiring a catalog refresh. sql """drop database `ns1.ns2.ns3` force""" - qt_sql17 """show databases like "ns1.ns2"""" // 1 + qt_sql17 """show databases like "ns1.ns2"""" // empty qt_sql18 """show databases like "ns1.ns2.ns3"""" // empty sql """refresh catalog ${catalog_name}""" qtWithRefreshRetry("sql19", """show databases like "ns1.ns2"""", "") // empty From 8e4f3a06ce033214d9da2e14958069469520bb1e Mon Sep 17 00:00:00 2001 From: 924060929 Date: Fri, 23 Oct 2026 10:11:50 +0800 Subject: [PATCH 4/5] [fix](fe) Invalidate cached case-equivalent rename identities ### What problem does this PR solve? Issue Number: None Problem Summary: With case-insensitive external table lookup, a rename destination can have a different spelling and deterministic ID from a previously cached table at the same case-equivalent name. Invalidating only the new spelling leaves the old destination object and derived caches reachable by ID. Normalize LOCAL table resource identities at the catalog root using an immutable case policy, keeping keys and remote scopes unchanged. On mode 2 lookup invalidation, fence numeric derived state for the database to cover IDs published after planning. Retire the root when table-name mode changes. Live changes and replay share this publication barrier without remote I/O. ### Release note Fix stale external table metadata after a rename replaces a case-equivalent cached destination. ### Check List (For Author) - Test: Unit Test - Reproduced stale destination identities in leader, cold-name replay and refreshed-name replay tests before the final fix. - All 437 tests passed using run-fe-ut.sh: cache framework 173, connector SPI 82, related FE 182. - Full FE build and Checkstyle passed using build.sh --fe -j4 with JDK 17. - Behavior changed: Yes. Case-insensitive lookup invalidation clears case-equivalent local identities and recalculates row counts in the affected database; sibling objects and schemas remain cached. - Does this need documentation: No --- fe/fe-connector/README.md | 8 + .../connector/cache/CatalogMetaCache.java | 23 ++- .../connector/cache/MetaCacheDefinition.java | 15 +- .../doris/connector/cache/ScopePath.java | 11 ++ .../connector/cache/CatalogMetaCacheTest.java | 35 ++++ .../doris/datasource/ExternalCatalog.java | 9 +- .../metacache/CatalogMetadataChanges.java | 24 ++- .../RefreshManagerRenameReplayTest.java | 3 + .../datasource/FeCatalogCacheOwnerTest.java | 149 ++++++++++++++++++ 9 files changed, 260 insertions(+), 17 deletions(-) diff --git a/fe/fe-connector/README.md b/fe/fe-connector/README.md index 5dd6e4ae4a8c27..1d5849558c8f11 100644 --- a/fe/fe-connector/README.md +++ b/fe/fe-connector/README.md @@ -55,6 +55,14 @@ name lists; explicit lookup refresh also invalidates the target identity and par written journals persist local/remote identities and the operation's lookup flag. Old records widen only the remote scope when a cache-only lookup cannot recover the remote identity. +The root captures an immutable resource normalization policy. With case-insensitive table lookup (mode 2), +LOCAL table scopes fold case while cache keys and displayed names retain their original spelling; REMOTE +scopes retain the connector's case semantics. Registration, bulk-load parents and invalidation use the same +projection, so an old spelling cannot publish after its case-equivalent name is invalidated. A mode 2 lookup invalidation (including rename and its event records) +also fences the database's numeric derived state: IDs published after scope planning must not leave a stale +row count behind. Sibling objects and schemas remain cached, while row counts in that database are reloaded. +Changing table-name mode retires the root rather than changing the policy of live registrations. + Gateways with different physical identities implement the pure `Connector.cacheDependencies` projection; it must not perform I/O or construct siblings. Partition names are translated by `partitionCacheScopes`. The host applies the resulting scopes; gateways do not maintain a second list of invalidation callbacks. diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java index 641ca7e535cab0..d308ace3fc48f4 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java @@ -27,6 +27,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.UnaryOperator; /** * Catalog-local owner of all framework metadata caches. @@ -43,6 +44,7 @@ public final class CatalogMetaCache implements AutoCloseable { private final boolean managed; private final CatalogMetaCache parent; private final ScopePath.Coordinates coordinates; + private final UnaryOperator scopeNormalizer; private final CacheCallbackContext callbacks; private final CatalogMetaCache rootOwner; private final String registrationPrefix; @@ -85,9 +87,15 @@ public static CatalogMetaCache managed(long catalogId, String engine, public static CatalogMetaCache managed(long catalogId, String engine, Map catalogProperties, ClassLoader classLoader) { + return managed(catalogId, engine, catalogProperties, classLoader, UnaryOperator.identity()); + } + + /** The pure, immutable normalizer defines resource equivalence for the lifetime of this root. */ + public static CatalogMetaCache managed(long catalogId, String engine, + Map catalogProperties, ClassLoader classLoader, UnaryOperator normalizer) { MetaCacheBudgetManager manager = MetaCacheGovernance.budgetManager(); CatalogMetaCache cache = new CatalogMetaCache(new ScopedMetaCacheRegistry(), manager, - catalogId, engine, manager.parseCatalogMaxWeight(catalogProperties), true, classLoader); + catalogId, engine, manager.parseCatalogMaxWeight(catalogProperties), true, classLoader, normalizer); MetaCacheGovernance.register(cache); return cache; } @@ -100,6 +108,14 @@ public static CatalogMetaCache managed(long catalogId, String engine, private CatalogMetaCache(ScopedMetaCacheRegistry registry, MetaCacheBudgetManager budgetManager, long catalogId, String engine, OptionalLong catalogMaxWeight, boolean managed, ClassLoader classLoader) { + this(registry, budgetManager, catalogId, engine, catalogMaxWeight, managed, classLoader, + UnaryOperator.identity()); + } + + private CatalogMetaCache(ScopedMetaCacheRegistry registry, MetaCacheBudgetManager budgetManager, + long catalogId, String engine, OptionalLong catalogMaxWeight, boolean managed, ClassLoader classLoader, + UnaryOperator normalizer) { + this.scopeNormalizer = Objects.requireNonNull(normalizer, "scope normalizer can not be null"); this.registry = Objects.requireNonNull(registry, "registry can not be null"); this.budgetManager = Objects.requireNonNull(budgetManager, "budgetManager can not be null"); this.catalogId = catalogId; @@ -118,6 +134,7 @@ private CatalogMetaCache(ScopedMetaCacheRegistry registry, MetaCacheBudgetManage private CatalogMetaCache(CatalogMetaCache parent, String engine, ScopePath.Coordinates coordinates, ClassLoader classLoader) { this.coordinates = coordinates; + this.scopeNormalizer = parent.scopeNormalizer; this.callbacks = new CacheCallbackContext(classLoader); this.registry = parent.registry; this.budgetManager = parent.budgetManager; @@ -150,7 +167,7 @@ public CatalogMetaCache openDomain(String engine, ScopePath.Coordinates coordina public MetaCache create(MetaCacheDefinition definition) { MetaCacheDefinition nonNullDefinition = - Objects.requireNonNull(definition, "definition can not be null").forDomain(coordinates, callbacks); + Objects.requireNonNull(definition, "definition can not be null").forDomain(this::project, callbacks); checkOpen(); if (!names.add(nonNullDefinition.name())) { throw new IllegalArgumentException("Duplicate meta cache name: " + nonNullDefinition.name()); @@ -243,7 +260,7 @@ public void invalidateScopes(Collection scopes) { } private ScopePath project(ScopePath scope) { - return coordinates == null ? scope : scope.in(coordinates); + return scopeNormalizer.apply(coordinates == null ? scope : scope.in(coordinates)); } public boolean isClosed() { diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java index 539824232fbd70..c0be274b30d8d5 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java @@ -22,6 +22,7 @@ import java.util.concurrent.Executor; import java.util.function.BiConsumer; import java.util.function.Function; +import java.util.function.UnaryOperator; /** * Declarative definition of one physical metadata cache in a catalog. @@ -47,10 +48,10 @@ public enum BudgetPolicy { private final Executor refreshExecutor; private final MetaCacheSizeEstimator sizeEstimator; private final BudgetPolicy budgetPolicy; - private final ScopePath.Coordinates coordinates; + private final UnaryOperator scopeProjection; private MetaCacheDefinition(Builder builder) { - coordinates = builder.coordinates; + scopeProjection = builder.scopeProjection; name = requireName(builder.name); budgetGroup = requireName(builder.budgetGroup == null ? builder.name : builder.budgetGroup); cacheSpec = Objects.requireNonNull(builder.cacheSpec, "cacheSpec can not be null"); @@ -118,10 +119,10 @@ BudgetPolicy budgetPolicy() { return budgetPolicy; } - MetaCacheDefinition forDomain(ScopePath.Coordinates coordinates, CacheCallbackContext callbacks) { + MetaCacheDefinition forDomain(UnaryOperator projection, CacheCallbackContext callbacks) { Builder copy = builder(name, cacheSpec, key -> callbacks.call(() -> { ScopePath path = scope(key); - return coordinates == null ? path : path.in(coordinates); + return projection.apply(path); })); copy.budgetGroup = budgetGroup; copy.budgetPolicy = budgetPolicy; @@ -135,12 +136,12 @@ MetaCacheDefinition forDomain(ScopePath.Coordinates coordinates, CacheCall : (key, value) -> callbacks.run(() -> discardListener.accept(key, value)); copy.sizeEstimator = sizeEstimator == null ? null : (key, value) -> callbacks.call(() -> sizeEstimator.estimate(key, value)); - copy.coordinates = coordinates; + copy.scopeProjection = projection; return copy.build(); } ScopePath project(ScopePath path) { - return coordinates == null ? path : path.in(coordinates); + return scopeProjection.apply(path); } private static String requireName(String name) { @@ -162,7 +163,7 @@ public static final class Builder { private Executor refreshExecutor; private MetaCacheSizeEstimator sizeEstimator; private String budgetGroup; - private ScopePath.Coordinates coordinates; + private UnaryOperator scopeProjection = UnaryOperator.identity(); private BudgetPolicy budgetPolicy = BudgetPolicy.WEIGHTED_PAYLOAD; private Builder(String name, CacheSpec cacheSpec, Function scopeResolver) { diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java index 8324225215e8a2..799b6127f01927 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java @@ -18,6 +18,7 @@ package org.apache.doris.connector.cache; import java.util.Objects; +import java.util.function.UnaryOperator; /** * Hierarchical invalidation path within one catalog-owned scoped cache registry. @@ -137,6 +138,12 @@ public ScopePath in(Coordinates coordinates) { database, table, partition, nameCollection, coordinates); } + /** Changes only a real table identity, preserving collection markers, coordinates and partition identity. */ + public ScopePath mapTableName(UnaryOperator mapper) { + return table == null ? this : new ScopePath(level, database, mapper.apply(table), + partition, nameCollection, coordinates); + } + int depth() { return level == Level.NAMESPACE ? 1 : level.ordinal() + (coordinates == null ? 0 : 1); } @@ -178,6 +185,10 @@ public Level level() { return level; } + public Coordinates coordinates() { + return coordinates; + } + public String database() { return database; } diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java index ae5363e7f15c00..96c48b522d7b8a 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java @@ -25,6 +25,7 @@ import java.util.ArrayDeque; import java.util.Arrays; import java.util.Collections; +import java.util.Locale; import java.util.OptionalLong; import java.util.Queue; import java.util.concurrent.CountDownLatch; @@ -38,6 +39,40 @@ class CatalogMetaCacheTest { private static final CacheSpec SPEC = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 100); + @Test + void resourceNormalizationCoversDomainsAndBulkLoadsWithoutChangingRemoteNames() { + try (CatalogMetaCache root = CatalogMetaCache.managed(987654321L, "test", Collections.emptyMap(), + getClass().getClassLoader(), scope -> scope.coordinates() == ScopePath.Coordinates.LOCAL + ? scope.mapTableName(name -> name.toLowerCase(Locale.ROOT)) : scope)) { + CatalogMetaCache local = root.openDomain("local", ScopePath.Coordinates.LOCAL, getClass().getClassLoader()); + CatalogMetaCache remote = root.openDomain("remote", ScopePath.Coordinates.REMOTE, + getClass().getClassLoader()); + MetaCache objects = local.create(MetaCacheDefinition.builder( + "objects", SPEC, key -> ScopePath.table("db", key)).build()); + MetaCache metadata = local.create(MetaCacheDefinition.builder( + "metadata", SPEC, key -> ScopePath.tableMetadata("db", key)).build()); + MetaCache remoteObjects = remote.create(MetaCacheDefinition.builder( + "objects", SPEC, key -> ScopePath.table("db", key)).build()); + objects.put("Dest", "old"); + metadata.put("Dest", "schema"); + objects.put("b", "sibling"); + remoteObjects.put("Dest", "remote"); + try (MetaCache.BulkLoad load = objects.beginBulkLoad(ScopePath.table("db", "Dest"))) { + root.invalidateScopes(Collections.singletonList( + ScopePath.table("db", "DEST").in(ScopePath.Coordinates.LOCAL))); + Assertions.assertFalse(load.publish("Dest", "late")); + } + Assertions.assertNull(objects.getIfPresent("Dest")); + Assertions.assertNull(metadata.getIfPresent("Dest")); + Assertions.assertEquals("sibling", objects.getIfPresent("b")); + Assertions.assertEquals("remote", remoteObjects.getIfPresent("Dest")); + remote.invalidateTable("db", "DEST"); + Assertions.assertEquals("remote", remoteObjects.getIfPresent("Dest")); + remote.invalidateTable("db", "Dest"); + Assertions.assertNull(remoteObjects.getIfPresent("Dest")); + } + } + @Test void unmanagedOwnershipMustBeSelectedExplicitly() throws Exception { Assertions.assertTrue(Modifier.isPrivate(CatalogMetaCache.class.getDeclaredConstructor().getModifiers())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index ea4f6b9907acfe..3d5bcf530eef31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -419,7 +419,10 @@ public synchronized CatalogMetaCache getFeMetaCacheOwner() { overlayMetaCacheConfig(properties); String engine = this instanceof RemoteDorisExternalCatalog ? "doris" : "default"; - feMetaCacheOwner = CatalogMetaCache.managed(id, engine, properties, ExternalCatalog.class.getClassLoader()); + boolean caseInsensitiveTables = getLowerCaseTableNames() == 2; + feMetaCacheOwner = CatalogMetaCache.managed(id, engine, properties, ExternalCatalog.class.getClassLoader(), + scope -> caseInsensitiveTables && scope.coordinates() == ScopePath.Coordinates.LOCAL + ? scope.mapTableName(table -> table.toLowerCase(Locale.ROOT)) : scope); dbIdNameIndex.clear(); if (replacingClosedOwner) { initialized = false; @@ -999,7 +1002,9 @@ public void tryModifyCatalogProps(Map props) { private void invalidateCachesAfterPropertyUpdate(Map updatedProps) { ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); if (updatedProps.get(SCHEMA_CACHE_TTL_SECOND) != null - || updatedProps.containsKey(MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY)) { + || updatedProps.containsKey(MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY) + || updatedProps.containsKey(LOWER_CASE_TABLE_NAMES) + || updatedProps.containsKey(ONLY_TEST_LOWER_CASE_TABLE_NAMES)) { cacheMgr.removeCatalog(id); return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java index 127ca9ec71fd87..1daec40246280c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java @@ -146,16 +146,18 @@ private static void plan(ExternalCatalog catalog, ExternalObjectLog change, List }); scopes.add(id(ScopePath.table(Long.toString(dbId), Long.toString(tableId)))); if (!partitions) { - if (db.isPresent() && change.isInvalidateTableLookup()) { - String canonical = db.get().resolveTableNameForInvalidation(change.getTableName()); - scopes.add(local(ScopePath.table(db.get().getFullName(), canonical))); - scopes.add(id(ScopePath.table(Long.toString(dbId), Long.toString(Util.genIdByName( - catalog.getName(), db.get().getFullName(), canonical))))); + if (db.isPresent() && (change.isInvalidateTableLookup() || change.getNewTableName() != null)) { + addCanonicalTableScopes(catalog, db.get(), dbId, change.getTableName(), scopes); } scopes.add(local(change.isInvalidateTableLookup() ? ScopePath.table(change.getDbName(), change.getTableName()) : ScopePath.tableMetadata(change.getDbName(), change.getTableName()))); if (change.isInvalidateTableLookup() || change.getNewTableName() != null) { + // A first object/row-count load can publish an old spelling after scope planning. + // LOCAL scopes share case-folded identities; numeric derived caches need this DB fence. + if (catalog.getLowerCaseTableNames() == 2) { + scopes.add(id(ScopePath.database(Long.toString(dbId)))); + } scopes.add(local(ScopePath.tableCollection(change.getDbName()))); } } @@ -169,6 +171,9 @@ private static void plan(ExternalCatalog catalog, ExternalObjectLog change, List remote.add(ScopePath.tableCollection(remoteDb)); } if (change.getNewTableName() != null) { + if (db.isPresent()) { + addCanonicalTableScopes(catalog, db.get(), dbId, change.getNewTableName(), scopes); + } scopes.add(local(ScopePath.table(change.getDbName(), change.getTableName()))); scopes.add(local(ScopePath.table(change.getDbName(), change.getNewTableName()))); scopes.add(id(ScopePath.table(Long.toString(dbId), Long.toString(Util.genIdByName( @@ -205,6 +210,15 @@ private static void plan(ExternalCatalog catalog, ExternalObjectLog change, List } } + private static void addCanonicalTableScopes(ExternalCatalog catalog, ExternalDatabase database, + long dbId, String tableName, List scopes) { + // Case-insensitive lookup can retain a differently cased owner with a different deterministic ID. + String canonical = database.resolveTableNameForInvalidation(tableName); + scopes.add(local(ScopePath.table(database.getFullName(), canonical))); + scopes.add(id(ScopePath.table(Long.toString(dbId), Long.toString(Util.genIdByName( + catalog.getName(), database.getFullName(), canonical))))); + } + private static ScopePath local(ScopePath scope) { return scope.in(ScopePath.Coordinates.LOCAL); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerRenameReplayTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerRenameReplayTest.java index c5028794961da5..e0640855e98ba3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerRenameReplayTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerRenameReplayTest.java @@ -79,7 +79,10 @@ public void setUp() { Mockito.when(catalog.getMetaCacheOwnerIfPresent()).thenReturn(root); Mockito.when(catalog.resolveDatabaseNameForInvalidation("db1")).thenReturn("db1"); Mockito.doReturn(Optional.of(db)).when(catalog).getDbForReplay("db1"); + Mockito.when(db.getFullName()).thenReturn("db1"); Mockito.when(db.getRemoteName()).thenReturn("DB1"); + Mockito.when(db.resolveTableNameForInvalidation(Mockito.anyString())) + .thenAnswer(invocation -> invocation.getArgument(0)); dbId = Util.genIdByName("c", "db1"); sourceTableId = Util.genIdByName("c", "db1", "t1"); destinationTableId = Util.genIdByName("c", "db1", "t2"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java index de530783963056..9106511e37a5d1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java @@ -26,9 +26,11 @@ import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.util.Util; import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheGovernance; import org.apache.doris.datasource.log.CatalogLog; import org.apache.doris.datasource.log.ExternalObjectLog; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.datasource.metacache.FeMetaCacheEntry; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.datasource.test.TestExternalDatabase; @@ -132,6 +134,143 @@ public void catalogRenameAndReplayReactivateOnlyTheRegisteredInstance() throws E Assertions.assertEquals(1, MetaCacheGovernance.catalogCaches(catalog.getId()).size()); } + @Test + public void renameInvalidatesCaseEquivalentDestinationAndDerivedCaches() { + verifyCaseEquivalentRenameDestination(false, false, true); + } + + @Test + public void renameReplayInvalidatesCaseEquivalentDestinationWithColdNames() { + verifyCaseEquivalentRenameDestination(true, false, true); + } + + @Test + public void renameReplayInvalidatesPreviousDestinationAfterNamesRefresh() { + verifyCaseEquivalentRenameDestination(true, true, true); + } + + @Test + public void renameEventRefreshReplayInvalidatesPreviousDestinationAfterNamesRefresh() { + verifyCaseEquivalentRenameDestination(true, true, false); + } + + @SuppressWarnings("unchecked") + private void verifyCaseEquivalentRenameDestination(boolean replay, boolean refreshedNames, boolean rename) { + prepareCaseInsensitiveTables(); + putTable("Dest", "previous_destination"); + TestExternalTable source = database.getTableNullable("a"); + TestExternalTable destination = database.getTableNullable("Dest"); + TestExternalTable sibling = database.getTableNullable("b"); + source.getFullSchema(); + destination.getFullSchema(); + List siblingSchema = sibling.getFullSchema(); + CatalogMetaCache root = catalog.getFeMetaCacheOwner(); + MetaCache schemas = + (MetaCache) root.entries().get("schema"); + SchemaCacheKey schemaKey = new SchemaCacheKey(destination.getOrBuildNameMapping()); + ExternalRowCountCache counts = Env.getCurrentEnv().getExtMetaCacheMgr().getRowCountCache(); + counts.getCachedRowCount(catalog.getId(), database.getId(), destination.getId(), false); + counts.getCachedRowCount(catalog.getId(), database.getId(), sibling.getId(), false); + MetaCache rows = + (MetaCache) root.entries().get("row_count"); + ExternalRowCountCache.RowCountKey rowKey = new ExternalRowCountCache.RowCountKey( + catalog.getId(), database.getId(), destination.getId()); + ExternalRowCountCache.RowCountKey siblingRowKey = new ExternalRowCountCache.RowCountKey( + catalog.getId(), database.getId(), sibling.getId()); + Assertions.assertNotNull(schemas.getIfPresent(schemaKey)); + Assertions.assertNotNull(rows.getIfPresent(rowKey)); + Assertions.assertNotNull(rows.getIfPresent(siblingRowKey)); + Assertions.assertEquals("Dest", database.getCachedTableNameByIdForTest(destination.getId())); + + // The destination was removed outside Doris; rename recreates its case-equivalent remote name. + Provider.TABLES.remove("Dest"); + Provider.TABLES.remove("a"); + putTable("DEST", "renamed_source"); + if (replay) { + database.resetMetaCacheNames(); + } + if (refreshedNames) { + Assertions.assertEquals(Set.of("DEST", "b"), database.getTableNamesWithLock()); + } + int namesBeforeChange = catalog.nameLoads.get(); + ExternalObjectLog change = rename + ? ExternalObjectLog.createForRenameTable(catalog.getId(), "db", "a", "DEST") + .withRemoteNames("db", "a") + : ExternalObjectLog.createForExplicitRefreshTable(catalog.getId(), "db", "DEST", 0) + .withRemoteNames("db", "DEST"); + if (rename) { + change.setRemoteNewTableName("DEST"); + } + if (replay) { + ExternalObjectLog restored = GsonUtils.GSON.fromJson( + GsonUtils.GSON.toJson(change), ExternalObjectLog.class); + new RefreshManager().replayRefreshTable(restored); + } else { + CatalogMetadataChanges.apply(catalog, change); + } + Assertions.assertEquals(namesBeforeChange, catalog.nameLoads.get(), "invalidation must not list remotely"); + Assertions.assertNull(database.getCachedTableNameByIdForTest(destination.getId())); + Assertions.assertNull(database.getTableNullable(destination.getId())); + if (rename) { + Assertions.assertNull(database.getTableNullable(source.getId())); + } else { + Assertions.assertSame(source, database.getTableNullable(source.getId())); + } + Assertions.assertNull(schemas.getIfPresent(schemaKey)); + Assertions.assertNull(rows.getIfPresent(rowKey)); + Assertions.assertNull(rows.getIfPresent(siblingRowKey), "mode 2 identity changes fence numeric state for this DB"); + TestExternalTable renamed = database.getTableNullable("DEST"); + Assertions.assertNotSame(destination, renamed); + Assertions.assertEquals("renamed_source", renamed.getFullSchema().get(0).getName()); + Assertions.assertSame(sibling, database.getTableNullable("b")); + Assertions.assertSame(siblingSchema, sibling.getFullSchema()); + } + + private void prepareCaseInsensitiveTables() { + catalog.modifyCatalogProps(Collections.singletonMap(ExternalCatalog.LOWER_CASE_TABLE_NAMES, "2")); + catalog.setInitializedForTest(true); + database = new CountingDatabase(catalog); + database.setInitializedForTest(true); + catalog.addDatabaseForTest(database); + } + + @Test + public void tableNameModeChangesRetireTheScopePolicy() { + CatalogMetaCache sensitive = catalog.getFeMetaCacheOwner(); + prepareCaseInsensitiveTables(); + CatalogMetaCache insensitive = catalog.getFeMetaCacheOwner(); + Assertions.assertTrue(sensitive.isClosed()); + Assertions.assertNotSame(sensitive, insensitive); + catalog.modifyCatalogProps(Collections.singletonMap(ExternalCatalog.LOWER_CASE_TABLE_NAMES, "0")); + Assertions.assertTrue(insensitive.isClosed()); + Assertions.assertNotSame(insensitive, catalog.getFeMetaCacheOwner()); + } + + @Test + public void renameFencesFirstDestinationLoadAfterNamesRefresh() throws Exception { + prepareCaseInsensitiveTables(); + putTable("Dest", "previous_destination"); + objectsGate.arm(); + Future old = workers.submit(() -> database.getTableNullable("Dest")); + objectsGate.awaitStarted(); + Provider.TABLES.remove("Dest"); + Provider.TABLES.remove("a"); + putTable("DEST", "renamed_source"); + database.resetMetaCacheNames(); + Assertions.assertEquals(Set.of("DEST", "b"), database.getTableNamesWithLock()); + CatalogMetadataChanges.apply(catalog, + ExternalObjectLog.createForRenameTable(catalog.getId(), "db", "a", "DEST") + .withRemoteNames("db", "a")); + objectsGate.release.countDown(); + TestExternalTable previous = old.get(10, TimeUnit.SECONDS); + Assertions.assertNotNull(previous); + Assertions.assertNull(database.getCachedTableNameByIdForTest(previous.getId())); + Assertions.assertNull(database.getTableNullable(previous.getId())); + TestExternalTable renamed = database.getTableNullable("DEST"); + Assertions.assertNotSame(previous, renamed); + Assertions.assertEquals("renamed_source", renamed.getFullSchema().get(0).getName()); + } + @Test public void disabledDatabaseCacheSupportsCaseInsensitiveDropAndRenameEvents() { long originalTtl = Config.external_cache_expire_time_seconds_after_access; @@ -172,6 +311,16 @@ public void fullRefreshSharesOwnerAndPreservesSibling() throws Exception { @Test public void committedWritesAndTheirReplayDoNotReloadNames() { + verifyCommittedWritesDoNotReloadNames(); + } + + @Test + public void caseInsensitiveCommittedWritesAndReplayDoNotReloadNames() { + prepareCaseInsensitiveTables(); + verifyCommittedWritesDoNotReloadNames(); + } + + private void verifyCommittedWritesDoNotReloadNames() { TestExternalTable a = database.getTableNullable("a"); database.getTableNamesWithLock(); RefreshManager manager = new RefreshManager(); From 77b60ac4e47159920ea7937b83daa7fc59097f39 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Fri, 23 Oct 2026 15:42:30 +0800 Subject: [PATCH 5/5] [fix](fe) Repair observed missing tables without clearing replacements ### What problem does this PR solve? Issue Number: None Related PR: #69458 Problem Summary: A remote lookup could report a missing external table while FE continued to list its cached name. Observe the exact retained table and matching name snapshot before handle lookup, then conditionally invalidate their dependencies under one catalog publication writer. Reject observations after replacement, names refresh, content invalidation or registration retirement so a late missing response cannot evict a recreated table. Reuse the common dependency planner without writing a DDL journal. Preserve Paimon and Elasticsearch lookup failures instead of treating them as table absence. ### Release note Clear stale external-table names and metadata after a lookup confirms that the table is missing, while protecting same-name replacements from delayed responses. ### Check List (For Author) - Test: Unit Test / Regression test - Cache framework: 176 tests passed; focused FE missing-table, lifecycle and owner tests: 56 passed. - ES/Paimon failure classification and related FE system-table, row-count and metadata-change tests passed. - Full FE build and Checkstyle passed. Real HMS owner-refresh regression was generated and compared successfully (1 suite, zero failed/fatal/skipped). - Behavior changed: Yes. Confirmed missing-table reads repair this FE's cached lookup state. - Does this need documentation: No. Internal cache ownership contract is documented in the connector README. --- fe/fe-connector/README.md | 8 + .../connector/cache/CatalogMetaCache.java | 13 +- .../doris/connector/cache/MetaCache.java | 46 +- .../connector/cache/ScopedMetaCache.java | 10 + .../cache/ScopedMetaCacheRegistry.java | 93 +++- .../connector/cache/CatalogMetaCacheTest.java | 94 ++++ .../connector/es/EsConnectorRestClient.java | 12 +- .../es/EsConnectorRestClientTest.java | 29 ++ .../paimon/PaimonConnectorMetadata.java | 3 +- .../paimon/PaimonConnectorMetadataTest.java | 9 + .../paimon/RecordingPaimonCatalogOps.java | 4 + .../spi/ConnectorTableMetadataOps.java | 5 +- .../doris/datasource/ExternalDatabase.java | 16 + .../metacache/CatalogMetadataChanges.java | 54 +- .../metacache/FeMetaCacheEntry.java | 5 + .../plugin/PluginDrivenExternalTable.java | 9 +- .../metacache/FeMetaCacheEntryTest.java | 33 ++ .../plugin/PluginDrivenMissingTableTest.java | 472 ++++++++++++++++++ .../hive/test_hive_catalog_owner_refresh.out | 8 + .../test_hive_catalog_owner_refresh.groovy | 23 + 20 files changed, 906 insertions(+), 40 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenMissingTableTest.java diff --git a/fe/fe-connector/README.md b/fe/fe-connector/README.md index 1d5849558c8f11..f550baae1b8bb0 100644 --- a/fe/fe-connector/README.md +++ b/fe/fe-connector/README.md @@ -55,6 +55,14 @@ name lists; explicit lookup refresh also invalidates the target identity and par written journals persist local/remote identities and the operation's lookup flag. Old records widen only the remote scope when a cache-only lookup cannot recover the remote identity. +A definitive absent base-table handle can repair this FE's stale lookup state through the same dependency +planner, without writing a DDL journal. Before the remote read, FE observes both the exact retained table +and its matching names snapshot. Checking those observations and detaching dependencies share one publication +write phase. Replacement, name refresh, content invalidation or logical registration retirement makes the +old observation ineligible, so a delayed negative result cannot clear a recreated table. Cold names, +system/session-bypassed reads and remote failures do not authorize shared-cache repair. Connector handle +lookups must return absence only for a missing table and propagate other failures. + The root captures an immutable resource normalization policy. With case-insensitive table lookup (mode 2), LOCAL table scopes fold case while cache keys and displayed names retain their original spelling; REMOTE scopes retain the connector's case semantics. Registration, bulk-load parents and invalidation use the same diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java index d308ace3fc48f4..f7dd5930d907f4 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java @@ -192,7 +192,7 @@ public MetaCache create(MetaCacheDefinition definition) { nonNullDefinition.cacheSpec(), nonNullDefinition.removalListener(), nonNullDefinition.discardListener(), nonNullDefinition.refreshAfterWrite(), nonNullDefinition.refreshExecutor(), - entryBudget == null ? null : nonNullDefinition.sizeEstimator(), entryBudget)); + entryBudget == null ? null : nonNullDefinition.sizeEstimator(), entryBudget), rootOwner); entries.put(nonNullDefinition.name(), created); allEntries.put(created.registrationName(), created); if (isClosed()) { @@ -254,9 +254,18 @@ public void invalidateScope(ScopePath scope) { /** One barrier covers local and remote dependencies, even when their object caches are cold. */ public void invalidateScopes(Collection scopes) { + registry.invalidate(projectScopes(scopes)); + } + + boolean invalidateScopesIfCurrent(Collection scopes, + ScopedMetaCacheRegistry.CacheObservation observation) { + return registry.invalidateIfCurrent(projectScopes(scopes), observation); + } + + private List projectScopes(Collection scopes) { List projected = new ArrayList<>(scopes.size()); scopes.forEach(scope -> projected.add(project(scope))); - registry.invalidate(projected); + return projected; } private ScopePath project(ScopePath scope) { diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java index 74ab6d496b4716..f6145b84e3f7f1 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java @@ -17,8 +17,11 @@ package org.apache.doris.connector.cache; +import java.util.Collection; import java.util.Objects; +import java.util.Optional; import java.util.function.BiConsumer; +import java.util.function.BooleanSupplier; import java.util.function.Consumer; import java.util.function.Function; @@ -29,10 +32,12 @@ public final class MetaCache { private final String engine; private final String registrationName; private final CacheCallbackContext callbacks; + private final CatalogMetaCache rootOwner; MetaCache(MetaCacheDefinition definition, String engine, String registrationName, - CacheCallbackContext callbacks, ScopedMetaCache delegate) { + CacheCallbackContext callbacks, ScopedMetaCache delegate, CatalogMetaCache rootOwner) { this.callbacks = callbacks; + this.rootOwner = rootOwner; this.engine = engine; this.registrationName = registrationName; this.definition = Objects.requireNonNull(definition, "definition can not be null"); @@ -77,6 +82,45 @@ public V getWithPublicationAction(K key, Function loader, keyToLoad -> callbacks.call(() -> loader.apply(keyToLoad)), publicationCoordinator); } + /** + * Observes an exact retained instance without loading it. The token also tracks scope invalidations below + * its resource, exact-key replacement and physical registration retirement. It holds no publication lock. + */ + public Optional observe(K key, Object expectedValue) { + return observe(key, expectedValue, () -> true); + } + + /** + * Also fences logical registration retirement. The predicate is evaluated under the publication writer; + * it must only read lifecycle state, never block, acquire locks, call user code or re-enter a cache. + */ + public Optional observe(K key, Object expectedValue, BooleanSupplier registrationCurrent) { + return delegate.observe(Objects.requireNonNull(key, "key"), definition.scope(key), + Objects.requireNonNull(expectedValue, "expectedValue"), + Objects.requireNonNull(registrationCurrent, "registrationCurrent")) + .map(observation -> new Observation(rootOwner, observation)); + } + + public static final class Observation { + private final CatalogMetaCache rootOwner; + private final ScopedMetaCacheRegistry.CacheObservation observation; + + private Observation(CatalogMetaCache rootOwner, ScopedMetaCacheRegistry.CacheObservation observation) { + this.rootOwner = rootOwner; + this.observation = observation; + } + + /** Requires both exact instances to remain current in the same publication write phase. */ + public Observation and(Observation other) { + return new Observation(rootOwner, observation.and(other.observation)); + } + + /** Checks the observed state and detaches all dependencies in one publication write phase. */ + public boolean invalidateScopes(Collection scopes) { + return rootOwner.invalidateScopesIfCurrent(scopes, observation); + } + } + public V getIfPresent(K key) { K nonNullKey = Objects.requireNonNull(key, "key can not be null"); return delegate.getIfPresent(nonNullKey, definition.scope(nonNullKey)); diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java index 5fa04ea067a5f6..b26d5729866619 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java @@ -276,6 +276,16 @@ public V getWithPublicationAction(K key, ScopePath path, Function loader, } } + Optional observe(K key, ScopePath path, Object expectedValue, + BooleanSupplier registrationCurrent) { + VersionedValue value = currentVersionedValue(key, path); + if (value == null || value.value != expectedValue) { + return Optional.empty(); + } + return Optional.of(registry.observe(value.scopeSnapshot, + () -> registrationCurrent.getAsBoolean() && !closed.get() && value.isCurrent(registry, keyNodes))); + } + public V getIfPresent(K key, ScopePath path) { Objects.requireNonNull(key, "key can not be null"); Objects.requireNonNull(path, "path can not be null"); diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java index ba04642aa6ef6a..6168805fbc1eb3 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java @@ -216,32 +216,85 @@ void invalidate(List paths, Runnable afterStateReplacement) { Objects.requireNonNull(paths, "paths can not be null"); Objects.requireNonNull(afterStateReplacement, "afterStateReplacement can not be null"); checkOpen(); - List invalidated = publicationGate.write(() -> { - List> resolvedScopes = new ArrayList<>(paths.size()); - Set targets = Collections.newSetFromMap(new IdentityHashMap<>()); - Set publicationNodes = Collections.newSetFromMap(new IdentityHashMap<>()); - for (ScopePath path : paths) { - Objects.requireNonNull(path, "path can not be null"); - List existing = resolveExisting(path); - publicationNodes.addAll(existing); - if (existing.size() == path.depth() + 1) { - ScopeNode target = existing.get(existing.size() - 1); - if (targets.add(target)) { - resolvedScopes.add(existing); - } + List invalidated = publicationGate.write(() -> detachScopes(paths)); + afterStateReplacement.run(); + cleanInvalidatedScopes(invalidated); + } + + CacheObservation observe(ScopeSnapshot snapshot, BooleanSupplier exactValueCurrent) { + return new CacheObservation(this, snapshot, exactValueCurrent); + } + + boolean invalidateIfCurrent(List paths, CacheObservation observation) { + Objects.requireNonNull(paths, "paths can not be null"); + if (observation.registry != this) { + throw new IllegalArgumentException("Observation belongs to another metadata cache registry"); + } + // The predicate reads only framework state: no cache lookup, callbacks, locks or remote I/O. + List invalidated = publicationGate.write(() -> observation.isCurrent() + ? detachScopes(paths) : null); + if (invalidated == null) { + return false; + } + cleanInvalidatedScopes(invalidated); + return true; + } + + private List detachScopes(List paths) { + List> resolvedScopes = new ArrayList<>(paths.size()); + Set targets = Collections.newSetFromMap(new IdentityHashMap<>()); + Set publicationNodes = Collections.newSetFromMap(new IdentityHashMap<>()); + for (ScopePath path : paths) { + Objects.requireNonNull(path, "path can not be null"); + List existing = resolveExisting(path); + publicationNodes.addAll(existing); + if (existing.size() == path.depth() + 1) { + ScopeNode target = existing.get(existing.size() - 1); + if (targets.add(target)) { + resolvedScopes.add(existing); } } - bumpPublicationStates(new ArrayList<>(publicationNodes)); - List result = new ArrayList<>(resolvedScopes.size()); - resolvedScopes.forEach(existing -> result.add(new InvalidatedScope( - existing, replaceState(existing.get(existing.size() - 1))))); - return result; - }); - afterStateReplacement.run(); + } + bumpPublicationStates(new ArrayList<>(publicationNodes)); + List result = new ArrayList<>(resolvedScopes.size()); + resolvedScopes.forEach(existing -> result.add(new InvalidatedScope( + existing, replaceState(existing.get(existing.size() - 1))))); + return result; + } + + private void cleanInvalidatedScopes(List invalidated) { invalidated.forEach(scope -> cleanDetachedState(scope.oldState)); invalidated.forEach(scope -> tryPrune(scope.existing)); } + static final class CacheObservation { + private final ScopedMetaCacheRegistry registry; + private final ScopeSnapshot snapshot; + private final PublicationState publicationState; + private final BooleanSupplier exactValueCurrent; + + private CacheObservation(ScopedMetaCacheRegistry registry, ScopeSnapshot snapshot, + BooleanSupplier exactValueCurrent) { + this.registry = registry; + this.snapshot = snapshot; + this.publicationState = snapshot.leafNode().descendantPublication.get(); + this.exactValueCurrent = exactValueCurrent; + } + + CacheObservation and(CacheObservation other) { + if (registry != other.registry) { + throw new IllegalArgumentException("Observations must belong to the same registry"); + } + return registry.observe(snapshot, () -> isCurrent() && other.isCurrent()); + } + + private boolean isCurrent() { + return snapshot.isCurrent(registry) + && snapshot.leafNode().descendantPublication.get() == publicationState + && exactValueCurrent.getAsBoolean(); + } + } + public ScopeMetrics metrics() { ScopeMetricsAccumulator accumulator = new ScopeMetricsAccumulator(); collectMetrics(root, accumulator); diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java index 96c48b522d7b8a..068e0f773b6932 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java @@ -73,6 +73,100 @@ void resourceNormalizationCoversDomainsAndBulkLoadsWithoutChangingRemoteNames() } } + @Test + void conditionalInvalidationRejectsExactReplacementAndNewScopeGeneration() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + ScopePath table = ScopePath.table("db", "a").in(ScopePath.Coordinates.LOCAL); + ScopePath names = ScopePath.tableCollection("db").in(ScopePath.Coordinates.LOCAL); + MetaCache objects = root.create(MetaCacheDefinition.builder( + "objects", SPEC, key -> table).build()); + MetaCache listings = root.create(MetaCacheDefinition.builder( + "names", SPEC, key -> names).build()); + Object old = new Object(); + objects.put("a", old); + listings.put("all", "a,b"); + MetaCache.Observation replaced = objects.observe("a", old).orElseThrow(); + Object replacement = new Object(); + objects.put("a", replacement); + Assertions.assertFalse(replaced.invalidateScopes(Arrays.asList(table, names))); + Assertions.assertSame(replacement, objects.getIfPresent("a")); + Assertions.assertEquals("a,b", listings.getIfPresent("all")); + MetaCache.Observation refreshed = objects.observe("a", replacement).orElseThrow(); + root.invalidateScope(table); + objects.put("a", old); + Assertions.assertFalse(refreshed.invalidateScopes(Arrays.asList(table, names))); + Assertions.assertSame(old, objects.getIfPresent("a")); + Assertions.assertEquals("a,b", listings.getIfPresent("all")); + Assertions.assertTrue(objects.observe("a", old).orElseThrow() + .invalidateScopes(Arrays.asList(table, names))); + Assertions.assertNull(objects.getIfPresent("a")); + Assertions.assertNull(listings.getIfPresent("all")); + } + } + + @Test + void conditionalInvalidationRejectsNewContentAndRetiredRegistration() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + CatalogMetaCache domain = root.openDomain("fe", ScopePath.Coordinates.LOCAL, + getClass().getClassLoader()); + MetaCache objects = domain.create(MetaCacheDefinition.builder( + "objects", SPEC, key -> ScopePath.table("db", key)).build()); + Object table = new Object(); + objects.put("a", table); + MetaCache.Observation beforeContent = objects.observe("a", table).orElseThrow(); + root.invalidateScope(ScopePath.tableMetadata("db", "a").in(ScopePath.Coordinates.LOCAL)); + Assertions.assertFalse(beforeContent.invalidateScopes(Collections.singletonList( + ScopePath.table("db", "a").in(ScopePath.Coordinates.LOCAL)))); + Assertions.assertSame(table, objects.getIfPresent("a")); + MetaCache.Observation retired = objects.observe("a", table).orElseThrow(); + domain.close(); + MetaCache replacement = root.openDomain("fe", ScopePath.Coordinates.LOCAL, + getClass().getClassLoader()).create(MetaCacheDefinition.builder( + "objects", SPEC, key -> ScopePath.table("db", key)).build()); + replacement.put("a", table); + Assertions.assertFalse(retired.invalidateScopes(Collections.singletonList( + ScopePath.table("db", "a").in(ScopePath.Coordinates.LOCAL)))); + Assertions.assertSame(table, replacement.getIfPresent("a")); + } + } + + @Test + void conditionalInvalidationCoversDependenciesAndFencesAnOldLoad() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + ScopePath local = ScopePath.table("db", "a").in(ScopePath.Coordinates.LOCAL); + ScopePath remote = ScopePath.table("DB", "A").in(ScopePath.Coordinates.REMOTE); + MetaCache objects = root.create(MetaCacheDefinition.builder( + "objects", SPEC, key -> local).build()); + Object table = new Object(); + objects.put("a", table); + MetaCache dependencies = root.create(MetaCacheDefinition.builder( + "remote", SPEC, key -> remote).build()); + java.util.concurrent.Future old = executor.submit(() -> dependencies.get("a", key -> { + started.countDown(); + try { + Assertions.assertTrue(release.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + return "old"; + })); + Assertions.assertTrue(started.await(10, TimeUnit.SECONDS)); + Assertions.assertTrue(objects.observe("a", table).orElseThrow() + .invalidateScopes(Arrays.asList(local, remote))); + release.countDown(); + Assertions.assertEquals("old", old.get(10, TimeUnit.SECONDS)); + Assertions.assertNull(dependencies.getIfPresent("a")); + } finally { + release.countDown(); + executor.shutdownNow(); + Assertions.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + } + } + @Test void unmanagedOwnershipMustBeSelectedExplicitly() throws Exception { Assertions.assertTrue(Modifier.isPrivate(CatalogMetaCache.class.getDeclaredConstructor().getModifiers())); diff --git a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorRestClient.java b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorRestClient.java index c509f45004fce2..d90c821f48b6c0 100644 --- a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorRestClient.java +++ b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorRestClient.java @@ -106,10 +106,16 @@ public boolean existIndex(String indexName) { OkHttpClient client = httpSslEnable ? getOrCreateSslClient() : PLAIN_CLIENT; String node = currentNode(); try (Response response = executeResponse(client, node, path, null)) { - return response.isSuccessful(); + if (response.isSuccessful()) { + return true; + } + if (response.code() == 404) { + return false; + } + throw new DorisConnectorException("Failed to check index " + indexName + + ": HTTP " + response.code()); } catch (IOException e) { - LOG.warn("existIndex error", e); - return false; + throw new DorisConnectorException("Failed to check index " + indexName, e); } } diff --git a/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsConnectorRestClientTest.java b/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsConnectorRestClientTest.java index 16439e765bd05b..d1354c93744c2f 100644 --- a/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsConnectorRestClientTest.java +++ b/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsConnectorRestClientTest.java @@ -17,10 +17,14 @@ package org.apache.doris.connector.es; +import org.apache.doris.connector.spi.DorisConnectorException; + +import com.sun.net.httpserver.HttpServer; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.lang.reflect.Field; +import java.net.InetSocketAddress; import java.util.Collections; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -34,6 +38,31 @@ */ public class EsConnectorRestClientTest { + @Test + public void indexAbsenceRequiresNotFoundResponse() throws Exception { + AtomicInteger status = new AtomicInteger(200); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/t/_mapping", exchange -> { + exchange.sendResponseHeaders(status.get(), -1); + exchange.close(); + }); + server.start(); + EsConnectorRestClient client = new EsConnectorRestClient( + new String[]{"http://127.0.0.1:" + server.getAddress().getPort()}, null, null, false, null); + try { + Assertions.assertTrue(client.existIndex("t")); + status.set(404); + Assertions.assertFalse(client.existIndex("t")); + for (int failure : new int[]{401, 403, 500}) { + status.set(failure); + Assertions.assertThrows(DorisConnectorException.class, () -> client.existIndex("t")); + } + } finally { + server.stop(0); + } + Assertions.assertThrows(DorisConnectorException.class, () -> client.existIndex("t")); + } + @Test public void testConstructorWithAuth() throws Exception { EsConnectorRestClient client = new EsConnectorRestClient( diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java index 245da688dd84f6..b7b365e81ebc4b 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java @@ -285,8 +285,7 @@ public Optional getTableHandle( return Optional.of(handle); }); } catch (Exception e) { - LOG.warn("Failed to get Paimon table handle: {}.{}", dbName, tableName, e); - return Optional.empty(); + throw new DorisConnectorException("Failed to get Paimon table handle: " + dbName + "." + tableName, e); } } diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataTest.java index 4a7c5316f9f903..564b4809509e05 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataTest.java @@ -181,6 +181,15 @@ public void getTableHandleEmptyWhenTableMissing() { Assertions.assertFalse(handleOpt.isPresent()); } + @Test + public void getTableHandlePropagatesLookupFailure() { + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.tableLookupFailure = new IllegalStateException("metastore timeout"); + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> metadataWith(ops).getTableHandle(null, "db1", "t1")); + Assertions.assertSame(ops.tableLookupFailure, failure.getCause()); + } + @Test public void getColumnHandlesReloadFallbackReloadsWhenTransientTableNull() { RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingPaimonCatalogOps.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingPaimonCatalogOps.java index 73147f40620a0d..a6465faf6c62ca 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingPaimonCatalogOps.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingPaimonCatalogOps.java @@ -69,6 +69,7 @@ final class RecordingPaimonCatalogOps implements PaimonCatalogOps { boolean throwDatabaseNotExist; boolean throwTableNotExist; + RuntimeException tableLookupFailure; // ---- B3 DDL capture fields (inputs the metadata layer passed to the seam) ---- Schema lastCreatedSchema; @@ -163,6 +164,9 @@ public List listTables(String databaseName) throws Catalog.DatabaseNotEx @Override public Table getTable(Identifier identifier) throws Catalog.TableNotExistException { + if (tableLookupFailure != null) { + throw tableLookupFailure; + } log.add("getTable:" + identifier.getFullName()); lastGetTableId = identifier; if (throwTableNotExist) { diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorTableMetadataOps.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorTableMetadataOps.java index 4d29727e1b84d9..64263b639f59a4 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorTableMetadataOps.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorTableMetadataOps.java @@ -62,7 +62,10 @@ */ public interface ConnectorTableMetadataOps { - /** Retrieves a table handle for the given database and table name. */ + /** + * Retrieves a table handle for the given database and table name. Empty means the table does not exist; + * authentication, transport and other lookup failures must propagate rather than report absence. + */ @ConnectorMustImplement default Optional getTableHandle( ConnectorSession session, String dbName, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 951be1d72225b5..0952859c318487 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -33,6 +33,7 @@ import org.apache.doris.common.util.Util; import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase; @@ -403,6 +404,21 @@ public Optional getTableForReplay(long tableId) { return Optional.ofNullable(tables.getIfPresent(tableName)); } + /** Captures only the shared base-table instance; transient and bypassed reads cannot repair it. */ + public Optional observeTableLookup(ExternalTable table) { + if (!initialized || cacheEntriesReleased || extCatalog.shouldBypassSchemaCache(SessionContext.current())) { + return Optional.empty(); + } + // Transient system/session tables are not this database's retained base-table instance. + NameCacheValue names = getTableNamesValue(false); + if (names == null || !table.getRemoteName().equals(names.remoteNameOfLocalName(table.getName()))) { + return Optional.empty(); + } + // Name refresh can change the canonical spelling without replacing the old exact table key. + return tableNames.observe("", names).flatMap(nameObservation -> + tables.observe(table.getName(), table).map(nameObservation::and)); + } + /** * Resolve the retained local table name for replay without loading table metadata. * diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java index 1daec40246280c..2cf8faa3003293 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java @@ -21,6 +21,7 @@ import org.apache.doris.common.util.Util; import org.apache.doris.connector.cache.CacheCallbackContext; import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; import org.apache.doris.datasource.ExternalCatalog; @@ -57,6 +58,34 @@ public static ExternalObjectLog table(ExternalTable table, boolean lookup, long return change.withRemoteNames(table.getRemoteDbName(), table.getRemoteName()); } + public static Optional observeTableLookup(ExternalTable table) { + ExternalDatabase database = table.getDb(); + return database.observeTableLookup(table) + .map(observation -> new MissingTableObservation(table, observation)); + } + + /** A read observation repairs only this FE; it is not a committed external DDL journal record. */ + public static final class MissingTableObservation { + private final ExternalCatalog catalog; + private final ExternalObjectLog change; + private final MetaCache.Observation observation; + + private MissingTableObservation(ExternalTable table, MetaCache.Observation observation) { + this.catalog = table.getCatalog(); + this.change = table(table, true, 0); + this.observation = observation; + } + + public void missing() { + List scopes = new ArrayList<>(); + try { + plan(catalog, change, scopes); + } finally { + observation.invalidateScopes(scopes); + } + } + } + public static void apply(ExternalCatalog catalog, ExternalObjectLog change) { apply(catalog, java.util.Collections.singletonList(change)); } @@ -68,7 +97,7 @@ public static void apply(ExternalCatalog catalog, List change try { for (ExternalObjectLog change : changes) { try { - plan(catalog, change, scopes); + updateCachedTable(plan(catalog, change, scopes), change); } catch (RuntimeException error) { if (failure == null) { failure = error; @@ -106,7 +135,19 @@ private static void invalidateSortedPartitions(ExternalCatalog catalog, External } } - private static void plan(ExternalCatalog catalog, ExternalObjectLog change, List scopes) { + private static void updateCachedTable(Optional table, ExternalObjectLog change) { + table.ifPresent(value -> { + if (change.getPartitionNames() == null) { + value.unsetObjectCreated(); + } + if (change.getLastUpdateTime() > 0) { + value.setUpdateTime(change.getLastUpdateTime()); + } + }); + } + + private static Optional plan( + ExternalCatalog catalog, ExternalObjectLog change, List scopes) { Optional> db = catalog.getDbForReplay(change.getDbName()); Optional table = Strings.isNullOrEmpty(change.getTableName()) ? Optional.empty() : db.flatMap(database -> database.getTableForReplay(change.getTableName())); @@ -136,14 +177,6 @@ private static void plan(ExternalCatalog catalog, ExternalObjectLog change, List remote.add(remoteDb == null ? ScopePath.catalog() : ScopePath.database(remoteDb)); remote.add(ScopePath.databaseCollection()); } else { - table.ifPresent(value -> { - if (!partitions) { - value.unsetObjectCreated(); - } - if (change.getLastUpdateTime() > 0) { - value.setUpdateTime(change.getLastUpdateTime()); - } - }); scopes.add(id(ScopePath.table(Long.toString(dbId), Long.toString(tableId)))); if (!partitions) { if (db.isPresent() && (change.isInvalidateTableLookup() || change.getNewTableName() != null)) { @@ -208,6 +241,7 @@ private static void plan(ExternalCatalog catalog, ExternalObjectLog change, List } finally { remote.forEach(scope -> scopes.add(scope.in(ScopePath.Coordinates.REMOTE))); } + return table; } private static void addCanonicalTableScopes(ExternalCatalog catalog, ExternalDatabase database, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java index 94e5b4147325c8..ce04bf52aa85fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java @@ -35,6 +35,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -313,6 +314,10 @@ private V getAndRunIfCurrentCached(K key, BiPredicate required, } } + public Optional observe(K key, Object expectedValue) { + return useCache(() -> data.observe(key, expectedValue, () -> !retired), Optional::empty); + } + public V getIfPresent(K key) { return useCache(() -> data.getIfPresent(key), () -> null); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java index f424fde948aff0..c5bdbdf54ab54e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java @@ -50,6 +50,7 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.TablePartitionValues; import org.apache.doris.datasource.connector.converter.ConnectorColumnConverter; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.PluginDrivenMvccSnapshot; import org.apache.doris.datasource.systable.PartitionsSysTable; @@ -120,8 +121,14 @@ public PluginDrivenExternalTable(long id, String name, String remoteName, */ public Optional resolveConnectorTableHandle( ConnectorSession session, ConnectorMetadata metadata) { + Optional observation = + CatalogMetadataChanges.observeTableLookup(this); String dbName = db != null ? db.getRemoteName() : ""; - return metadata.getTableHandle(session, dbName, getRemoteName()); + Optional handle = metadata.getTableHandle(session, dbName, getRemoteName()); + if (!handle.isPresent()) { + observation.ifPresent(CatalogMetadataChanges.MissingTableObservation::missing); + } + return handle; } /** diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java index b2a997ea713ae5..53d4e80cd7987b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java @@ -19,6 +19,7 @@ import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.ScopePath; import org.junit.jupiter.api.Assertions; @@ -125,6 +126,38 @@ public void testRetirementPinsAuxiliaryStateUntilAcceptedActionFinishes() throws } } + @Test + @SuppressWarnings("unchecked") + public void testObservationRejectsLogicalRetirementBeforePhysicalRelease() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch loadStarted = new CountDownLatch(1); + CountDownLatch finishLoad = new CountDownLatch(1); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + FeMetaCacheEntry entry = FeMetaCacheEntry.sharedLookup(owner, + ScopePath::database, ScopePath.catalog(), "objects", + key -> new Object(), ENABLED, executor, false, 8, null); + Object original = entry.get("db"); + MetaCache.Observation observation = entry.observe("db", original).get(); + Future load = executor.submit(() -> entry.get("other", key -> { + loadStarted.countDown(); + await(finishLoad); + return new Object(); + })); + Assertions.assertTrue(loadStarted.await(3L, TimeUnit.SECONDS)); + entry.close(); + Assertions.assertFalse(owner.entries().isEmpty(), "accepted load pins the physical registration"); + Assertions.assertFalse(observation.invalidateScopes(List.of( + ScopePath.database("db").in(ScopePath.Coordinates.LOCAL)))); + Assertions.assertSame(original, ((MetaCache) owner.entries().get("objects")).getIfPresent("db")); + finishLoad.countDown(); + load.get(3L, TimeUnit.SECONDS); + Assertions.assertTrue(owner.entries().isEmpty()); + } finally { + finishLoad.countDown(); + executor.shutdownNow(); + } + } + @Test public void testRejectedWeightedMutationReturnsWithoutRetryingForever() throws Exception { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenMissingTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenMissingTableTest.java new file mode 100644 index 00000000000000..479527c84e3db0 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenMissingTableTest.java @@ -0,0 +1,472 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.plugin; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.common.util.Util; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorColumn; +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorTableSchema; +import org.apache.doris.connector.spi.ConnectorType; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.datasource.CacheException; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.SchemaCacheKey; +import org.apache.doris.datasource.SchemaCacheValue; +import org.apache.doris.datasource.SessionContext; +import org.apache.doris.datasource.log.ExternalObjectLog; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; +import org.apache.doris.datasource.metacache.FeMetaCacheEntry; +import org.apache.doris.datasource.metacache.NameCacheValue; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** Tests read-observed absence through real FE object/schema caches and a controllable connector. */ +public class PluginDrivenMissingTableTest extends TestWithFeService { + private static final AtomicLong IDS = new AtomicLong(910000); + private final Map remoteTables = new ConcurrentHashMap<>(); + private Connector connector; + private ConnectorMetadata metadata; + private TestCatalog catalog; + private TestDatabase database; + private MetaCache remoteCache; + private ExecutorService worker; + private CountDownLatch missingStarted; + private CountDownLatch releaseMissing; + private AtomicBoolean blockMissing; + private boolean failLookup; + + @Override + protected void runBeforeAll() throws Exception { + FeConstants.runningUnitTest = true; + createDefaultCtx().setThreadLocalInfo(); + } + + @BeforeEach + void setup() { + failLookup = false; + remoteTables.put("a", "old_column"); + remoteTables.put("b", "sibling_column"); + missingStarted = new CountDownLatch(1); + releaseMissing = new CountDownLatch(1); + blockMissing = new AtomicBoolean(); + worker = Executors.newFixedThreadPool(2); + connector = Mockito.mock(Connector.class); + metadata = Mockito.mock(ConnectorMetadata.class); + Mockito.when(connector.getMetadata(Mockito.any())).thenReturn(metadata); + Mockito.when(metadata.getTableHandle(Mockito.any(), Mockito.anyString(), Mockito.anyString())) + .thenAnswer(call -> { + if (failLookup) { + throw new IllegalStateException("metastore timeout"); + } + String name = call.getArgument(2); + String column = remoteTables.get(name); + if (column == null) { + if (blockMissing.compareAndSet(true, false)) { + missingStarted.countDown(); + Assertions.assertTrue(releaseMissing.await(10, TimeUnit.SECONDS)); + } + return Optional.empty(); + } + return Optional.of(new Handle(name, column)); + }); + Mockito.when(metadata.getTableSchema(Mockito.any(), Mockito.any())).thenAnswer(call -> { + Handle handle = call.getArgument(1); + return new ConnectorTableSchema(handle.name, Collections.singletonList(new ConnectorColumn( + handle.column, ConnectorType.of("INT", -1, -1), "", true, null, true)), + null, Collections.emptyMap()); + }); + Mockito.when(metadata.fromRemoteColumnName(Mockito.any(), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString())).thenAnswer(call -> call.getArgument(3)); + Mockito.when(metadata.fromRemoteTableName(Mockito.any(), Mockito.anyString(), Mockito.anyString())) + .thenAnswer(call -> call.getArgument(2)); + Mockito.when(metadata.fromRemoteDatabaseName(Mockito.any(), Mockito.anyString())) + .thenAnswer(call -> call.getArgument(1)); + catalog = new TestCatalog(IDS.incrementAndGet()); + CatalogMgr manager = Env.getCurrentEnv().getCatalogMgr(); + manager.getIdToCatalog().put(catalog.getId(), catalog); + Map> byName = Deencapsulation.getField(manager, "nameToCatalog"); + byName.put(catalog.getName(), catalog); + installDatabase(); + } + + @AfterEach + void cleanup() throws Exception { + releaseMissing.countDown(); + worker.shutdownNow(); + Assertions.assertTrue(worker.awaitTermination(10, TimeUnit.SECONDS)); + catalog.closeFeMetaCacheOwner(); + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalog(catalog.getId()); + CatalogMgr manager = Env.getCurrentEnv().getCatalogMgr(); + manager.getIdToCatalog().remove(catalog.getId()); + Map> byName = Deencapsulation.getField(manager, "nameToCatalog"); + byName.remove(catalog.getName()); + remoteTables.clear(); + } + + private void installDatabase() { + catalog.setInitializedForTest(true); + database = new TestDatabase(); + database.setInitializedForTest(true); + catalog.addDatabaseForTest(database); + remoteCache = catalog.getFeMetaCacheOwner().openDomain("missing-test", ScopePath.Coordinates.REMOTE, + getClass().getClassLoader()).create(MetaCacheDefinition.builder( + "remote", CacheSpec.of(true, -1, 100), key -> ScopePath.table("db", key)).build()); + } + + @SuppressWarnings("unchecked") + private MetaCache schemas() { + return (MetaCache) catalog.getFeMetaCacheOwner().entries().get("schema"); + } + + private PluginDrivenExternalTable warmTarget() { + Assertions.assertEquals(Set.of("a", "b"), database.getTableNamesWithLock()); + PluginDrivenExternalTable table = database.getTableNullable("a"); + table.getFullSchema(); + database.getTableNullable("b").getFullSchema(); + remoteCache.put("a", "old"); + remoteCache.put("b", "sibling"); + // Keep membership/object entries hot while forcing the next schema read to consult the source. + schemas().invalidateKey(new SchemaCacheKey(table.getOrBuildNameMapping())); + return table; + } + + @Test + void missingTableRepairsNamesObjectsIdsAndConnectorMetadata() { + PluginDrivenExternalTable old = warmTarget(); + PluginDrivenExternalTable sibling = database.getTableNullable("b"); + List siblingSchema = sibling.getFullSchema(); + remoteTables.remove("a"); + Assertions.assertThrows(CacheException.class, old::getFullSchema); + Assertions.assertEquals(Set.of("b"), database.getTableNamesWithLock()); + Assertions.assertNull(database.getTableNullable(old.getId())); + Assertions.assertNull(remoteCache.getIfPresent("a")); + Assertions.assertEquals("sibling", remoteCache.getIfPresent("b")); + Assertions.assertSame(sibling, database.getTableNullable("b")); + Assertions.assertSame(siblingSchema, sibling.getFullSchema()); + } + + @Test + void missingHandleRepairsNamesBeforeSchemaLoading() { + PluginDrivenExternalTable table = warmTarget(); + remoteTables.remove("a"); + ConnectorSession session = catalog.buildCrossStatementSession(); + try { + Assertions.assertFalse(table.resolveConnectorTableHandle(session, metadata).isPresent()); + } finally { + session.getStatementScope().closeAll(); + } + Assertions.assertEquals(Set.of("b"), database.getTableNamesWithLock()); + Assertions.assertNull(database.getTableNullable(table.getId())); + Assertions.assertNull(remoteCache.getIfPresent("a")); + } + + @Test + void delayedMissingDoesNotInvalidateRecreatedTable() throws Exception { + verifyDelayedMissing(false); + } + + @Test + void delayedMissingDoesNotInvalidateExactKeyReplacement() throws Exception { + verifyDelayedMissing(true); + } + + private void verifyDelayedMissing(boolean replaceWithoutScopeInvalidation) throws Exception { + PluginDrivenExternalTable old = warmTarget(); + remoteTables.remove("a"); + blockMissing.set(true); + Future result = worker.submit(() -> Assertions.assertThrows(CacheException.class, + old::getFullSchema)); + Assertions.assertTrue(missingStarted.await(10, TimeUnit.SECONDS)); + remoteTables.put("a", "new_column"); + PluginDrivenExternalTable replacement; + if (replaceWithoutScopeInvalidation) { + replacement = database.buildTableInternal("a", "a", old.getId(), catalog, database); + database.addTableForTest(replacement); + schemas().put(new SchemaCacheKey(replacement.getOrBuildNameMapping()), replacement.initSchema().get()); + } else { + CatalogMetadataChanges.apply(catalog, ExternalObjectLog.createForExplicitRefreshTable( + catalog.getId(), "db", "a", 0).withRemoteNames("db", "a")); + replacement = database.getTableNullable("a"); + } + List newSchema = replacement.getFullSchema(); + remoteCache.put("a", "new"); + database.getTableNamesWithLock(); + int nameLoads = catalog.nameLoads.get(); + releaseMissing.countDown(); + Assertions.assertTrue(result.get(10, TimeUnit.SECONDS).getMessage().contains("does not exist")); + Assertions.assertSame(replacement, database.getTableNullable("a")); + Assertions.assertSame(replacement, database.getTableNullable(replacement.getId())); + Assertions.assertSame(newSchema, replacement.getFullSchema()); + Assertions.assertTrue(replacement.isObjectCreated()); + Assertions.assertEquals("new_column", newSchema.get(0).getName()); + Assertions.assertEquals("new", remoteCache.getIfPresent("a")); + Assertions.assertEquals(Set.of("a", "b"), database.getTableNamesWithLock()); + Assertions.assertEquals(nameLoads, catalog.nameLoads.get()); + } + + @Test + void delayedMissingDoesNotInvalidateReplacementCatalogRoot() throws Exception { + PluginDrivenExternalTable old = warmTarget(); + remoteTables.remove("a"); + blockMissing.set(true); + Future result = worker.submit(() -> Assertions.assertThrows(CacheException.class, + old::getFullSchema)); + Assertions.assertTrue(missingStarted.await(10, TimeUnit.SECONDS)); + CatalogMetaCache retired = catalog.getFeMetaCacheOwner(); + catalog.closeFeMetaCacheOwner(); + remoteTables.put("a", "new_root_column"); + installDatabase(); + PluginDrivenExternalTable replacement = database.getTableNullable("a"); + List schema = replacement.getFullSchema(); + remoteCache.put("a", "new_root"); + Assertions.assertEquals(Set.of("a", "b"), database.getTableNamesWithLock()); + int nameLoads = catalog.nameLoads.get(); + releaseMissing.countDown(); + result.get(10, TimeUnit.SECONDS); + Assertions.assertTrue(retired.isClosed()); + Assertions.assertSame(replacement, database.getTableNullable("a")); + Assertions.assertSame(schema, replacement.getFullSchema()); + Assertions.assertTrue(replacement.isObjectCreated()); + Assertions.assertEquals("new_root", remoteCache.getIfPresent("a")); + Assertions.assertEquals(Set.of("a", "b"), database.getTableNamesWithLock()); + Assertions.assertEquals(nameLoads, catalog.nameLoads.get()); + } + + @Test + void retiredDatabaseWithAnActiveLoadCannotClearReplacement() throws Exception { + PluginDrivenExternalTable old = warmTarget(); + remoteTables.remove("a"); + blockMissing.set(true); + Future missing = worker.submit(() -> { + ConnectorSession session = catalog.buildCrossStatementSession(); + try { + Assertions.assertFalse(old.resolveConnectorTableHandle(session, metadata).isPresent()); + } finally { + session.getStatementScope().closeAll(); + } + }); + Assertions.assertTrue(missingStarted.await(10, TimeUnit.SECONDS)); + FeMetaCacheEntry oldTables = Deencapsulation.getField(database, "tables"); + CountDownLatch loadStarted = new CountDownLatch(1); + CountDownLatch releaseLoad = new CountDownLatch(1); + Future activeLoad = worker.submit(() -> oldTables.get("loading", key -> { + loadStarted.countDown(); + try { + Assertions.assertTrue(releaseLoad.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + return null; + })); + try { + Assertions.assertTrue(loadStarted.await(10, TimeUnit.SECONDS)); + database.releaseCachedEntries(); + remoteTables.put("a", "replacement_column"); + database = new TestDatabase(); + database.setInitializedForTest(true); + catalog.addDatabaseForTest(database); + PluginDrivenExternalTable replacement = database.getTableNullable("a"); + List schema = replacement.getFullSchema(); + remoteCache.put("a", "replacement"); + database.getTableNamesWithLock(); + int loads = catalog.nameLoads.get(); + releaseMissing.countDown(); + missing.get(10, TimeUnit.SECONDS); + Assertions.assertSame(replacement, database.getTableNullable("a")); + Assertions.assertSame(schema, replacement.getFullSchema()); + Assertions.assertEquals("replacement", remoteCache.getIfPresent("a")); + Assertions.assertEquals(loads, catalog.nameLoads.get()); + } finally { + releaseLoad.countDown(); + activeLoad.get(10, TimeUnit.SECONDS); + } + } + + @Test + void changedCanonicalSpellingFencesBothExistingAndNewOldObjectReads() throws Exception { + catalog.lowerCaseMode = 2; + try { + catalog.closeFeMetaCacheOwner(); + installDatabase(); + remoteTables.remove("a"); + remoteTables.put("Foo", "old_column"); + PluginDrivenExternalTable old = database.getTableNullable("foo"); + old.getFullSchema(); + schemas().invalidateKey(new SchemaCacheKey(old.getOrBuildNameMapping())); + remoteTables.remove("Foo"); + blockMissing.set(true); + Future missing = worker.submit(() -> Assertions.assertThrows(CacheException.class, + old::getFullSchema)); + Assertions.assertTrue(missingStarted.await(10, TimeUnit.SECONDS)); + remoteTables.put("FOO", "new_column"); + FeMetaCacheEntry names = Deencapsulation.getField(database, "tableNames"); + names.invalidateKey(""); + PluginDrivenExternalTable replacement = database.getTableNullable("foo"); + Assertions.assertEquals("FOO", replacement.getName()); + List schema = replacement.getFullSchema(); + remoteCache.put("FOO", "new"); + int loads = catalog.nameLoads.get(); + releaseMissing.countDown(); + missing.get(10, TimeUnit.SECONDS); + Assertions.assertSame(replacement, database.getTableNullable("foo")); + Assertions.assertSame(schema, replacement.getFullSchema()); + Assertions.assertEquals("new", remoteCache.getIfPresent("FOO")); + Assertions.assertEquals(loads, catalog.nameLoads.get()); + // A fresh request holding the old object must not combine it with the new names snapshot. + ConnectorSession session = catalog.buildCrossStatementSession(); + try { + Assertions.assertFalse(old.resolveConnectorTableHandle(session, metadata).isPresent()); + } finally { + session.getStatementScope().closeAll(); + } + Assertions.assertSame(replacement, database.getTableNullable("foo")); + Assertions.assertSame(schema, replacement.getFullSchema()); + Assertions.assertEquals("new", remoteCache.getIfPresent("FOO")); + } finally { + catalog.lowerCaseMode = 0; + } + } + + @Test + void lookupFailureDoesNotRepairNamesOrEvictObjects() { + PluginDrivenExternalTable table = warmTarget(); + remoteTables.remove("a"); + failLookup = true; + Assertions.assertThrows(IllegalStateException.class, table::getFullSchema); + Assertions.assertEquals(Set.of("a", "b"), database.getTableNamesWithLock()); + Assertions.assertSame(table, database.getTableNullable(table.getId())); + Assertions.assertEquals("old", remoteCache.getIfPresent("a")); + Assertions.assertEquals(1, catalog.nameLoads.get()); + } + + @Test + void sessionBypassDoesNotInvalidateSharedNames() { + PluginDrivenExternalTable table = warmTarget(); + remoteTables.remove("a"); + catalog.bypassSchema = true; + Assertions.assertThrows(CacheException.class, table::getFullSchema); + Assertions.assertEquals(Set.of("a", "b"), database.getTableNamesWithLock()); + Assertions.assertSame(table, database.getTableNullable(table.getId())); + Assertions.assertEquals("old", remoteCache.getIfPresent("a")); + } + + private static class Handle implements ConnectorTableHandle { + private final String name; + private final String column; + + Handle(String name, String column) { + this.name = name; + this.column = column; + } + } + + private class TestCatalog extends PluginDrivenExternalCatalog { + private final AtomicInteger nameLoads = new AtomicInteger(); + private boolean bypassSchema; + private int lowerCaseMode; + + TestCatalog(long id) { + super(id, "missing_test_" + id, "", new HashMap<>(), "", connector); + } + + @Override + protected void initLocalObjectsImpl() { + // The fixture injects the connector directly instead of loading a production plugin. + } + + @Override + public int getLowerCaseTableNames() { + return lowerCaseMode; + } + + @Override + public Connector getConnector() { + return connector; + } + + @Override + protected List listDatabaseNames() { + return Collections.singletonList("db"); + } + + @Override + protected List listTableNamesFromRemote(SessionContext context, String dbName) { + nameLoads.incrementAndGet(); + return new ArrayList<>(remoteTables.keySet()); + } + + @Override + public boolean shouldBypassSchemaCache(SessionContext context) { + return bypassSchema; + } + } + + private class TestDatabase extends ExternalDatabase { + TestDatabase() { + super(catalog, Util.genIdByName(catalog.getName(), "db"), "db", "db"); + } + + @Override + protected PluginDrivenExternalTable buildTableInternal(String remoteName, String localName, long id, + ExternalCatalog owner, ExternalDatabase db) { + return new PluginDrivenExternalTable(id, localName, remoteName, owner, db) { + @Override + protected synchronized void makeSureInitialized() { + // The fixture registers this database explicitly; connector schema loading remains real. + objectCreated = true; + } + }; + } + } +} diff --git a/regression-test/data/external_table_p0/hive/test_hive_catalog_owner_refresh.out b/regression-test/data/external_table_p0/hive/test_hive_catalog_owner_refresh.out index 0f94eda759ea1d..7730655d8510a0 100644 --- a/regression-test/data/external_table_p0/hive/test_hive_catalog_owner_refresh.out +++ b/regression-test/data/external_table_p0/hive/test_hive_catalog_owner_refresh.out @@ -25,3 +25,11 @@ after_col text Yes true \N -- !schema_after_recreation -- recreated_col bigint Yes true \N +-- !observed_names_before -- +owner_observed + +-- !observed_schema_before -- +observed_col int Yes true \N + +-- !observed_names_after -- + diff --git a/regression-test/suites/external_table_p0/hive/test_hive_catalog_owner_refresh.groovy b/regression-test/suites/external_table_p0/hive/test_hive_catalog_owner_refresh.groovy index 7a9ae6e65742b3..13cb4cf6db544f 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_catalog_owner_refresh.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_catalog_owner_refresh.groovy @@ -67,4 +67,27 @@ suite("test_hive_catalog_owner_refresh", "p0,external") { sql "CREATE TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_target (recreated_col BIGINT)" sql "REFRESH TABLE hive_owner_target.fe_cache_owner_refresh_db.owner_target" qt_schema_after_recreation "DESC hive_owner_target.fe_cache_owner_refresh_db.owner_target" + + sql "DROP CATALOG IF EXISTS hive_owner_observed" + sql """CREATE CATALOG hive_owner_observed PROPERTIES ( + 'type' = 'hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfsPort}', + 'schema.cache.ttl-second' = '0' + )""" + sql "CREATE TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_observed (observed_col INT)" + // Schema/HMS table reads are live, while the FE table-name and object caches stay enabled. + order_qt_observed_names_before """SHOW TABLES FROM hive_owner_observed.fe_cache_owner_refresh_db + LIKE 'owner_observed'""" + qt_observed_schema_before "DESC hive_owner_observed.fe_cache_owner_refresh_db.owner_observed" + sql "DROP TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_observed" + test { + sql "SELECT * FROM hive_owner_observed.fe_cache_owner_refresh_db.owner_observed" + // MVCC observes absence before a later schema lookup reports the now-missing cached object. + exception "failed to load schema cache value" + } + order_qt_observed_names_after """SHOW TABLES FROM hive_owner_observed.fe_cache_owner_refresh_db + LIKE 'owner_observed'""" + }