diff --git a/fe/fe-connector/README.md b/fe/fe-connector/README.md index b455e1c0bfcda9..f550baae1b8bb0 100644 --- a/fe/fe-connector/README.md +++ b/fe/fe-connector/README.md @@ -31,6 +31,51 @@ Three design rules shape everything in this directory: reached only through the SPI, and optional behavior is opt-in via capabilities (see "Reading the API"). +## Catalog cache ownership + +`ExternalCatalog` owns one `CatalogMetaCache` root. Each connector instance acquires a domain through +`ConnectorContext.createCacheDomain`; embedded siblings acquire child domains. FE entries and connector +entries share the root's registry and catalog/global budgets. Domains qualify physical registration names, +wrap plugin callbacks with the registering classloader, and retire only their own registrations. Closing an +old instance never invalidates a replacement instance's resource scope. + +Resource dependencies are independent of domain lifetime: + +``` +Catalog root + LOCAL -> database -> table -> schema/content + REMOTE -> database -> table -> partition + ID -> database id -> table id -> derived row count +``` + +Each cache value declares one most-specific `ScopePath`. Membership snapshots have typed collection scopes. +`CatalogMetadataChanges` projects a semantic refresh, mutation, DDL, event or replay into these coordinates +and detaches all affected scopes at one publication barrier. Content changes preserve lookup objects and +name lists; explicit lookup refresh also invalidates the target identity and parent membership. Newly +written journals persist local/remote identities and the operation's lookup flag. Old records widen only +the remote scope when a cache-only lookup cannot recover the remote identity. + +A definitive absent base-table handle can repair this FE's stale lookup state through the same dependency +planner, without writing a DDL journal. Before the remote read, FE observes both the exact retained table +and its matching names snapshot. Checking those observations and detaching dependencies share one publication +write phase. Replacement, name refresh, content invalidation or logical registration retirement makes the +old observation ineligible, so a delayed negative result cannot clear a recreated table. Cold names, +system/session-bypassed reads and remote failures do not authorize shared-cache repair. Connector handle +lookups must return absence only for a missing table and propagate other failures. + +The root captures an immutable resource normalization policy. With case-insensitive table lookup (mode 2), +LOCAL table scopes fold case while cache keys and displayed names retain their original spelling; REMOTE +scopes retain the connector's case semantics. Registration, bulk-load parents and invalidation use the same +projection, so an old spelling cannot publish after its case-equivalent name is invalidated. A mode 2 lookup invalidation (including rename and its event records) +also fences the database's numeric derived state: IDs published after scope planning must not leave a stale +row count behind. Sibling objects and schemas remain cached, while row counts in that database are reloaded. +Changing table-name mode retires the root rather than changing the policy of live registrations. + +Gateways with different physical identities implement the pure `Connector.cacheDependencies` projection; +it must not perform I/O or construct siblings. Partition names are translated by `partitionCacheScopes`. +The host applies the resulting scopes; gateways do not maintain a second list of invalidation callbacks. +Immutable path-addressed caches can stay at catalog scope and survive table refreshes. + ## Module Map Roles only — one line each. For anything deeper, read the module's javadoc @@ -61,7 +106,7 @@ endpoint properties) | Module | Role | |---|---| -| `fe-connector-cache` | Self-contained caching framework used by several connectors. No fe-core dependency; it is bundled into each consuming plugin, so shared third-party libraries stay at the consumers' lowest common version (see the version notes in consumer poms). | +| `fe-connector-cache` | Shared metadata-cache framework with no fe-core dependency. The parent-first `org.apache.doris.connector.*` loader policy gives FE and plugins one runtime identity; plugin packaging does not create private cache owners. ConnectorContext supplies instance domains under the catalog root. | | `fe-connector-hms-hive-shade` | Slim, relocated HMS metastore-client closure for connectors that speak HMS thrift. The pom comments say exactly what relocates where and why. | | `fe-connector-paimon-hive-shade` | Paimon-private relocated HMS-thrift closure; same idea, different owner. | diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java index 97ba7288d673cf..d51c3140e243da 100644 --- a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcConnector.java @@ -67,7 +67,7 @@ public AdbcConnector(Map properties, ConnectorContext context) { // The raw map, because the cache knobs are the shared framework's keys rather than this // connector's: CacheSpec owns their names, and mirroring them as fields here would give the // validator and the reader two things to drift apart. - this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "adbc", props.getRaw()); + this.metaCache = context.createCacheDomain("adbc", props.getRaw()); try { this.metadataCache = new AdbcMetadataCache(metaCache, props.getRaw()); } catch (RuntimeException | Error e) { diff --git a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java index 27a7a3b2a4fc80..90bdbfcc8f6c90 100644 --- a/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java +++ b/fe/fe-connector/fe-connector-adbc/src/main/java/org/apache/doris/connector/adbc/AdbcMetadataCache.java @@ -105,13 +105,14 @@ public AdbcMetadataCache(Map properties) { this.owner = Objects.requireNonNull(owner, "owner can not be null"); CacheSpec spec = cacheSpec(properties); this.namespaces = owner.create(MetaCacheDefinition - .>builder("adbc-namespaces", spec, ignored -> ScopePath.catalog()) + .>builder("adbc-namespaces", spec, + ignored -> ScopePath.databaseCollection()) .budgetGroup(ENTRY) .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); this.tableNames = owner.create(MetaCacheDefinition .>builder("adbc-table-names", spec, - key -> ScopePath.database(key.dorisDbName)) + key -> ScopePath.tableCollection(key.dorisDbName)) .budgetGroup(ENTRY) .sizeEstimator(MetaCacheSizeEstimators.reflective()) .build()); @@ -174,11 +175,7 @@ Schema tableSchema(AdbcTableHandle handle, Supplier loader) { * new name in, and the one the user tried would appear to do nothing. */ void invalidateTable(String dbName, String tableName) { - owner.invalidateTable(dbName, tableName); - // A table refresh must also forget the parent name listing, so a remotely-created table can be found. - // This is an ancestor materialization, not a sibling-cache dependency, and therefore cannot be selected - // by the table's descendant scope. - tableNames.invalidateKey(TableNameKey.forDatabase(dbName)); + owner.invalidateTableAndNames(dbName, tableName); } /** {@code REFRESH DATABASE}: that database's table listing and every schema in it. */ diff --git a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcMetadataCacheTest.java b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcMetadataCacheTest.java index 5bc224778584ed..58537f96615e01 100644 --- a/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcMetadataCacheTest.java +++ b/fe/fe-connector/fe-connector-adbc/src/test/java/org/apache/doris/connector/adbc/AdbcMetadataCacheTest.java @@ -20,6 +20,7 @@ import org.apache.doris.connector.cache.CacheSpec; import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCacheGovernance; +import org.apache.doris.connector.cache.ScopePath; import org.apache.arrow.vector.types.pojo.ArrowType; import org.apache.arrow.vector.types.pojo.Field; @@ -41,6 +42,45 @@ */ class AdbcMetadataCacheTest { + @Test + void rootResourceInvalidationIncludesRememberedMemberships() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + CatalogMetaCache domain = root.openDomain("adbc", ScopePath.Coordinates.REMOTE, + getClass().getClassLoader()); + AdbcMetadataCache cache = new AdbcMetadataCache(domain, Map.of()); + Counting> namespaces = new Counting<>(List.of(MAIN, OTHER)); + Counting> main = new Counting<>(List.of("t1")); + Counting> other = new Counting<>(List.of("t9")); + Counting schema = new Counting<>(SCHEMA); + cache.namespaces(namespaces); + cache.tableNames(MAIN, main); + cache.tableNames(OTHER, other); + cache.tableSchema(MAIN_T1, schema); + + root.invalidateScopes(List.of( + ScopePath.table("main", "t1").in(ScopePath.Coordinates.REMOTE), + ScopePath.tableCollection("main").in(ScopePath.Coordinates.REMOTE))); + cache.namespaces(namespaces); + cache.tableNames(MAIN, main); + cache.tableNames(OTHER, other); + cache.tableSchema(MAIN_T1, schema); + Assertions.assertEquals(1, namespaces.calls); + Assertions.assertEquals(2, main.calls); + Assertions.assertEquals(1, other.calls); + Assertions.assertEquals(2, schema.calls); + + root.invalidateScopes(List.of( + ScopePath.database("main").in(ScopePath.Coordinates.REMOTE), + ScopePath.databaseCollection().in(ScopePath.Coordinates.REMOTE))); + cache.namespaces(namespaces); + cache.tableNames(MAIN, main); + cache.tableNames(OTHER, other); + Assertions.assertEquals(2, namespaces.calls); + Assertions.assertEquals(3, main.calls); + Assertions.assertEquals(1, other.calls); + } + } + @Test void managedCachesAccountAllPayloadsAndReleaseReservations() { long baseline = MetaCacheGovernance.globalEstimatedWeight(); diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheCallbackContext.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheCallbackContext.java new file mode 100644 index 00000000000000..aa5b0c8ac5e475 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CacheCallbackContext.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import java.util.function.Supplier; + +/** Restores the registering plugin's loader around callbacks, including asynchronous cleanup. */ +public final class CacheCallbackContext { + private final ClassLoader loader; + + public CacheCallbackContext(ClassLoader loader) { + this.loader = loader; + } + + public T call(Supplier callback) { + Thread thread = Thread.currentThread(); + ClassLoader previous = thread.getContextClassLoader(); + if (previous == loader) { + return callback.get(); + } + try { + thread.setContextClassLoader(loader); + return callback.get(); + } finally { + thread.setContextClassLoader(previous); + } + } + + public void run(Runnable callback) { + call(() -> { + callback.run(); + return null; + }); + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java index d84c389b04b186..f7dd5930d907f4 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/CatalogMetaCache.java @@ -26,6 +26,8 @@ import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; +import java.util.function.UnaryOperator; /** * Catalog-local owner of all framework metadata caches. @@ -40,6 +42,15 @@ public final class CatalogMetaCache implements AutoCloseable { private final String engine; private final OptionalLong catalogMaxWeight; private final boolean managed; + private final CatalogMetaCache parent; + private final ScopePath.Coordinates coordinates; + private final UnaryOperator scopeNormalizer; + private final CacheCallbackContext callbacks; + private final CatalogMetaCache rootOwner; + private final String registrationPrefix; + private final AtomicLong domainSequence; + private final Set domains = ConcurrentHashMap.newKeySet(); + private final Map> allEntries; private final Set names = ConcurrentHashMap.newKeySet(); private final Map> entries = new ConcurrentHashMap<>(); private final AtomicBoolean closed = new AtomicBoolean(false); @@ -71,34 +82,101 @@ public static CatalogMetaCache unmanaged() { public static CatalogMetaCache managed(long catalogId, String engine, Map catalogProperties) { + return managed(catalogId, engine, catalogProperties, Thread.currentThread().getContextClassLoader()); + } + + public static CatalogMetaCache managed(long catalogId, String engine, + Map catalogProperties, ClassLoader classLoader) { + return managed(catalogId, engine, catalogProperties, classLoader, UnaryOperator.identity()); + } + + /** The pure, immutable normalizer defines resource equivalence for the lifetime of this root. */ + public static CatalogMetaCache managed(long catalogId, String engine, + Map catalogProperties, ClassLoader classLoader, UnaryOperator normalizer) { MetaCacheBudgetManager manager = MetaCacheGovernance.budgetManager(); CatalogMetaCache cache = new CatalogMetaCache(new ScopedMetaCacheRegistry(), manager, - catalogId, engine, manager.parseCatalogMaxWeight(catalogProperties), true); + catalogId, engine, manager.parseCatalogMaxWeight(catalogProperties), true, classLoader, normalizer); MetaCacheGovernance.register(cache); return cache; } CatalogMetaCache(ScopedMetaCacheRegistry registry, MetaCacheBudgetManager budgetManager, long catalogId, String engine, OptionalLong catalogMaxWeight, boolean managed) { + this(registry, budgetManager, catalogId, engine, catalogMaxWeight, managed, + Thread.currentThread().getContextClassLoader()); + } + + private CatalogMetaCache(ScopedMetaCacheRegistry registry, MetaCacheBudgetManager budgetManager, + long catalogId, String engine, OptionalLong catalogMaxWeight, boolean managed, ClassLoader classLoader) { + this(registry, budgetManager, catalogId, engine, catalogMaxWeight, managed, classLoader, + UnaryOperator.identity()); + } + + private CatalogMetaCache(ScopedMetaCacheRegistry registry, MetaCacheBudgetManager budgetManager, + long catalogId, String engine, OptionalLong catalogMaxWeight, boolean managed, ClassLoader classLoader, + UnaryOperator normalizer) { + this.scopeNormalizer = Objects.requireNonNull(normalizer, "scope normalizer can not be null"); this.registry = Objects.requireNonNull(registry, "registry can not be null"); this.budgetManager = Objects.requireNonNull(budgetManager, "budgetManager can not be null"); this.catalogId = catalogId; this.engine = Objects.requireNonNull(engine, "engine can not be null"); this.catalogMaxWeight = Objects.requireNonNull(catalogMaxWeight, "catalogMaxWeight can not be null"); this.managed = managed; + this.parent = null; + this.coordinates = null; + this.callbacks = new CacheCallbackContext(classLoader); + this.rootOwner = this; + this.registrationPrefix = ""; + this.domainSequence = new AtomicLong(); + this.allEntries = new ConcurrentHashMap<>(); + } + + private CatalogMetaCache(CatalogMetaCache parent, String engine, + ScopePath.Coordinates coordinates, ClassLoader classLoader) { + this.coordinates = coordinates; + this.scopeNormalizer = parent.scopeNormalizer; + this.callbacks = new CacheCallbackContext(classLoader); + this.registry = parent.registry; + this.budgetManager = parent.budgetManager; + this.catalogId = parent.catalogId; + this.engine = Objects.requireNonNull(engine, "engine"); + this.catalogMaxWeight = parent.catalogMaxWeight; + this.managed = false; + this.parent = parent; + this.rootOwner = parent.rootOwner; + this.domainSequence = rootOwner.domainSequence; + this.registrationPrefix = engine + "@" + domainSequence.incrementAndGet() + "."; + this.allEntries = rootOwner.allEntries; + } + + /** Opens one instance-owned domain. Resource invalidation is shared; close only retires this instance. */ + public CatalogMetaCache openDomain(String engine) { + return openDomain(engine, coordinates, Thread.currentThread().getContextClassLoader()); + } + + public CatalogMetaCache openDomain(String engine, ScopePath.Coordinates coordinates, ClassLoader classLoader) { + checkOpen(); + CatalogMetaCache domain = new CatalogMetaCache(this, engine, coordinates, classLoader); + domains.add(domain); + if (isClosed()) { + domain.close(); + throw new IllegalStateException("Parent metadata cache domain was retired during construction"); + } + return domain; } public MetaCache create(MetaCacheDefinition definition) { MetaCacheDefinition nonNullDefinition = - Objects.requireNonNull(definition, "definition can not be null"); + Objects.requireNonNull(definition, "definition can not be null").forDomain(this::project, callbacks); checkOpen(); if (!names.add(nonNullDefinition.name())) { throw new IllegalArgumentException("Duplicate meta cache name: " + nonNullDefinition.name()); } MetaCacheBudgetManager.EntryBudget entryBudget = null; try { - boolean weightLimited = budgetManager.hasLimit( - catalogMaxWeight, nonNullDefinition.cacheSpec().getMaxWeight()); + boolean weightLimited = nonNullDefinition.budgetPolicy() + == MetaCacheDefinition.BudgetPolicy.WEIGHTED_PAYLOAD + && budgetManager.hasLimit(catalogMaxWeight, nonNullDefinition.cacheSpec().getMaxWeight()); if (weightLimited && nonNullDefinition.sizeEstimator() == null) { throw new IllegalArgumentException("Weighted metadata cache requires a size estimator: " + nonNullDefinition.name()); @@ -108,26 +186,25 @@ public MetaCache create(MetaCacheDefinition definition) { nonNullDefinition.name(), nonNullDefinition.budgetGroup(), catalogMaxWeight, nonNullDefinition.cacheSpec().getMaxWeight()); } - MetaCache created = new MetaCache<>(nonNullDefinition, - registry.createCacheWithMetaRemovalListener(nonNullDefinition.name(), + String registrationName = registrationPrefix + nonNullDefinition.name(); + MetaCache created = new MetaCache<>(nonNullDefinition, engine, registrationName, callbacks, + registry.createCacheWithMetaRemovalListener(registrationName, nonNullDefinition.cacheSpec(), nonNullDefinition.removalListener(), nonNullDefinition.discardListener(), nonNullDefinition.refreshAfterWrite(), nonNullDefinition.refreshExecutor(), - entryBudget == null ? null : nonNullDefinition.sizeEstimator(), entryBudget)); + entryBudget == null ? null : nonNullDefinition.sizeEstimator(), entryBudget), rootOwner); entries.put(nonNullDefinition.name(), created); + allEntries.put(created.registrationName(), created); + if (isClosed()) { + remove(created); + throw new IllegalStateException("Metadata cache domain was retired during entry construction"); + } return created; } catch (RuntimeException | Error throwable) { if (entryBudget != null) { entryBudget.close(); } names.remove(nonNullDefinition.name()); - if (managed) { - try { - close(); - } catch (RuntimeException | Error closeFailure) { - throwable.addSuppressed(closeFailure); - } - } throw throwable; } } @@ -139,6 +216,7 @@ public MetaCache create(MetaCacheDefinition definition) { public void remove(MetaCache cache) { MetaCache nonNullCache = Objects.requireNonNull(cache, "cache can not be null"); if (entries.remove(nonNullCache.name(), nonNullCache)) { + allEntries.remove(nonNullCache.registrationName(), nonNullCache); try { nonNullCache.closeFromOwner(); } finally { @@ -148,23 +226,62 @@ public void remove(MetaCache cache) { } public void invalidateCatalog() { - registry.invalidate(ScopePath.catalog()); + invalidateScope(ScopePath.catalog()); } public void invalidateDatabase(String database) { - registry.invalidate(ScopePath.database(database)); + invalidateScope(ScopePath.database(database)); } public void invalidateTable(String database, String table) { - registry.invalidate(ScopePath.table(database, table)); + invalidateScope(ScopePath.table(database, table)); + } + + /** Invalidates an object and its parent membership view at one logical publication barrier. */ + public void invalidateTableAndNames(String database, String table) { + invalidateScopes(java.util.Arrays.asList( + ScopePath.table(database, table), ScopePath.tableCollection(database))); + } + + public void invalidateDatabaseAndNames(String database) { + invalidateScopes(java.util.Arrays.asList( + ScopePath.database(database), ScopePath.databaseCollection())); + } + + public void invalidateScope(ScopePath scope) { + registry.invalidate(project(scope)); + } + + /** One barrier covers local and remote dependencies, even when their object caches are cold. */ + public void invalidateScopes(Collection scopes) { + registry.invalidate(projectScopes(scopes)); + } + + boolean invalidateScopesIfCurrent(Collection scopes, + ScopedMetaCacheRegistry.CacheObservation observation) { + return registry.invalidateIfCurrent(projectScopes(scopes), observation); + } + + private List projectScopes(Collection scopes) { + List projected = new ArrayList<>(scopes.size()); + scopes.forEach(scope -> projected.add(project(scope))); + return projected; + } + + private ScopePath project(ScopePath scope) { + return scopeNormalizer.apply(coordinates == null ? scope : scope.in(coordinates)); + } + + public boolean isClosed() { + return closed.get() || (parent != null && parent.isClosed()); } public void invalidatePartition(String database, String table, Object partition) { - registry.invalidate(ScopePath.partition(database, table, partition)); + invalidateScope(ScopePath.partition(database, table, partition)); } public void invalidatePartitionCollection(String database, String table) { - registry.invalidate(ScopePath.partitionCollection(database, table)); + invalidateScope(ScopePath.partitionCollection(database, table)); } /** @@ -177,7 +294,7 @@ public void invalidatePartitions(String database, String table, Collection pa List paths = new ArrayList<>(partitions.size() + 1); paths.add(ScopePath.partitionCollection(database, table)); partitions.forEach(partition -> paths.add(ScopePath.partition(database, table, partition))); - registry.invalidate(paths); + invalidateScopes(paths); } public ScopedMetaCacheRegistry.ScopeMetrics metrics() { @@ -188,6 +305,11 @@ public ScopedMetaCacheRegistry.ScopeMetrics metrics() { return java.util.Collections.unmodifiableMap(entries); } + /** Catalog-wide registrations, including every live connector instance domain. */ + public Map> allEntries() { + return java.util.Collections.unmodifiableMap(allEntries); + } + public long catalogId() { return catalogId; } @@ -206,22 +328,49 @@ public boolean hasEnclosingWeightLimit() { @Override public void close() { - if (closed.compareAndSet(false, true)) { - try { - registry.close(); - } finally { - names.clear(); - entries.clear(); - if (managed) { - MetaCacheGovernance.unregister(this); - } + if (!closed.compareAndSet(false, true)) { + return; + } + Throwable failure = null; + for (CatalogMetaCache domain : new ArrayList<>(domains)) { + failure = closeAndCollect(domain::close, failure); + } + for (MetaCache entry : new ArrayList<>(entries.values())) { + failure = closeAndCollect(() -> remove(entry), failure); + } + domains.clear(); + if (parent == null) { + failure = closeAndCollect(registry::close, failure); + allEntries.clear(); + if (managed) { + MetaCacheGovernance.unregister(this); + } + } else { + parent.domains.remove(this); + } + if (failure instanceof Error) { + throw (Error) failure; + } + if (failure != null) { + throw (RuntimeException) failure; + } + } + + private static Throwable closeAndCollect(Runnable cleanup, Throwable failure) { + try { + cleanup.run(); + } catch (RuntimeException | Error error) { + if (failure == null) { + return error; } + failure.addSuppressed(error); } + return failure; } private void checkOpen() { - if (closed.get()) { - throw new IllegalStateException("Catalog meta cache is closed"); + if (isClosed()) { + throw new IllegalStateException("Catalog meta cache domain is closed"); } } } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java index 07afd00576d4bf..f6145b84e3f7f1 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCache.java @@ -17,8 +17,11 @@ package org.apache.doris.connector.cache; +import java.util.Collection; import java.util.Objects; +import java.util.Optional; import java.util.function.BiConsumer; +import java.util.function.BooleanSupplier; import java.util.function.Consumer; import java.util.function.Function; @@ -26,8 +29,17 @@ public final class MetaCache { private final MetaCacheDefinition definition; private final ScopedMetaCache delegate; - - MetaCache(MetaCacheDefinition definition, ScopedMetaCache delegate) { + private final String engine; + private final String registrationName; + private final CacheCallbackContext callbacks; + private final CatalogMetaCache rootOwner; + + MetaCache(MetaCacheDefinition definition, String engine, String registrationName, + CacheCallbackContext callbacks, ScopedMetaCache delegate, CatalogMetaCache rootOwner) { + this.callbacks = callbacks; + this.rootOwner = rootOwner; + this.engine = engine; + this.registrationName = registrationName; this.definition = Objects.requireNonNull(definition, "definition can not be null"); this.delegate = Objects.requireNonNull(delegate, "delegate can not be null"); } @@ -36,6 +48,14 @@ public String name() { return definition.name(); } + public String engine() { + return engine; + } + + public String registrationName() { + return registrationName; + } + public V get(K key) { Function loader = definition.loader(); if (loader == null) { @@ -46,7 +66,8 @@ public V get(K key) { public V get(K key, Function loader) { K nonNullKey = Objects.requireNonNull(key, "key can not be null"); - return delegate.get(nonNullKey, definition.scope(nonNullKey), loader); + return delegate.get(nonNullKey, definition.scope(nonNullKey), + keyToLoad -> callbacks.call(() -> loader.apply(keyToLoad))); } /** @@ -57,8 +78,47 @@ public V get(K key, Function loader) { public V getWithPublicationAction(K key, Function loader, BiConsumer> publicationCoordinator) { K nonNullKey = Objects.requireNonNull(key, "key can not be null"); - return delegate.getWithPublicationAction(nonNullKey, definition.scope(nonNullKey), loader, - publicationCoordinator); + return delegate.getWithPublicationAction(nonNullKey, definition.scope(nonNullKey), + keyToLoad -> callbacks.call(() -> loader.apply(keyToLoad)), publicationCoordinator); + } + + /** + * Observes an exact retained instance without loading it. The token also tracks scope invalidations below + * its resource, exact-key replacement and physical registration retirement. It holds no publication lock. + */ + public Optional observe(K key, Object expectedValue) { + return observe(key, expectedValue, () -> true); + } + + /** + * Also fences logical registration retirement. The predicate is evaluated under the publication writer; + * it must only read lifecycle state, never block, acquire locks, call user code or re-enter a cache. + */ + public Optional observe(K key, Object expectedValue, BooleanSupplier registrationCurrent) { + return delegate.observe(Objects.requireNonNull(key, "key"), definition.scope(key), + Objects.requireNonNull(expectedValue, "expectedValue"), + Objects.requireNonNull(registrationCurrent, "registrationCurrent")) + .map(observation -> new Observation(rootOwner, observation)); + } + + public static final class Observation { + private final CatalogMetaCache rootOwner; + private final ScopedMetaCacheRegistry.CacheObservation observation; + + private Observation(CatalogMetaCache rootOwner, ScopedMetaCacheRegistry.CacheObservation observation) { + this.rootOwner = rootOwner; + this.observation = observation; + } + + /** Requires both exact instances to remain current in the same publication write phase. */ + public Observation and(Observation other) { + return new Observation(rootOwner, observation.and(other.observation)); + } + + /** Checks the observed state and detaches all dependencies in one publication write phase. */ + public boolean invalidateScopes(Collection scopes) { + return rootOwner.invalidateScopesIfCurrent(scopes, observation); + } } public V getIfPresent(K key) { @@ -89,6 +149,10 @@ public boolean compareAndSet(K key, V expectedValue, V updatedValue, Runnable co Objects.requireNonNull(commitAction, "commitAction can not be null")); } + public boolean runIfCurrent(K key, V expected, Runnable action) { + return delegate.runIfCurrent(key, definition.scope(key), expected, action); + } + public void invalidateKey(K key) { delegate.invalidateKey(Objects.requireNonNull(key, "key can not be null")); } @@ -99,7 +163,7 @@ public void invalidateKey(K key) { * returns a currentness fence, but {@link BulkLoad#publish(Object, Object)} retains no values. */ public BulkLoad beginBulkLoad(ScopePath parentScope) { - return new BulkLoad<>(this, delegate.beginBulkLoad(parentScope)); + return new BulkLoad<>(this, delegate.beginBulkLoad(definition.project(parentScope))); } public ScopedMetaCache.CacheMetrics metrics() { diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java index a461b8879e44a6..c0be274b30d8d5 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/MetaCacheDefinition.java @@ -22,6 +22,7 @@ import java.util.concurrent.Executor; import java.util.function.BiConsumer; import java.util.function.Function; +import java.util.function.UnaryOperator; /** * Declarative definition of one physical metadata cache in a catalog. @@ -31,6 +32,11 @@ * automatically covers it without a connector-maintained list of sibling caches. */ public final class MetaCacheDefinition { + public enum BudgetPolicy { + WEIGHTED_PAYLOAD, + COUNT_ONLY + } + private final String name; private final String budgetGroup; private final CacheSpec cacheSpec; @@ -41,8 +47,11 @@ public final class MetaCacheDefinition { private final Duration refreshAfterWrite; private final Executor refreshExecutor; private final MetaCacheSizeEstimator sizeEstimator; + private final BudgetPolicy budgetPolicy; + private final UnaryOperator scopeProjection; private MetaCacheDefinition(Builder builder) { + scopeProjection = builder.scopeProjection; name = requireName(builder.name); budgetGroup = requireName(builder.budgetGroup == null ? builder.name : builder.budgetGroup); cacheSpec = Objects.requireNonNull(builder.cacheSpec, "cacheSpec can not be null"); @@ -53,6 +62,7 @@ private MetaCacheDefinition(Builder builder) { refreshAfterWrite = builder.refreshAfterWrite; refreshExecutor = builder.refreshExecutor; sizeEstimator = builder.sizeEstimator; + budgetPolicy = builder.budgetPolicy; if (refreshAfterWrite != null && loader == null) { throw new IllegalArgumentException("refresh-after-write requires a default loader"); } @@ -105,6 +115,35 @@ MetaCacheSizeEstimator sizeEstimator() { return sizeEstimator; } + BudgetPolicy budgetPolicy() { + return budgetPolicy; + } + + MetaCacheDefinition forDomain(UnaryOperator projection, CacheCallbackContext callbacks) { + Builder copy = builder(name, cacheSpec, key -> callbacks.call(() -> { + ScopePath path = scope(key); + return projection.apply(path); + })); + copy.budgetGroup = budgetGroup; + copy.budgetPolicy = budgetPolicy; + copy.refreshAfterWrite = refreshAfterWrite; + copy.refreshExecutor = refreshExecutor == null ? null + : command -> refreshExecutor.execute(() -> callbacks.run(command)); + copy.loader = loader == null ? null : key -> callbacks.call(() -> loader.apply(key)); + copy.removalListener = removalListener == null ? null + : (key, value, reason) -> callbacks.run(() -> removalListener.onRemoval(key, value, reason)); + copy.discardListener = discardListener == null ? null + : (key, value) -> callbacks.run(() -> discardListener.accept(key, value)); + copy.sizeEstimator = sizeEstimator == null ? null + : (key, value) -> callbacks.call(() -> sizeEstimator.estimate(key, value)); + copy.scopeProjection = projection; + return copy.build(); + } + + ScopePath project(ScopePath path) { + return scopeProjection.apply(path); + } + private static String requireName(String name) { String nonNullName = Objects.requireNonNull(name, "name can not be null"); if (nonNullName.isEmpty()) { @@ -124,6 +163,8 @@ public static final class Builder { private Executor refreshExecutor; private MetaCacheSizeEstimator sizeEstimator; private String budgetGroup; + private UnaryOperator scopeProjection = UnaryOperator.identity(); + private BudgetPolicy budgetPolicy = BudgetPolicy.WEIGHTED_PAYLOAD; private Builder(String name, CacheSpec cacheSpec, Function scopeResolver) { this.name = name; @@ -165,6 +206,16 @@ public Builder sizeEstimator(MetaCacheSizeEstimator estimator) { return this; } + /** + * Mutable lookup handles may use count/TTL retention instead of payload-byte accounting. Their owner + * back-references are not an owned payload graph. Such entries still share invalidation and lifecycle + * with this catalog, but are explicitly excluded from global/catalog payload-byte usage statistics. + */ + public Builder budgetPolicy(BudgetPolicy policy) { + this.budgetPolicy = Objects.requireNonNull(policy, "budgetPolicy can not be null"); + return this; + } + /** Shares one configured weight limit across physical caches in the same logical entry group. */ public Builder budgetGroup(String budgetGroup) { this.budgetGroup = requireName(budgetGroup); diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java index 98d9a7abdfed3f..799b6127f01927 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopePath.java @@ -18,6 +18,7 @@ package org.apache.doris.connector.cache; import java.util.Objects; +import java.util.function.UnaryOperator; /** * Hierarchical invalidation path within one catalog-owned scoped cache registry. @@ -30,14 +31,27 @@ */ public final class ScopePath { private enum SyntheticPartition { - COLLECTION + COLLECTION, + METADATA + } + + private enum NameCollection { + DATABASES, + TABLES } public enum Level { CATALOG, DATABASE, TABLE, - PARTITION + PARTITION, + NAMESPACE + } + + public enum Coordinates { + LOCAL, + REMOTE, + ID } private static final ScopePath CATALOG = new ScopePath(Level.CATALOG, null, null, null); @@ -46,12 +60,25 @@ public enum Level { private final String database; private final String table; private final Object partition; + private final NameCollection nameCollection; + private final Coordinates coordinates; private ScopePath(Level level, String database, String table, Object partition) { + this(level, database, table, partition, null); + } + + private ScopePath(Level level, String database, String table, Object partition, NameCollection nameCollection) { + this(level, database, table, partition, nameCollection, null); + } + + private ScopePath(Level level, String database, String table, Object partition, + NameCollection nameCollection, Coordinates coordinates) { this.level = level; + this.coordinates = coordinates; this.database = database; this.table = table; this.partition = partition; + this.nameCollection = nameCollection; } public static ScopePath catalog() { @@ -88,10 +115,80 @@ public static ScopePath partitionCollection(String database, String table) { return partition(database, table, SyntheticPartition.COLLECTION); } + /** The catalog's database-name snapshot, distinct from every user database. */ + public static ScopePath databaseCollection() { + return new ScopePath(Level.DATABASE, null, null, null, NameCollection.DATABASES); + } + + /** A database's table-name snapshot, distinct from every user table. */ + public static ScopePath tableCollection(String database) { + return new ScopePath(Level.TABLE, Objects.requireNonNull(database, "database"), + null, null, NameCollection.TABLES); + } + + /** FE schema/content entries; invalidating them preserves the FE table object and name snapshot. */ + public static ScopePath tableMetadata(String database, String table) { + return partition(database, table, SyntheticPartition.METADATA); + } + + /** Selects a name coordinate system without embedding names or plugin instances in string prefixes. */ + public ScopePath in(Coordinates coordinates) { + Objects.requireNonNull(coordinates, "coordinates"); + return new ScopePath(level == Level.CATALOG ? Level.NAMESPACE : level, + database, table, partition, nameCollection, coordinates); + } + + /** Changes only a real table identity, preserving collection markers, coordinates and partition identity. */ + public ScopePath mapTableName(UnaryOperator mapper) { + return table == null ? this : new ScopePath(level, database, mapper.apply(table), + partition, nameCollection, coordinates); + } + + int depth() { + return level == Level.NAMESPACE ? 1 : level.ordinal() + (coordinates == null ? 0 : 1); + } + + Object keyAtDepth(int depth) { + int resourceDepth = depth; + if (coordinates != null) { + if (depth == 1) { + return coordinates; + } + resourceDepth--; + } + switch (resourceDepth) { + case 1: + return databaseKey(); + case 2: + return tableKey(); + case 3: + return partition; + default: + throw new IllegalArgumentException("Invalid scope depth: " + depth); + } + } + + Level levelAtDepth(int depth) { + return coordinates != null && depth == 1 ? Level.NAMESPACE + : Level.values()[depth - (coordinates == null ? 0 : 1)]; + } + + Object databaseKey() { + return nameCollection == NameCollection.DATABASES ? NameCollection.DATABASES : database; + } + + Object tableKey() { + return nameCollection == NameCollection.TABLES ? NameCollection.TABLES : table; + } + public Level level() { return level; } + public Coordinates coordinates() { + return coordinates; + } + public String database() { return database; } @@ -106,22 +203,18 @@ public Object partition() { public boolean contains(ScopePath other) { Objects.requireNonNull(other, "other can not be null"); - if (level.ordinal() > other.level.ordinal()) { - return false; - } if (level == Level.CATALOG) { return true; } - if (!database.equals(other.database)) { + if (coordinates != other.coordinates || depth() > other.depth()) { return false; } - if (level == Level.DATABASE) { - return true; - } - if (!table.equals(other.table)) { - return false; + for (int i = 1; i <= depth(); i++) { + if (!Objects.equals(keyAtDepth(i), other.keyAtDepth(i))) { + return false; + } } - return level == Level.TABLE || partition.equals(other.partition); + return true; } @Override @@ -133,9 +226,9 @@ public boolean equals(Object obj) { return false; } ScopePath other = (ScopePath) obj; - return level == other.level - && Objects.equals(database, other.database) - && Objects.equals(table, other.table) + return level == other.level && coordinates == other.coordinates + && Objects.equals(databaseKey(), other.databaseKey()) + && Objects.equals(tableKey(), other.tableKey()) && Objects.equals(partition, other.partition); } @@ -143,16 +236,23 @@ public boolean equals(Object obj) { public int hashCode() { int pathHashCode = 1; pathHashCode = 31 * pathHashCode + Objects.hashCode(level); - pathHashCode = 31 * pathHashCode + Objects.hashCode(database); - pathHashCode = 31 * pathHashCode + Objects.hashCode(table); + pathHashCode = 31 * pathHashCode + Objects.hashCode(coordinates); + pathHashCode = 31 * pathHashCode + Objects.hashCode(databaseKey()); + pathHashCode = 31 * pathHashCode + Objects.hashCode(tableKey()); return 31 * pathHashCode + Objects.hashCode(partition); } @Override public String toString() { + if (nameCollection != null) { + return nameCollection == NameCollection.DATABASES + ? "databaseCollection" : "tableCollection[" + database + "]"; + } switch (level) { case CATALOG: return "catalog"; + case NAMESPACE: + return "namespace[" + coordinates + "]"; case DATABASE: return "database[" + database + "]"; case TABLE: diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java index b58ad13bd9dfa5..b26d5729866619 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCache.java @@ -186,26 +186,32 @@ public V getWithPublicationAction(K key, ScopePath path, Function loader, BiConsumer> coordinator = Objects.requireNonNull( publicationCoordinator, "publicationCoordinator can not be null"); checkOpen(); + if (!effectiveEnabled) { recordAccess(false); - V loaded = loadAndRecord(key, loadFunction); - if (loaded != null) { - AtomicBoolean commitInvoked = new AtomicBoolean(false); - try { - coordinator.accept(loaded, beforePublication -> { - if (!commitInvoked.compareAndSet(false, true)) { - throw new IllegalStateException("Metadata cache publication callback was invoked twice"); + // Disabled retention keeps the original per-request load behavior, but auxiliary publication still + // captures its resource generation before I/O and is rejected if an invalidation races the load. + try (PublicationLease lease = acquirePublicationLease(key, path, true)) { + V loaded = loadAndRecord(key, loadFunction); + if (loaded != null) { + AtomicBoolean invoked = new AtomicBoolean(); + try { + coordinator.accept(loaded, action -> { + if (!invoked.compareAndSet(false, true)) { + throw new IllegalStateException( + "Metadata cache publication callback was invoked twice"); + } + commitLoaded(lease, key, loaded, action); + }); + if (!invoked.get()) { + throw new IllegalStateException("Metadata cache publication callback was not invoked"); } - Objects.requireNonNull(beforePublication, "beforePublication can not be null").run(); - }); - if (!commitInvoked.get()) { - throw new IllegalStateException("Metadata cache publication callback was not invoked"); + } finally { + notifyDiscarded(key, loaded); } - } finally { - notifyDiscarded(key, loaded); } + return loaded; } - return loaded; } VersionedValue presentVersioned = currentVersionedValue(key, path); @@ -270,6 +276,16 @@ public V getWithPublicationAction(K key, ScopePath path, Function loader, } } + Optional observe(K key, ScopePath path, Object expectedValue, + BooleanSupplier registrationCurrent) { + VersionedValue value = currentVersionedValue(key, path); + if (value == null || value.value != expectedValue) { + return Optional.empty(); + } + return Optional.of(registry.observe(value.scopeSnapshot, + () -> registrationCurrent.getAsBoolean() && !closed.get() && value.isCurrent(registry, keyNodes))); + } + public V getIfPresent(K key, ScopePath path) { Objects.requireNonNull(key, "key can not be null"); Objects.requireNonNull(path, "path can not be null"); @@ -349,6 +365,21 @@ public void put(K key, ScopePath path, V value) { } } + /** Runs a short side effect only while the exact retained value and its resource scope are current. */ + public boolean runIfCurrent(K key, ScopePath path, V expected, Runnable action) { + checkOpen(); + try (PublicationLease lease = acquirePublicationLease(key, path, false)) { + return guardedCommit(lease, () -> { + VersionedValue current = currentVersionedValue(key, path); + if (current == null || current.value != expected) { + return false; + } + action.run(); + return true; + }); + } + } + public boolean compareAndSet(K key, ScopePath path, V expectedValue, V updatedValue) { return compareAndSet(key, path, expectedValue, updatedValue, NO_OP); } @@ -360,8 +391,12 @@ public boolean compareAndSet( Runnable action = Objects.requireNonNull(commitAction, "commitAction can not be null"); checkOpen(); if (!effectiveEnabled) { - action.run(); - return true; + try (PublicationLease lease = acquirePublicationLease(key, path, false)) { + return guardedCommit(lease, () -> { + action.run(); + return true; + }); + } } try (PublicationLease lease = acquirePublicationLease(key, path, false)) { AtomicReference> preparedRef = new AtomicReference<>(); @@ -711,6 +746,9 @@ private boolean guardedCommit(PublicationLease lease, BooleanSupplier comm private VersionedValue prepareVersionedValue( PublicationLease lease, K key, V value) { + if (!effectiveEnabled) { + return null; + } MetaCacheBudgetManager.AdmissionReservation reservation = reserve(key, value); if (weightBounded && reservation == null) { return null; diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java index 918b2829a7927b..6168805fbc1eb3 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/ScopedMetaCacheRegistry.java @@ -198,7 +198,7 @@ void invalidate(ScopePath path, Runnable afterStateReplacement) { InvalidatedScope invalidated = publicationGate.write(() -> { List existing = resolveExisting(path); bumpPublicationStates(existing); - if (existing.size() != path.level().ordinal() + 1) { + if (existing.size() != path.depth() + 1) { return null; } ScopeNode target = existing.get(existing.size() - 1); @@ -216,32 +216,85 @@ void invalidate(List paths, Runnable afterStateReplacement) { Objects.requireNonNull(paths, "paths can not be null"); Objects.requireNonNull(afterStateReplacement, "afterStateReplacement can not be null"); checkOpen(); - List invalidated = publicationGate.write(() -> { - List> resolvedScopes = new ArrayList<>(paths.size()); - Set targets = Collections.newSetFromMap(new IdentityHashMap<>()); - Set publicationNodes = Collections.newSetFromMap(new IdentityHashMap<>()); - for (ScopePath path : paths) { - Objects.requireNonNull(path, "path can not be null"); - List existing = resolveExisting(path); - publicationNodes.addAll(existing); - if (existing.size() == path.level().ordinal() + 1) { - ScopeNode target = existing.get(existing.size() - 1); - if (targets.add(target)) { - resolvedScopes.add(existing); - } + List invalidated = publicationGate.write(() -> detachScopes(paths)); + afterStateReplacement.run(); + cleanInvalidatedScopes(invalidated); + } + + CacheObservation observe(ScopeSnapshot snapshot, BooleanSupplier exactValueCurrent) { + return new CacheObservation(this, snapshot, exactValueCurrent); + } + + boolean invalidateIfCurrent(List paths, CacheObservation observation) { + Objects.requireNonNull(paths, "paths can not be null"); + if (observation.registry != this) { + throw new IllegalArgumentException("Observation belongs to another metadata cache registry"); + } + // The predicate reads only framework state: no cache lookup, callbacks, locks or remote I/O. + List invalidated = publicationGate.write(() -> observation.isCurrent() + ? detachScopes(paths) : null); + if (invalidated == null) { + return false; + } + cleanInvalidatedScopes(invalidated); + return true; + } + + private List detachScopes(List paths) { + List> resolvedScopes = new ArrayList<>(paths.size()); + Set targets = Collections.newSetFromMap(new IdentityHashMap<>()); + Set publicationNodes = Collections.newSetFromMap(new IdentityHashMap<>()); + for (ScopePath path : paths) { + Objects.requireNonNull(path, "path can not be null"); + List existing = resolveExisting(path); + publicationNodes.addAll(existing); + if (existing.size() == path.depth() + 1) { + ScopeNode target = existing.get(existing.size() - 1); + if (targets.add(target)) { + resolvedScopes.add(existing); } } - bumpPublicationStates(new ArrayList<>(publicationNodes)); - List result = new ArrayList<>(resolvedScopes.size()); - resolvedScopes.forEach(existing -> result.add(new InvalidatedScope( - existing, replaceState(existing.get(existing.size() - 1))))); - return result; - }); - afterStateReplacement.run(); + } + bumpPublicationStates(new ArrayList<>(publicationNodes)); + List result = new ArrayList<>(resolvedScopes.size()); + resolvedScopes.forEach(existing -> result.add(new InvalidatedScope( + existing, replaceState(existing.get(existing.size() - 1))))); + return result; + } + + private void cleanInvalidatedScopes(List invalidated) { invalidated.forEach(scope -> cleanDetachedState(scope.oldState)); invalidated.forEach(scope -> tryPrune(scope.existing)); } + static final class CacheObservation { + private final ScopedMetaCacheRegistry registry; + private final ScopeSnapshot snapshot; + private final PublicationState publicationState; + private final BooleanSupplier exactValueCurrent; + + private CacheObservation(ScopedMetaCacheRegistry registry, ScopeSnapshot snapshot, + BooleanSupplier exactValueCurrent) { + this.registry = registry; + this.snapshot = snapshot; + this.publicationState = snapshot.leafNode().descendantPublication.get(); + this.exactValueCurrent = exactValueCurrent; + } + + CacheObservation and(CacheObservation other) { + if (registry != other.registry) { + throw new IllegalArgumentException("Observations must belong to the same registry"); + } + return registry.observe(snapshot, () -> isCurrent() && other.isCurrent()); + } + + private boolean isCurrent() { + return snapshot.isCurrent(registry) + && snapshot.leafNode().descendantPublication.get() == publicationState + && exactValueCurrent.getAsBoolean(); + } + } + public ScopeMetrics metrics() { ScopeMetricsAccumulator accumulator = new ScopeMetricsAccumulator(); collectMetrics(root, accumulator); @@ -329,61 +382,28 @@ private ScopeState replaceState(ScopeNode node) { } private ScopeSnapshot resolveOrCreateSnapshot(ScopePath path) { - ScopeNode catalogNode = root; - if (path.level() == ScopePath.Level.CATALOG) { - return ScopeSnapshot.capture(path, catalogNode, null, null, null); - } - ScopeState catalogState = catalogNode.current.get(); - ScopeNode dbNode = child(catalogNode, catalogState, path.database(), ScopePath.Level.DATABASE); - if (dbNode == null) { - return null; - } - if (path.level() == ScopePath.Level.DATABASE) { - return ScopeSnapshot.capture(path, catalogNode, dbNode, null, null); - } - ScopeState dbState = dbNode.current.get(); - ScopeNode tableNode = child(dbNode, dbState, path.table(), ScopePath.Level.TABLE); - if (tableNode == null) { - return null; - } - if (path.level() == ScopePath.Level.TABLE) { - return ScopeSnapshot.capture(path, catalogNode, dbNode, tableNode, null); - } - ScopeState tableState = tableNode.current.get(); - ScopeNode partitionNode = child( - tableNode, tableState, path.partition(), ScopePath.Level.PARTITION); - if (partitionNode == null) { - return null; + ScopeNode[] nodes = new ScopeNode[path.depth() + 1]; + nodes[0] = root; + for (int depth = 1; depth < nodes.length; depth++) { + ScopeNode parent = nodes[depth - 1]; + nodes[depth] = child(parent, parent.current.get(), path.keyAtDepth(depth), path.levelAtDepth(depth)); + if (nodes[depth] == null) { + return null; + } } - return ScopeSnapshot.capture(path, catalogNode, dbNode, tableNode, partitionNode); + return ScopeSnapshot.capture(path, nodes); } private List resolveExisting(ScopePath path) { - List nodes = new ArrayList<>(path.level().ordinal() + 1); - ScopeNode catalogNode = root; - nodes.add(catalogNode); - if (path.level() == ScopePath.Level.CATALOG) { - return nodes; - } - ScopeNode dbNode = catalogNode.current.get().children.get(path.database()); - if (dbNode == null) { - return nodes; - } - nodes.add(dbNode); - if (path.level() == ScopePath.Level.DATABASE) { - return nodes; - } - ScopeNode tableNode = dbNode.current.get().children.get(path.table()); - if (tableNode == null) { - return nodes; - } - nodes.add(tableNode); - if (path.level() == ScopePath.Level.TABLE) { - return nodes; - } - ScopeNode partitionNode = tableNode.current.get().children.get(path.partition()); - if (partitionNode != null) { - nodes.add(partitionNode); + List nodes = new ArrayList<>(path.depth() + 1); + ScopeNode node = root; + nodes.add(node); + for (int depth = 1; depth <= path.depth(); depth++) { + node = node.current.get().children.get(path.keyAtDepth(depth)); + if (node == null) { + break; + } + nodes.add(node); } return nodes; } @@ -446,30 +466,16 @@ private boolean retain(ScopeSnapshot snapshot) { if (!snapshot.leafNode().tryRetain()) { return false; } - retainedActiveLoads[ScopePath.Level.CATALOG.ordinal()].increment(); - if (snapshot.databaseNode != null) { - retainedActiveLoads[ScopePath.Level.DATABASE.ordinal()].increment(); - } - if (snapshot.tableNode != null) { - retainedActiveLoads[ScopePath.Level.TABLE.ordinal()].increment(); - } - if (snapshot.partitionNode != null) { - retainedActiveLoads[ScopePath.Level.PARTITION.ordinal()].increment(); + for (ScopeNode node : snapshot.nodes) { + retainedActiveLoads[node.level.ordinal()].increment(); } return true; } private void release(ScopeSnapshot snapshot) { snapshot.leafNode().releaseRetain(); - retainedActiveLoads[ScopePath.Level.CATALOG.ordinal()].decrement(); - if (snapshot.databaseNode != null) { - retainedActiveLoads[ScopePath.Level.DATABASE.ordinal()].decrement(); - } - if (snapshot.tableNode != null) { - retainedActiveLoads[ScopePath.Level.TABLE.ordinal()].decrement(); - } - if (snapshot.partitionNode != null) { - retainedActiveLoads[ScopePath.Level.PARTITION.ordinal()].decrement(); + for (ScopeNode node : snapshot.nodes) { + retainedActiveLoads[node.level.ordinal()].decrement(); } tryPrune(snapshot); } @@ -481,14 +487,8 @@ private void tryPrune(List nodes) { } private void tryPrune(ScopeSnapshot snapshot) { - if (snapshot.partitionNode != null) { - tryPrune(snapshot.partitionNode); - } - if (snapshot.tableNode != null) { - tryPrune(snapshot.tableNode); - } - if (snapshot.databaseNode != null) { - tryPrune(snapshot.databaseNode); + for (int i = snapshot.nodes.length - 1; i > 0; i--) { + tryPrune(snapshot.nodes[i]); } } @@ -525,6 +525,8 @@ private void collectMetrics(ScopeNode node, ScopeMetricsAccumulator accumulator) accumulator.registrations += state.entries.size(); state.children.values().forEach(child -> { switch (child.level) { + case NAMESPACE: + break; case DATABASE: accumulator.databaseNodes++; break; @@ -551,7 +553,7 @@ private static LongAdder[] newCounters(int count) { void forceGenerationForTest(ScopePath path, long generation) { List existing = resolveExisting(path); - if (existing.size() != path.level().ordinal() + 1) { + if (existing.size() != path.depth() + 1) { throw new IllegalStateException("Scope does not exist: " + path); } ScopeNode node = existing.get(existing.size() - 1); @@ -568,7 +570,7 @@ void forceGenerationForTest(ScopePath path, long generation) { long generationForTest(ScopePath path) { List existing = resolveExisting(path); - if (existing.size() != path.level().ordinal() + 1) { + if (existing.size() != path.depth() + 1) { throw new IllegalStateException("Scope does not exist: " + path); } return existing.get(existing.size() - 1).current.get().generation; @@ -626,114 +628,48 @@ private InvalidatedScope(List existing, ScopeState oldState) { static final class ScopeSnapshot { private final ScopePath path; - private final ScopeNode catalogNode; - private final ScopeState catalogState; - private final ScopeNode databaseNode; - private final ScopeState databaseState; - private final ScopeNode tableNode; - private final ScopeState tableState; - private final ScopeNode partitionNode; - private final ScopeState partitionState; + private final ScopeNode[] nodes; + private final ScopeState[] states; private final int pathHashCode; - private ScopeSnapshot( - ScopePath path, - ScopeNode catalogNode, - ScopeState catalogState, - ScopeNode databaseNode, - ScopeState databaseState, - ScopeNode tableNode, - ScopeState tableState, - ScopeNode partitionNode, - ScopeState partitionState) { + private ScopeSnapshot(ScopePath path, ScopeNode[] nodes, ScopeState[] states) { this.path = path; - this.catalogNode = catalogNode; - this.catalogState = catalogState; - this.databaseNode = databaseNode; - this.databaseState = databaseState; - this.tableNode = tableNode; - this.tableState = tableState; - this.partitionNode = partitionNode; - this.partitionState = partitionState; + this.nodes = nodes; + this.states = states; this.pathHashCode = path.hashCode(); } - static ScopeSnapshot capture( - ScopePath path, - ScopeNode catalogNode, - ScopeNode databaseNode, - ScopeNode tableNode, - ScopeNode partitionNode) { - ScopeState catalogState = catalogNode.current.get(); - ScopeState databaseState = databaseNode == null ? null : databaseNode.current.get(); - ScopeState tableState = tableNode == null ? null : tableNode.current.get(); - ScopeState partitionState = partitionNode == null ? null : partitionNode.current.get(); - ScopeState leafState = partitionState != null - ? partitionState - : tableState != null ? tableState : databaseState != null ? databaseState : catalogState; - ScopeSnapshot currentSnapshot = leafState.scopeSnapshot; - if (currentSnapshot != null && currentSnapshot.matches( - catalogNode, - catalogState, - databaseNode, - databaseState, - tableNode, - tableState, - partitionNode, - partitionState)) { - return currentSnapshot; - } - synchronized (leafState) { - currentSnapshot = leafState.scopeSnapshot; - if (currentSnapshot == null || !currentSnapshot.matches( - catalogNode, - catalogState, - databaseNode, - databaseState, - tableNode, - tableState, - partitionNode, - partitionState)) { - currentSnapshot = new ScopeSnapshot( - path, - catalogNode, - catalogState, - databaseNode, - databaseState, - tableNode, - tableState, - partitionNode, - partitionState); - leafState.scopeSnapshot = currentSnapshot; + static ScopeSnapshot capture(ScopePath path, ScopeNode[] nodes) { + ScopeState[] states = new ScopeState[nodes.length]; + for (int i = 0; i < nodes.length; i++) { + states[i] = nodes[i].current.get(); + } + ScopeState leaf = states[states.length - 1]; + ScopeSnapshot current = leaf.scopeSnapshot; + if (current != null && current.matches(nodes, states)) { + return current; + } + synchronized (leaf) { + current = leaf.scopeSnapshot; + if (current == null || !current.matches(nodes, states)) { + current = new ScopeSnapshot(path, nodes, states); + leaf.scopeSnapshot = current; } - return currentSnapshot; + return current; } } boolean isCurrent(ScopedMetaCacheRegistry registry) { - if (registry.closed.get() || registry.root != catalogNode - || catalogNode.current.get() != catalogState) { - return false; - } - if (databaseNode == null) { - return true; - } - if (catalogState.children.get(databaseNode.childKey) != databaseNode - || databaseNode.current.get() != databaseState) { - return false; - } - if (tableNode == null) { - return true; - } - if (databaseState.children.get(tableNode.childKey) != tableNode - || tableNode.current.get() != tableState) { + if (registry.closed.get() || registry.root != nodes[0]) { return false; } - if (partitionNode == null) { - return true; + for (int i = 0; i < nodes.length; i++) { + if (nodes[i].current.get() != states[i] + || (i > 0 && states[i - 1].children.get(nodes[i].childKey) != nodes[i])) { + return false; + } } - return tableState.children.get(partitionNode.childKey) == partitionNode - && partitionNode.current.get() == partitionState; + return true; } ScopePath path() { @@ -741,64 +677,31 @@ ScopePath path() { } ScopeNode leafNode() { - if (partitionNode != null) { - return partitionNode; - } - if (tableNode != null) { - return tableNode; - } - if (databaseNode != null) { - return databaseNode; - } - return catalogNode; + return nodes[nodes.length - 1]; } ScopeState leafState() { - if (partitionState != null) { - return partitionState; - } - if (tableState != null) { - return tableState; - } - if (databaseState != null) { - return databaseState; - } - return catalogState; + return states[states.length - 1]; } boolean sameGeneration(ScopeSnapshot other) { - return matches( - other.catalogNode, - other.catalogState, - other.databaseNode, - other.databaseState, - other.tableNode, - other.tableState, - other.partitionNode, - other.partitionState); + return matches(other.nodes, other.states); } int pathHashCode() { return pathHashCode; } - private boolean matches( - ScopeNode expectedCatalogNode, - ScopeState expectedCatalogState, - ScopeNode expectedDatabaseNode, - ScopeState expectedDatabaseState, - ScopeNode expectedTableNode, - ScopeState expectedTableState, - ScopeNode expectedPartitionNode, - ScopeState expectedPartitionState) { - return catalogNode == expectedCatalogNode - && catalogState == expectedCatalogState - && databaseNode == expectedDatabaseNode - && databaseState == expectedDatabaseState - && tableNode == expectedTableNode - && tableState == expectedTableState - && partitionNode == expectedPartitionNode - && partitionState == expectedPartitionState; + private boolean matches(ScopeNode[] expectedNodes, ScopeState[] expectedStates) { + if (nodes.length != expectedNodes.length) { + return false; + } + for (int i = 0; i < nodes.length; i++) { + if (nodes[i] != expectedNodes[i] || states[i] != expectedStates[i]) { + return false; + } + } + return true; } } diff --git a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/StripedPhaseGate.java b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/StripedPhaseGate.java index b585e8f020b64f..e69d09ea34e3bd 100644 --- a/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/StripedPhaseGate.java +++ b/fe/fe-connector/fe-connector-cache/src/main/java/org/apache/doris/connector/cache/StripedPhaseGate.java @@ -24,7 +24,8 @@ /** * Writer-preferred phase gate for short cache-publication critical sections. * - *

The gate is intentionally non-reentrant: an action must not enter either phase of the same gate. In particular, + *

Read phases are reentrant so one guarded publication can update a second cache in the same catalog. + * Read-to-write upgrade is forbidden. In particular, * synchronous removal callbacks must run after the read phase. Reader actions must also avoid remote I/O or other * unbounded work because a waiting writer and newly arriving readers spin until the current reader phase drains. */ @@ -37,13 +38,14 @@ final class StripedPhaseGate { private final AtomicIntegerArray activeReaders = new AtomicIntegerArray(READER_STRIPE_COUNT * READER_STRIDE); private volatile boolean writerActive; + private final ThreadLocal readDepth = new ThreadLocal<>(); boolean readBoolean(BooleanSupplier action) { int readerIndex = enterRead(); try { return action.getAsBoolean(); } finally { - activeReaders.decrementAndGet(readerIndex); + exitRead(readerIndex); } } @@ -52,7 +54,7 @@ T read(Supplier action) { try { return action.get(); } finally { - activeReaders.decrementAndGet(readerIndex); + exitRead(readerIndex); } } @@ -64,6 +66,9 @@ void write(Runnable action) { } T write(Supplier action) { + if (readDepth.get() != null) { + throw new IllegalStateException("A cache publication action cannot invalidate its own scope"); + } synchronized (writerLock) { writerActive = true; try { @@ -76,6 +81,11 @@ T write(Supplier action) { } private int enterRead() { + Integer depth = readDepth.get(); + if (depth != null) { + readDepth.set(depth + 1); + return -1; + } int readerIndex = readerIndex(); while (true) { while (writerActive) { @@ -83,12 +93,22 @@ private int enterRead() { } activeReaders.incrementAndGet(readerIndex); if (!writerActive) { + readDepth.set(1); return readerIndex; } activeReaders.decrementAndGet(readerIndex); } } + private void exitRead(int readerIndex) { + if (readerIndex < 0) { + readDepth.set(readDepth.get() - 1); + } else { + readDepth.remove(); + activeReaders.decrementAndGet(readerIndex); + } + } + private void awaitReaders() { while (hasReaders()) { Thread.onSpinWait(); diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogCacheDomainTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogCacheDomainTest.java new file mode 100644 index 00000000000000..ca47b1bb2dca15 --- /dev/null +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogCacheDomainTest.java @@ -0,0 +1,192 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.cache; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Arrays; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; + +class CatalogCacheDomainTest { + private static final CacheSpec SPEC = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 100); + + private MetaCache entry(CatalogMetaCache domain) { + return domain.create(MetaCacheDefinition.builder("table", SPEC, + key -> ScopePath.table("db", key)).build()); + } + + @Test + void sameNameInstancesShareInvalidationButNotRetirement() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + CatalogMetaCache old = root.openDomain("hive"); + CatalogMetaCache replacement = root.openDomain("hive"); + MetaCache first = entry(old); + MetaCache second = entry(replacement); + first.put("a", "old"); + second.put("a", "new"); + Assertions.assertEquals(2, root.allEntries().size()); + old.close(); + Assertions.assertEquals("new", second.getIfPresent("a")); + Assertions.assertEquals(1, root.allEntries().size()); + Assertions.assertThrows(IllegalStateException.class, () -> old.openDomain("iceberg")); + root.invalidateTable("db", "a"); + Assertions.assertNull(second.getIfPresent("a")); + } + } + + @Test + void constructionRollbackDoesNotRetireTheRootOrSiblings() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + MetaCache sibling = entry(root.openDomain("iceberg")); + sibling.put("a", "live"); + CatalogMetaCache failing = root.openDomain("hive"); + entry(failing).put("a", "partial"); + Assertions.assertThrows(IllegalArgumentException.class, () -> failing.create(MetaCacheDefinition + .builder("bad", CacheSpec.of(true, -1L, -1L), + key -> ScopePath.table("db", key)).build())); + failing.close(); + Assertions.assertFalse(root.isClosed()); + Assertions.assertEquals("live", sibling.getIfPresent("a")); + Assertions.assertEquals(1, root.allEntries().size()); + } + } + + @Test + void localAndRemoteCoordinatesDoNotCollideAndCanBeFencedTogether() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + CatalogMetaCache local = root.openDomain("fe", ScopePath.Coordinates.LOCAL, getClass().getClassLoader()); + CatalogMetaCache remote = root.openDomain("hive", ScopePath.Coordinates.REMOTE, getClass().getClassLoader()); + MetaCache fe = entry(local); + MetaCache source = entry(remote); + fe.put("a", "local"); + source.put("a", "remote"); + local.invalidateTable("db", "a"); + Assertions.assertNull(fe.getIfPresent("a")); + Assertions.assertEquals("remote", source.getIfPresent("a")); + fe.put("a", "local2"); + root.invalidateScopes(Arrays.asList(ScopePath.table("db", "a").in(ScopePath.Coordinates.LOCAL), + ScopePath.table("db", "a").in(ScopePath.Coordinates.REMOTE))); + Assertions.assertNull(fe.getIfPresent("a")); + Assertions.assertNull(source.getIfPresent("a")); + } + } + + @Test + void oneRemoteInvalidationFencesAnInFlightSiblingLoad() throws Exception { + ExecutorService worker = Executors.newSingleThreadExecutor(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finish = new CountDownLatch(1); + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + CatalogMetaCache hive = root.openDomain("hive", ScopePath.Coordinates.REMOTE, getClass().getClassLoader()); + MetaCache sibling = entry(hive.openDomain("iceberg")); + Future old = worker.submit(() -> sibling.get("a", key -> { + started.countDown(); + try { + Assertions.assertTrue(finish.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new RuntimeException(e); + } + return "old"; + })); + Assertions.assertTrue(started.await(10, TimeUnit.SECONDS)); + hive.invalidateTable("db", "a"); + sibling.put("a", "new"); + finish.countDown(); + Assertions.assertEquals("old", old.get(10, TimeUnit.SECONDS)); + Assertions.assertEquals("new", sibling.getIfPresent("a")); + } finally { + finish.countDown(); + worker.shutdownNow(); + } + } + + @Test + void bulkLoadsProjectTheirParentIntoTheDomainCoordinates() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + CatalogMetaCache domain = root.openDomain("hive", ScopePath.Coordinates.REMOTE, + getClass().getClassLoader()); + MetaCache cache = entry(domain); + try (MetaCache.BulkLoad load = cache.beginBulkLoad(ScopePath.database("db"))) { + Assertions.assertTrue(load.publish("a", "first")); + domain.invalidateDatabase("db"); + Assertions.assertFalse(load.publish("b", "stale")); + Assertions.assertNull(cache.getIfPresent("a")); + Assertions.assertNull(cache.getIfPresent("b")); + } + } + } + + @Test + void disabledCacheStillFencesPublicationSideEffectsBeforeLoading() throws Exception { + ExecutorService worker = Executors.newSingleThreadExecutor(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch finish = new CountDownLatch(1); + java.util.concurrent.atomic.AtomicInteger actions = new java.util.concurrent.atomic.AtomicInteger(); + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + MetaCache cache = root.create(MetaCacheDefinition.builder("disabled", + CacheSpec.of(false, -1, 10), key -> ScopePath.table("db", key)).build()); + Future load = worker.submit(() -> cache.getWithPublicationAction("a", key -> { + started.countDown(); + try { + Assertions.assertTrue(finish.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + return "old"; + }, (value, commit) -> commit.accept(actions::incrementAndGet))); + Assertions.assertTrue(started.await(10, TimeUnit.SECONDS)); + root.invalidateTable("db", "a"); + finish.countDown(); + Assertions.assertEquals("old", load.get(10, TimeUnit.SECONDS)); + Assertions.assertEquals(0, actions.get()); + Assertions.assertNull(cache.getIfPresent("a")); + } finally { + finish.countDown(); + worker.shutdownNow(); + } + } + + @Test + void pluginCallbacksRestoreTheirRegisteringClassLoader() { + ClassLoader original = Thread.currentThread().getContextClassLoader(); + ClassLoader plugin = new ClassLoader(original) { }; + AtomicReference loaded = new AtomicReference<>(); + AtomicReference removed = new AtomicReference<>(); + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + CatalogMetaCache domain = root.openDomain("hive", ScopePath.Coordinates.REMOTE, plugin); + MetaCache cache = domain.create(MetaCacheDefinition.builder("table", SPEC, + key -> ScopePath.table("db", key)) + .loader(key -> { + loaded.set(Thread.currentThread().getContextClassLoader()); + return "v"; + }).removalListener((key, value, cause) -> + removed.set(Thread.currentThread().getContextClassLoader())).build()); + cache.get("a"); + domain.close(); + Assertions.assertSame(plugin, loaded.get()); + Assertions.assertSame(plugin, removed.get()); + Assertions.assertSame(original, Thread.currentThread().getContextClassLoader()); + } + } +} diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java index 3bb6ad66e80d68..068e0f773b6932 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/CatalogMetaCacheTest.java @@ -24,14 +24,149 @@ import java.time.Duration; import java.util.ArrayDeque; import java.util.Arrays; +import java.util.Collections; +import java.util.Locale; +import java.util.OptionalLong; import java.util.Queue; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; class CatalogMetaCacheTest { private static final CacheSpec SPEC = CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 100); + @Test + void resourceNormalizationCoversDomainsAndBulkLoadsWithoutChangingRemoteNames() { + try (CatalogMetaCache root = CatalogMetaCache.managed(987654321L, "test", Collections.emptyMap(), + getClass().getClassLoader(), scope -> scope.coordinates() == ScopePath.Coordinates.LOCAL + ? scope.mapTableName(name -> name.toLowerCase(Locale.ROOT)) : scope)) { + CatalogMetaCache local = root.openDomain("local", ScopePath.Coordinates.LOCAL, getClass().getClassLoader()); + CatalogMetaCache remote = root.openDomain("remote", ScopePath.Coordinates.REMOTE, + getClass().getClassLoader()); + MetaCache objects = local.create(MetaCacheDefinition.builder( + "objects", SPEC, key -> ScopePath.table("db", key)).build()); + MetaCache metadata = local.create(MetaCacheDefinition.builder( + "metadata", SPEC, key -> ScopePath.tableMetadata("db", key)).build()); + MetaCache remoteObjects = remote.create(MetaCacheDefinition.builder( + "objects", SPEC, key -> ScopePath.table("db", key)).build()); + objects.put("Dest", "old"); + metadata.put("Dest", "schema"); + objects.put("b", "sibling"); + remoteObjects.put("Dest", "remote"); + try (MetaCache.BulkLoad load = objects.beginBulkLoad(ScopePath.table("db", "Dest"))) { + root.invalidateScopes(Collections.singletonList( + ScopePath.table("db", "DEST").in(ScopePath.Coordinates.LOCAL))); + Assertions.assertFalse(load.publish("Dest", "late")); + } + Assertions.assertNull(objects.getIfPresent("Dest")); + Assertions.assertNull(metadata.getIfPresent("Dest")); + Assertions.assertEquals("sibling", objects.getIfPresent("b")); + Assertions.assertEquals("remote", remoteObjects.getIfPresent("Dest")); + remote.invalidateTable("db", "DEST"); + Assertions.assertEquals("remote", remoteObjects.getIfPresent("Dest")); + remote.invalidateTable("db", "Dest"); + Assertions.assertNull(remoteObjects.getIfPresent("Dest")); + } + } + + @Test + void conditionalInvalidationRejectsExactReplacementAndNewScopeGeneration() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + ScopePath table = ScopePath.table("db", "a").in(ScopePath.Coordinates.LOCAL); + ScopePath names = ScopePath.tableCollection("db").in(ScopePath.Coordinates.LOCAL); + MetaCache objects = root.create(MetaCacheDefinition.builder( + "objects", SPEC, key -> table).build()); + MetaCache listings = root.create(MetaCacheDefinition.builder( + "names", SPEC, key -> names).build()); + Object old = new Object(); + objects.put("a", old); + listings.put("all", "a,b"); + MetaCache.Observation replaced = objects.observe("a", old).orElseThrow(); + Object replacement = new Object(); + objects.put("a", replacement); + Assertions.assertFalse(replaced.invalidateScopes(Arrays.asList(table, names))); + Assertions.assertSame(replacement, objects.getIfPresent("a")); + Assertions.assertEquals("a,b", listings.getIfPresent("all")); + MetaCache.Observation refreshed = objects.observe("a", replacement).orElseThrow(); + root.invalidateScope(table); + objects.put("a", old); + Assertions.assertFalse(refreshed.invalidateScopes(Arrays.asList(table, names))); + Assertions.assertSame(old, objects.getIfPresent("a")); + Assertions.assertEquals("a,b", listings.getIfPresent("all")); + Assertions.assertTrue(objects.observe("a", old).orElseThrow() + .invalidateScopes(Arrays.asList(table, names))); + Assertions.assertNull(objects.getIfPresent("a")); + Assertions.assertNull(listings.getIfPresent("all")); + } + } + + @Test + void conditionalInvalidationRejectsNewContentAndRetiredRegistration() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + CatalogMetaCache domain = root.openDomain("fe", ScopePath.Coordinates.LOCAL, + getClass().getClassLoader()); + MetaCache objects = domain.create(MetaCacheDefinition.builder( + "objects", SPEC, key -> ScopePath.table("db", key)).build()); + Object table = new Object(); + objects.put("a", table); + MetaCache.Observation beforeContent = objects.observe("a", table).orElseThrow(); + root.invalidateScope(ScopePath.tableMetadata("db", "a").in(ScopePath.Coordinates.LOCAL)); + Assertions.assertFalse(beforeContent.invalidateScopes(Collections.singletonList( + ScopePath.table("db", "a").in(ScopePath.Coordinates.LOCAL)))); + Assertions.assertSame(table, objects.getIfPresent("a")); + MetaCache.Observation retired = objects.observe("a", table).orElseThrow(); + domain.close(); + MetaCache replacement = root.openDomain("fe", ScopePath.Coordinates.LOCAL, + getClass().getClassLoader()).create(MetaCacheDefinition.builder( + "objects", SPEC, key -> ScopePath.table("db", key)).build()); + replacement.put("a", table); + Assertions.assertFalse(retired.invalidateScopes(Collections.singletonList( + ScopePath.table("db", "a").in(ScopePath.Coordinates.LOCAL)))); + Assertions.assertSame(table, replacement.getIfPresent("a")); + } + } + + @Test + void conditionalInvalidationCoversDependenciesAndFencesAnOldLoad() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + ScopePath local = ScopePath.table("db", "a").in(ScopePath.Coordinates.LOCAL); + ScopePath remote = ScopePath.table("DB", "A").in(ScopePath.Coordinates.REMOTE); + MetaCache objects = root.create(MetaCacheDefinition.builder( + "objects", SPEC, key -> local).build()); + Object table = new Object(); + objects.put("a", table); + MetaCache dependencies = root.create(MetaCacheDefinition.builder( + "remote", SPEC, key -> remote).build()); + java.util.concurrent.Future old = executor.submit(() -> dependencies.get("a", key -> { + started.countDown(); + try { + Assertions.assertTrue(release.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + return "old"; + })); + Assertions.assertTrue(started.await(10, TimeUnit.SECONDS)); + Assertions.assertTrue(objects.observe("a", table).orElseThrow() + .invalidateScopes(Arrays.asList(local, remote))); + release.countDown(); + Assertions.assertEquals("old", old.get(10, TimeUnit.SECONDS)); + Assertions.assertNull(dependencies.getIfPresent("a")); + } finally { + release.countDown(); + executor.shutdownNow(); + Assertions.assertTrue(executor.awaitTermination(10, TimeUnit.SECONDS)); + } + } + @Test void unmanagedOwnershipMustBeSelectedExplicitly() throws Exception { Assertions.assertTrue(Modifier.isPrivate(CatalogMetaCache.class.getDeclaredConstructor().getModifiers())); @@ -41,6 +176,37 @@ void unmanagedOwnershipMustBeSelectedExplicitly() throws Exception { } } + @Test + void countOnlyHandlesShareInvalidationWhilePayloadsRemainWeightBounded() { + MetaCacheBudgetManager manager = new MetaCacheBudgetManager(OptionalLong.of(1024L)); + try (CatalogMetaCache owner = new CatalogMetaCache(manager, 1L, "test", Collections.emptyMap())) { + AtomicInteger loads = new AtomicInteger(); + MetaCache handles = owner.create(MetaCacheDefinition + .builder("lookup", SPEC, key -> ScopePath.table("db", key)) + .budgetPolicy(MetaCacheDefinition.BudgetPolicy.COUNT_ONLY) + .loader(key -> { + loads.incrementAndGet(); + return () -> { }; + }).build()); + Runnable first = handles.get("a"); + Assertions.assertSame(first, handles.get("a")); + Assertions.assertEquals(1, loads.get()); + Assertions.assertFalse(handles.isWeightBounded()); + MetaCache payload = owner.create(MetaCacheDefinition + .builder("payload", SPEC, key -> ScopePath.table("db", key)) + .sizeEstimator(MetaCacheSizeEstimators.reflective()).build()); + Assertions.assertTrue(payload.isWeightBounded()); + payload.put("a", "x".repeat(2048)); + Assertions.assertNull(payload.getIfPresent("a")); + payload.put("a", "small"); + Assertions.assertTrue(manager.getGlobalUsedWeight() > 0L); + owner.invalidateTable("db", "a"); + Assertions.assertNull(handles.getIfPresent("a")); + Assertions.assertNull(payload.getIfPresent("a")); + } + Assertions.assertEquals(0L, manager.getGlobalUsedWeight()); + } + @Test void tableInvalidationAutomaticallyCoversEveryPhysicalCache() { try (CatalogMetaCache catalog = CatalogMetaCache.unmanaged()) { @@ -104,6 +270,93 @@ void partitionInvalidationCoversCollectionAndSelectedPartitionsOnly() { } } + @Test + void fullTableInvalidationAlsoCoversMembershipWithoutClearingSiblings() { + try (CatalogMetaCache catalog = CatalogMetaCache.unmanaged()) { + MetaCache names = catalog.create(MetaCacheDefinition + .builder("table-names", SPEC, ScopePath::tableCollection).build()); + MetaCache objects = catalog.create(tableDefinition("objects")); + MetaCache schema = catalog.create(MetaCacheDefinition + .builder("schema", SPEC, + key -> ScopePath.tableMetadata(key.database, key.table)).build()); + TableKey first = new TableKey("db", "a"); + TableKey sibling = new TableKey("db", "b"); + names.put("db", "a,b"); + names.put("other", "unrelated"); + objects.put(first, "object-a"); + objects.put(sibling, "object-b"); + schema.put(first, "schema-a"); + schema.put(sibling, "schema-b"); + + catalog.invalidateScope(ScopePath.tableMetadata("db", "a")); + Assertions.assertEquals("a,b", names.getIfPresent("db")); + Assertions.assertEquals("object-a", objects.getIfPresent(first)); + Assertions.assertNull(schema.getIfPresent(first)); + schema.put(first, "schema-a-new"); + + catalog.invalidateTableAndNames("db", "a"); + Assertions.assertNull(names.getIfPresent("db")); + Assertions.assertNull(objects.getIfPresent(first)); + Assertions.assertNull(schema.getIfPresent(first)); + Assertions.assertEquals("object-b", objects.getIfPresent(sibling)); + Assertions.assertEquals("schema-b", schema.getIfPresent(sibling)); + Assertions.assertEquals("unrelated", names.getIfPresent("other")); + } + } + + @Test + void absentChildStillInvalidatesMembershipAndCollectionsCannotAliasUserNames() { + try (CatalogMetaCache catalog = CatalogMetaCache.unmanaged()) { + MetaCache names = catalog.create(MetaCacheDefinition + .builder("names", SPEC, ScopePath::tableCollection).build()); + names.put("db", "cold-table"); + catalog.invalidateTableAndNames("db", "cold-table"); + Assertions.assertNull(names.getIfPresent("db")); + Assertions.assertNotEquals(ScopePath.tableCollection("db"), ScopePath.table("db", "TABLES")); + Assertions.assertNotEquals(ScopePath.databaseCollection(), ScopePath.database("DATABASES")); + Assertions.assertTrue(ScopePath.database("db").contains(ScopePath.tableCollection("db"))); + Assertions.assertFalse(ScopePath.tableCollection("db").contains(ScopePath.table("db", "TABLES"))); + } + } + + @Test + void lateAsyncRefreshCannotReplaceRecreatedTableAfterFullInvalidation() throws Exception { + ExecutorService refreshes = Executors.newSingleThreadExecutor(); + CountDownLatch started = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicInteger loads = new AtomicInteger(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + MetaCache schema = owner.create(MetaCacheDefinition + .builder("schema", SPEC, key -> ScopePath.tableMetadata("db", key)) + .loader(key -> { + int load = loads.incrementAndGet(); + if (load == 2) { + started.countDown(); + try { + Assertions.assertTrue(release.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + return "old-schema"; + }) + .refreshAfterWrite(Duration.ofNanos(1), refreshes).build()); + Assertions.assertEquals("old-schema", schema.get("a")); + Assertions.assertEquals("old-schema", schema.get("a")); + Assertions.assertTrue(started.await(10, TimeUnit.SECONDS)); + owner.invalidateTableAndNames("db", "a"); + schema.put("a", "recreated-schema"); + release.countDown(); + refreshes.submit(() -> { }).get(10, TimeUnit.SECONDS); + Assertions.assertEquals("recreated-schema", schema.getIfPresent("a")); + } finally { + release.countDown(); + refreshes.shutdownNow(); + Assertions.assertTrue(refreshes.awaitTermination(10, TimeUnit.SECONDS)); + } + } + @Test void defaultAndContextualLoadersUseDefinitionScope() { AtomicInteger loads = new AtomicInteger(); diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java index 59acec37a200c2..804d8fd6c3c5c8 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/MetaCacheWeightGovernanceTest.java @@ -247,7 +247,7 @@ void budgetPressureReclaimsOneColdEntryBeforeTheNextAdmission() { } @Test - void managedOwnerClosesWhenPhysicalCacheConstructionFails() { + void failedEntryConstructionDoesNotCloseTheCatalogRoot() { long catalogId = Long.MIN_VALUE + 31L; Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalogId).isEmpty()); CatalogMetaCache owner = CatalogMetaCache.managed(catalogId, "iceberg", Collections.emptyMap()); @@ -258,6 +258,9 @@ void managedOwnerClosesWhenPhysicalCacheConstructionFails() { .sizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(512L)) .build())); + Assertions.assertFalse(owner.isClosed()); + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalogId).contains(owner)); + owner.close(); Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalogId).isEmpty()); } diff --git a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/StripedPhaseGateTest.java b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/StripedPhaseGateTest.java index 1e562ec85ec3a6..5cce549c3f1283 100644 --- a/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/StripedPhaseGateTest.java +++ b/fe/fe-connector/fe-connector-cache/src/test/java/org/apache/doris/connector/cache/StripedPhaseGateTest.java @@ -147,6 +147,38 @@ public void exceptionReleasesReaderAndWriterPhases() { Assertions.assertTrue(gate.readBoolean(() -> true)); } + @Test + public void nestedPublicationCanFinishWhileAnInvalidationWriterWaits() throws Exception { + StripedPhaseGate gate = new StripedPhaseGate(); + java.lang.reflect.Field activeWriter = StripedPhaseGate.class.getDeclaredField("writerActive"); + activeWriter.setAccessible(true); + ExecutorService workers = Executors.newFixedThreadPool(2); + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch nest = new CountDownLatch(1); + try { + Future publication = workers.submit(() -> gate.readBoolean(() -> { + entered.countDown(); + await(nest); + return gate.readBoolean(() -> true); + })); + await(entered); + Future invalidation = workers.submit(() -> gate.write(() -> { })); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(TIMEOUT_SECONDS); + while (!activeWriter.getBoolean(gate) && System.nanoTime() < deadline) { + Thread.onSpinWait(); + } + Assertions.assertTrue(activeWriter.getBoolean(gate)); + nest.countDown(); + Assertions.assertTrue(publication.get(TIMEOUT_SECONDS, TimeUnit.SECONDS)); + invalidation.get(TIMEOUT_SECONDS, TimeUnit.SECONDS); + Assertions.assertThrows(IllegalStateException.class, + () -> gate.read(() -> gate.write(() -> true))); + } finally { + nest.countDown(); + workers.shutdownNow(); + } + } + private static void await(CountDownLatch latch) { try { if (!latch.await(TIMEOUT_SECONDS, TimeUnit.SECONDS)) { diff --git a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorRestClient.java b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorRestClient.java index c509f45004fce2..d90c821f48b6c0 100644 --- a/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorRestClient.java +++ b/fe/fe-connector/fe-connector-es/src/main/java/org/apache/doris/connector/es/EsConnectorRestClient.java @@ -106,10 +106,16 @@ public boolean existIndex(String indexName) { OkHttpClient client = httpSslEnable ? getOrCreateSslClient() : PLAIN_CLIENT; String node = currentNode(); try (Response response = executeResponse(client, node, path, null)) { - return response.isSuccessful(); + if (response.isSuccessful()) { + return true; + } + if (response.code() == 404) { + return false; + } + throw new DorisConnectorException("Failed to check index " + indexName + + ": HTTP " + response.code()); } catch (IOException e) { - LOG.warn("existIndex error", e); - return false; + throw new DorisConnectorException("Failed to check index " + indexName, e); } } diff --git a/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsConnectorRestClientTest.java b/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsConnectorRestClientTest.java index 16439e765bd05b..d1354c93744c2f 100644 --- a/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsConnectorRestClientTest.java +++ b/fe/fe-connector/fe-connector-es/src/test/java/org/apache/doris/connector/es/EsConnectorRestClientTest.java @@ -17,10 +17,14 @@ package org.apache.doris.connector.es; +import org.apache.doris.connector.spi.DorisConnectorException; + +import com.sun.net.httpserver.HttpServer; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import java.lang.reflect.Field; +import java.net.InetSocketAddress; import java.util.Collections; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; @@ -34,6 +38,31 @@ */ public class EsConnectorRestClientTest { + @Test + public void indexAbsenceRequiresNotFoundResponse() throws Exception { + AtomicInteger status = new AtomicInteger(200); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/t/_mapping", exchange -> { + exchange.sendResponseHeaders(status.get(), -1); + exchange.close(); + }); + server.start(); + EsConnectorRestClient client = new EsConnectorRestClient( + new String[]{"http://127.0.0.1:" + server.getAddress().getPort()}, null, null, false, null); + try { + Assertions.assertTrue(client.existIndex("t")); + status.set(404); + Assertions.assertFalse(client.existIndex("t")); + for (int failure : new int[]{401, 403, 500}) { + status.set(failure); + Assertions.assertThrows(DorisConnectorException.class, () -> client.existIndex("t")); + } + } finally { + server.stop(0); + } + Assertions.assertThrows(DorisConnectorException.class, () -> client.existIndex("t")); + } + @Test public void testConstructorWithAuth() throws Exception { EsConnectorRestClient client = new EsConnectorRestClient( diff --git a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java index 1d3d7f364cca84..4bd721640a35f7 100644 --- a/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java +++ b/fe/fe-connector/fe-connector-fluss/src/main/java/org/apache/doris/connector/fluss/FlussConnector.java @@ -17,6 +17,8 @@ package org.apache.doris.connector.fluss; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorCapability; import org.apache.doris.connector.spi.ConnectorContext; @@ -37,7 +39,6 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -92,12 +93,14 @@ public class FlussConnector implements Connector { // Source Fluss identity -> physical lake identity. The latter may be overridden independently for the // database and table, and targeted invalidation must follow the same path as lookup. + private final CatalogMetaCache metaCache; private final Map lakeTablePaths = new LinkedHashMap<>(); public FlussConnector(FlussCatalogProperties properties, ConnectorContext context) { this.properties = properties; this.catalogName = context.getCatalogName(); this.context = context; + this.metaCache = context.createCacheDomain("fluss", properties.getRawCatalogProperties()); } @Override @@ -229,53 +232,50 @@ synchronized void rememberLakePath(TablePath sourcePath, TablePath lakePath) { } @Override - public synchronized void invalidateTable(String dbName, String tableName) { - if (lakeSibling != null) { - // Keep the mapping until the table is resolved again (which replaces it) or the whole catalog - // is invalidated. A single engine refresh may emit more than one targeted invalidation, and a - // second one must not fall back to the source name merely because the first already ran. - TablePath lakePath = lakeTablePaths.get(TablePath.of(dbName, tableName)); - lakeSibling.invalidateTable( - lakePath == null ? dbName : lakePath.getDatabaseName(), - lakePath == null ? tableName : lakePath.getTableName()); + public synchronized List cacheDependencies(ScopePath scope) { + List dependencies = new java.util.ArrayList<>(); + if (scope.database() == null) { + return dependencies; } - } - - @Override - public synchronized void invalidateDb(String dbName) { - if (lakeSibling != null) { - Set lakeDatabases = new LinkedHashSet<>(); - // An unresolved table cannot have populated the sibling's table cache, but the sibling may - // still hold database-level metadata for the source namespace. Invalidating it as well as every - // known physical override is conservative and prevents one overridden table from accidentally - // suppressing invalidation of ordinary lake tables in the same Fluss database. - lakeDatabases.add(dbName); + if (scope.table() != null) { + TablePath lake = lakeTablePaths.get(TablePath.of(scope.database(), scope.table())); + if (lake != null) { + dependencies.add(ScopePath.table(lake.getDatabaseName(), lake.getTableName())); + } + } else { lakeTablePaths.forEach((source, lake) -> { - if (source.getDatabaseName().equals(dbName)) { - lakeDatabases.add(lake.getDatabaseName()); + if (source.getDatabaseName().equals(scope.database())) { + dependencies.add(scope.level() == ScopePath.Level.TABLE + ? ScopePath.tableCollection(lake.getDatabaseName()) + : ScopePath.database(lake.getDatabaseName())); } }); - lakeDatabases.forEach(lakeSibling::invalidateDb); } + return dependencies; } @Override - public synchronized void invalidateAll() { - lakeTablePaths.clear(); - if (lakeSibling != null) { - lakeSibling.invalidateAll(); - } + public void invalidateTable(String database, String table) { + List scopes = new java.util.ArrayList<>(cacheDependencies(ScopePath.table(database, table))); + scopes.add(ScopePath.table(database, table)); + metaCache.invalidateScopes(scopes); } @Override - public synchronized void invalidatePartition( - String dbName, String tableName, List partitionNames) { - if (lakeSibling != null) { - TablePath lakePath = lakeTablePaths.get(TablePath.of(dbName, tableName)); - lakeSibling.invalidatePartition( - lakePath == null ? dbName : lakePath.getDatabaseName(), - lakePath == null ? tableName : lakePath.getTableName(), partitionNames); - } + public void invalidateDb(String database) { + List scopes = new java.util.ArrayList<>(cacheDependencies(ScopePath.database(database))); + scopes.add(ScopePath.database(database)); + metaCache.invalidateScopes(scopes); + } + + @Override + public void invalidateAll() { + metaCache.invalidateCatalog(); + } + + @Override + public void invalidatePartition(String database, String table, List partitions) { + invalidateTable(database, table); } @Override @@ -296,6 +296,7 @@ public ConnectorTestResult testConnection(ConnectorSession session) { */ @Override public void close() throws IOException { + metaCache.close(); Connector sibling; Connection toClose; synchronized (this) { diff --git a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java index 5081030231c5f4..9675cc69af0857 100644 --- a/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java +++ b/fe/fe-connector/fe-connector-fluss/src/test/java/org/apache/doris/connector/fluss/FlussConnectorLakeSiblingTest.java @@ -186,8 +186,7 @@ public void invalidationsReachOnlyAnAlreadyBuiltLakeSibling() { connector.invalidatePartition("db", "tbl", Collections.singletonList("dt=20260101")); connector.invalidateAll(); - Assertions.assertEquals(Arrays.asList("invalidateTable:db.tbl", "invalidateDb:db", - "invalidatePartition:db.tbl:[dt=20260101]", "invalidateAll"), sibling.calls); + Assertions.assertTrue(sibling.calls.isEmpty(), "shared scopes replace manual forwarding"); } @Test @@ -208,16 +207,13 @@ public void targetedInvalidationsUseThePhysicalLakeIdentity() { connector.invalidateDb("db"); connector.invalidateDb("db"); - Assertions.assertEquals(Arrays.asList( - "invalidatePartition:archive.orders_lake:[dt=20260101]", - "invalidateTable:archive.orders_lake", - "invalidateTable:archive.orders_lake", - "invalidateDb:db", - "invalidateDb:archive", - "invalidateDb:cold", - "invalidateDb:db", - "invalidateDb:archive", - "invalidateDb:cold"), sibling.calls); + Assertions.assertTrue(sibling.calls.isEmpty()); + Assertions.assertEquals(Collections.singletonList( + org.apache.doris.connector.cache.ScopePath.table("archive", "orders_lake")), + connector.cacheDependencies(org.apache.doris.connector.cache.ScopePath.table("db", "orders"))); + Assertions.assertEquals(Arrays.asList(org.apache.doris.connector.cache.ScopePath.database("archive"), + org.apache.doris.connector.cache.ScopePath.database("cold")), + connector.cacheDependencies(org.apache.doris.connector.cache.ScopePath.database("db"))); } @Test diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java index 7b6c5656806bfe..de67dbfa6a2b52 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnector.java @@ -53,15 +53,14 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.util.ArrayList; import java.util.Collections; import java.util.EnumSet; -import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; import java.util.Set; import java.util.concurrent.Callable; -import java.util.function.Consumer; /** * Hive connector implementation. Manages the lifecycle of @@ -151,7 +150,7 @@ public HiveConnector(Map properties, ConnectorContext context) { this.props = HiveCatalogProperties.of(properties); this.properties = props.getRaw(); this.context = context; - this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "hive", this.properties); + this.metaCache = context.createCacheDomain("hive", this.properties); this.fileListingCache = new HiveFileListingCache(metaCache, props); // Reads its own meta.cache.hive.partition_view.(enable|ttl-second|capacity) from the catalog properties // via the framework's CacheSpec (default ON / 24h / 1000). @@ -378,12 +377,12 @@ public Set getCapabilities() { * schema, partitions AND files are all mutable, so a REFRESH must re-read all of them (unlike iceberg, whose * manifests are immutable and kept across REFRESH TABLE). fe-core already routes {@code REFRESH TABLE} to * {@code connector.invalidateTable} for a plugin-driven catalog. - * Also forwarded to the already-built embedded siblings — see {@link #forEachBuiltSibling}. + * Sibling domains share the REMOTE scope tree and participate without explicit forwarding. */ @Override public void invalidateTable(String dbName, String tableName) { metaCache.invalidateTable(dbName, tableName); - forEachBuiltSibling(sibling -> sibling.invalidateTable(dbName, tableName)); + } /** @@ -391,23 +390,23 @@ public void invalidateTable(String dbName, String tableName) { * layers (metastore-metadata + {@link HiveFileListingCache#invalidateDb} * directory listings). Same no-force-build read of the client as {@link #invalidateTable(String, String)}. * fe-core routes {@code REFRESH DATABASE} to {@code connector.invalidateDb} for a plugin-driven catalog. - * Also forwarded to the already-built embedded siblings — see {@link #forEachBuiltSibling}. + * Sibling domains share the REMOTE scope tree and participate without explicit forwarding. */ @Override public void invalidateDb(String dbName) { metaCache.invalidateDatabase(dbName); - forEachBuiltSibling(sibling -> sibling.invalidateDb(dbName)); + } /** * REFRESH CATALOG hook: drop ALL of this catalog's connector-owned scan caches — every metastore-metadata * entry and every directory listing. - * Also forwarded to the already-built embedded siblings — see {@link #forEachBuiltSibling}. + * Sibling domains share the REMOTE scope tree and participate without explicit forwarding. */ @Override public void invalidateAll() { metaCache.invalidateCatalog(); - forEachBuiltSibling(Connector::invalidateAll); + } /** @@ -418,17 +417,21 @@ public void invalidateAll() { * partition-metadata cache. Deriving values from the name * (rather than looking the partition up) is exactly what stops an evicted partition-metadata entry from * leaving a stale file listing — the #65334 failure mode. Table-level column stats and the table object are - * intentionally left intact (legacy did not drop them on a partition-level refresh). Also forwarded to the - * already-built embedded siblings. + * intentionally left intact. Embedded siblings participate through the shared remote resource scopes. */ @Override public void invalidatePartition(String dbName, String tableName, List partitionNames) { - Set> partitionValues = new HashSet<>(); - for (String name : partitionNames) { - partitionValues.add(HiveWriteUtils.toPartitionValues(name)); + metaCache.invalidateScopes(partitionCacheScopes(dbName, tableName, partitionNames)); + } + + @Override + public List partitionCacheScopes(String database, String table, List partitions) { + List scopes = new ArrayList<>(); + scopes.add(ScopePath.partitionCollection(database, table)); + for (String name : partitions) { + scopes.add(ScopePath.partition(database, table, HiveWriteUtils.toPartitionValues(name))); } - metaCache.invalidatePartitions(dbName, tableName, partitionValues); - forEachBuiltSibling(sibling -> sibling.invalidatePartition(dbName, tableName, partitionNames)); + return scopes; } /** @@ -448,26 +451,6 @@ public ConnectorEventSource getEventSource() { return new HmsEventSource(getOrCreateClient(), props.getHmsEventsBatchSizePerRpc()); } - /** - * Runs one invalidation action on each ALREADY-BUILT embedded sibling (iceberg, hudi). fe-core routes - * {@code REFRESH TABLE/DATABASE/CATALOG} only to a catalog's PRIMARY connector, so the gateway must - * propagate invalidation to the sibling-owned caches — e.g. the iceberg sibling's latest-snapshot pin, - * whose ACCESS-based expiry keeps a continuously-queried stale entry alive indefinitely, making an explicit - * REFRESH the only way to unpin an iceberg-on-HMS table's staleness. Mirrors {@link #close()}: plain - * volatile reads, never force-builds (a never-built sibling has no cache to drop, and building one just to - * flush it could fail-loud spuriously on a catalog whose sibling plugin is absent). - */ - private void forEachBuiltSibling(Consumer action) { - Connector iceberg = icebergSibling; - if (iceberg != null) { - action.accept(iceberg); - } - Connector hudi = hudiSibling; - if (hudi != null) { - action.accept(hudi); - } - } - /** The connector-owned directory-listing cache, exposed for unit tests (mirrors iceberg manifestCacheForTest). */ HiveFileListingCache fileListingCacheForTest() { return fileListingCache; diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java index 05570d79db6e98..42a1d879d3a083 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorSiblingTest.java @@ -387,34 +387,40 @@ public Connector createSiblingConnector(String catalogType, Map } @Test - public void refreshHooksForwardToBothBuiltSiblings() { - // WHY: fe-core routes REFRESH TABLE/DATABASE/CATALOG only to a catalog's PRIMARY connector. If the - // gateway did not forward its invalidate hooks, the iceberg sibling's latest-snapshot pin could NEVER - // be dropped by an explicit REFRESH — and its access-based expiry keeps a continuously-queried stale - // entry alive indefinitely, so staleness would be unbounded (breaks "bounded by TTL + REFRESH"). - FakeSibling icebergSibling = new FakeSibling(); - FakeSibling hudiSibling = new FakeSibling(); - FakeConnectorContext context = new FakeConnectorContext() { - @Override - public Connector createSiblingConnector(String catalogType, Map properties) { - return "hudi".equals(catalogType) ? hudiSibling : icebergSibling; - } - }; - HiveConnector connector = new HiveConnector(HiveTestProperties.minimalMap(), context); - connector.getOrCreateIcebergSibling(); - connector.getOrCreateHudiSibling(); - - connector.invalidateTable("db1", "t1"); - connector.invalidateDb("db2"); - connector.invalidateAll(); - - for (FakeSibling sibling : new FakeSibling[] {icebergSibling, hudiSibling}) { - Assertions.assertEquals("db1.t1", sibling.lastInvalidatedTable, - "invalidateTable must forward the (db, table) pair to each built sibling"); - Assertions.assertEquals("db2", sibling.lastInvalidatedDb, - "invalidateDb must forward the db to each built sibling"); - Assertions.assertEquals(1, sibling.invalidateAllCount, - "invalidateAll must forward to each built sibling exactly once"); + public void refreshHooksInvalidateBuiltSiblingDomainsWithoutForwarding() { + try (org.apache.doris.connector.cache.CatalogMetaCache root = + org.apache.doris.connector.cache.CatalogMetaCache.unmanaged()) { + FakeSibling icebergSibling = new FakeSibling(); + FakeConnectorContext context = new FakeConnectorContext() { + @Override + public org.apache.doris.connector.cache.CatalogMetaCache createCacheDomain( + String engine, Map properties) { + return root.openDomain(engine); + } + + @Override + public Connector createSiblingConnector(String type, Map properties) { + return icebergSibling; + } + }; + HiveConnector connector = new HiveConnector(HiveTestProperties.minimalMap(), context); + connector.getOrCreateIcebergSibling(); + org.apache.doris.connector.cache.MetaCache siblingCache = root.openDomain("iceberg") + .create(org.apache.doris.connector.cache.MetaCacheDefinition.builder("table", + org.apache.doris.connector.cache.CacheSpec.of(true, -1, 10), + key -> org.apache.doris.connector.cache.ScopePath.table("db", key)).build()); + siblingCache.put("a", "old"); + siblingCache.put("b", "other"); + connector.invalidateTable("db", "a"); + Assertions.assertNull(siblingCache.getIfPresent("a")); + Assertions.assertEquals("other", siblingCache.getIfPresent("b")); + connector.invalidateDb("db"); + Assertions.assertNull(siblingCache.getIfPresent("b")); + siblingCache.put("a", "reloaded"); + connector.invalidateAll(); + Assertions.assertNull(siblingCache.getIfPresent("a")); + Assertions.assertEquals(0, icebergSibling.invalidateAllCount); + Assertions.assertNull(icebergSibling.lastInvalidatedTable); } } diff --git a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java index 9c031bea67e7f7..5b65934b93557a 100644 --- a/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java +++ b/fe/fe-connector/fe-connector-hudi/src/main/java/org/apache/doris/connector/hudi/HudiConnector.java @@ -191,7 +191,7 @@ public HudiConnector(Map properties, ConnectorContext context) { this.props = HudiCatalogProperties.of(properties); this.properties = props.getRaw(); this.context = context; - this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "hudi", this.properties); + this.metaCache = context.createCacheDomain("hudi", this.properties); } @Override diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java index afc79d395440d9..07fe4844b8d2cd 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java @@ -269,7 +269,7 @@ public IcebergConnector(Map properties, ConnectorContext context // authenticator never logs in — so without this the DDL/read hits secured HDFS as SIMPLE auth. this.context = new TcclPinningConnectorContext(context, getClass().getClassLoader(), this::pluginAuthenticator); - this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "iceberg", this.properties); + this.metaCache = context.createCacheDomain("iceberg", this.properties); this.manifestCache = new IcebergManifestCache(metaCache, this.properties); // The latest-snapshot, format and comment caches have no meta.cache.iceberg. keys of their own; they // take the table entry's enable / ttl-second / capacity (the legacy IcebergExternalMetaCache served the diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java index 5526a52587dcb8..2b4fd30f37f9d7 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTableCache.java @@ -215,7 +215,7 @@ void invalidateAll() { /** Seals new borrows and retires every cache owner, including loaders that publish after invalidation. */ void close() { if (closed.compareAndSet(false, true)) { - owner.invalidateCatalog(); + owner.remove(entry); } } diff --git a/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeDorisConnector.java b/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeDorisConnector.java index 4b1095ae5af769..56acd8062097f2 100644 --- a/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeDorisConnector.java +++ b/fe/fe-connector/fe-connector-maxcompute/src/main/java/org/apache/doris/connector/maxcompute/MaxComputeDorisConnector.java @@ -78,7 +78,7 @@ public MaxComputeDorisConnector(Map properties, ConnectorContext context) { this.props = MCCatalogProperties.of(properties); this.context = context; - this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "max_compute", props.getRaw()); + this.metaCache = context.createCacheDomain("max_compute", props.getRaw()); try { this.partitionCache = new MaxComputePartitionCache(metaCache, props.getRaw(), (db, t) -> structureHelper.getPartitions(odps, db, t)); diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java index a5154b81f86049..3b33f6b8042d76 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnector.java @@ -178,7 +178,7 @@ public PaimonConnector(Map properties, ConnectorContext context) // this a DDL/read against secured HDFS negotiates SIMPLE auth. See TcclPinningConnectorContext. this.context = new TcclPinningConnectorContext(context, getClass().getClassLoader(), this::pluginAuthenticator); - this.metaCache = CatalogMetaCache.managed(context.getCatalogId(), "paimon", properties); + this.metaCache = context.createCacheDomain("paimon", properties); this.schemaAtMemo = new PaimonSchemaAtMemo(metaCache, PaimonSchemaAtMemo.DEFAULT_MAX_SIZE); this.latestSnapshotCache = new PaimonLatestSnapshotCache( diff --git a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java index 245da688dd84f6..b7b365e81ebc4b 100644 --- a/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java @@ -285,8 +285,7 @@ public Optional getTableHandle( return Optional.of(handle); }); } catch (Exception e) { - LOG.warn("Failed to get Paimon table handle: {}.{}", dbName, tableName, e); - return Optional.empty(); + throw new DorisConnectorException("Failed to get Paimon table handle: " + dbName + "." + tableName, e); } } diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataTest.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataTest.java index 4a7c5316f9f903..564b4809509e05 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataTest.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorMetadataTest.java @@ -181,6 +181,15 @@ public void getTableHandleEmptyWhenTableMissing() { Assertions.assertFalse(handleOpt.isPresent()); } + @Test + public void getTableHandlePropagatesLookupFailure() { + RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); + ops.tableLookupFailure = new IllegalStateException("metastore timeout"); + DorisConnectorException failure = Assertions.assertThrows(DorisConnectorException.class, + () -> metadataWith(ops).getTableHandle(null, "db1", "t1")); + Assertions.assertSame(ops.tableLookupFailure, failure.getCause()); + } + @Test public void getColumnHandlesReloadFallbackReloadsWhenTransientTableNull() { RecordingPaimonCatalogOps ops = new RecordingPaimonCatalogOps(); diff --git a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingPaimonCatalogOps.java b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingPaimonCatalogOps.java index 73147f40620a0d..a6465faf6c62ca 100644 --- a/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingPaimonCatalogOps.java +++ b/fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/RecordingPaimonCatalogOps.java @@ -69,6 +69,7 @@ final class RecordingPaimonCatalogOps implements PaimonCatalogOps { boolean throwDatabaseNotExist; boolean throwTableNotExist; + RuntimeException tableLookupFailure; // ---- B3 DDL capture fields (inputs the metadata layer passed to the seam) ---- Schema lastCreatedSchema; @@ -163,6 +164,9 @@ public List listTables(String databaseName) throws Catalog.DatabaseNotEx @Override public Table getTable(Identifier identifier) throws Catalog.TableNotExistException { + if (tableLookupFailure != null) { + throw tableLookupFailure; + } log.add("getTable:" + identifier.getFullName()); lastGetTableId = identifier; if (throwTableNotExist) { diff --git a/fe/fe-connector/fe-connector-spi/pom.xml b/fe/fe-connector/fe-connector-spi/pom.xml index d64936b49fcd92..a0a60fa57d69a3 100644 --- a/fe/fe-connector/fe-connector-spi/pom.xml +++ b/fe/fe-connector/fe-connector-spi/pom.xml @@ -55,6 +55,11 @@ under the License. + + ${project.groupId} + fe-connector-cache + ${project.version} + ${project.groupId} fe-thrift diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/Connector.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/Connector.java index f7675ba6d0a24b..407a79d424c923 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/Connector.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/Connector.java @@ -17,6 +17,7 @@ package org.apache.doris.connector.spi; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.event.ConnectorEventSource; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; import org.apache.doris.connector.spi.procedure.ConnectorProcedureOps; @@ -53,6 +54,19 @@ */ public interface Connector extends Closeable { + /** + * Additional physical resource identities used by this gateway's already-resolved tables. Pure projection: + * no loading, connector construction or invalidation. The host invalidates all returned scopes together. + */ + default List cacheDependencies(ScopePath remoteScope) { + return Collections.emptyList(); + } + + /** Translate event partition names into the identities used by this connector's caches, without I/O. */ + default List partitionCacheScopes(String database, String table, List partitions) { + return Collections.singletonList(ScopePath.table(database, table)); + } + /** * Returns the metadata interface for the given session. The engine calls this exactly once per catalog per * statement through its own single entry point and closes the result when the statement ends, so an @@ -220,29 +234,30 @@ default void close() throws IOException { /** * Invalidates any connector-side per-table cache (e.g. a latest-snapshot/version cache) so a subsequent - * read reflects the latest external state. Called by the engine on {@code REFRESH TABLE} and before applying - * structural table events (create, drop, rename, and same-name view recreation). The names are the REMOTE - * db/table names (as seen by the connector). Default no-op for connectors that cache nothing. + * read reflects the latest external state. This is a direct connector/standalone entry point; engine refresh, + * DDL and events invalidate shared resource scopes instead, including {@link #cacheDependencies(ScopePath)}. + * The names are the REMOTE db/table names. Default no-op for connectors that cache nothing. */ default void invalidateTable(String dbName, String tableName) { } - /** Invalidates all connector-side per-table caches. Default no-op. */ + /** Direct connector invalidation entry point. Engine changes use the shared root. Default no-op. */ default void invalidateAll() { } /** - * Invalidates the connector-side caches for every table in one database. Called by the engine on - * {@code REFRESH DATABASE} and before applying structural database events (create, drop, and rename). - * The name is the REMOTE db name (as seen by the connector). Default no-op for connectors that cache nothing. + * Directly invalidates the connector-side caches for every table in one database. Engine refresh, DDL and + * events use the shared root's resource scopes. The name is the REMOTE db name. + * Default no-op for connectors that cache nothing. */ default void invalidateDb(String dbName) { } /** * Invalidates the connector-side caches for specific partitions of a table so a subsequent read - * reflects the latest external state. Driven by the engine's metastore-event sync when partitions - * are added/dropped/altered. The names are the REMOTE db/table names and canonical partition names + * reflects the latest external state. Engine events project partition scopes through + * {@link #partitionCacheScopes(String, String, List)} and invalidate them at the shared root instead. + * The names are the REMOTE db/table names and canonical partition names * ({@code "col=val/.../colN=valN"}); an empty/whole-table drop is expressed by * {@link #invalidateTable(String, String)}. A connector whose partition cache cannot target a single * name may degrade to invalidating the whole table's partition caches (correctness-safe when the diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorContext.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorContext.java index e0a6245733d1b9..c5ec2632ce89ac 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorContext.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorContext.java @@ -17,6 +17,8 @@ package org.apache.doris.connector.spi; +import org.apache.doris.connector.cache.CatalogMetaCache; + import java.util.Collections; import java.util.Map; import java.util.concurrent.Callable; @@ -32,6 +34,15 @@ */ public interface ConnectorContext { + /** + * Obtains this connector instance's cache domain. Engine contexts bind it to the catalog root before + * construction; standalone/test contexts own an independent root and close it with their connector. + * Decorators must forward this service. A sibling receives a new instance context and a child domain. + */ + default CatalogMetaCache createCacheDomain(String engine, Map properties) { + return CatalogMetaCache.managed(getCatalogId(), engine, properties); + } + /** Returns the catalog name. */ String getCatalogName(); diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorTableMetadataOps.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorTableMetadataOps.java index 4d29727e1b84d9..64263b639f59a4 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorTableMetadataOps.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorTableMetadataOps.java @@ -62,7 +62,10 @@ */ public interface ConnectorTableMetadataOps { - /** Retrieves a table handle for the given database and table name. */ + /** + * Retrieves a table handle for the given database and table name. Empty means the table does not exist; + * authentication, transport and other lookup failures must propagate rather than report absence. + */ @ConnectorMustImplement default Optional getTableHandle( ConnectorSession session, String dbName, diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ForwardingConnectorContext.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ForwardingConnectorContext.java index d65d9f4c4313b3..06fbe3bd0e844f 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ForwardingConnectorContext.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ForwardingConnectorContext.java @@ -17,6 +17,8 @@ package org.apache.doris.connector.spi; +import org.apache.doris.connector.cache.CatalogMetaCache; + import java.util.Map; import java.util.Objects; import java.util.concurrent.Callable; @@ -65,6 +67,11 @@ protected final ConnectorContext delegate() { return delegate; } + @Override + public CatalogMetaCache createCacheDomain(String engine, Map properties) { + return delegate.createCacheDomain(engine, properties); + } + @Override public String getCatalogName() { return delegate.getCatalogName(); diff --git a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java index cce4173053f2f7..fc3d710b433d2c 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java +++ b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java @@ -90,7 +90,7 @@ public void connectorApiMajorTracksTheRecordedSurfaceChange() throws IOException } // Major 19 adds ConnectorColumn#isSourceNullable()/withSourceNullable(boolean) - the nullability the // source declares, kept separate from the nullability the engine reads the column with. - Assertions.assertEquals("19.0", version.getProperty("api.version")); + Assertions.assertEquals("20.0", version.getProperty("api.version")); } /** Root entry points plus provider/handle types returned to connector plugins. */ diff --git a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt index 81918e6daa1431..c925bf1fd03533 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt +++ b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt @@ -1,3 +1,4 @@ +org.apache.doris.connector.spi.Connector#cacheDependencies(org.apache.doris.connector.cache.ScopePath):java.util.List org.apache.doris.connector.spi.Connector#close():void org.apache.doris.connector.spi.Connector#defaultTestConnection():boolean org.apache.doris.connector.spi.Connector#deriveStorageProperties(java.util.Map):java.util.Map @@ -16,6 +17,7 @@ org.apache.doris.connector.spi.Connector#invalidateDb(java.lang.String):void org.apache.doris.connector.spi.Connector#invalidatePartition(java.lang.String,java.lang.String,java.util.List):void org.apache.doris.connector.spi.Connector#invalidateTable(java.lang.String,java.lang.String):void org.apache.doris.connector.spi.Connector#ownsHandle(org.apache.doris.connector.spi.handle.ConnectorTableHandle):boolean +org.apache.doris.connector.spi.Connector#partitionCacheScopes(java.lang.String,java.lang.String,java.util.List):java.util.List org.apache.doris.connector.spi.Connector#preCreateValidation(org.apache.doris.connector.spi.ConnectorValidationContext):void org.apache.doris.connector.spi.Connector#schemaCacheTtlSecondOverride():java.util.OptionalLong org.apache.doris.connector.spi.Connector#testConnection(org.apache.doris.connector.spi.ConnectorSession):org.apache.doris.connector.spi.ConnectorTestResult @@ -63,6 +65,7 @@ org.apache.doris.connector.spi.ConnectorColumn#withSpecified(boolean,boolean):or org.apache.doris.connector.spi.ConnectorColumn#withStringWriteType(org.apache.doris.connector.spi.ConnectorType):org.apache.doris.connector.spi.ConnectorColumn org.apache.doris.connector.spi.ConnectorColumn#withTimeZone():org.apache.doris.connector.spi.ConnectorColumn org.apache.doris.connector.spi.ConnectorColumn#withUniqueId(int):org.apache.doris.connector.spi.ConnectorColumn +org.apache.doris.connector.spi.ConnectorContext#createCacheDomain(java.lang.String,java.util.Map):org.apache.doris.connector.cache.CatalogMetaCache org.apache.doris.connector.spi.ConnectorContext#createSiblingConnector(java.lang.String,java.util.Map):org.apache.doris.connector.spi.Connector org.apache.doris.connector.spi.ConnectorContext#executeAuthenticated(java.util.concurrent.Callable):java.lang.Object org.apache.doris.connector.spi.ConnectorContext#getCatalogId():long diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml index 50adfde9967a05..d1d8f2fb4c445f 100644 --- a/fe/fe-connector/pom.xml +++ b/fe/fe-connector/pom.xml @@ -55,7 +55,7 @@ under the License. of the latter two means bumping this property as well (and fe-extension-spi means bumping all five families). --> - 19.0 + 20.0 diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 1b8c36a751e93c..7ff06e47f3a751 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -107,6 +107,7 @@ import org.apache.doris.datasource.ExternalMetaIdMgr; import org.apache.doris.datasource.InternalCatalog; import org.apache.doris.datasource.MetastoreEventSyncDriver; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; import org.apache.doris.datasource.plugin.PluginDrivenSysExternalTable; import org.apache.doris.datasource.split.SplitSourceManager; @@ -3737,7 +3738,12 @@ public void replayCreateDb(CreateDbInfo dbInfo) { } else { ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(dbInfo.getCtlName()); if (externalCatalog != null) { - externalCatalog.replayCreateDb(dbInfo.getDbName()); + if (dbInfo.getExternalCacheChange() != null) { + CatalogMetadataChanges.apply( + externalCatalog, dbInfo.getExternalCacheChange()); + } else { + externalCatalog.replayCreateDb(dbInfo.getDbName()); + } } } } @@ -3759,7 +3765,12 @@ public void replayDropDb(DropDbInfo info) throws DdlException { } else { ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(info.getCtlName()); if (externalCatalog != null) { - externalCatalog.replayDropDb(info.getDbName()); + if (info.getExternalCacheChange() != null) { + CatalogMetadataChanges.apply( + externalCatalog, info.getExternalCacheChange()); + } else { + externalCatalog.replayDropDb(info.getDbName()); + } } } } @@ -5102,7 +5113,12 @@ public void replayCreateTable(org.apache.doris.persist.CreateTableInfo info) thr } else { ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(info.getCtlName()); if (externalCatalog != null) { - externalCatalog.replayCreateTable(info.getDbName(), info.getTblName()); + if (info.getExternalCacheChange() != null) { + CatalogMetadataChanges.apply( + externalCatalog, info.getExternalCacheChange()); + } else { + externalCatalog.replayCreateTable(info.getDbName(), info.getTblName()); + } } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index 8e5586b344f79c..4208b7c9d72d72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -21,15 +21,13 @@ import org.apache.doris.common.DdlException; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.UserException; -import org.apache.doris.common.util.Util; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; -import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.log.CatalogLog; import org.apache.doris.datasource.log.ExternalObjectLog; -import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.persist.OperationType; import com.google.common.base.Strings; @@ -39,7 +37,6 @@ import java.util.List; import java.util.Map; -import java.util.Optional; import java.util.concurrent.ScheduledThreadPoolExecutor; import java.util.concurrent.TimeUnit; @@ -92,7 +89,8 @@ public void handleRefreshDb(String catalogName, String dbName) throws DdlExcepti DatabaseIf db = catalog.getDbOrDdlException(dbName); refreshDbInternal((ExternalDatabase) db); - ExternalObjectLog log = ExternalObjectLog.createForRefreshDb(catalog.getId(), db.getFullName()); + ExternalObjectLog log = ExternalObjectLog.createForRefreshDb(catalog.getId(), db.getFullName()) + .withRemoteNames(((ExternalDatabase) db).getRemoteName(), null); Env.getCurrentEnv().getEditLog().logRefreshExternalDb(log); } @@ -107,38 +105,13 @@ public void replayRefreshDb(ExternalObjectLog log) { LOG.warn("refresh database replay log has no local name: {}", log.debugForRefreshDb()); return; } - long dbId = Util.genIdByName(catalog.getName(), localDbName); - Optional> db = catalog.getDbForReplay(localDbName); - - if (!db.isPresent()) { - ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); - try { - invalidateAllConnectorCachesIfPresent(catalog); - } finally { - cacheMgr.invalidateDb(log.getCatalogId(), dbId, localDbName); - } - LOG.info("database object cache is cold when replaying refresh database; invalidated caches by " - + "local name {}: {}", - localDbName, log.debugForRefreshDb()); - return; - } - refreshDbInternal(db.get()); + CatalogMetadataChanges.apply(catalog, log); } private void refreshDbInternal(ExternalDatabase db) { - // Connector metadata is the row-count source, so invalidate it before engine and row-count caches. - try { - if (db.getCatalog() instanceof PluginDrivenExternalCatalog) { - ((PluginDrivenExternalCatalog) db.getCatalog()).getConnector().invalidateDb(db.getRemoteName()); - } - } finally { - try { - db.resetMetaToUninitialized(); - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr() - .invalidateDbRowCountCache(db.getCatalog().getId(), db.getId()); - } - } + CatalogMetadataChanges.apply((ExternalCatalog) db.getCatalog(), + ExternalObjectLog.createForRefreshDb(db.getCatalog().getId(), db.getFullName()) + .withRemoteNames(db.getRemoteName(), null)); LOG.info("refresh database {} in catalog {}", db.getFullName(), db.getCatalog().getName()); } @@ -170,9 +143,8 @@ public void handleRefreshTable(String catalogName, String dbName, String tableNa return; } long updateTime = System.currentTimeMillis(); - refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, updateTime); - ExternalObjectLog log = ExternalObjectLog.createForRefreshTable( - catalog.getId(), db.getFullName(), table.getName(), updateTime); + refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, updateTime, true); + ExternalObjectLog log = CatalogMetadataChanges.table((ExternalTable) table, true, updateTime); env.getEditLog().logRefreshExternalTable(log); } @@ -180,8 +152,7 @@ public void handleRefreshTable(String catalogName, String dbName, String tableNa public void refreshTableAfterExternalMutation(ExternalTable table) { ExternalDatabase db = table.getDb(); long updateTime = System.currentTimeMillis(); - ExternalObjectLog log = ExternalObjectLog.createForRefreshTable( - table.getCatalog().getId(), db.getFullName(), table.getName(), updateTime); + ExternalObjectLog log = CatalogMetadataChanges.table(table, false, updateTime); Env.getCurrentEnv().getEditLog().logRefreshExternalTable(log); refreshTableInternal(db, table, updateTime); } @@ -198,84 +169,11 @@ public void replayRefreshTable(ExternalObjectLog log) { LOG.warn("refresh table replay log has no local name: {}", log.debugForRefreshTable()); return; } - long dbId = Util.genIdByName(catalog.getName(), localDbName); - long tableId = Util.genIdByName(catalog.getName(), localDbName, localTableName); - Optional> db = catalog.getDbForReplay(localDbName); - if (!Strings.isNullOrEmpty(log.getNewTableName())) { - replayRenameTable(log, catalog, db); - return; - } - if (!db.isPresent()) { - try { - invalidateAllConnectorCachesIfPresent(catalog); - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTable( - log.getCatalogId(), dbId, localDbName, tableId, localTableName); - } - LOG.info("database object cache is cold when replaying refresh table; " - + "invalidated caches by local names {}.{}: {}", - localDbName, localTableName, log.debugForRefreshTable()); - return; - } - Optional table = db.get().getTableForReplay(localTableName); - if (!table.isPresent()) { - try { - if (catalog instanceof PluginDrivenExternalCatalog) { - // A cold table object cannot provide the mapped remote table name. - ((PluginDrivenExternalCatalog) catalog).getConnector().invalidateDb(db.get().getRemoteName()); - } - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTable( - log.getCatalogId(), dbId, db.get().getFullName(), tableId, localTableName); - } - LOG.info("table object cache is cold when replaying refresh table; invalidated caches by local name {}: {}", - localTableName, log.debugForRefreshTable()); - return; - } - refreshTableInternal(db.get(), table.get(), log.getLastUpdateTime()); - } - - private void replayRenameTable(ExternalObjectLog log, ExternalCatalog catalog, - Optional> db) { - ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); - String localDbName = log.getDbName(); - String localTableName = log.getTableName(); - long dbId = Util.genIdByName(catalog.getName(), localDbName); - long sourceTableId = Util.genIdByName(catalog.getName(), localDbName, localTableName); - long destinationTableId = Util.genIdByName(catalog.getName(), localDbName, log.getNewTableName()); - try { - if (!db.isPresent()) { - invalidateAllConnectorCachesIfPresent(catalog); - LOG.info("database object cache is cold when replaying rename table; invalidated connector caches for " - + "{}: {}", - localDbName, log.debugForRefreshTable()); - } else { - ExternalDatabase database = db.get(); - try { - if (catalog instanceof PluginDrivenExternalCatalog) { - // Persisted names are local identities, so database scope is the replay-safe connector key. - ((PluginDrivenExternalCatalog) catalog).getConnector() - .invalidateDb(database.getRemoteName()); - } - } finally { - database.invalidateTableRename(localTableName, log.getNewTableName()); - } - } - } finally { - try { - Env.getCurrentEnv().getConstraintManager().renameTable( - new TableNameInfo(catalog.getName(), localDbName, localTableName), - new TableNameInfo(catalog.getName(), localDbName, log.getNewTableName())); - } finally { - cacheMgr.invalidateTableRename(log.getCatalogId(), dbId, localDbName, - sourceTableId, localTableName, destinationTableId, log.getNewTableName()); - } - } - } - - private void invalidateAllConnectorCachesIfPresent(ExternalCatalog catalog) { - if (catalog instanceof PluginDrivenExternalCatalog) { - ((PluginDrivenExternalCatalog) catalog).invalidateAllConnectorCachesIfPresent(); + CatalogMetadataChanges.apply(catalog, log); + if (log.getNewTableName() != null) { + Env.getCurrentEnv().getConstraintManager().renameTable( + new TableNameInfo(catalog.getName(), localDbName, localTableName), + new TableNameInfo(catalog.getName(), localDbName, log.getNewTableName())); } } @@ -301,19 +199,17 @@ public void refreshExternalTableFromEvent(String catalogName, String localDbName } public void refreshTableInternal(ExternalDatabase db, ExternalTable table, long updateTime) { + refreshTableInternal(db, table, updateTime, false); + } + + private void refreshTableInternal(ExternalDatabase db, ExternalTable table, long updateTime, + boolean invalidateTableLookup) { table.unsetObjectCreated(); if (updateTime > 0) { table.setUpdateTime(updateTime); } - // Connector metadata is the row-count source, so invalidate it before engine and row-count caches. - try { - if (table.getCatalog() instanceof PluginDrivenExternalCatalog) { - ((PluginDrivenExternalCatalog) table.getCatalog()).getConnector() - .invalidateTable(db.getRemoteName(), table.getRemoteName()); - } - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTable(table); - } + CatalogMetadataChanges.apply((ExternalCatalog) table.getCatalog(), + CatalogMetadataChanges.table(table, invalidateTableLookup, updateTime)); LOG.info("refresh table {}, id {} from db {} in catalog {}, update time: {}", table.getName(), table.getId(), db.getFullName(), db.getCatalog().getName(), updateTime); } @@ -339,15 +235,10 @@ public void refreshPartitionsFromEvent(String catalogName, String localDbName, S } ExternalTable externalTable = (ExternalTable) table; - try { - if (externalTable.getCatalog() instanceof PluginDrivenExternalCatalog) { - ((PluginDrivenExternalCatalog) externalTable.getCatalog()).getConnector().invalidatePartition( - ((ExternalDatabase) db).getRemoteName(), externalTable.getRemoteName(), partitionNames); - } - externalTable.setUpdateTime(updateTime); - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableRowCountCache(externalTable); - } + CatalogMetadataChanges.apply((ExternalCatalog) externalTable.getCatalog(), + ExternalObjectLog.createForRefreshPartitions(catalog.getId(), localDbName, localTableName, + partitionNames, null, updateTime).withRemoteNames( + ((ExternalDatabase) db).getRemoteName(), externalTable.getRemoteName())); } public void addToRefreshMap(long catalogId, Integer[] sec) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorInstanceContext.java b/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorInstanceContext.java new file mode 100644 index 00000000000000..f249ae24d6bad3 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorInstanceContext.java @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector; + +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.ScopePath; +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ForwardingConnectorContext; + +import java.util.Map; + +/** One construction/instance boundary, shared by its decorators but never by another connector instance. */ +final class ConnectorInstanceContext extends ForwardingConnectorContext { + private final String provider; + private final Map properties; + private final ClassLoader loader; + private CatalogMetaCache domain; + + ConnectorInstanceContext(ConnectorContext delegate, String provider, Map properties, + ClassLoader loader) { + super(delegate); + this.provider = provider; + this.properties = properties; + this.loader = loader; + } + + @Override + public synchronized CatalogMetaCache createCacheDomain(String engine, Map properties) { + if (domain == null) { + domain = delegate() instanceof ConnectorInstanceContext + ? ((ConnectorInstanceContext) delegate()).childDomain(engine, loader) + : delegate().createCacheDomain(engine, properties); + } + if (domain.isClosed()) { + throw new IllegalStateException("Connector cache domain has been retired"); + } + return domain; + } + + private synchronized CatalogMetaCache childDomain(String engine, ClassLoader loader) { + return createCacheDomain(provider, properties).openDomain(engine, ScopePath.Coordinates.REMOTE, loader); + } + + @Override + public Connector createSiblingConnector(String type, Map properties) { + synchronized (this) { + createCacheDomain(provider, this.properties); + } + return ConnectorFactory.createConnector(type, properties, this); + } + + void rollback() { + CatalogMetaCache captured; + synchronized (this) { + captured = domain; + } + if (captured != null) { + captured.close(); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorPluginManager.java b/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorPluginManager.java index 4eddb0563460d3..dd704162486406 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorPluginManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/connector/ConnectorPluginManager.java @@ -414,7 +414,23 @@ private Connector createConnector(String catalogType, Map proper } LOG.info("Creating connector via provider '{}' for catalogType='{}'", provider.getType(), catalogType); - return provider.create(properties, withConnectorConfig(context, provider)); + ConnectorInstanceContext instance = new ConnectorInstanceContext(context, provider.getType(), + properties, provider.getClass().getClassLoader()); + Thread thread = Thread.currentThread(); + ClassLoader previous = thread.getContextClassLoader(); + try { + thread.setContextClassLoader(provider.getClass().getClassLoader()); + return provider.create(properties, withConnectorConfig(instance, provider)); + } catch (RuntimeException | Error failure) { + try { + instance.rollback(); + } catch (RuntimeException | Error cleanup) { + failure.addSuppressed(cleanup); + } + throw failure; + } finally { + thread.setContextClassLoader(previous); + } } } LOG.debug("No ConnectorProvider supports catalogType='{}' (standaloneOnly={}). Registered: {}", diff --git a/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java b/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java index f519f6de28b368..1e993ac022a5d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/connector/DefaultConnectorContext.java @@ -25,6 +25,8 @@ import org.apache.doris.common.EnvUtils; import org.apache.doris.common.Version; import org.apache.doris.common.util.LocationPath; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorBrokerAddress; import org.apache.doris.connector.spi.ConnectorContext; @@ -110,6 +112,8 @@ public class DefaultConnectorContext implements ConnectorContext, ConnectorStora private final Object fsLock = new Object(); private volatile FileSystem catalogFileSystem; private volatile boolean closed; + private Supplier cacheRootSupplier; + private CatalogMetaCache standaloneCacheRoot; private final ConnectorHttpSecurityHook httpSecurityHook = new ConnectorHttpSecurityHook() { @Override @@ -166,6 +170,36 @@ public DefaultConnectorContext(String catalogName, long catalogId, this.environment = buildEnvironment(); } + /** Runtime catalogs inject their root; validation contexts own their standalone connector root. */ + public synchronized void bindCacheRoot(Supplier supplier) { + this.cacheRootSupplier = Objects.requireNonNull(supplier, "cacheRootSupplier"); + } + + @Override + public CatalogMetaCache createCacheDomain(String engine, Map properties) { + Supplier supplier; + synchronized (this) { + if (closed) { + throw new IllegalStateException("Connector context has been closed"); + } + supplier = cacheRootSupplier; + if (supplier == null) { + if (standaloneCacheRoot == null) { + standaloneCacheRoot = CatalogMetaCache.managed(catalogId, engine, properties); + } + return standaloneCacheRoot; + } + } + // Catalog lookup may take the catalog monitor; never hold this context's retirement monitor here. + CatalogMetaCache domain = supplier.get().openDomain(engine, ScopePath.Coordinates.REMOTE, + Thread.currentThread().getContextClassLoader()); + if (closed) { + domain.close(); + throw new IllegalStateException("Connector context was retired during cache domain construction"); + } + return domain; + } + @Override public String getCatalogName() { return catalogName; @@ -395,17 +429,27 @@ FileSystem buildCatalogFileSystem(Map storageProp */ @Override public void close() throws IOException { - FileSystem fs; - synchronized (fsLock) { + CatalogMetaCache standalone; + synchronized (this) { if (closed) { return; } closed = true; + standalone = standaloneCacheRoot; + } + FileSystem fs; + synchronized (fsLock) { fs = catalogFileSystem; catalogFileSystem = null; } - if (fs != null) { - fs.close(); + try { + if (fs != null) { + fs.close(); + } + } finally { + if (standalone != null) { + standalone.close(); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 00300420aa1514..8e6651875f1336 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -38,7 +38,8 @@ import org.apache.doris.common.util.TimeUtils; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.log.CatalogLog; -import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; +import org.apache.doris.datasource.log.ExternalObjectLog; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.plans.commands.CreateCatalogCommand; import org.apache.doris.persist.OperationType; @@ -111,12 +112,14 @@ private void initInternalCatalog() { } private void addCatalog(CatalogIf catalog) { - nameToCatalog.put(catalog.getName(), catalog); - idToCatalog.put(catalog.getId(), catalog); String catalogName = catalog.getName(); if (!catalogName.equals(InternalCatalog.INTERNAL_CATALOG_NAME)) { - ((ExternalCatalog) catalog).resetToUninitialized(false); + ExternalCatalog external = (ExternalCatalog) catalog; + external.resetToUninitialized(false); + external.onRegistered(); } + nameToCatalog.put(catalogName, catalog); + idToCatalog.put(catalog.getId(), catalog); } private RemovedCatalog removeCatalog(long catalogId) { @@ -892,16 +895,10 @@ public void addExternalPartitionsFromEvent(String catalogName, if (table == null) { return; } - try { - if (catalog instanceof PluginDrivenExternalCatalog) { - ((PluginDrivenExternalCatalog) catalog).getConnector().invalidatePartition( - ((ExternalDatabase) db).getRemoteName(), ((ExternalTable) table).getRemoteName(), - partitionNames); - ((ExternalTable) table).setUpdateTime(updateTime); - } - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableRowCountCache((ExternalTable) table); - } + CatalogMetadataChanges.apply((ExternalCatalog) catalog, + ExternalObjectLog.createForRefreshPartitions(catalog.getId(), localDbName, localTableName, + partitionNames, null, updateTime).withRemoteNames( + ((ExternalDatabase) db).getRemoteName(), ((ExternalTable) table).getRemoteName())); } public void dropExternalPartitionsFromEvent(String catalogName, @@ -924,16 +921,10 @@ public void dropExternalPartitionsFromEvent(String catalogName, return; } - try { - if (catalog instanceof PluginDrivenExternalCatalog) { - ((PluginDrivenExternalCatalog) catalog).getConnector().invalidatePartition( - ((ExternalDatabase) db).getRemoteName(), ((ExternalTable) table).getRemoteName(), - partitionNames); - ((ExternalTable) table).setUpdateTime(updateTime); - } - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableRowCountCache((ExternalTable) table); - } + CatalogMetadataChanges.apply((ExternalCatalog) catalog, + ExternalObjectLog.createForRefreshPartitions(catalog.getId(), localDbName, localTableName, + partitionNames, null, updateTime).withRemoteNames( + ((ExternalDatabase) db).getRemoteName(), ((ExternalTable) table).getRemoteName())); } public void registerCatalogRefreshListener(Env env) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 3934483b97bf48..3d5bcf530eef31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -40,7 +40,10 @@ import org.apache.doris.common.util.ExceptionDiagnosticUtils; import org.apache.doris.common.util.Util; import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCacheBudgetManager; +import org.apache.doris.connector.cache.ScopePath; +import org.apache.doris.datasource.doris.RemoteDorisExternalCatalog; import org.apache.doris.datasource.doris.RemoteDorisExternalDatabase; import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase; @@ -82,6 +85,7 @@ import java.util.Set; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.stream.Collectors; @@ -169,6 +173,12 @@ public abstract class ExternalCatalog private boolean objectCreated = false; protected TransactionManager transactionManager; + private transient volatile CatalogMetaCache feMetaCacheOwner; + private transient CatalogMetaCache namingCacheOwner; + private transient boolean feCacheRetired; + private transient AtomicLong cacheEntrySequence = + new AtomicLong(); + protected FeMetaCacheEntry databaseNames; protected FeMetaCacheEntry> databases; protected transient IdNameIndex dbIdNameIndex = new IdNameIndex("external database"); @@ -343,7 +353,8 @@ public final synchronized void makeSureInitialized() { } try { initLocalObjects(); - if (!initialized) { + if (!initialized || feMetaCacheOwner == null || feMetaCacheOwner.isClosed() + || namingCacheOwner != feMetaCacheOwner) { buildMetaCache(); setLastUpdateTime(System.currentTimeMillis()); initialized = true; @@ -399,38 +410,113 @@ public boolean isInitialized() { return this.initialized; } - private void buildMetaCache() { - if (databaseNames != null && databases != null) { + /** Catalog root shared by FE entries and connector instance domains. */ + public synchronized CatalogMetaCache getFeMetaCacheOwner() { + Preconditions.checkState(!feCacheRetired, "External catalog cache owner has been retired: %s", name); + if (feMetaCacheOwner == null || feMetaCacheOwner.isClosed()) { + boolean replacingClosedOwner = feMetaCacheOwner != null; + Map properties = Maps.newHashMap(getProperties()); + overlayMetaCacheConfig(properties); + String engine = this instanceof RemoteDorisExternalCatalog + ? "doris" : "default"; + boolean caseInsensitiveTables = getLowerCaseTableNames() == 2; + feMetaCacheOwner = CatalogMetaCache.managed(id, engine, properties, ExternalCatalog.class.getClassLoader(), + scope -> caseInsensitiveTables && scope.coordinates() == ScopePath.Coordinates.LOCAL + ? scope.mapTableName(table -> table.toLowerCase(Locale.ROOT)) : scope); + dbIdNameIndex.clear(); + if (replacingClosedOwner) { + initialized = false; + } + } + return feMetaCacheOwner; + } + + public CatalogMetaCache getMetaCacheOwnerIfPresent() { + CatalogMetaCache owner = feMetaCacheOwner; + return owner == null || owner.isClosed() ? null : owner; + } + + /** Cache retirement must also work before the first schema entry has been prepared. */ + public void closeFeMetaCacheOwner() { + CatalogMetaCache owner; + FeMetaCacheEntry names; + FeMetaCacheEntry> objects; + synchronized (this) { + owner = feMetaCacheOwner; + names = databaseNames; + objects = databases; + dbIdNameIndex.clear(); + initialized = false; + } + if (names != null) { + names.close(); + } + if (objects != null) { + objects.close(); + } + if (owner != null) { + owner.close(); + } + } + + /** A full database refresh also fences the catalog's database-name snapshot. */ + public void invalidateDatabaseLookup(String localDbName) { + if (databases != null) { + databases.invalidateKeyAndRun(localDbName, () -> dbIdNameIndex.removeName(localDbName)); + } else { + dbIdNameIndex.removeName(localDbName); + } + // Replay must not resurrect a retired owner or initialize a remote catalog after ALTER. + CatalogMetaCache owner = feMetaCacheOwner; + if (owner != null && !owner.isClosed()) { + owner.invalidateScopes(java.util.Arrays.asList( + ScopePath.database(localDbName).in(ScopePath.Coordinates.LOCAL), + ScopePath.databaseCollection().in(ScopePath.Coordinates.LOCAL))); + } + } + + /** Instance-qualified registration permits old database cleanup without touching its replacement. */ + public String nextFeCacheEntryName(String prefix) { + return prefix + "." + cacheEntrySequence.getAndIncrement(); + } + + private synchronized void buildMetaCache() { + CatalogMetaCache owner = getFeMetaCacheOwner(); + if (namingCacheOwner == owner && databaseNames != null && databases != null) { return; } if (LOG.isDebugEnabled()) { LOG.debug("buildMetaCache for catalog: {}:{}", this.name, this.id, new Exception()); } + dbIdNameIndex.bind(owner, "database_ids", ScopePath::database); CacheSpec namesSpec = CacheSpec.of( true, Config.external_cache_expire_time_seconds_after_access, 1); // Build one immutable names snapshot so list and lower-case index share the same cache version. - databaseNames = new FeMetaCacheEntry<>( - name + ".database_names", + databaseNames = FeMetaCacheEntry.shared(owner, + ignored -> ScopePath.databaseCollection(), ScopePath.databaseCollection(), + "database_names", ignored -> NameCacheValue.of(getFilteredDatabaseNames()), namesSpec, Env.getCurrentEnv().getExtMetaCacheMgr().commonRefreshExecutor(), true, - FeMetaCacheEntry.singleKeyStripeCount()); + FeMetaCacheEntry.singleKeyStripeCount(), null); CacheSpec objectSpec = CacheSpec.of( true, Config.external_cache_expire_time_seconds_after_access, Math.max(Config.max_meta_object_cache_num, 1)); // Object entries keep the sync removal listener semantics and therefore do not enable auto refresh. - databases = FeMetaCacheEntry.withSyncRemovalListener( - name + ".databases", + databases = FeMetaCacheEntry.sharedLookup(owner, + ScopePath::database, ScopePath.catalog(), + "databases", localDbName -> buildDbForInit(null, localDbName, Util.genIdByName(name, localDbName), logType, true), objectSpec, - Env.getCurrentEnv().getExtMetaCacheMgr().commonRefreshExecutor(), + Env.getCurrentEnv().getExtMetaCacheMgr().commonRefreshExecutor(), false, FeMetaCacheEntry.defaultObjectStripeCount(), - (key, value, cause) -> value.resetMetaToUninitialized()); + (key, value, cause) -> value.releaseCachedEntries()); + namingCacheOwner = owner; } /** @@ -835,7 +921,11 @@ private ExternalDatabase getDbNullableWithoutCache(Stri String localDbName = dbNamePair.get().value(); // checkExists=false: the pair came from the live listing, so re-listing (getDbNames) is both // unnecessary and would recurse back through this bypass — build the db object directly. - return buildDbForInit(remoteDbName, localDbName, Util.genIdByName(name, localDbName), logType, false); + ExternalDatabase database = + buildDbForInit(remoteDbName, localDbName, Util.genIdByName(name, localDbName), logType, false); + // A per-session object has no catalog-cache lifecycle to retire its child registrations. + database.releaseCachedEntries(); + return database; } /** Matches a live listing's LOCAL db name against a requested name, honoring the db-name case modes. */ @@ -912,7 +1002,9 @@ public void tryModifyCatalogProps(Map props) { private void invalidateCachesAfterPropertyUpdate(Map updatedProps) { ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); if (updatedProps.get(SCHEMA_CACHE_TTL_SECOND) != null - || updatedProps.containsKey(MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY)) { + || updatedProps.containsKey(MetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY) + || updatedProps.containsKey(LOWER_CASE_TABLE_NAMES) + || updatedProps.containsKey(ONLY_TEST_LOWER_CASE_TABLE_NAMES)) { cacheMgr.removeCatalog(id); return; } @@ -957,11 +1049,23 @@ public void setLastUpdateTime(long lastUpdateTime) { this.lastUpdateTime = lastUpdateTime; } + /** Only registry publication may reactivate an instance retired by rename cleanup. */ + synchronized void onRegistered() { + feCacheRetired = false; + } + @Override public final void onClose() { + synchronized (this) { + feCacheRetired = true; + } Runnable accessControllerCleanup = detachAccessController(); try { - closeResourcesQuietly("registered catalog"); + try { + closeResourcesQuietly("registered catalog"); + } finally { + closeFeMetaCacheOwner(); + } } finally { accessControllerCleanup.run(); } @@ -1188,6 +1292,7 @@ public void gsonPostProcess() throws IOException { tableAutoAnalyzePolicy = Maps.newHashMap(); } this.dbIdNameIndex = new IdNameIndex("external database"); + this.cacheEntrySequence = new AtomicLong(); } public void addDatabaseForTest(ExternalDatabase db) { @@ -1501,7 +1606,7 @@ private List listLocalDatabaseNamesFromCache() { return namesValue.localNames(); } - private String resolveDatabaseNameForInvalidation(String dbName) { + public String resolveDatabaseNameForInvalidation(String dbName) { int mode = getLowerCaseDatabaseNames(); if (mode == 1) { return dbName.toLowerCase(Locale.ROOT); @@ -1593,6 +1698,10 @@ private void invalidateDatabaseNamesOnly() { } private void invalidateAllDatabaseCache() { + if (feMetaCacheOwner != null && feMetaCacheOwner.isClosed()) { + dbIdNameIndex.clear(); + return; + } if (databaseNames != null) { databaseNames.invalidateAll(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 3d24d0c3b78ce9..0952859c318487 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -32,6 +32,9 @@ import org.apache.doris.common.util.DebugPointUtil; import org.apache.doris.common.util.Util; import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.datasource.infoschema.ExternalInfoSchemaDatabase; import org.apache.doris.datasource.infoschema.ExternalMysqlDatabase; import org.apache.doris.datasource.metacache.FeMetaCacheEntry; @@ -93,6 +96,7 @@ public abstract class ExternalDatabase private transient IdNameIndex tableIdNameIndex = new IdNameIndex("external table"); private volatile boolean isInitializing = false; + private transient volatile boolean cacheEntriesReleased; /** * Create external database. @@ -124,12 +128,47 @@ public void resetMetaToUninitialized() { } synchronized (this) { this.initialized = false; - invalidateAllTableCache(); + if (!cacheEntriesReleased) { + invalidateAllTableCache(); + } } Env.getCurrentEnv().getExtMetaCacheMgr() .invalidateDbMetadataCache(extCatalog.getId(), getFullName()); } + /** Eviction retires only this database instance's entries, never the current shared database scope. */ + public synchronized void releaseCachedEntries() { + cacheEntriesReleased = true; + initialized = false; + if (tableNames != null) { + tableNames.close(); + } + if (tables != null) { + tables.close(); + } + // The table adapter pins its auxiliary identity index until accepted table operations finish. + if (tables == null) { + CatalogMetaCache owner = extCatalog.getMetaCacheOwnerIfPresent(); + if (owner != null) { + tableIdNameIndex.close(owner); + } else { + tableIdNameIndex.clear(); + } + } + } + + /** Full refresh also invalidates the parent membership snapshot at the shared publication barrier. */ + public void invalidateTableLookup(String localTableName) { + if (tables != null) { + tables.invalidateKeyAndRun(localTableName, () -> tableIdNameIndex.removeName(localTableName)); + } else { + tableIdNameIndex.removeName(localTableName); + } + extCatalog.getFeMetaCacheOwner().invalidateScopes(java.util.Arrays.asList( + ScopePath.table(name, localTableName).in(ScopePath.Coordinates.LOCAL), + ScopePath.tableCollection(name).in(ScopePath.Coordinates.LOCAL))); + } + public boolean isInitialized() { return initialized; } @@ -137,14 +176,20 @@ public boolean isInitialized() { public final void makeSureInitialized() { // Must call this method before any operation on the database to avoid deadlock of synchronized block extCatalog.makeSureInitialized(); + CatalogMetaCache owner = extCatalog.getFeMetaCacheOwner(); synchronized (this) { + if (cacheEntriesReleased) { + // A database returned without cache admission must remain usable by the current caller. + // Its children are read through the existing transient path, never registered again. + return; + } if (isInitializing) { return; } isInitializing = true; try { if (!initialized) { - buildMetaCache(); + buildMetaCache(owner); setLastUpdateTime(System.currentTimeMillis()); initialized = true; } @@ -159,24 +204,34 @@ public final void makeSureInitialized() { } private void buildMetaCache() { + buildMetaCache(extCatalog.getFeMetaCacheOwner()); + } + + private void buildMetaCache(CatalogMetaCache owner) { if (tableNames != null && tables != null) { return; } if (LOG.isDebugEnabled()) { LOG.debug("buildMetaCache for database: {}:{}", this.name, this.id, new Exception()); } + tableIdNameIndex.bind(owner, extCatalog.nextFeCacheEntryName("table_ids"), + tableName -> ScopePath.table(name, tableName)); CacheSpec namesSpec = CacheSpec.of( true, Config.external_cache_expire_time_seconds_after_access, 1); // Build one immutable names snapshot so list and lower-case index share the same cache version. - tableNames = new FeMetaCacheEntry<>( - name + ".table_names", + // Keep each database's existing capacity/TTL policy while sharing the catalog's invalidation tree. + // The database object's removal/discard hook closes these instance-qualified registrations. + String registrationName = extCatalog.nextFeCacheEntryName(name); + tableNames = FeMetaCacheEntry.shared(owner, + ignored -> ScopePath.tableCollection(name), ScopePath.tableCollection(name), + registrationName + ".table_names", ignored -> NameCacheValue.of(listTableNames()), namesSpec, Env.getCurrentEnv().getExtMetaCacheMgr().commonRefreshExecutor(), true, - FeMetaCacheEntry.singleKeyStripeCount()); + FeMetaCacheEntry.singleKeyStripeCount(), null); CacheSpec objectSpec = CacheSpec.of( true, @@ -184,8 +239,9 @@ private void buildMetaCache() { Math.max(Config.max_meta_object_cache_num, 1)); // Keep table object entries on the normal manual-miss/refresh path because they do not need // the database-level reset callback used by catalog object caches. - tables = new FeMetaCacheEntry<>( - name + ".tables", + tables = FeMetaCacheEntry.sharedLookup(owner, + localTableName -> ScopePath.table(name, localTableName), ScopePath.database(name), + registrationName + ".tables", localTableName -> buildTableForInit( null, localTableName, @@ -196,7 +252,8 @@ private void buildMetaCache() { objectSpec, Env.getCurrentEnv().getExtMetaCacheMgr().commonRefreshExecutor(), false, - FeMetaCacheEntry.defaultObjectStripeCount()); + FeMetaCacheEntry.defaultObjectStripeCount(), null); + tables.onRetired(() -> tableIdNameIndex.close(owner)); } private List> listTableNames() { @@ -347,6 +404,21 @@ public Optional getTableForReplay(long tableId) { return Optional.ofNullable(tables.getIfPresent(tableName)); } + /** Captures only the shared base-table instance; transient and bypassed reads cannot repair it. */ + public Optional observeTableLookup(ExternalTable table) { + if (!initialized || cacheEntriesReleased || extCatalog.shouldBypassSchemaCache(SessionContext.current())) { + return Optional.empty(); + } + // Transient system/session tables are not this database's retained base-table instance. + NameCacheValue names = getTableNamesValue(false); + if (names == null || !table.getRemoteName().equals(names.remoteNameOfLocalName(table.getName()))) { + return Optional.empty(); + } + // Name refresh can change the canonical spelling without replacing the old exact table key. + return tableNames.observe("", names).flatMap(nameObservation -> + tables.observe(table.getName(), table).map(nameObservation::and)); + } + /** * Resolve the retained local table name for replay without loading table metadata. * @@ -472,7 +544,8 @@ public DatabaseProperty getDbProperties() { public boolean isTableExist(String tableName) { SessionContext sessionContext = SessionContext.current(); String remoteTblName = tableName; - if (extCatalog.shouldBypassTableNameCache(sessionContext) && requiresRemoteTableNameResolution()) { + if ((cacheEntriesReleased || extCatalog.shouldBypassTableNameCache(sessionContext)) + && requiresRemoteTableNameResolution()) { Optional> matched = findTableNamePairWithoutCache(sessionContext, tableName); remoteTblName = matched.map(Pair::key).orElse(null); } else if (this.isTableNamesCaseInsensitive()) { @@ -525,7 +598,7 @@ public List getTablesOnIdOrderOrThrowException(List tableIdList) throws public Set getTableNamesWithLock() { makeSureInitialized(); SessionContext sessionContext = SessionContext.current(); - if (extCatalog.shouldBypassTableNameCache(sessionContext)) { + if (cacheEntriesReleased || extCatalog.shouldBypassTableNameCache(sessionContext)) { return Sets.newHashSet(listLocalTableNamesWithoutCache(sessionContext)); } return Sets.newHashSet(listLocalTableNamesFromCache()); @@ -535,7 +608,7 @@ public Set getTableNamesWithLock() { public T getTableNullable(String tableName) { makeSureInitialized(); SessionContext sessionContext = SessionContext.current(); - if (extCatalog.shouldBypassTableNameCache(sessionContext)) { + if (cacheEntriesReleased || extCatalog.shouldBypassTableNameCache(sessionContext)) { return getTableNullableWithoutCache(sessionContext, tableName); } String finalName = getLocalTableName(tableName, false); @@ -669,11 +742,11 @@ private boolean matchesLocalTableName(String localTableName, String requestedTab return localTableName.equals(requestedTableName); } - private String resolveTableNameForInvalidation(String tableName) { + public String resolveTableNameForInvalidation(String tableName) { if (isStoredTableNamesLowerCase()) { return tableName.toLowerCase(Locale.ROOT); } - if (!isTableNamesCaseInsensitive()) { + if (cacheEntriesReleased || !isTableNamesCaseInsensitive() || !isInitialized()) { return tableName; } @@ -756,6 +829,9 @@ private void invalidateAllTableCache() { @Override public T getTableNullable(long tableId) { makeSureInitialized(); + if (cacheEntriesReleased) { + return null; + } String tableName = tableIdNameIndex.getName(tableId); if (tableName == null) { return null; @@ -789,7 +865,9 @@ public void unregisterTable(String tableName) { setLastUpdateTime(System.currentTimeMillis()); localTableName = resolveTableNameForInvalidation(tableName); // Always clean local names/object/ID state, even when the object entry is cold or already evicted. - invalidateTableCache(localTableName); + if (!cacheEntriesReleased) { + invalidateTableCache(localTableName); + } } finally { Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTable( extCatalog.getId(), id, getFullName(), @@ -947,7 +1025,7 @@ public String getCachedTableNameByIdForTest(long tableId) { } public void resetMetaCacheNames() { - if (tableNames != null) { + if (!cacheEntriesReleased && tableNames != null) { tableNames.invalidateAll(); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index f6f4045e570285..14874a6b49a35b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -198,6 +198,13 @@ public void prepareCatalogByEngine(long catalogId, String engine) { } public void invalidateCatalog(long catalogId) { + CatalogIf catalog = getCatalog(catalogId); + if (catalog instanceof ExternalCatalog) { + CatalogMetaCache owner = ((ExternalCatalog) catalog).getMetaCacheOwnerIfPresent(); + if (owner != null) { + owner.invalidateCatalog(); + } + } try { try { routeCatalogEngines(catalogId, cache -> safeInvalidate( @@ -219,6 +226,10 @@ public void invalidateCatalogByEngine(long catalogId, String engine) { } public void removeCatalog(long catalogId) { + CatalogIf catalog = getCatalog(catalogId); + if (catalog instanceof ExternalCatalog) { + ((ExternalCatalog) catalog).closeFeMetaCacheOwner(); + } try { try { routeCatalogEngines(catalogId, cache -> safeInvalidate( @@ -350,18 +361,19 @@ public void invalidatePartitions(long catalogId, public List getCatalogCacheStats(long catalogId) { List stats = new ArrayList<>(); - allCacheTypes().forEach(externalMetaCache -> externalMetaCache.stats(catalogId) - .forEach((entryName, entryStats) -> stats.add( - new CatalogMetaCacheStats(externalMetaCache.engine(), entryName, entryStats)))); + Map> byEngine = Maps.newHashMap(); + allCacheTypes().forEach(externalMetaCache -> byEngine + .computeIfAbsent(externalMetaCache.engine(), ignored -> Maps.newHashMap()) + .putAll(externalMetaCache.stats(catalogId))); for (CatalogMetaCache cache : MetaCacheGovernance.catalogCaches(catalogId)) { - if (ENGINE_DEFAULT.equals(cache.engine()) || ENGINE_DORIS.equals(cache.engine())) { - continue; - } - for (Map.Entry> entry : cache.entries().entrySet()) { - stats.add(new CatalogMetaCacheStats( - cache.engine(), entry.getKey(), MetaCacheEntryStats.from(entry.getValue()))); + for (MetaCache entry : cache.allEntries().values()) { + Map entries = + byEngine.computeIfAbsent(entry.engine(), ignored -> Maps.newHashMap()); + entries.put(entry.registrationName(), MetaCacheEntryStats.from(entry)); } } + byEngine.forEach((engine, entries) -> entries.forEach((entryName, entryStats) -> + stats.add(new CatalogMetaCacheStats(engine, entryName, entryStats)))); stats.sort(Comparator.comparing(CatalogMetaCacheStats::getEngineName) .thenComparing(CatalogMetaCacheStats::getEntryName)); return stats; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java index 85d3f2be12cb49..fb19659b48d3c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java @@ -17,42 +17,155 @@ package org.apache.doris.datasource; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; -import org.apache.doris.common.CacheFactory; import org.apache.doris.common.Config; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.MetaCacheRemovalReason; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.qe.ConnectContext; import org.apache.doris.statistics.cache.BasicAsyncCacheLoader; import org.apache.doris.statistics.util.StatisticsUtil; -import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import lombok.Getter; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.time.Duration; +import java.util.Map; import java.util.Optional; -import java.util.OptionalLong; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; -import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.LongFunction; public class ExternalRowCountCache { private static final Logger LOG = LogManager.getLogger(ExternalRowCountCache.class); - private final AsyncLoadingCache> rowCountCache; - // Serialize future publication with explicit invalidation. The read lock ends as soon as Caffeine has - // published the future, before this class waits for the row-count result. - private final ReentrantReadWriteLock publicationLock = new ReentrantReadWriteLock(); + private final ExecutorService executor; + private final LongFunction rootResolver; + private final RowCountCacheLoader loader = new RowCountCacheLoader(); + private final Map entries = new ConcurrentHashMap<>(); + // A process-wide retention policy, not a second metadata store. Values/futures belong exclusively to + // catalog roots. Keep the existing global count bound while invalidation uses root resource scopes. + private final Cache retention = Caffeine.newBuilder() + .maximumSize(Config.max_external_table_row_count_cache_num) + .removalListener((RowCountKey key, Retention token, + com.github.benmanes.caffeine.cache.RemovalCause cause) -> retire(token)) + .build(); public ExternalRowCountCache(ExecutorService executor) { - // 1. set expireAfterWrite to 1 day, avoid too many entries - // 2. set refreshAfterWrite to 10min(default), so that the cache will be refreshed after 10min - CacheFactory rowCountCacheFactory = new CacheFactory( - OptionalLong.of(Config.external_cache_expire_time_seconds_after_access), - OptionalLong.of(Config.external_cache_refresh_time_minutes * 60), - Config.max_external_table_row_count_cache_num, - false, - null); - rowCountCache = rowCountCacheFactory.buildAsyncCache(new RowCountCacheLoader(), executor); + this(executor, id -> { + CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(id); + return catalog instanceof ExternalCatalog ? ((ExternalCatalog) catalog).getFeMetaCacheOwner() : null; + }); + } + + ExternalRowCountCache(ExecutorService executor, LongFunction rootResolver) { + this.executor = executor; + this.rootResolver = rootResolver; + } + + private Entry entry(long catalogId) { + CatalogMetaCache root = rootResolver.apply(catalogId); + if (root == null) { + throw new IllegalStateException("No external catalog for row count: " + catalogId); + } + return entries.compute(catalogId, (id, old) -> old != null && old.owner == root + ? old : new Entry(root)); + } + + private void retire(Retention token) { + if (token.entry.owner.isClosed() || retention.getIfPresent(token.key) == token) { + return; + } + try { + while (true) { + RowCountValue current = token.entry.cache.getIfPresent(token.key); + if (current == null || current.retention != token + || token.entry.cache.compareAndSet(token.key, current, null)) { + return; + } + } + } catch (IllegalStateException failure) { + // Catalog close may retire the physical entry between the owner check and the exact removal. + if (!token.entry.owner.isClosed()) { + throw failure; + } + } + } + + private final class Entry { + private final CatalogMetaCache owner; + private final MetaCache cache; + + private Entry(CatalogMetaCache owner) { + this.owner = owner; + MetaCacheDefinition.Builder definition = + MetaCacheDefinition.builder("row_count", + CacheSpec.of(true, Config.external_cache_expire_time_seconds_after_access, + Config.max_external_table_row_count_cache_num), + key -> ScopePath.table(Long.toString(key.dbId), Long.toString(key.tableId)) + .in(ScopePath.Coordinates.ID)) + .budgetPolicy(MetaCacheDefinition.BudgetPolicy.COUNT_ONLY) + .loader(key -> load(key, false)) + .removalListener((key, value, reason) -> { + // Automatic refresh retains the same policy token across future replacement. + if (reason != MetaCacheRemovalReason.REPLACED) { + retention.asMap().remove(key, value.retention); + } + }); + if (Config.external_cache_refresh_time_minutes > 0) { + definition.refreshAfterWrite(Duration.ofMinutes(Config.external_cache_refresh_time_minutes), executor); + } + cache = owner.create(definition.build()); + } + + private RowCountValue load(RowCountKey key, boolean fillMetaCache) { + RowCountValue current = cache.getIfPresent(key); + Retention token = current == null ? new Retention(this, key) : current.retention; + if (current != null) { + // The framework runs refreshes on the background executor. Keep the previous successful + // future visible until this load succeeds; submitting and waiting on the same pool can deadlock. + // Like the original async-cache loader, automatic refresh does not fill metadata caches. + Optional refreshed = loader.doLoad(key); + return refreshed == null ? null + : new RowCountValue(token, CompletableFuture.completedFuture(refreshed)); + } + CompletableFuture> future = fillMetaCache + ? CompletableFuture.supplyAsync(() -> loadRowCount(key, true), executor) + : loader.asyncLoad(key, executor); + return new RowCountValue(token, future); + } + + private void touch(RowCountKey key, RowCountValue value) { + cache.runIfCurrent(key, value, () -> retention.put(key, value.retention)); + } + } + + private static final class Retention { + private final Entry entry; + private final RowCountKey key; + + private Retention(Entry entry, RowCountKey key) { + this.entry = entry; + this.key = key; + } + } + + private static final class RowCountValue { + private final Retention retention; + private final CompletableFuture> future; + + private RowCountValue(Retention retention, CompletableFuture> future) { + this.retention = retention; + this.future = future; + } } @Getter @@ -126,23 +239,17 @@ static Optional loadRowCount(RowCountKey rowCountKey, boolean fillMetaCach public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean fillMetaCache) { RowCountKey key = new RowCountKey(catalogId, dbId, tableId); try { - CompletableFuture> f; - publicationLock.readLock().lock(); - try { - f = fillMetaCache - ? rowCountCache.get(key, (rowCountKey, executor) -> CompletableFuture.supplyAsync( - () -> loadRowCount(rowCountKey, true), executor)) - : rowCountCache.get(key); - } finally { - publicationLock.readLock().unlock(); - } + Entry entry = entry(catalogId); + RowCountValue value = entry.cache.get(key, rowCountKey -> entry.load(rowCountKey, fillMetaCache)); + entry.touch(key, value); + CompletableFuture> f = value.future; // Get row count synchronously by default. if (ConnectContext.get() == null || ConnectContext.get().getSessionVariable().fetchHiveRowCountSync) { - return f.get().orElse(TableIf.UNKNOWN_ROW_COUNT); + return result(entry, key, value); } else { if (f.isDone()) { - return f.get().orElse(TableIf.UNKNOWN_ROW_COUNT); + return result(entry, key, value); } LOG.info("Row count for table {}.{}.{} is still processing.", catalogId, dbId, tableId); } @@ -160,11 +267,16 @@ public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean f public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) { RowCountKey key = new RowCountKey(catalogId, dbId, tableId); try { - CompletableFuture> f = rowCountCache.getIfPresent(key); - if (f == null) { + Entry entry = entries.get(catalogId); + if (entry == null || entry.owner.isClosed()) { + return TableIf.UNKNOWN_ROW_COUNT; + } + RowCountValue value = entry.cache.getIfPresent(key); + if (value == null) { return -1; - } else if (f.isDone()) { - return f.get().orElse(-1L); + } else if (value.future.isDone()) { + entry.touch(key, value); + return result(entry, key, value); } } catch (Exception e) { LOG.warn("Unexpected exception while returning row count if present", e); @@ -172,33 +284,42 @@ public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) return -1; } - // Catalog/db invalidation is O(N): row-count keys are numeric ids, and Caffeine - // does not support prefix invalidation by catalog or database id. - void invalidateCatalog(long catalogId) { - publicationLock.writeLock().lock(); + private long result(Entry entry, RowCountKey key, RowCountValue value) throws Exception { try { - rowCountCache.asMap().keySet().removeIf(key -> key.catalogId == catalogId); + Optional loaded = value.future.get(); + return loaded == null ? TableIf.UNKNOWN_ROW_COUNT : loaded.orElse(TableIf.UNKNOWN_ROW_COUNT); } finally { - publicationLock.writeLock().unlock(); + if (value.future.isDone() && (value.future.isCompletedExceptionally() + || value.future.getNow(null) == null)) { + entry.cache.compareAndSet(key, value, null); + } + } + } + + // These compatibility entry points share the same resource generations as the unified change plan. + void invalidateCatalog(long catalogId) { + Entry entry = entries.get(catalogId); + if (entry != null) { + if (entry.owner.isClosed()) { + entries.remove(catalogId, entry); + } else { + entry.owner.invalidateScope(ScopePath.catalog().in(ScopePath.Coordinates.ID)); + } } } void invalidateDb(long catalogId, long dbId) { - publicationLock.writeLock().lock(); - try { - rowCountCache.asMap().keySet().removeIf(key -> key.catalogId == catalogId && key.dbId == dbId); - } finally { - publicationLock.writeLock().unlock(); + Entry entry = entries.get(catalogId); + if (entry != null && !entry.owner.isClosed()) { + entry.owner.invalidateScope(ScopePath.database(Long.toString(dbId)).in(ScopePath.Coordinates.ID)); } } void invalidateTable(long catalogId, long dbId, long tableId) { - publicationLock.writeLock().lock(); - try { - rowCountCache.synchronous().invalidate(new RowCountKey(catalogId, dbId, tableId)); - } finally { - publicationLock.writeLock().unlock(); + Entry entry = entries.get(catalogId); + if (entry != null && !entry.owner.isClosed()) { + entry.owner.invalidateScope(ScopePath.table(Long.toString(dbId), Long.toString(tableId)) + .in(ScopePath.Coordinates.ID)); } } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/MetastoreEventSyncDriver.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/MetastoreEventSyncDriver.java index 53dc1c9142debb..d7005f55e14d6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/MetastoreEventSyncDriver.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/MetastoreEventSyncDriver.java @@ -30,7 +30,9 @@ import org.apache.doris.connector.spi.event.EventPollResult; import org.apache.doris.connector.spi.event.MetastoreChangeDescriptor; import org.apache.doris.datasource.log.CatalogLog; +import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.datasource.log.MetaIdMappingsLog; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.MasterOpExecutor; @@ -234,7 +236,7 @@ private void applyDescriptors(PluginDrivenExternalCatalog catalog, Connector con List descriptors) { for (MetastoreChangeDescriptor descriptor : descriptors) { try { - applyOne(catalog, connector, descriptor); + applyOne(catalog, descriptor); } catch (Exception e) { throw new RuntimeException( "Failed to apply metastore change " + descriptor + " on catalog " @@ -245,15 +247,14 @@ private void applyDescriptors(PluginDrivenExternalCatalog catalog, Connector con // Applies one neutral descriptor via the engine's own (connector-agnostic) mutators — the same ones the // legacy event.process() bodies called, now generalized to work on a flipped catalog. - private void applyOne(PluginDrivenExternalCatalog catalog, Connector connector, - MetastoreChangeDescriptor descriptor) throws Exception { + private void applyOne(PluginDrivenExternalCatalog catalog, MetastoreChangeDescriptor descriptor) throws Exception { EventIdentity before = EventIdentity.from( catalog, descriptor.getDbName(), descriptor.getTableName()); EventIdentity after = descriptor.getDbNameAfter() == null ? null : EventIdentity.from( catalog, descriptor.getDbNameAfter(), descriptor.getTableNameAfter()); String catalogName = catalog.getName(); CatalogMgr catalogMgr = Env.getCurrentEnv().getCatalogMgr(); - invalidateStructuralEventCaches(connector, descriptor.getOp(), before, after); + invalidateStructuralEventCaches(catalog, descriptor.getOp(), before, after); switch (descriptor.getOp()) { case REGISTER_DATABASE: catalogMgr.registerExternalDatabaseFromEvent( @@ -310,39 +311,36 @@ private void applyOne(PluginDrivenExternalCatalog catalog, Connector connector, } } - /** - * Invalidates connector-owned caches for structural events before publishing the corresponding FE mutation. - * Connector cache keys use remote identities; FE mutators below use the local identities from the same values. - */ - private void invalidateStructuralEventCaches(Connector connector, + /** Structural events invalidate both coordinates before the incremental FE snapshot mutation. */ + private void invalidateStructuralEventCaches(PluginDrivenExternalCatalog catalog, MetastoreChangeDescriptor.Op op, EventIdentity before, EventIdentity after) { switch (op) { case REGISTER_DATABASE: case UNREGISTER_DATABASE: - connector.invalidateDb(before.remoteDbName); - break; - case RENAME_DATABASE: - connector.invalidateDb(before.remoteDbName); - if (!Objects.equals(before.remoteDbName, after.remoteDbName)) { - connector.invalidateDb(after.remoteDbName); - } - break; case REGISTER_TABLE: case UNREGISTER_TABLE: - connector.invalidateTable(before.remoteDbName, before.remoteTableName); + CatalogMetadataChanges.apply(catalog, structuralChange(catalog, before)); break; + case RENAME_DATABASE: case RENAME_TABLE: - connector.invalidateTable(before.remoteDbName, before.remoteTableName); - if (!Objects.equals(before.remoteDbName, after.remoteDbName) - || !Objects.equals(before.remoteTableName, after.remoteTableName)) { - connector.invalidateTable(after.remoteDbName, after.remoteTableName); - } + ExternalObjectLog source = structuralChange(catalog, before); + ExternalObjectLog destination = structuralChange(catalog, after); + CatalogMetadataChanges.apply(catalog, source.equals(destination) + ? java.util.Collections.singletonList(source) : java.util.Arrays.asList(source, destination)); break; default: break; } } + private ExternalObjectLog structuralChange(PluginDrivenExternalCatalog catalog, EventIdentity identity) { + ExternalObjectLog change = identity.localTableName == null + ? ExternalObjectLog.createForRefreshDb(catalog.getId(), identity.localDbName) + : ExternalObjectLog.createForExplicitRefreshTable( + catalog.getId(), identity.localDbName, identity.localTableName, 0); + return change.withRemoteNames(identity.remoteDbName, identity.remoteTableName); + } + /** One remote connector identity and its canonical FE-local counterpart. */ private static final class EventIdentity { private final String remoteDbName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/log/ExternalObjectLog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/log/ExternalObjectLog.java index 30464457e60a80..881df2ac1070ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/log/ExternalObjectLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/log/ExternalObjectLog.java @@ -52,6 +52,15 @@ public class ExternalObjectLog implements Writable { @SerializedName(value = "ntn") private String newTableName; // for rename table op + @SerializedName(value = "remoteDbName") + private String remoteDbName; + + @SerializedName(value = "remoteTableName") + private String remoteTableName; + + @SerializedName(value = "remoteNewTableName") + private String remoteNewTableName; + @SerializedName(value = "invalidCache") private boolean invalidCache; @@ -61,6 +70,9 @@ public class ExternalObjectLog implements Writable { @SerializedName(value = "newPartitionNames") private List newPartitionNames; + @SerializedName(value = "invalidateTableLookup") + private boolean invalidateTableLookup; + @SerializedName(value = "lastUpdateTime") private long lastUpdateTime; @@ -85,6 +97,13 @@ public static ExternalObjectLog createForRefreshTable(long catalogId, String dbN return externalObjectLog; } + public static ExternalObjectLog createForExplicitRefreshTable(long catalogId, String dbName, String tblName, + long updateTime) { + ExternalObjectLog log = createForRefreshTable(catalogId, dbName, tblName, updateTime); + log.setInvalidateTableLookup(true); + return log; + } + public static ExternalObjectLog createForRefreshPartitions(long catalogId, String dbName, String tblName, List modifiedPartNames, List newPartNames, long updateTime) { ExternalObjectLog externalObjectLog = new ExternalObjectLog(); @@ -107,6 +126,13 @@ public static ExternalObjectLog createForRenameTable(long catalogId, String dbNa return externalObjectLog; } + /** Persist both coordinates; replay must not need a warm object cache to recover remote identities. */ + public ExternalObjectLog withRemoteNames(String database, String table) { + setRemoteDbName(database); + setRemoteTableName(table); + return this; + } + @Override public void write(DataOutput out) throws IOException { Text.writeString(out, GsonUtils.GSON.toJson(this)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java index 9f9dbfe674086a..97c853b5ea13d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetaCacheRuntime.java @@ -31,10 +31,20 @@ */ final class CatalogMetaCacheRuntime { private final CatalogMetaCache owner; + private final boolean ownsOwner; private final Map> entries = new ConcurrentHashMap<>(); CatalogMetaCacheRuntime(long catalogId, String engine, Map catalogProperties) { - owner = CatalogMetaCache.managed(catalogId, engine, catalogProperties); + this(CatalogMetaCache.managed(catalogId, engine, catalogProperties), true); + } + + CatalogMetaCacheRuntime(CatalogMetaCache owner) { + this(owner, false); + } + + private CatalogMetaCacheRuntime(CatalogMetaCache owner, boolean ownsOwner) { + this.owner = Objects.requireNonNull(owner, "owner"); + this.ownsOwner = ownsOwner; } CatalogMetaCache owner() { @@ -51,7 +61,7 @@ void put(String entryName, MetaCache entry) { Map stats() { Map result = Maps.newHashMap(); - entries.forEach((name, entry) -> result.put(name, stats(entry))); + owner.entries().forEach((name, entry) -> result.put(name, stats(entry))); return result; } @@ -60,7 +70,14 @@ void invalidateAll() { } void close() { - owner.close(); + // An entry group borrows the catalog's FE owner. Reconfiguring schema entries must not close + // sibling naming/object adapters; only ExternalCatalog retires that shared owner. + if (ownsOwner) { + owner.close(); + } else { + entries.values().forEach(owner::remove); + } + entries.clear(); } private MetaCacheEntryStats stats(MetaCache entry) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java new file mode 100644 index 00000000000000..2cf8faa3003293 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogMetadataChanges.java @@ -0,0 +1,263 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.util.Util; +import org.apache.doris.connector.cache.CacheCallbackContext; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.ScopePath; +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.log.ExternalObjectLog; +import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; + +import com.google.common.base.Strings; + +import java.util.ArrayList; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Optional; +import java.util.Set; + +/** + * The engine's semantic change boundary. FE names, remote connector identities and numeric derived state are + * projected once and detached at one catalog publication barrier. Projection never loads remote metadata. + * Connector domains own registrations/lifetime; resource scopes own invalidation, independently of domains. + */ +public final class CatalogMetadataChanges { + private CatalogMetadataChanges() { + } + + public static ExternalObjectLog table(ExternalTable table, boolean lookup, long updateTime) { + ExternalObjectLog change = lookup + ? ExternalObjectLog.createForExplicitRefreshTable(table.getCatalog().getId(), + table.getDb().getFullName(), table.getName(), updateTime) + : ExternalObjectLog.createForRefreshTable(table.getCatalog().getId(), + table.getDb().getFullName(), table.getName(), updateTime); + change.setDbId(table.getDb().getId()); + change.setTableId(table.getId()); + return change.withRemoteNames(table.getRemoteDbName(), table.getRemoteName()); + } + + public static Optional observeTableLookup(ExternalTable table) { + ExternalDatabase database = table.getDb(); + return database.observeTableLookup(table) + .map(observation -> new MissingTableObservation(table, observation)); + } + + /** A read observation repairs only this FE; it is not a committed external DDL journal record. */ + public static final class MissingTableObservation { + private final ExternalCatalog catalog; + private final ExternalObjectLog change; + private final MetaCache.Observation observation; + + private MissingTableObservation(ExternalTable table, MetaCache.Observation observation) { + this.catalog = table.getCatalog(); + this.change = table(table, true, 0); + this.observation = observation; + } + + public void missing() { + List scopes = new ArrayList<>(); + try { + plan(catalog, change, scopes); + } finally { + observation.invalidateScopes(scopes); + } + } + } + + public static void apply(ExternalCatalog catalog, ExternalObjectLog change) { + apply(catalog, java.util.Collections.singletonList(change)); + } + + /** A rename event can span databases; all old/new coordinates share one publication barrier. */ + public static void apply(ExternalCatalog catalog, List changes) { + List scopes = new ArrayList<>(); + RuntimeException failure = null; + try { + for (ExternalObjectLog change : changes) { + try { + updateCachedTable(plan(catalog, change, scopes), change); + } catch (RuntimeException error) { + if (failure == null) { + failure = error; + } else { + failure.addSuppressed(error); + } + } + } + } finally { + CatalogMetaCache owner = catalog.getMetaCacheOwnerIfPresent(); + if (owner != null) { + owner.invalidateScopes(scopes); + } + if (Env.getCurrentEnv().getSortedPartitionsCacheManager() != null) { + for (ExternalObjectLog change : changes) { + invalidateSortedPartitions(catalog, change); + } + } + } + if (failure != null) { + throw failure; + } + } + + private static void invalidateSortedPartitions(ExternalCatalog catalog, ExternalObjectLog change) { + if (change.getTableName() == null) { + Env.getCurrentEnv().getSortedPartitionsCacheManager().invalidateAll(); + } else { + Env.getCurrentEnv().getSortedPartitionsCacheManager().invalidateTable( + catalog.getName(), change.getDbName(), change.getTableName()); + if (change.getNewTableName() != null) { + Env.getCurrentEnv().getSortedPartitionsCacheManager().invalidateTable( + catalog.getName(), change.getDbName(), change.getNewTableName()); + } + } + } + + private static void updateCachedTable(Optional table, ExternalObjectLog change) { + table.ifPresent(value -> { + if (change.getPartitionNames() == null) { + value.unsetObjectCreated(); + } + if (change.getLastUpdateTime() > 0) { + value.setUpdateTime(change.getLastUpdateTime()); + } + }); + } + + private static Optional plan( + ExternalCatalog catalog, ExternalObjectLog change, List scopes) { + Optional> db = catalog.getDbForReplay(change.getDbName()); + Optional table = Strings.isNullOrEmpty(change.getTableName()) + ? Optional.empty() : db.flatMap(database -> database.getTableForReplay(change.getTableName())); + long dbId = change.getDbId() != 0 ? change.getDbId() + : Util.genIdByName(catalog.getName(), change.getDbName()); + long tableId = change.getTableId() != 0 ? change.getTableId() + : change.getTableName() == null ? 0L + : Util.genIdByName(catalog.getName(), change.getDbName(), change.getTableName()); + String remoteDb = change.getRemoteDbName() != null ? change.getRemoteDbName() + : db.map(ExternalDatabase::getRemoteName).orElse(null); + String remoteTable = change.getRemoteTableName() != null ? change.getRemoteTableName() + : table.map(ExternalTable::getRemoteName).orElse(null); + Connector connector = catalog instanceof PluginDrivenExternalCatalog + ? ((PluginDrivenExternalCatalog) catalog).getConnectorIfPresent() : null; + Set remote = new LinkedHashSet<>(); + boolean database = Strings.isNullOrEmpty(change.getTableName()); + boolean partitions = change.getPartitionNames() != null; + String canonicalDb = catalog.resolveDatabaseNameForInvalidation(change.getDbName()); + if (!canonicalDb.equals(change.getDbName())) { + scopes.add(local(ScopePath.database(canonicalDb))); + scopes.add(id(ScopePath.database(Long.toString(Util.genIdByName(catalog.getName(), canonicalDb))))); + } + if (database) { + scopes.add(local(ScopePath.database(change.getDbName()))); + scopes.add(local(ScopePath.databaseCollection())); + scopes.add(id(ScopePath.database(Long.toString(dbId)))); + remote.add(remoteDb == null ? ScopePath.catalog() : ScopePath.database(remoteDb)); + remote.add(ScopePath.databaseCollection()); + } else { + scopes.add(id(ScopePath.table(Long.toString(dbId), Long.toString(tableId)))); + if (!partitions) { + if (db.isPresent() && (change.isInvalidateTableLookup() || change.getNewTableName() != null)) { + addCanonicalTableScopes(catalog, db.get(), dbId, change.getTableName(), scopes); + } + scopes.add(local(change.isInvalidateTableLookup() + ? ScopePath.table(change.getDbName(), change.getTableName()) + : ScopePath.tableMetadata(change.getDbName(), change.getTableName()))); + if (change.isInvalidateTableLookup() || change.getNewTableName() != null) { + // A first object/row-count load can publish an old spelling after scope planning. + // LOCAL scopes share case-folded identities; numeric derived caches need this DB fence. + if (catalog.getLowerCaseTableNames() == 2) { + scopes.add(id(ScopePath.database(Long.toString(dbId)))); + } + scopes.add(local(ScopePath.tableCollection(change.getDbName()))); + } + } + if (remoteDb == null || remoteTable == null) { + // Old logs carry only local identities. Widen only the remote coordinate, without I/O. + remote.add(remoteDb == null ? ScopePath.catalog() : ScopePath.database(remoteDb)); + } else { + remote.add(ScopePath.table(remoteDb, remoteTable)); + } + if (remoteDb != null && (change.isInvalidateTableLookup() || change.getNewTableName() != null)) { + remote.add(ScopePath.tableCollection(remoteDb)); + } + if (change.getNewTableName() != null) { + if (db.isPresent()) { + addCanonicalTableScopes(catalog, db.get(), dbId, change.getNewTableName(), scopes); + } + scopes.add(local(ScopePath.table(change.getDbName(), change.getTableName()))); + scopes.add(local(ScopePath.table(change.getDbName(), change.getNewTableName()))); + scopes.add(id(ScopePath.table(Long.toString(dbId), Long.toString(Util.genIdByName( + catalog.getName(), change.getDbName(), change.getNewTableName()))))); + remote.add(remoteDb != null && change.getRemoteNewTableName() != null + ? ScopePath.table(remoteDb, change.getRemoteNewTableName()) + : remoteDb == null ? ScopePath.catalog() : ScopePath.database(remoteDb)); + } + } + try { + if (connector != null) { + new CacheCallbackContext(connector.getClass().getClassLoader()).run(() -> { + if (partitions && remoteDb != null && remoteTable != null) { + remote.clear(); + remote.addAll(connector.partitionCacheScopes( + remoteDb, remoteTable, change.getPartitionNames())); + } + List dependencies = new ArrayList<>(); + for (ScopePath scope : remote) { + if (scope.database() != null) { + dependencies.addAll(connector.cacheDependencies(scope)); + } + } + remote.addAll(dependencies); + }); + } + } catch (RuntimeException | Error failure) { + // A gateway failed to project its dependencies. Fence every remote cache rather than retain a + // dependency whose identity is unknown, then report the failure to the caller. + remote.add(ScopePath.catalog()); + throw failure; + } finally { + remote.forEach(scope -> scopes.add(scope.in(ScopePath.Coordinates.REMOTE))); + } + return table; + } + + private static void addCanonicalTableScopes(ExternalCatalog catalog, ExternalDatabase database, + long dbId, String tableName, List scopes) { + // Case-insensitive lookup can retain a differently cased owner with a different deterministic ID. + String canonical = database.resolveTableNameForInvalidation(tableName); + scopes.add(local(ScopePath.table(database.getFullName(), canonical))); + scopes.add(id(ScopePath.table(Long.toString(dbId), Long.toString(Util.genIdByName( + catalog.getName(), database.getFullName(), canonical))))); + } + + private static ScopePath local(ScopePath scope) { + return scope.in(ScopePath.Coordinates.LOCAL); + } + + private static ScopePath id(ScopePath scope) { + return scope.in(ScopePath.Coordinates.ID); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java index b7f9809e264934..c9f945ab59dc87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalCatalogMetaCache.java @@ -23,6 +23,7 @@ import org.apache.doris.connector.cache.MetaCache; import org.apache.doris.connector.cache.MetaCacheDefinition; import org.apache.doris.connector.cache.MetaCacheSizeEstimators; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; @@ -88,8 +89,9 @@ public Collection aliases() { public void initCatalog(long catalogId, Map catalogProperties) { Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( catalogProperties, catalogPropertyCompatibilityMap()); - catalogEntries.computeIfAbsent(catalogId, - id -> buildCatalogRuntime(id, safeCatalogProperties)); + catalogEntries.compute(catalogId, (id, current) -> + current != null && !current.owner().isClosed() + ? current : buildCatalogRuntime(id, safeCatalogProperties)); } @Override @@ -148,7 +150,7 @@ public void invalidateCatalogEntries(long catalogId) { public void invalidateDb(long catalogId, String dbName) { CatalogMetaCacheRuntime group = catalogEntries.get(catalogId); if (group != null) { - group.owner().invalidateDatabase(dbName); + group.owner().invalidateScope(ScopePath.database(dbName).in(ScopePath.Coordinates.LOCAL)); } } @@ -156,13 +158,13 @@ public void invalidateDb(long catalogId, String dbName) { public void invalidateTable(long catalogId, String dbName, String tableName) { CatalogMetaCacheRuntime group = catalogEntries.get(catalogId); if (group != null) { - group.owner().invalidateTable(dbName, tableName); + group.owner().invalidateScope(ScopePath.tableMetadata(dbName, tableName).in(ScopePath.Coordinates.LOCAL)); } } @Override public void invalidatePartitions(long catalogId, String dbName, String tableName, List partitions) { - // FE schema entries are table-scoped: any partition metadata change can affect the table view. + // Partition metadata changes can also affect the table's schema/content view. invalidateTable(catalogId, dbName, tableName); } @@ -274,7 +276,10 @@ private void validateRegisteredMetaCacheEntryDef(MetaCacheEntryDef } private CatalogMetaCacheRuntime buildCatalogRuntime(long catalogId, Map catalogProperties) { - CatalogMetaCacheRuntime group = new CatalogMetaCacheRuntime(catalogId, engine, catalogProperties); + CatalogIf catalog = getCatalog(catalogId); + CatalogMetaCacheRuntime group = catalog instanceof ExternalCatalog + ? new CatalogMetaCacheRuntime(((ExternalCatalog) catalog).getFeMetaCacheOwner()) + : new CatalogMetaCacheRuntime(catalogId, engine, catalogProperties); metaCacheEntryDefs.values() .forEach(entryDef -> group.put(entryDef.getName(), newMetaCacheEntry(group, entryDef, catalogProperties))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java index 1cb7f0dcf9c444..ce04bf52aa85fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/FeMetaCacheEntry.java @@ -26,6 +26,7 @@ import org.apache.doris.connector.cache.MetaCacheSizeEstimators; import org.apache.doris.connector.cache.ScopePath; +import com.github.benmanes.caffeine.cache.RemovalCause; import com.github.benmanes.caffeine.cache.RemovalListener; import java.time.Duration; @@ -34,6 +35,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicReferenceArray; @@ -42,6 +44,7 @@ import java.util.function.BiPredicate; import java.util.function.Function; import java.util.function.Predicate; +import java.util.function.Supplier; import javax.annotation.Nullable; /** @@ -84,8 +87,15 @@ private ActionToken(K key, ActionState state) { private final boolean autoRefresh; private final int stripeCount; private final AtomicReferenceArray> stripeStates; - private final CatalogMetaCache owner = CatalogMetaCache.unmanaged(); + private final CatalogMetaCache owner; + private final ScopePath invalidationScope; private final MetaCache data; + private final BiConsumer discardedValueAction; + private final Object lifecycleLock = new Object(); + private Runnable retirementCleanup; + private final AtomicBoolean registrationReleased = new AtomicBoolean(); + private volatile boolean retired; + private int activeUses; public FeMetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor) { this(name, loader, cacheSpec, refreshExecutor, true, false, defaultObjectStripeCount(), null); @@ -128,6 +138,43 @@ public static FeMetaCacheEntry withSyncRemovalListener(String name, FeMetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, int stripeCount, @Nullable RemovalListener removalListener) { + this(CatalogMetaCache.unmanaged(), ignored -> ScopePath.catalog(), ScopePath.catalog(), + name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, + stripeCount, removalListener, false, MetaCacheDefinition.BudgetPolicy.WEIGHTED_PAYLOAD); + } + + public static FeMetaCacheEntry shared(CatalogMetaCache owner, + Function scopeResolver, ScopePath invalidationScope, + String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, + boolean autoRefresh, int stripeCount, @Nullable RemovalListener removalListener) { + return new FeMetaCacheEntry<>(owner, scopeResolver, invalidationScope.in(ScopePath.Coordinates.LOCAL), + name, loader, cacheSpec, + refreshExecutor, autoRefresh, false, stripeCount, removalListener, true, + MetaCacheDefinition.BudgetPolicy.WEIGHTED_PAYLOAD); + } + + /** Lookup handles retain their existing count/TTL policy; schema payloads use the owner's byte budget. */ + public static FeMetaCacheEntry sharedLookup(CatalogMetaCache owner, + Function scopeResolver, ScopePath invalidationScope, + String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, + boolean autoRefresh, int stripeCount, @Nullable RemovalListener removalListener) { + return new FeMetaCacheEntry<>(owner, scopeResolver, invalidationScope.in(ScopePath.Coordinates.LOCAL), + name, loader, cacheSpec, + refreshExecutor, autoRefresh, false, stripeCount, removalListener, true, + MetaCacheDefinition.BudgetPolicy.COUNT_ONLY); + } + + private FeMetaCacheEntry(CatalogMetaCache owner, Function scopeResolver, + ScopePath invalidationScope, String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, int stripeCount, + @Nullable RemovalListener removalListener, boolean retireDiscardedValues, + MetaCacheDefinition.BudgetPolicy budgetPolicy) { + this.owner = Objects.requireNonNull(owner, "owner"); + discardedValueAction = retireDiscardedValues && removalListener != null + ? (key, value) -> removalListener.onRemoval(key, value, RemovalCause.EXPLICIT) : null; + this.invalidationScope = Objects.requireNonNull(invalidationScope, "invalidationScope"); + Function entryScope = retireDiscardedValues + ? key -> scopeResolver.apply(key).in(ScopePath.Coordinates.LOCAL) : scopeResolver; this.name = Objects.requireNonNull(name, "name can not be null"); this.loader = loader; this.cacheSpec = Objects.requireNonNull(cacheSpec, "cacheSpec can not be null"); @@ -156,14 +203,20 @@ public static FeMetaCacheEntry withSyncRemovalListener(String name, effectiveEnabled = CacheSpec.isCacheEnabled( cacheSpec.isEnable(), cacheSpec.getTtlSecond(), cacheSpec.getCapacity()); MetaCacheDefinition.Builder builder = MetaCacheDefinition.builder( - name, cacheSpec, ignored -> ScopePath.catalog()) - .sizeEstimator(MetaCacheSizeEstimators.reflective()); + name, cacheSpec, entryScope) + .sizeEstimator(MetaCacheSizeEstimators.reflective()) + .budgetPolicy(budgetPolicy); if (loader != null) { builder.loader(key -> loadAndPause(key, loader)); } if (removalListener != null) { builder.removalListener((key, value, reason) -> removalListener.onRemoval(key, value, toCaffeineRemovalCause(reason))); + // An uncached database is still returned to its caller. Retire its cache registrations too, + // so the caller uses the transient lookup path rather than leaking a new set of child entries. + if (discardedValueAction != null) { + builder.discardListener(discardedValueAction); + } } if (autoRefresh && Config.external_cache_refresh_time_minutes > 0) { builder.refreshAfterWrite( @@ -181,12 +234,13 @@ public V get(K key) { throw new UnsupportedOperationException(String.format( "Entry '%s' requires a contextual miss loader.", name)); } - return data.get(key); + return useCache(() -> data.get(key), () -> loadWithoutCaching(key, loader)); } public V get(K key, Function missLoader) { Function nonNullLoader = Objects.requireNonNull(missLoader, "missLoader can not be null"); - return data.get(key, loadKey -> loadAndPause(loadKey, nonNullLoader)); + return useCache(() -> data.get(key, loadKey -> loadAndPause(loadKey, nonNullLoader)), + () -> loadWithoutCaching(key, nonNullLoader)); } public V getAndRunIfCurrent(K key, BiConsumer currentValueAction) { @@ -198,6 +252,12 @@ public V getAndRunIfCurrent(K key, BiPredicate actionRequired, BiPredicate required = Objects.requireNonNull(actionRequired, "actionRequired can not be null"); BiConsumer action = Objects.requireNonNull(currentValueAction, "currentValueAction can not be null"); Function loadFunction = Objects.requireNonNull(loader, "loader can not be null"); + return useCache(() -> getAndRunIfCurrentCached(key, required, action, loadFunction), + () -> loadWithoutCaching(key, loadFunction)); + } + + private V getAndRunIfCurrentCached(K key, BiPredicate required, + BiConsumer action, Function loadFunction) { V cached = data.getIfPresent(key); if (cached != null && !required.test(key, cached)) { return cached; @@ -239,7 +299,7 @@ public V getAndRunIfCurrent(K key, BiPredicate actionRequired, && (!effectiveEnabled || data.getIfPresent(key) == value); if (current && required.test(key, value) && isCurrent(stripe, token)) { try { - action.accept(key, value); + data.runIfCurrent(key, value, () -> action.accept(key, value)); } catch (RuntimeException | Error throwable) { data.invalidateKey(key); throw throwable; @@ -254,12 +314,20 @@ public V getAndRunIfCurrent(K key, BiPredicate actionRequired, } } + public Optional observe(K key, Object expectedValue) { + return useCache(() -> data.observe(key, expectedValue, () -> !retired), Optional::empty); + } + public V getIfPresent(K key) { - return data.getIfPresent(key); + return useCache(() -> data.getIfPresent(key), () -> null); } @Nullable public V findIfPresent(Predicate keyPredicate) { + return useCache(() -> findIfPresentCached(keyPredicate), () -> null); + } + + private V findIfPresentCached(Predicate keyPredicate) { Objects.requireNonNull(keyPredicate, "keyPredicate can not be null"); List result = new ArrayList<>(1); data.forEach((key, value) -> { @@ -271,6 +339,10 @@ public V findIfPresent(Predicate keyPredicate) { } public void put(K key, V value) { + useCache(() -> putCached(key, value)); + } + + private void putCached(K key, V value) { Objects.requireNonNull(value, "value can not be null"); StripeState stripe = stripeState(key); synchronized (stripe) { @@ -286,6 +358,10 @@ public V compute(K key, BiFunction remappingFunction) { } public V computeAndRun(K key, BiFunction remappingFunction, Runnable afterMutation) { + return useCache(() -> computeAndRunCached(key, remappingFunction, afterMutation), () -> null); + } + + private V computeAndRunCached(K key, BiFunction remappingFunction, Runnable afterMutation) { BiFunction remapper = Objects.requireNonNull(remappingFunction, "remappingFunction can not be null"); Runnable action = Objects.requireNonNull(afterMutation, "afterMutation can not be null"); StripeState stripe = stripeState(key); @@ -304,6 +380,10 @@ public V computeAndRun(K key, BiFunction remappingFunction, Runnable af } public V computeAfterValidation(K key, BiFunction remappingFunction, Runnable validationAction) { + return useCache(() -> computeAfterValidationCached(key, remappingFunction, validationAction), () -> null); + } + + private V computeAfterValidationCached(K key, BiFunction remappingFunction, Runnable validationAction) { BiFunction remapper = Objects.requireNonNull(remappingFunction, "remappingFunction can not be null"); Runnable validation = Objects.requireNonNull(validationAction, "validationAction can not be null"); StripeState stripe = stripeState(key); @@ -328,6 +408,10 @@ public V computeAfterValidation(K key, BiFunction remappingFunction, Ru * entry. If it throws, the value is not updated, but completed external side effects are not rolled back. */ public V computeWithCommitAction(K key, BiFunction remappingFunction, Runnable commitAction) { + return useCache(() -> computeWithCommitActionCached(key, remappingFunction, commitAction), () -> null); + } + + private V computeWithCommitActionCached(K key, BiFunction remappingFunction, Runnable commitAction) { BiFunction remapper = Objects.requireNonNull(remappingFunction, "remappingFunction can not be null"); Runnable action = Objects.requireNonNull(commitAction, "commitAction can not be null"); StripeState stripe = stripeState(key); @@ -351,15 +435,25 @@ public void invalidateKey(K key) { public void invalidateKeyAndRun(K key, Runnable afterInvalidation) { Runnable action = Objects.requireNonNull(afterInvalidation, "afterInvalidation can not be null"); - StripeState stripe = stripeState(key); - synchronized (stripe) { - bumpAction(stripe, key); - data.invalidateKey(key); + useCache(() -> { + StripeState stripe = stripeState(key); + synchronized (stripe) { + bumpAction(stripe, key); + data.invalidateKey(key); + action.run(); + } + return null; + }, () -> { action.run(); - } + return null; + }); } public void invalidateAll() { + useCache(this::invalidateAllCached); + } + + private void invalidateAllCached() { for (int i = 0; i < stripeCount; i++) { StripeState stripe = stripeStates.get(i); if (stripe != null) { @@ -370,11 +464,81 @@ public void invalidateAll() { } } } - owner.invalidateCatalog(); + owner.invalidateScope(invalidationScope); + } + + /** Auxiliary state has the same accepted-operation lifetime as this exact physical registration. */ + public void onRetired(Runnable cleanup) { + synchronized (lifecycleLock) { + if (retired || retirementCleanup != null) { + throw new IllegalStateException("Retirement cleanup must be bound once before retirement"); + } + retirementCleanup = Objects.requireNonNull(cleanup, "cleanup"); + } + } + + private void releaseRegistration() { + if (registrationReleased.compareAndSet(false, true)) { + try { + owner.remove(data); + } finally { + if (retirementCleanup != null) { + retirementCleanup.run(); + } + } + } + } + + /** + * Stops accepting cached operations without waiting for a remote load. Accepted operations pin the physical + * entry until they finish; later reads use the loader without publishing cache values or ID side effects. + * Cleanup removes this exact registration, never the current database scope or a replacement instance. + */ + public void close() { + boolean remove; + synchronized (lifecycleLock) { + retired = true; + remove = activeUses == 0; + } + if (remove) { + releaseRegistration(); + } } public void forEach(BiConsumer consumer) { - data.forEach(consumer); + useCache(() -> data.forEach(consumer)); + } + + private void useCache(Runnable operation) { + useCache(() -> { + operation.run(); + return null; + }, () -> null); + } + + private R useCache(Supplier operation, Supplier uncached) { + boolean accepted; + synchronized (lifecycleLock) { + accepted = !retired; + if (accepted) { + activeUses++; + } + } + if (!accepted) { + return uncached.get(); + } + try { + return operation.get(); + } finally { + boolean remove; + synchronized (lifecycleLock) { + activeUses--; + remove = retired && activeUses == 0; + } + if (remove) { + releaseRegistration(); + } + } } public MetaCacheEntryStats stats() { @@ -441,6 +605,14 @@ private V loadAndPause(K key, Function loadFunction) { return loaded; } + private V loadWithoutCaching(K key, Function loadFunction) { + V loaded = loadFunction.apply(key); + if (loaded != null && discardedValueAction != null) { + discardedValueAction.accept(key, loaded); + } + return loaded; + } + private static com.github.benmanes.caffeine.cache.RemovalCause toCaffeineRemovalCause( MetaCacheRemovalReason reason) { return com.github.benmanes.caffeine.cache.RemovalCause.valueOf(reason.name()); @@ -470,7 +642,7 @@ private ActionToken beginAction(StripeState stripe, K key) { } private boolean isCurrent(StripeState stripe, ActionToken token) { - return stripe.activeActions != null + return !retired && stripe.activeActions != null && stripe.activeActions.get(token.key) == token.state && token.state.generation == token.generation; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/IdNameIndex.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/IdNameIndex.java index fcc075b80fe799..0426685bf1ce87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/IdNameIndex.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/IdNameIndex.java @@ -17,136 +17,210 @@ package org.apache.doris.datasource.metacache; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; + import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; -import com.google.common.collect.Maps; import org.jetbrains.annotations.Nullable; import java.util.Objects; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; +import java.util.function.Function; /** - * Runtime-only one-to-one index between metadata IDs and canonical local names. - * - *

Reads by ID are lock-free. All mutations are serialized so the two maps have a stable one-to-one relationship - * after each mutation completes. The reverse map is intentionally kept internal: production callers use it only - * indirectly for uniqueness validation and constant-time removal by name. + * Runtime-only identity index. Identities survive object TTL/eviction, but share their semantic resource scope. + * Physical cleanup removes the exact identity record, so delayed cleanup cannot erase a same-name recreation. */ public final class IdNameIndex { private final String scope; - private final ConcurrentMap idToName = Maps.newConcurrentMap(); - private final ConcurrentMap nameToId = Maps.newConcurrentMap(); + private final ConcurrentMap byId = new ConcurrentHashMap<>(); + private final ConcurrentMap byName = new ConcurrentHashMap<>(); + private volatile Binding binding; public IdNameIndex(String scope) { - this.scope = Objects.requireNonNull(scope, "scope can not be null"); + this.scope = Objects.requireNonNull(scope, "scope"); + } + + /** Bind once per owning catalog/database instance; standalone indexes retain the same validation contract. */ + public synchronized void bind(CatalogMetaCache owner, String entryName, Function resolver) { + if (binding != null && binding.owner == owner) { + return; + } + if (binding != null) { + close(binding.owner); + } + MetaCache cache = owner.create(MetaCacheDefinition.builder(entryName, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, Long.MAX_VALUE), + name -> resolver.apply(name).in(ScopePath.Coordinates.LOCAL)) + .budgetPolicy(MetaCacheDefinition.BudgetPolicy.COUNT_ONLY) + .removalListener((name, identity, reason) -> removeExact(identity)) + .build()); + binding = new Binding(owner, cache); } @Nullable public String getName(long id) { - return idToName.get(id); + Identity identity = byId.get(id); + return isCurrent(identity) ? identity.name : null; + } + + private boolean isCurrent(Identity identity) { + if (identity == null || byId.get(identity.id) != identity || byName.get(identity.name) != identity) { + return false; + } + Binding current = binding; + if (current == null) { + return true; + } + try { + return !current.owner.isClosed() && current.cache.getIfPresent(identity.name) == identity; + } catch (IllegalStateException failure) { + // The catalog root may close the physical registration concurrently with this index read. + if (binding != current || current.owner.isClosed()) { + return false; + } + throw failure; + } } - /** - * Validate one identity without publishing it. - * - *

Callers use this before mutating names/object caches so a pre-existing identity conflict fails without - * leaving those caches partially updated. The final {@link #put(long, String)} must still run inside the - * FeMetaCacheEntry publication action to keep the ID side effect ordered with object invalidation. - */ public void checkCanPut(long id, String name) { - Objects.requireNonNull(name, "name can not be null"); - Long existingId = nameToId.get(name); - String existingName = idToName.get(id); - - // Keep the common exact-match and cold-index paths lock-free. A later put remains the authoritative - // validation inside the caller's FeMetaCacheEntry publication window. - if ((existingId == null && existingName == null) - || (Long.valueOf(id).equals(existingId) && name.equals(existingName))) { + Identity named = byName.get(name); + Identity identified = byId.get(id); + if (named == identified && (named == null || (named.id == id && named.name.equals(name)))) { return; } - - // Re-read suspicious snapshots under the mutation lock. This both confirms real conflicts and avoids - // rejecting a reader that observed the two maps between the writes of one synchronized mutation. synchronized (this) { checkCanPutLocked(id, name); } } private void checkCanPutLocked(long id, String name) { - Long existingId = nameToId.get(name); - String existingName = idToName.get(id); - - Preconditions.checkState(existingId == null || existingId == id, + Objects.requireNonNull(name, "name"); + Identity named = byName.get(name); + Identity identified = byId.get(id); + if (!isCurrent(named)) { + removeExact(named); + named = null; + } + if (!isCurrent(identified)) { + removeExact(identified); + identified = null; + } + Preconditions.checkState(named == null || named.id == id, "%s name '%s' is already mapped to id %s, can not map to id %s", - scope, name, existingId, id); - Preconditions.checkState(existingName == null || existingName.equals(name), + scope, name, named == null ? null : named.id, id); + Preconditions.checkState(identified == null || identified.name.equals(name), "%s id %s is already mapped to name '%s', can not map to name '%s'", - scope, id, existingName, name); - Preconditions.checkState((existingId == null) == (existingName == null), - "%s index is inconsistent for id %s and name '%s': existing id %s, existing name '%s'", - scope, id, name, existingId, existingName); + scope, id, identified == null ? null : identified.name, name); + Preconditions.checkState(named == identified, "%s index is inconsistent for %s:%s", scope, id, name); + } + + /** Called inside the object cache's guarded publication action. */ + public void put(long id, String name) { + checkCanPut(id, name); + Binding captured = binding; + MetaCache cache = captured == null ? null : captured.cache; + Identity identity = new Identity(id, name); + // Enter cache publication before taking the index monitor. Other guarded object publications may + // hold the catalog read phase while waiting for this monitor; the reverse order would deadlock. + if (cache != null) { + while (true) { + Identity current = cache.getIfPresent(name); + if (current != null) { + Preconditions.checkState(current.id == id, "%s identity conflict for %s", scope, name); + identity = current; + break; + } + if (cache.compareAndSet(name, null, identity)) { + break; + } + } + } + synchronized (this) { + if (binding != captured || (cache != null && cache.getIfPresent(name) != identity)) { + return; + } + checkCanPutLocked(id, name); + byId.put(id, identity); + byName.put(name, identity); + } } - /** - * Publish one identity. Re-publishing the same pair is idempotent; every other conflict is a correctness error. - */ - public synchronized void put(long id, String name) { - Objects.requireNonNull(name, "name can not be null"); - checkCanPutLocked(id, name); - idToName.put(id, name); - nameToId.put(name, id); + private synchronized void removeExact(Identity identity) { + if (identity != null) { + byId.remove(identity.id, identity); + byName.remove(identity.name, identity); + } } - /** - * Remove the identity associated with a canonical local name in constant time. - */ public synchronized void removeName(String name) { - Objects.requireNonNull(name, "name can not be null"); - Long id = nameToId.get(name); - if (id == null) { - return; + Objects.requireNonNull(name, "name"); + Identity identity = byName.get(name); + Binding current = binding; + if (current != null && !current.owner.isClosed()) { + current.cache.invalidateKey(name); } - String indexedName = idToName.get(id); - Preconditions.checkState(name.equals(indexedName), - "%s index is inconsistent for name '%s' and id %s: indexed name '%s'", - scope, name, id, indexedName); - - nameToId.remove(name); - boolean removed = idToName.remove(id, name); - Preconditions.checkState(removed, - "%s index failed to remove id %s and name '%s'", scope, id, name); + removeExact(identity); } - /** - * Find a retained canonical name for the mode-2 cache-only invalidation fallback. - * - *

This deliberately preserves the existing O(N) fallback behavior. Normal lookup and exact-name cleanup do - * not use this method. - */ @Nullable public String findNameIgnoreCase(String requestedName) { - Objects.requireNonNull(requestedName, "requestedName can not be null"); - return idToName.values().stream() - .filter(name -> name.equalsIgnoreCase(requestedName)) - .findFirst() - .orElse(null); + return byId.values().stream().filter(this::isCurrent).map(identity -> identity.name) + .filter(name -> name.equalsIgnoreCase(requestedName)).findFirst().orElse(null); } public synchronized void clear() { - idToName.clear(); - nameToId.clear(); + Binding current = binding; + if (current != null && !current.owner.isClosed()) { + byName.keySet().forEach(current.cache::invalidateKey); + } + byId.clear(); + byName.clear(); + } + + /** Retires this instance's registration without invalidating a replacement resource. */ + public synchronized void close(CatalogMetaCache owner) { + Binding current = binding; + binding = null; + if (current != null) { + owner.remove(current.cache); + } + byId.clear(); + byName.clear(); } @VisibleForTesting - public synchronized boolean containsMappingForTest(long id, String name) { - return name.equals(idToName.get(id)) && Long.valueOf(id).equals(nameToId.get(name)); + public boolean containsMappingForTest(long id, String name) { + return name.equals(getName(id)); } @VisibleForTesting public synchronized int sizeForTest() { - Preconditions.checkState(idToName.size() == nameToId.size(), - "%s index size mismatch: idToName=%s, nameToId=%s", - scope, idToName.size(), nameToId.size()); - return idToName.size(); + return (int) byId.values().stream().filter(this::isCurrent).count(); + } + + private static final class Binding { + private final CatalogMetaCache owner; + private final MetaCache cache; + + private Binding(CatalogMetaCache owner, MetaCache cache) { + this.owner = owner; + this.cache = cache; + } + } + + private static final class Identity { + private final long id; + private final String name; + + private Identity(long id, String name) { + this.id = id; + this.name = name; + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryInvalidation.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryInvalidation.java index 7d68fbd51ff4ee..d2e69593281d84 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryInvalidation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryInvalidation.java @@ -75,8 +75,10 @@ public static MetaCacheEntryInvalidation forTableIdentity( Function dbNameExtractor, Function tableNameExtractor) { Objects.requireNonNull(dbNameExtractor, "dbNameExtractor"); Objects.requireNonNull(tableNameExtractor, "tableNameExtractor"); + // Metadata is a child of the identity scope: ordinary writes must not evict FE lookup objects. return new MetaCacheEntryInvalidation<>( - key -> ScopePath.table(dbNameExtractor.apply(key), tableNameExtractor.apply(key)), + key -> ScopePath.tableMetadata(dbNameExtractor.apply(key), tableNameExtractor.apply(key)) + .in(ScopePath.Coordinates.LOCAL), dbName -> key -> dbNameExtractor.apply(key).equals(dbName), (dbName, tableName) -> key -> dbNameExtractor.apply(key).equals(dbName) && tableNameExtractor.apply(key).equals(tableName), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java index 60c2a5422e4334..92616b7312aecc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalog.java @@ -61,7 +61,6 @@ import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalFunctionRules; -import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SessionContext; import org.apache.doris.datasource.connector.converter.ConnectorBranchTagConverter; @@ -69,6 +68,7 @@ import org.apache.doris.datasource.connector.converter.ConnectorPartitionFieldConverter; import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.datasource.log.InitCatalogLog; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.foundation.property.StoragePropertiesException; import org.apache.doris.foundation.security.JdbcDriverUrlSecurity; import org.apache.doris.fs.FileSystemFactory; @@ -215,6 +215,7 @@ protected Connector createConnectorFromProperties() { DefaultConnectorContext context = new DefaultConnectorContext(name, id, this::getExecutionAuthenticator, () -> catalogProperty.getStorageAdaptersMap(), catalogProperty::getEffectiveRawStorageProperties); + context.bindCacheRoot(this::getFeMetaCacheOwner); this.connectorContext = context; // The standalone entry point, same as CatalogFactory uses: this is the second door onto a catalog (the // lazy build after image deserialization), and both doors must agree on what may become a catalog. @@ -441,33 +442,9 @@ public void notifyPropertiesUpdated(Map updatedProps) { super.notifyPropertiesUpdated(updatedProps); } - /** - * Invalidates connector-owned metadata before FE caches because connector metadata feeds row-count loading. - * The connector field is read directly so refreshing an uninitialized catalog does not initialize it. - */ - @Override - public void onRefreshCache(boolean invalidCache) { - if (!invalidCache) { - super.onRefreshCache(false); - return; - } - try { - invalidateAllConnectorCachesIfPresent(); - } finally { - super.onRefreshCache(true); - } - } - - /** - * Invalidates connector-owned caches without initializing or rebuilding the connector. - * - *

Replay also uses this when no database object is cached and remote cache keys cannot be recovered. - */ - public void invalidateAllConnectorCachesIfPresent() { - Connector localConnector = connector; - if (localConnector != null) { - localConnector.invalidateAll(); - } + /** Cache coordination is cache-only; replay must never initialize a connector. */ + public Connector getConnectorIfPresent() { + return connector; } @Override @@ -646,7 +623,7 @@ public void overlayMetaCacheConfig(Map metaCacheProperties) { /** * Routes CREATE TABLE through the connector SPI. Returning {@code true} for an existing table preserves * the caller's CTAS short-circuit contract. A newly created table persists canonical local names; every - * successful path invalidates connector, metadata, and row-count caches in that order. + * successful path applies the same catalog cache change to connector metadata and derived FE state. */ @Override public boolean createTable(CreateTableInfo createTableInfo) throws UserException { @@ -702,7 +679,10 @@ public boolean createTable(CreateTableInfo createTableInfo) throws UserException } org.apache.doris.persist.CreateTableInfo persistInfo = new org.apache.doris.persist.CreateTableInfo( - getName(), db.getFullName(), localTableName); + getName(), db.getFullName(), localTableName).withExternalCacheChange( + ExternalObjectLog.createForExplicitRefreshTable(getId(), db.getFullName(), + localTableName, 0L).withRemoteNames( + db.getRemoteName(), createTableInfo.getTableName())); Env.getCurrentEnv().getEditLog().logCreateTable(persistInfo); invalidateCreatedTableCaches(db, createTableInfo.getTableName(), localTableName); LOG.info("finished to create table {}.{}.{}", getName(), @@ -751,7 +731,9 @@ public void createDb(String dbName, boolean ifNotExists, Map pro } throw new DdlException(e.getMessage(), e); } - Env.getCurrentEnv().getEditLog().logCreateDb(new CreateDbInfo(getName(), localDbName, null)); + Env.getCurrentEnv().getEditLog().logCreateDb(new CreateDbInfo(getName(), localDbName, null) + .withExternalCacheChange(ExternalObjectLog.createForRefreshDb(getId(), localDbName) + .withRemoteNames(dbName, null))); invalidateCreatedDatabaseCaches(dbName, localDbName); LOG.info("finished to create database {}.{}", getName(), dbName); } @@ -788,12 +770,11 @@ public void dropDb(String dbName, boolean ifExists, boolean force) throws DdlExc } catch (DorisConnectorException e) { throw new DdlException(e.getMessage(), e); } - Env.getCurrentEnv().getEditLog().logDropDb(new DropDbInfo(getName(), db.getFullName())); - try { - connector.invalidateDb(db.getRemoteName()); - } finally { - unregisterDatabase(db.getFullName()); - } + Env.getCurrentEnv().getEditLog().logDropDb(new DropDbInfo(getName(), db.getFullName()) + .withExternalCacheChange(ExternalObjectLog.createForRefreshDb(getId(), db.getFullName()) + .withRemoteNames(db.getRemoteName(), null))); + CatalogMetadataChanges.apply(this, ExternalObjectLog.createForRefreshDb(getId(), db.getFullName()) + .withRemoteNames(db.getRemoteName(), null)); LOG.info("finished to drop database {}.{}", getName(), dbName); } @@ -946,7 +927,8 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa } long updateTime = System.currentTimeMillis(); Env.getCurrentEnv().getEditLog().logTruncateTable( - new TruncateTableInfo(getName(), db.getFullName(), dorisTable.getName(), partitions, updateTime)); + new TruncateTableInfo(getName(), db.getFullName(), dorisTable.getName(), partitions, updateTime) + .withExternalCacheChange(CatalogMetadataChanges.table(dorisTable, false, updateTime))); Env.getCurrentEnv().getRefreshManager().refreshTableInternal(db, dorisTable, updateTime); LOG.info("finished to truncate table {}.{}.{}", getName(), dbName, tableName); } @@ -958,95 +940,29 @@ public void truncateTable(String dbName, String tableName, PartitionNamesInfo pa */ @Override public void replayTruncateTable(TruncateTableInfo info) { - Optional> db = getDbForReplay(info.getDb()); - Optional table = db.flatMap(database -> database.getTableForReplay(info.getTable())); - if (table.isPresent()) { - Env.getCurrentEnv().getRefreshManager() - .refreshTableInternal(db.get(), table.get(), info.getUpdateTime()); - return; - } - ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); - try { - Connector replayConnector = connector; - if (isInitialized() && replayConnector != null) { - if (db.isPresent()) { - replayConnector.invalidateDb(db.get().getRemoteName()); - } else { - replayConnector.invalidateAll(); - } - } - } finally { - long dbId = Util.genIdByName(getName(), info.getDb()); - cacheMgr.invalidateTable(getId(), dbId, info.getDb(), - Util.genIdByName(getName(), info.getDb(), info.getTable()), info.getTable()); - } + CatalogMetadataChanges.apply(this, info.getExternalCacheChange() != null ? info.getExternalCacheChange() + : ExternalObjectLog.createForRefreshTable( + getId(), info.getDb(), info.getTable(), info.getUpdateTime())); } @Override - public void replayDropTable(String dbName, String tblName) { - try { - invalidateTableConnectorCacheForReplay(dbName, tblName); - } finally { - super.replayDropTable(dbName, tblName); - } + public void replayDropTable(String dbName, String tableName) { + applyTableLookupChange(dbName, tableName, null, null); } @Override public void replayDropDb(String dbName) { - try { - invalidateDatabaseConnectorCacheForReplay(dbName); - } finally { - super.replayDropDb(dbName); - } + CatalogMetadataChanges.apply(this, ExternalObjectLog.createForRefreshDb(getId(), dbName)); } @Override - public void replayCreateTable(String dbName, String tblName) { - try { - invalidateDatabaseConnectorCacheForReplay(dbName); - } finally { - super.replayCreateTable(dbName, tblName); - } + public void replayCreateTable(String dbName, String tableName) { + applyTableLookupChange(dbName, tableName, null, null); } @Override public void replayCreateDb(String dbName) { - try { - invalidateDatabaseConnectorCacheForReplay(dbName); - } finally { - super.replayCreateDb(dbName); - } - } - - // Replay never initializes a cold catalog; missing local-to-remote identity widens the live connector scope. - private void invalidateDatabaseConnectorCacheForReplay(String localDbName) { - Connector replayConnector = connector; - if (isInitialized() && replayConnector != null) { - Optional> db = getDbForReplay(localDbName); - if (db.isPresent()) { - replayConnector.invalidateDb(db.get().getRemoteName()); - } else { - replayConnector.invalidateAll(); - } - } - } - - private void invalidateTableConnectorCacheForReplay(String localDbName, String localTableName) { - Connector replayConnector = connector; - if (!isInitialized() || replayConnector == null) { - return; - } - Optional> db = getDbForReplay(localDbName); - if (!db.isPresent()) { - replayConnector.invalidateAll(); - return; - } - Optional table = db.get().getTableForReplay(localTableName); - if (table.isPresent()) { - replayConnector.invalidateTable(db.get().getRemoteName(), table.get().getRemoteName()); - } else { - replayConnector.invalidateDb(db.get().getRemoteName()); - } + CatalogMetadataChanges.apply(this, ExternalObjectLog.createForRefreshDb(getId(), dbName)); } /** @@ -1450,8 +1366,7 @@ private static ConnectorColumnPath toConnectorPath(ColumnPath columnPath) { /** Persists replay identity, then refreshes the cached table when present or the resolved transient table. */ protected void afterExternalDdl(ExternalTable externalTable, long updateTime) { Env.getCurrentEnv().getEditLog().logRefreshExternalTable( - ExternalObjectLog.createForRefreshTable( - getId(), externalTable.getDbName(), externalTable.getName(), updateTime)); + CatalogMetadataChanges.table(externalTable, false, updateTime)); ExternalTable refreshTarget = getDbForReplay(externalTable.getDbName()) .flatMap(db -> db.getTableForReplay(externalTable.getName())) .orElse(externalTable); @@ -1460,62 +1375,41 @@ protected void afterExternalDdl(ExternalTable externalTable, long updateTime) { } private void invalidateCreatedDatabaseCaches(String remoteDbName, String localDbName) { - try { - connector.invalidateDb(remoteDbName); - } finally { - invalidateCreatedDatabase(localDbName); - } + CatalogMetadataChanges.apply(this, ExternalObjectLog.createForRefreshDb(getId(), localDbName) + .withRemoteNames(remoteDbName, null)); } private void invalidateCreatedTableCaches(ExternalDatabase db, String remoteTableName, String localTableName) { - try { - connector.invalidateTable(db.getRemoteName(), remoteTableName); - } finally { - invalidateCreatedTable(db.getFullName(), localTableName); - } + applyTableLookupChange(db.getFullName(), localTableName, db.getRemoteName(), remoteTableName); + } + + private void applyTableLookupChange(String database, String table, String remoteDb, String remoteTable) { + CatalogMetadataChanges.apply(this, ExternalObjectLog.createForExplicitRefreshTable( + getId(), database, table, 0L).withRemoteNames(remoteDb, remoteTable)); } private void finishDropTable(ExternalDatabase db, ExternalTable table) { Env.getCurrentEnv().getEditLog().logDropTable( - new DropInfo(getName(), db.getFullName(), table.getName())); - try { - connector.invalidateTable(table.getRemoteDbName(), table.getRemoteName()); - } finally { - invalidateDroppedTable(db.getFullName(), table.getName()); - } + new DropInfo(getName(), db.getFullName(), table.getName()) + .withExternalCacheChange(CatalogMetadataChanges.table(table, true, 0L))); + CatalogMetadataChanges.apply(this, CatalogMetadataChanges.table(table, true, 0L)); } protected void afterExternalRename(ExternalDatabase db, ExternalTable table, String remoteNewTableName, String localNewTableName) { - String dbName = db.getFullName(); - String oldTableName = table.getName(); - // The remote rename has committed; persist replay identity before fallible local bookkeeping. - Env.getCurrentEnv().getEditLog().logRefreshExternalTable( - ExternalObjectLog.createForRenameTable( - getId(), dbName, oldTableName, localNewTableName)); - try { - try { - connector.invalidateTable(table.getRemoteDbName(), table.getRemoteName()); - } finally { - connector.invalidateTable(table.getRemoteDbName(), remoteNewTableName); - } - } finally { - try { - getDbForReplay(dbName).ifPresent( - cachedDb -> cachedDb.invalidateTableRename(oldTableName, localNewTableName)); - } finally { - try { - Env.getCurrentEnv().getConstraintManager().renameTable( - new TableNameInfo(getName(), dbName, oldTableName), - new TableNameInfo(getName(), dbName, localNewTableName)); - } finally { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableRename( - getId(), db.getId(), dbName, table.getId(), oldTableName, - Util.genIdByName(getName(), dbName, localNewTableName), localNewTableName); - } - } - } + ExternalObjectLog change = ExternalObjectLog.createForRenameTable( + getId(), db.getFullName(), table.getName(), localNewTableName) + .withRemoteNames(table.getRemoteDbName(), table.getRemoteName()); + change.setRemoteNewTableName(remoteNewTableName); + change.setDbId(db.getId()); + change.setTableId(table.getId()); + // Persist the committed remote identities before fallible local bookkeeping. + Env.getCurrentEnv().getEditLog().logRefreshExternalTable(change); + CatalogMetadataChanges.apply(this, change); + Env.getCurrentEnv().getConstraintManager().renameTable( + new TableNameInfo(getName(), db.getFullName(), table.getName()), + new TableNameInfo(getName(), db.getFullName(), localNewTableName)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java index 6cbea40af7505b..c5bdbdf54ab54e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java @@ -43,12 +43,14 @@ import org.apache.doris.connector.spi.write.ConnectorRowChangeStyle; import org.apache.doris.connector.spi.write.ConnectorRowLevelDmlRequest; import org.apache.doris.connector.spi.write.ConnectorWritePlanProvider; +import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.TablePartitionValues; import org.apache.doris.datasource.connector.converter.ConnectorColumnConverter; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.PluginDrivenMvccSnapshot; import org.apache.doris.datasource.systable.PartitionsSysTable; @@ -119,8 +121,14 @@ public PluginDrivenExternalTable(long id, String name, String remoteName, */ public Optional resolveConnectorTableHandle( ConnectorSession session, ConnectorMetadata metadata) { + Optional observation = + CatalogMetadataChanges.observeTableLookup(this); String dbName = db != null ? db.getRemoteName() : ""; - return metadata.getTableHandle(session, dbName, getRemoteName()); + Optional handle = metadata.getTableHandle(session, dbName, getRemoteName()); + if (!handle.isPresent()) { + observation.ifPresent(CatalogMetadataChanges.MissingTableObservation::missing); + } + return handle; } /** @@ -667,8 +675,7 @@ public Optional initSchema() { } Optional handleOpt = resolveConnectorTableHandle(session, metadata); if (!handleOpt.isPresent()) { - LOG.warn("Table handle not found for plugin-driven table: {}.{}", dbName, tableName); - return Optional.empty(); + throw new CacheException("Table %s.%s does not exist", null, dbName, tableName); } ConnectorTableSchema tableSchema = metadata.getTableSchema(session, handleOpt.get()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java index 522ae02142915a..5749e0438df555 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateDbInfo.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Database; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.persist.gson.GsonUtils; import com.google.gson.annotations.SerializedName; @@ -29,6 +30,18 @@ import java.io.IOException; public class CreateDbInfo implements Writable { + @SerializedName("externalCacheChange") + private ExternalObjectLog externalCacheChange; + + public ExternalObjectLog getExternalCacheChange() { + return externalCacheChange; + } + + public CreateDbInfo withExternalCacheChange(ExternalObjectLog change) { + this.externalCacheChange = change; + return this; + } + @SerializedName("ctl") private String ctlName; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java index 31a6bb0bc83e10..7b5faa62b521a1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/CreateTableInfo.java @@ -21,6 +21,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.persist.gson.GsonUtils; import com.google.common.base.Strings; @@ -34,6 +35,18 @@ import java.util.Objects; public class CreateTableInfo implements Writable { + @SerializedName("externalCacheChange") + private ExternalObjectLog externalCacheChange; + + public ExternalObjectLog getExternalCacheChange() { + return externalCacheChange; + } + + public CreateTableInfo withExternalCacheChange(ExternalObjectLog change) { + this.externalCacheChange = change; + return this; + } + public static final Logger LOG = LoggerFactory.getLogger(CreateTableInfo.class); @SerializedName(value = "ctl") diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java index c9d2009521ad24..de338649921ac3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java @@ -19,6 +19,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.persist.gson.GsonUtils; import com.google.gson.annotations.SerializedName; @@ -28,6 +29,18 @@ import java.io.IOException; public class DropDbInfo implements Writable { + @SerializedName("externalCacheChange") + private ExternalObjectLog externalCacheChange; + + public ExternalObjectLog getExternalCacheChange() { + return externalCacheChange; + } + + public DropDbInfo withExternalCacheChange(ExternalObjectLog change) { + this.externalCacheChange = change; + return this; + } + @SerializedName(value = "ctl") private String ctlName; @SerializedName(value = "dbName") diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java index c2082ecc22610d..a4d6e44259e779 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java @@ -20,6 +20,7 @@ import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.persist.gson.GsonUtils; import com.google.common.base.Strings; @@ -30,6 +31,18 @@ import java.io.IOException; public class DropInfo implements Writable { + @SerializedName("externalCacheChange") + private ExternalObjectLog externalCacheChange; + + public ExternalObjectLog getExternalCacheChange() { + return externalCacheChange; + } + + public DropInfo withExternalCacheChange(ExternalObjectLog change) { + this.externalCacheChange = change; + return this; + } + @SerializedName(value = "ctl") private String ctl; @SerializedName(value = "db") diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index 4530be5af9ae34..5cd90207998182 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -67,6 +67,7 @@ import org.apache.doris.datasource.log.InitCatalogLog; import org.apache.doris.datasource.log.InitDatabaseLog; import org.apache.doris.datasource.log.MetaIdMappingsLog; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; import org.apache.doris.dictionary.Dictionary; import org.apache.doris.ha.MasterInfo; import org.apache.doris.indexpolicy.DropIndexPolicyLog; @@ -415,7 +416,12 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { ExternalCatalog ctl = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr() .getCatalog(info.getCtl()); if (ctl != null) { - ctl.replayDropTable(info.getDb(), info.getTableName()); + if (info.getExternalCacheChange() != null) { + CatalogMetadataChanges.apply( + ctl, info.getExternalCacheChange()); + } else { + ctl.replayDropTable(info.getDb(), info.getTableName()); + } } } break; diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java index 7edffceaf0be2c..2513a1ff21f569 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/TruncateTableInfo.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Partition; import org.apache.doris.common.io.Text; import org.apache.doris.common.io.Writable; +import org.apache.doris.datasource.log.ExternalObjectLog; import org.apache.doris.persist.gson.GsonUtils; import com.google.common.collect.Lists; @@ -33,6 +34,18 @@ import java.util.Map; public class TruncateTableInfo implements Writable { + @SerializedName("externalCacheChange") + private ExternalObjectLog externalCacheChange; + + public ExternalObjectLog getExternalCacheChange() { + return externalCacheChange; + } + + public TruncateTableInfo withExternalCacheChange(ExternalObjectLog change) { + this.externalCacheChange = change; + return this; + } + @SerializedName(value = "ctl") private String ctl; @SerializedName(value = "dbId") diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java index 8699ff566201fc..3cbf693b77c8b4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshDbTest.java @@ -22,7 +22,6 @@ import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.FeConstants; import org.apache.doris.datasource.CatalogIf; -import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.datasource.test.TestExternalDatabase; import org.apache.doris.datasource.test.TestExternalTable; @@ -97,21 +96,22 @@ public void testRefreshDatabase() throws Exception { RefreshDatabaseCommand refreshDatabaseCommand = new RefreshDatabaseCommand("test1", "db1", null); refreshDatabaseCommand.run(connectContext, null); - long l3 = db1.getLastUpdateTime(); - Assertions.assertTrue(l3 == l2); - // the table will be recreated after refresh. - // so we need to get table again - table = db1.getTable("tbl11").get(); - Assertions.assertFalse(table.isObjectCreated()); - test1.getDbNullable("db1").getTables(); - Assertions.assertFalse(table.isObjectCreated()); + TestExternalDatabase replacement = (TestExternalDatabase) test1.getDbNullable("db1"); + Assertions.assertNotSame(db1, replacement); + Assertions.assertFalse(replacement.isInitialized()); + TestExternalTable replacementTable = replacement.getTable("tbl11").get(); + Assertions.assertNotSame(table, replacementTable); + Assertions.assertFalse(replacementTable.isObjectCreated()); + Assertions.assertTrue(replacement.getLastUpdateTime() > 0); + Assertions.assertEquals(l2, db1.getLastUpdateTime()); refreshDatabaseCommand.run(connectContext, null); - Assertions.assertFalse(((ExternalDatabase) test1.getDbNullable("db1")).isInitialized()); - table.makeSureInitialized(); - long l4 = db1.getLastUpdateTime(); - Assertions.assertTrue(l4 > l3); - Assertions.assertTrue(((ExternalDatabase) test1.getDbNullable("db1")).isInitialized()); + TestExternalDatabase current = (TestExternalDatabase) test1.getDbNullable("db1"); + Assertions.assertNotSame(replacement, current); + Assertions.assertFalse(current.isInitialized()); + current.getTable("tbl11").get().makeSureInitialized(); + Assertions.assertTrue(current.isInitialized()); + Assertions.assertTrue(current.getLastUpdateTime() > 0); refreshDatabaseCommand.run(connectContext, null); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerRenameReplayTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerRenameReplayTest.java index 16084561377fac..e0640855e98ba3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerRenameReplayTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerRenameReplayTest.java @@ -19,6 +19,11 @@ import org.apache.doris.catalog.constraint.ConstraintManager; import org.apache.doris.common.util.Util; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.ExternalDatabase; @@ -28,9 +33,9 @@ import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.mockito.InOrder; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -40,6 +45,7 @@ public class RefreshManagerRenameReplayTest { private MockedStatic mockedEnv; + private final CatalogMetaCache root = CatalogMetaCache.unmanaged(); private CatalogMgr catalogMgr; private ExternalMetaCacheMgr metaCacheMgr; private ConstraintManager constraintManager; @@ -69,9 +75,14 @@ public void setUp() { db = mockDb(); Mockito.doReturn(catalog).when(catalogMgr).getCatalog(7L); Mockito.when(catalog.getName()).thenReturn("c"); - Mockito.when(catalog.getConnector()).thenReturn(connector); + Mockito.when(catalog.getConnectorIfPresent()).thenReturn(connector); + Mockito.when(catalog.getMetaCacheOwnerIfPresent()).thenReturn(root); + Mockito.when(catalog.resolveDatabaseNameForInvalidation("db1")).thenReturn("db1"); Mockito.doReturn(Optional.of(db)).when(catalog).getDbForReplay("db1"); + Mockito.when(db.getFullName()).thenReturn("db1"); Mockito.when(db.getRemoteName()).thenReturn("DB1"); + Mockito.when(db.resolveTableNameForInvalidation(Mockito.anyString())) + .thenAnswer(invocation -> invocation.getArgument(0)); dbId = Util.genIdByName("c", "db1"); sourceTableId = Util.genIdByName("c", "db1", "t1"); destinationTableId = Util.genIdByName("c", "db1", "t2"); @@ -79,6 +90,7 @@ public void setUp() { @AfterEach public void tearDown() { + root.close(); if (mockedEnv != null) { mockedEnv.close(); } @@ -93,24 +105,26 @@ private static ExternalDatabase mockDb() { public void testRenameReplayInvalidatesConnectorDatabaseOnFollower() { replayRename(); - InOrder order = Mockito.inOrder(connector, db, metaCacheMgr); - order.verify(connector).invalidateDb("DB1"); - order.verify(db).invalidateTableRename("t1", "t2"); - order.verify(metaCacheMgr).invalidateTableRename( - 7L, dbId, "db1", sourceTableId, "t1", destinationTableId, "t2"); + Mockito.verify(connector).cacheDependencies(ScopePath.database("DB1")); + Mockito.verify(connector, Mockito.never()).invalidateDb(Mockito.anyString()); + Mockito.verify(constraintManager).renameTable(Mockito.any(), Mockito.any()); } @Test public void testRenameReplayWithoutDatabaseObjectInvalidatesBothIdentitiesWithoutConnectorInit() { Mockito.doReturn(Optional.empty()).when(catalog).getDbForReplay("db1"); - + MetaCache old = root.create(MetaCacheDefinition.builder("old", + CacheSpec.of(true, -1, 10), key -> ScopePath.table("db1", "t1") + .in(ScopePath.Coordinates.LOCAL)).build()); + MetaCache destination = root.create(MetaCacheDefinition.builder("new", + CacheSpec.of(true, -1, 10), key -> ScopePath.table("db1", "t2") + .in(ScopePath.Coordinates.LOCAL)).build()); + old.put("key", "old"); + destination.put("key", "previous incarnation"); replayRename(); - - Mockito.verify(metaCacheMgr).invalidateTableRename( - 7L, dbId, "db1", sourceTableId, "t1", destinationTableId, "t2"); - Mockito.verify(catalog).invalidateAllConnectorCachesIfPresent(); + Assertions.assertNull(old.getIfPresent("key")); + Assertions.assertNull(destination.getIfPresent("key")); Mockito.verify(catalog, Mockito.never()).getConnector(); - Mockito.verifyNoInteractions(connector); } private void replayRename() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java index 44059931620e82..6e8285b0ee27cf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java @@ -17,574 +17,170 @@ package org.apache.doris.catalog; -import org.apache.doris.catalog.constraint.ConstraintManager; -import org.apache.doris.catalog.info.TableNameInfo; -import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.util.Util; import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; -import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.CatalogMgr; -import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; -import org.apache.doris.datasource.ExternalMetaCacheMgr; -import org.apache.doris.datasource.ExternalRowCountCache; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.log.ExternalObjectLog; -import org.apache.doris.datasource.metacache.ExternalMetaCache; -import org.apache.doris.datasource.metacache.FeMetaCacheEntry; -import org.apache.doris.datasource.metacache.MetaCacheEntryStats; import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; -import org.apache.doris.datasource.test.TestExternalCatalog; -import org.apache.doris.datasource.test.TestExternalDatabase; -import org.apache.doris.datasource.test.TestExternalTable; import org.apache.doris.persist.EditLog; -import com.google.common.util.concurrent.MoreExecutors; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.mockito.ArgumentCaptor; import org.mockito.InOrder; -import org.mockito.MockedConstruction; import org.mockito.MockedStatic; import org.mockito.Mockito; -import java.util.Collections; -import java.util.List; -import java.util.Map; import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.concurrent.atomic.AtomicLong; +/** Refresh entry points use one resource change for FE, connector and derived cache state. */ public class RefreshManagerTest { - private static final long CATALOG_ID = 10L; - private static final String CATALOG_NAME = "test_catalog"; - private static final String DATABASE_NAME = "db1"; - private static final String TABLE_NAME = "tbl1"; - private static final String NEW_TABLE_NAME = "tbl2"; - private static final long DATABASE_ID = Util.genIdByName(CATALOG_NAME, DATABASE_NAME); - private static final long TABLE_ID = Util.genIdByName(CATALOG_NAME, DATABASE_NAME, TABLE_NAME); - - private MockedStatic mockedEnv; - private TestExternalCatalog catalog; - private CountingDatabase database; - private RecordingExternalMetaCache engineCache; - private ExternalMetaCacheMgr metaCacheMgr; - private RecordingConstraintManager constraintManager; + private final CatalogMetaCache root = CatalogMetaCache.unmanaged(); + private MockedStatic env; + private CatalogMgr catalogs; + private PluginDrivenExternalCatalog catalog; + private ExternalDatabase db; + private ExternalTable table; + private Connector connector; private EditLog editLog; - private AtomicInteger databaseObjectLoadCalls; - private TestingCatalogMgr testingCatalogMgr; @BeforeEach - public void setUp() { - Map properties = Collections.singletonMap( - "catalog_provider.class", EmptyCatalogProvider.class.getName()); - catalog = new TestExternalCatalog(CATALOG_ID, CATALOG_NAME, "", properties, ""); - catalog.setInitializedForTest(true); - - database = new CountingDatabase(catalog, DATABASE_ID, DATABASE_NAME, DATABASE_NAME); - database.setInitializedForTest(true); - catalog.addDatabaseForTest(database); - - engineCache = new RecordingExternalMetaCache(); - databaseObjectLoadCalls = new AtomicInteger(); - metaCacheMgr = Mockito.spy(new ExternalMetaCacheMgr(true)); - Map cacheTypes = Deencapsulation.getField(metaCacheMgr, "cacheTypes"); - cacheTypes.clear(); - cacheTypes.put(engineCache.engine(), engineCache); - constraintManager = new RecordingConstraintManager(); - testingCatalogMgr = new TestingCatalogMgr(catalog); - TestingEnv testingEnv = new TestingEnv(testingCatalogMgr, metaCacheMgr, constraintManager); + public void setup() { + Env current = Mockito.mock(Env.class); + env = Mockito.mockStatic(Env.class); + env.when(Env::getCurrentEnv).thenReturn(current); + catalogs = Mockito.mock(CatalogMgr.class); editLog = Mockito.mock(EditLog.class); - testingEnv.setEditLog(editLog); - mockedEnv = Mockito.mockStatic(Env.class); - mockedEnv.when(Env::getCurrentEnv).thenReturn(testingEnv); + Mockito.when(current.getCatalogMgr()).thenReturn(catalogs); + Mockito.when(current.getEditLog()).thenReturn(editLog); + catalog = Mockito.mock(PluginDrivenExternalCatalog.class); + db = Mockito.mock(ExternalDatabase.class); + table = Mockito.mock(ExternalTable.class); + connector = Mockito.mock(Connector.class); + Mockito.doReturn(catalog).when(catalogs).getCatalog(10L); + Mockito.when(catalog.getId()).thenReturn(10L); + Mockito.when(catalog.getName()).thenReturn("c"); + Mockito.when(catalog.getMetaCacheOwnerIfPresent()).thenReturn(root); + Mockito.when(catalog.getConnectorIfPresent()).thenReturn(connector); + Mockito.when(catalog.resolveDatabaseNameForInvalidation("db")).thenReturn("db"); + Mockito.doReturn(Optional.of(db)).when(catalog).getDbForReplay("db"); + Mockito.when(db.getCatalog()).thenReturn(catalog); + Mockito.when(db.getFullName()).thenReturn("db"); + Mockito.when(db.getRemoteName()).thenReturn("DB"); + Mockito.when(db.getId()).thenReturn(Util.genIdByName("c", "db")); + Mockito.when(db.resolveTableNameForInvalidation(Mockito.anyString())).thenAnswer(inv -> inv.getArgument(0)); + Mockito.doReturn(Optional.of(table)).when(db).getTableForReplay("t"); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.getDb()).thenReturn(db); + Mockito.when(table.getId()).thenReturn(Util.genIdByName("c", "db", "t")); + Mockito.when(table.getName()).thenReturn("t"); + Mockito.when(table.getRemoteDbName()).thenReturn("DB"); + Mockito.when(table.getRemoteName()).thenReturn("T"); } @AfterEach - public void tearDown() { - if (mockedEnv != null) { - mockedEnv.close(); - mockedEnv = null; - } + public void cleanup() { + root.close(); + env.close(); } - @Test - public void testReplayRefreshDbReturnsWhenCatalogIsMissing() { - testingCatalogMgr.setCatalog(null); - Mockito.clearInvocations(metaCacheMgr); - - new RefreshManager().replayRefreshDb( - ExternalObjectLog.createForRefreshDb(CATALOG_ID, DATABASE_NAME)); - - Mockito.verifyNoInteractions(metaCacheMgr); + private MetaCache cache(String name, ScopePath scope) { + MetaCache cache = root.create(MetaCacheDefinition.builder(name, + CacheSpec.of(true, -1, 10), key -> scope).build()); + cache.put("key", "value"); + return cache; } @Test - public void testReplayHotRefreshDbInvalidatesMetadataThenRowCountOnce() { - ExternalObjectLog log = ExternalObjectLog.createForRefreshDb(CATALOG_ID, DATABASE_NAME); - - new RefreshManager().replayRefreshDb(log); - - InOrder order = Mockito.inOrder(metaCacheMgr); - order.verify(metaCacheMgr).invalidateDbMetadataCache(CATALOG_ID, DATABASE_NAME); - order.verify(metaCacheMgr).invalidateDbRowCountCache(CATALOG_ID, DATABASE_ID); - Mockito.verify(metaCacheMgr, Mockito.never()).invalidateDb( - Mockito.anyLong(), Mockito.anyLong(), Mockito.anyString()); + public void replayMissingCatalogDoesNothing() { + Mockito.doReturn(null).when(catalogs).getCatalog(10L); + new RefreshManager().replayRefreshDb(ExternalObjectLog.createForRefreshDb(10, "db")); + Mockito.verifyNoInteractions(catalog, connector); } @Test - public void testReplayHotRefreshDbKeepsRowCountBarrierWhenMetadataInvalidationFails() { - engineCache.failDbInvalidation = true; - ExternalObjectLog log = ExternalObjectLog.createForRefreshDb(CATALOG_ID, DATABASE_NAME); - - Assertions.assertThrows(IllegalStateException.class, () -> new RefreshManager().replayRefreshDb(log)); - - InOrder order = Mockito.inOrder(metaCacheMgr); - order.verify(metaCacheMgr).invalidateDbMetadataCache(CATALOG_ID, DATABASE_NAME); - order.verify(metaCacheMgr).invalidateDbRowCountCache(CATALOG_ID, DATABASE_ID); + public void writeRefreshPreservesLookupAndMembershipWhileUpdatingContent() { + MetaCache names = cache("names", + ScopePath.tableCollection("db").in(ScopePath.Coordinates.LOCAL)); + MetaCache object = cache("object", + ScopePath.table("db", "t").in(ScopePath.Coordinates.LOCAL)); + MetaCache source = cache("source", + ScopePath.table("DB", "T").in(ScopePath.Coordinates.REMOTE)); + new RefreshManager().refreshTableInternal(db, table, 123L); + Assertions.assertEquals("value", names.getIfPresent("key")); + Assertions.assertEquals("value", object.getIfPresent("key")); + Assertions.assertNull(source.getIfPresent("key")); + Mockito.verify(table, Mockito.atLeastOnce()).unsetObjectCreated(); + Mockito.verify(table, Mockito.atLeastOnce()).setUpdateTime(123L); + Mockito.verify(connector, Mockito.never()).invalidateTable(Mockito.anyString(), Mockito.anyString()); } @Test - public void testReplayNameBasedRefreshInvalidatesColdTableByLogName() { - seedAndEvictTableObject(); - ExternalObjectLog log = ExternalObjectLog.createForRefreshTable( - CATALOG_ID, DATABASE_NAME, TABLE_NAME, 123L); - - new RefreshManager().replayRefreshTable(log); - - assertColdTableInvalidatedByName(); - Mockito.verify(metaCacheMgr).invalidateTable( - CATALOG_ID, DATABASE_ID, DATABASE_NAME, TABLE_ID, TABLE_NAME); - } - - @Test - public void testRefreshTableRemovesRowCountLoadedDuringConnectorInvalidation() throws Exception { - PluginCatalogFixture fixture = usePluginCatalog(); - ExternalTable table = Mockito.mock(ExternalTable.class); - Mockito.when(table.getCatalog()).thenReturn(fixture.catalog); - Mockito.when(table.getDb()).thenReturn(fixture.database); - Mockito.when(table.getId()).thenReturn(TABLE_ID); - Mockito.when(table.getDbName()).thenReturn(DATABASE_NAME); - Mockito.when(table.getName()).thenReturn(TABLE_NAME); - Mockito.when(table.getRemoteName()).thenReturn(TABLE_NAME); - AtomicLong sourceRowCount = new AtomicLong(100L); - Mockito.when(table.fetchRowCountWithMetaCache(false)).thenAnswer(inv -> sourceRowCount.get()); - - CountDownLatch connectorInvalidationStarted = new CountDownLatch(1); - CountDownLatch finishConnectorInvalidation = new CountDownLatch(1); - Mockito.doAnswer(inv -> { - connectorInvalidationStarted.countDown(); - Assertions.assertTrue(finishConnectorInvalidation.await(3L, TimeUnit.SECONDS)); - sourceRowCount.set(200L); - return null; - }).when(fixture.connector).invalidateTable(DATABASE_NAME, TABLE_NAME); - - ExecutorService executor = Executors.newSingleThreadExecutor(); - try (MockedConstruction mocked = - Mockito.mockConstruction(ExternalRowCountCache.RowCountCacheLoader.class, - (loader, context) -> Mockito.when(loader.asyncLoad(Mockito.any(), Mockito.any())) - .thenAnswer(inv -> CompletableFuture.completedFuture( - Optional.of(sourceRowCount.get()))))) { - ExternalRowCountCache rowCountCache = - new ExternalRowCountCache(MoreExecutors.newDirectExecutorService()); - Deencapsulation.setField(metaCacheMgr, "rowCountCache", rowCountCache); - - Future loadDuringConnectorInvalidation = executor.submit(() -> { - Assertions.assertTrue(connectorInvalidationStarted.await(3L, TimeUnit.SECONDS)); - try { - return rowCountCache.getCachedRowCount(CATALOG_ID, DATABASE_ID, TABLE_ID, false); - } finally { - finishConnectorInvalidation.countDown(); - } - }); - new RefreshManager().refreshTableInternal(fixture.database, table, 123L); - - Assertions.assertEquals(100L, loadDuringConnectorInvalidation.get(3L, TimeUnit.SECONDS).longValue()); - Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, - rowCountCache.getCachedRowCountIfPresent(CATALOG_ID, DATABASE_ID, TABLE_ID)); - Assertions.assertEquals(200L, - rowCountCache.getCachedRowCount(CATALOG_ID, DATABASE_ID, TABLE_ID, false)); - Mockito.verify(metaCacheMgr).invalidateTable( - CATALOG_ID, DATABASE_ID, DATABASE_NAME, TABLE_ID, TABLE_NAME); - } finally { - finishConnectorInvalidation.countDown(); - executor.shutdownNow(); - } - } - - @Test - public void testRefreshTableKeepsLocalBarrierWhenConnectorInvalidationFails() { - PluginCatalogFixture fixture = usePluginCatalog(); - ExternalTable table = Mockito.mock(ExternalTable.class); - Mockito.when(table.getCatalog()).thenReturn(fixture.catalog); - Mockito.when(table.getDb()).thenReturn(fixture.database); - Mockito.when(table.getId()).thenReturn(TABLE_ID); - Mockito.when(table.getDbName()).thenReturn(DATABASE_NAME); - Mockito.when(table.getName()).thenReturn(TABLE_NAME); - Mockito.when(table.getRemoteName()).thenReturn(TABLE_NAME); - Mockito.doThrow(new IllegalStateException("connector invalidation failed")) - .when(fixture.connector).invalidateTable(DATABASE_NAME, TABLE_NAME); - - Assertions.assertThrows(IllegalStateException.class, - () -> new RefreshManager().refreshTableInternal(fixture.database, table, 123L)); - - Mockito.verify(metaCacheMgr).invalidateTable(table); + public void coldExplicitReplayUsesPersistedCoordinatesWithoutInitialization() { + Mockito.doReturn(Optional.empty()).when(catalog).getDbForReplay("db"); + MetaCache names = cache("names", + ScopePath.tableCollection("db").in(ScopePath.Coordinates.LOCAL)); + MetaCache remote = cache("remote", + ScopePath.table("DB", "T").in(ScopePath.Coordinates.REMOTE)); + new RefreshManager().replayRefreshTable(ExternalObjectLog.createForExplicitRefreshTable(10, "db", "t", 0) + .withRemoteNames("DB", "T")); + Assertions.assertNull(names.getIfPresent("key")); + Assertions.assertNull(remote.getIfPresent("key")); + Mockito.verify(catalog, Mockito.never()).getConnector(); + Mockito.verify(catalog, Mockito.never()).getFeMetaCacheOwner(); } @Test - public void testRefreshTableAfterExternalMutationLogsBeforeFallibleInvalidation() { - ExternalCatalog externalCatalog = Mockito.mock(ExternalCatalog.class); - ExternalDatabase database = Mockito.mock(ExternalDatabase.class); - ExternalTable table = Mockito.mock(ExternalTable.class); - Mockito.when(externalCatalog.getId()).thenReturn(CATALOG_ID); - Mockito.when(database.getFullName()).thenReturn(DATABASE_NAME); - Mockito.when(table.getCatalog()).thenReturn(externalCatalog); - Mockito.when(table.getDb()).thenReturn(database); - Mockito.when(table.getName()).thenReturn(TABLE_NAME); - RefreshManager refreshManager = Mockito.spy(new RefreshManager()); - IllegalStateException refreshFailure = new IllegalStateException("cache invalidation failed"); - Mockito.doThrow(refreshFailure).when(refreshManager) - .refreshTableInternal(Mockito.eq(database), Mockito.eq(table), Mockito.anyLong()); - - IllegalStateException thrown = Assertions.assertThrows(IllegalStateException.class, - () -> refreshManager.refreshTableAfterExternalMutation(table)); - Assertions.assertSame(refreshFailure, thrown); - - InOrder order = Mockito.inOrder(editLog, refreshManager); - ArgumentCaptor logCaptor = ArgumentCaptor.forClass(ExternalObjectLog.class); - order.verify(editLog).logRefreshExternalTable(logCaptor.capture()); - ExternalObjectLog log = logCaptor.getValue(); - Assertions.assertEquals(CATALOG_ID, log.getCatalogId()); - Assertions.assertEquals(DATABASE_NAME, log.getDbName()); - Assertions.assertEquals(TABLE_NAME, log.getTableName()); - order.verify(refreshManager).refreshTableInternal(database, table, log.getLastUpdateTime()); + public void mutationJournalPrecedesFallibleInvalidationAndCarriesBothCoordinates() { + RefreshManager manager = Mockito.spy(new RefreshManager()); + IllegalStateException failure = new IllegalStateException("invalidation"); + Mockito.doThrow(failure).when(manager).refreshTableInternal(Mockito.eq(db), Mockito.eq(table), + Mockito.anyLong()); + Assertions.assertSame(failure, Assertions.assertThrows(IllegalStateException.class, + () -> manager.refreshTableAfterExternalMutation(table))); + InOrder order = Mockito.inOrder(editLog, manager); + ArgumentCaptor change = ArgumentCaptor.forClass(ExternalObjectLog.class); + order.verify(editLog).logRefreshExternalTable(change.capture()); + order.verify(manager).refreshTableInternal(db, table, change.getValue().getLastUpdateTime()); + Assertions.assertEquals("db", change.getValue().getDbName()); + Assertions.assertEquals("DB", change.getValue().getRemoteDbName()); + Assertions.assertEquals("T", change.getValue().getRemoteTableName()); + Assertions.assertFalse(change.getValue().isInvalidateTableLookup()); } @Test - public void testReplayRefreshDbKeepsLocalBarrierWhenConnectorInvalidationFails() { - PluginCatalogFixture fixture = usePluginCatalog(); - Mockito.doThrow(new IllegalStateException("connector invalidation failed")) - .when(fixture.connector).invalidateDb(DATABASE_NAME); - + public void projectionFailureStillInvalidatesKnownLocalAndNumericDependencies() { + MetaCache schema = cache("schema", + ScopePath.tableMetadata("db", "t").in(ScopePath.Coordinates.LOCAL)); + MetaCache rows = cache("rows", ScopePath.table( + Long.toString(db.getId()), Long.toString(table.getId())).in(ScopePath.Coordinates.ID)); + Mockito.when(connector.cacheDependencies(Mockito.any())) + .thenThrow(new IllegalStateException("projection")); Assertions.assertThrows(IllegalStateException.class, - () -> new RefreshManager().replayRefreshDb( - ExternalObjectLog.createForRefreshDb(CATALOG_ID, DATABASE_NAME))); - - Assertions.assertFalse(fixture.database.isInitialized()); - Mockito.verify(metaCacheMgr).invalidateDbMetadataCache(CATALOG_ID, DATABASE_NAME); - Mockito.verify(metaCacheMgr).invalidateDbRowCountCache(CATALOG_ID, DATABASE_ID); + () -> new RefreshManager().refreshTableInternal(db, table, 0)); + Assertions.assertNull(schema.getIfPresent("key")); + Assertions.assertNull(rows.getIfPresent("key")); } @Test - public void testReplayRenameEvictsPreexistingSourceAndDestinationRowCounts() { - long sourceTableId = Util.genIdByName(catalog.getName(), DATABASE_NAME, TABLE_NAME); - long destinationTableId = Util.genIdByName(catalog.getName(), DATABASE_NAME, NEW_TABLE_NAME); - try (MockedConstruction mocked = - Mockito.mockConstruction(ExternalRowCountCache.RowCountCacheLoader.class, - (loader, context) -> Mockito.when(loader.asyncLoad(Mockito.any(), Mockito.any())) - .thenAnswer(inv -> { - ExternalRowCountCache.RowCountKey key = inv.getArgument(0); - long rowCount = key.getTableId() == sourceTableId ? 100L : 200L; - return CompletableFuture.completedFuture(Optional.of(rowCount)); - }))) { - ExternalRowCountCache rowCountCache = - new ExternalRowCountCache(MoreExecutors.newDirectExecutorService()); - Deencapsulation.setField(metaCacheMgr, "rowCountCache", rowCountCache); - Assertions.assertEquals(100L, - rowCountCache.getCachedRowCount(CATALOG_ID, DATABASE_ID, sourceTableId, false)); - Assertions.assertEquals(200L, - rowCountCache.getCachedRowCount(CATALOG_ID, DATABASE_ID, destinationTableId, false)); - - database.addTableForTest( - new TestExternalTable(sourceTableId, TABLE_NAME, TABLE_NAME, catalog, database)); - database.addTableForTest( - new TestExternalTable(destinationTableId, NEW_TABLE_NAME, NEW_TABLE_NAME, catalog, database)); - Assertions.assertNotNull(database.getCachedTableForTest(NEW_TABLE_NAME)); - new RefreshManager().replayRefreshTable(ExternalObjectLog.createForRenameTable( - CATALOG_ID, DATABASE_NAME, TABLE_NAME, NEW_TABLE_NAME)); - - Assertions.assertNull(database.getCachedTableForTest(TABLE_NAME)); - Assertions.assertNull(database.getCachedTableForTest(NEW_TABLE_NAME)); - Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, - rowCountCache.getCachedRowCountIfPresent(CATALOG_ID, DATABASE_ID, sourceTableId)); - Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, - rowCountCache.getCachedRowCountIfPresent(CATALOG_ID, DATABASE_ID, destinationTableId)); - } - } - - @Test - public void testReplayNameBasedRenameMigratesColdLocalState() { - seedAndEvictTableObject(); - ExternalObjectLog log = ExternalObjectLog.createForRenameTable( - CATALOG_ID, DATABASE_NAME, TABLE_NAME, NEW_TABLE_NAME); - - new RefreshManager().replayRefreshTable(log); - - assertColdRenameMigrated(); - } - - private void seedAndEvictTableObject() { - TestExternalTable table = new TestExternalTable(TABLE_ID, TABLE_NAME, TABLE_NAME, catalog, database); - database.addTableForTest(table); - database.evictTableObjectForTest(TABLE_NAME); - Assertions.assertFalse(database.getTableForReplay(TABLE_NAME).isPresent()); - } - - private void disableDatabaseObjectCacheWithTtlZero() { - disableDatabaseObjectCacheWithTtlZero(catalog, database); - } - - private void disableDatabaseObjectCacheWithTtlZero( - ExternalCatalog targetCatalog, ExternalDatabase targetDatabase) { - FeMetaCacheEntry> disabledDatabases = new FeMetaCacheEntry<>( - "ttl_zero_databases", - ignored -> { - databaseObjectLoadCalls.incrementAndGet(); - return targetDatabase; - }, - CacheSpec.of(true, 0L, 10L), - Env.getCurrentEnv().getExtMetaCacheMgr().commonRefreshExecutor(), - false); - Deencapsulation.setField(targetCatalog, "databases", disabledDatabases); - Assertions.assertFalse(targetCatalog.getDbForReplay(DATABASE_NAME).isPresent()); - } - - private Connector usePluginCatalogWithDisabledDatabaseObjectCache() { - PluginCatalogFixture fixture = usePluginCatalog(); - disableDatabaseObjectCacheWithTtlZero(fixture.catalog, fixture.database); - return fixture.connector; - } - - private PluginCatalogFixture usePluginCatalog() { - Connector connector = Mockito.mock(Connector.class); - Map properties = Collections.singletonMap("type", "test"); - PluginDrivenExternalCatalog pluginCatalog = new PluginDrivenExternalCatalog( - CATALOG_ID, CATALOG_NAME, null, properties, "", connector); - pluginCatalog.setInitializedForTest(true); - PluginDatabase pluginDatabase = - new PluginDatabase(pluginCatalog, DATABASE_ID, DATABASE_NAME, DATABASE_NAME); - pluginDatabase.setInitializedForTest(true); - pluginCatalog.addDatabaseForTest(pluginDatabase); - testingCatalogMgr.setCatalog(pluginCatalog); - return new PluginCatalogFixture(connector, pluginCatalog, pluginDatabase); - } - - private void assertColdTableInvalidatedByName() { - Assertions.assertEquals(1, engineCache.invalidateTableCalls.get()); - Assertions.assertEquals(CATALOG_ID, engineCache.lastCatalogId); - Assertions.assertEquals(DATABASE_NAME, engineCache.lastDatabaseName); - Assertions.assertEquals(TABLE_NAME, engineCache.lastTableName); - Assertions.assertFalse(database.getTableForReplay(TABLE_NAME).isPresent()); - Assertions.assertEquals(0, database.buildTableCalls.get()); - } - - private void assertColdRenameMigrated() { - Assertions.assertEquals(CATALOG_ID, engineCache.lastCatalogId); - Assertions.assertEquals(DATABASE_NAME, engineCache.lastDatabaseName); - Assertions.assertEquals(NEW_TABLE_NAME, engineCache.lastTableName); - Assertions.assertNull(database.getCachedTableNameByIdForTest(TABLE_ID)); - Assertions.assertFalse(database.getTableForReplay(TABLE_NAME).isPresent()); - Assertions.assertNull(database.getCachedTableNamesForTest()); - Assertions.assertEquals(0, database.buildTableCalls.get()); - Assertions.assertEquals(1, constraintManager.renameTableCalls.get()); - Assertions.assertEquals("test_catalog", constraintManager.oldTableName.getCtl()); - Assertions.assertEquals(DATABASE_NAME, constraintManager.oldTableName.getDb()); - Assertions.assertEquals(TABLE_NAME, constraintManager.oldTableName.getTbl()); - Assertions.assertEquals("test_catalog", constraintManager.newTableName.getCtl()); - Assertions.assertEquals(DATABASE_NAME, constraintManager.newTableName.getDb()); - Assertions.assertEquals(NEW_TABLE_NAME, constraintManager.newTableName.getTbl()); - } - - public static class EmptyCatalogProvider implements TestExternalCatalog.TestCatalogProvider { - @Override - public Map>> getMetadata() { - return Collections.emptyMap(); - } - } - - private static class CountingDatabase extends TestExternalDatabase { - private final AtomicInteger buildTableCalls = new AtomicInteger(); - - CountingDatabase(ExternalCatalog catalog, long id, String name, String remoteName) { - super(catalog, id, name, remoteName); - } - - @Override - public TestExternalTable buildTableInternal(String remoteTableName, String localTableName, long tableId, - org.apache.doris.datasource.ExternalCatalog externalCatalog, ExternalDatabase db) { - buildTableCalls.incrementAndGet(); - return super.buildTableInternal(remoteTableName, localTableName, tableId, externalCatalog, db); - } - } - - private static class PluginCatalogFixture { - private final Connector connector; - private final PluginDrivenExternalCatalog catalog; - private final PluginDatabase database; - - PluginCatalogFixture( - Connector connector, PluginDrivenExternalCatalog catalog, PluginDatabase database) { - this.connector = connector; - this.catalog = catalog; - this.database = database; - } - } - - private static class PluginDatabase extends ExternalDatabase { - private final AtomicInteger buildTableCalls = new AtomicInteger(); - - PluginDatabase(ExternalCatalog catalog, long id, String name, String remoteName) { - super(catalog, id, name, remoteName); - } - - @Override - public ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tableId, - ExternalCatalog catalog, ExternalDatabase db) { - buildTableCalls.incrementAndGet(); - return new ExternalTable(tableId, localTableName, remoteTableName, catalog, db, - TableIf.TableType.PLUGIN_EXTERNAL_TABLE); - } - } - - private static class TestingCatalogMgr extends CatalogMgr { - private CatalogIf> catalog; - - TestingCatalogMgr(CatalogIf> catalog) { - this.catalog = catalog; - } - - void setCatalog(CatalogIf> catalog) { - this.catalog = catalog; - } - - @Override - public CatalogIf> getCatalog(long id) { - return catalog != null && id == catalog.getId() ? catalog : null; - } - } - - private static class TestingEnv extends Env { - private final CatalogMgr catalogMgr; - private final ExternalMetaCacheMgr metaCacheMgr; - private final ConstraintManager constraintManager; - - TestingEnv(CatalogMgr catalogMgr, ExternalMetaCacheMgr metaCacheMgr, - ConstraintManager constraintManager) { - super(true); - this.catalogMgr = catalogMgr; - this.metaCacheMgr = metaCacheMgr; - this.constraintManager = constraintManager; - } - - @Override - public CatalogMgr getCatalogMgr() { - return catalogMgr; - } - - @Override - public ExternalMetaCacheMgr getExtMetaCacheMgr() { - return metaCacheMgr; - } - - @Override - public ConstraintManager getConstraintManager() { - return constraintManager; - } - } - - private static class RecordingConstraintManager extends ConstraintManager { - private final AtomicInteger renameTableCalls = new AtomicInteger(); - private TableNameInfo oldTableName; - private TableNameInfo newTableName; - - @Override - public void renameTable(TableNameInfo oldTableInfo, TableNameInfo newTableInfo) { - oldTableName = oldTableInfo; - newTableName = newTableInfo; - renameTableCalls.incrementAndGet(); - } - } - - private static class RecordingExternalMetaCache implements ExternalMetaCache { - private final AtomicInteger invalidateDbCalls = new AtomicInteger(); - private final AtomicInteger invalidateTableCalls = new AtomicInteger(); - private boolean failDbInvalidation; - private long lastCatalogId; - private String lastDatabaseName; - private String lastTableName; - - @Override - public String engine() { - return "default"; - } - - @Override - public List aliases() { - return Collections.emptyList(); - } - - @Override - public void initCatalog(long catalogId, Map catalogProperties) { - } - - @Override - public MetaCache entry( - long catalogId, String entryName, Class keyType, Class valueType) { - throw new UnsupportedOperationException(); - } - - @Override - public void checkCatalogInitialized(long catalogId) { - } - - @Override - public boolean isCatalogInitialized(long catalogId) { - return true; - } - - @Override - public void invalidateCatalog(long catalogId) { - } - - @Override - public void invalidateDb(long catalogId, String dbName) { - lastCatalogId = catalogId; - lastDatabaseName = dbName; - invalidateDbCalls.incrementAndGet(); - if (failDbInvalidation) { - throw new IllegalStateException("database metadata invalidation failed"); - } - } - - @Override - public void invalidateTable(long catalogId, String dbName, String tableName) { - lastCatalogId = catalogId; - lastDatabaseName = dbName; - lastTableName = tableName; - invalidateTableCalls.incrementAndGet(); - } - - @Override - public void invalidatePartitions(long catalogId, String dbName, String tableName, List partitions) { - } - - @Override - public Map stats(long catalogId) { - return Collections.emptyMap(); - } - - @Override - public void close() { - } + public void databaseReplayInvalidatesBothCoordinatesAndParentMembership() { + MetaCache names = cache("names", + ScopePath.databaseCollection().in(ScopePath.Coordinates.LOCAL)); + MetaCache source = cache("source", + ScopePath.table("DB", "T").in(ScopePath.Coordinates.REMOTE)); + new RefreshManager().replayRefreshDb(ExternalObjectLog.createForRefreshDb(10, "db") + .withRemoteNames("DB", null)); + Assertions.assertNull(names.getIfPresent("key")); + Assertions.assertNull(source.getIfPresent("key")); + Mockito.verify(catalog, Mockito.never()).getConnector(); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java index 36edaacdf0197c..0f3b073cda6c8d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshTableTest.java @@ -95,22 +95,16 @@ public void testRefreshTable() throws Exception { long l2 = table.getUpdateTime(); Assertions.assertTrue(l2 >= l1); TableNameInfo tableNameInfo = new TableNameInfo("test1", "db1", "tbl11"); - try { - Env.getCurrentEnv().getRefreshManager() - .handleRefreshTable(tableNameInfo.getCtl(), tableNameInfo.getDb(), tableNameInfo.getTbl(), false); - } catch (Exception e) { - // Do nothing - } + Env.getCurrentEnv().getRefreshManager().handleRefreshTable( + tableNameInfo.getCtl(), tableNameInfo.getDb(), tableNameInfo.getTbl(), false); Assertions.assertFalse(table.isObjectCreated()); - long l3 = table.getUpdateTime(); - Assertions.assertTrue(l3 >= l2); - table.getFullSchema(); - // only table.getFullSchema() can change table.lastUpdateTime - long l4 = table.getUpdateTime(); - Assertions.assertTrue(l4 > l3); - // updateTime is equal to schema update time as default - long l5 = table.getUpdateTime(); - Assertions.assertTrue(l5 >= l4); + long retiredUpdateTime = table.getUpdateTime(); + TestExternalTable replacement = (TestExternalTable) test1.getDbNullable("db1").getTable("tbl11").get(); + Assertions.assertNotSame(table, replacement); + Assertions.assertFalse(replacement.isObjectCreated()); + replacement.getFullSchema(); + Assertions.assertTrue(replacement.getUpdateTime() > 0); + Assertions.assertEquals(retiredUpdateTime, table.getUpdateTime()); // external info schema db ExternalInfoSchemaDatabase infoDb = (ExternalInfoSchemaDatabase) test1.getDbNullable(InfoSchemaDb.DATABASE_NAME); diff --git a/fe/fe-core/src/test/java/org/apache/doris/connector/ConnectorCacheDomainTest.java b/fe/fe-core/src/test/java/org/apache/doris/connector/ConnectorCacheDomainTest.java new file mode 100644 index 00000000000000..4171f483cf5756 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/connector/ConnectorCacheDomainTest.java @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector; + +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorContext; +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorProvider; +import org.apache.doris.connector.spi.ConnectorSession; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; + +class ConnectorCacheDomainTest { + @AfterEach + void reset() { + ConnectorFactory.clearPluginManager(); + } + + private void providers() { + ConnectorPluginManager manager = new ConnectorPluginManager(); + for (String type : new String[] {"primary", "sibling"}) { + manager.registerProvider(new ConnectorProvider() { + @Override + public String getType() { + return type; + } + + @Override + public Connector create(Map properties, ConnectorContext context) { + CachedConnector connector = new CachedConnector(context, type, properties); + if (properties.containsKey("fail")) { + throw new IllegalArgumentException("partial construction"); + } + return connector; + } + }); + } + ConnectorFactory.initPluginManager(manager); + } + + @Test + void factoryInstancesShareRootAndResourceInvalidationButOwnTheirLifetimes() throws Exception { + providers(); + try (CatalogMetaCache root = CatalogMetaCache.unmanaged(); + DefaultConnectorContext context = new DefaultConnectorContext("c", 7L)) { + context.bindCacheRoot(() -> root); + CachedConnector first = (CachedConnector) ConnectorFactory.createConnector( + "primary", Collections.emptyMap(), context); + CachedConnector child = (CachedConnector) first.context.createSiblingConnector( + "sibling", Collections.emptyMap()); + CachedConnector replacement = (CachedConnector) ConnectorFactory.createConnector( + "primary", Collections.emptyMap(), context); + child.cache.put("t", "child"); + replacement.cache.put("t", "new"); + first.close(); + Assertions.assertTrue(child.domain.isClosed()); + Assertions.assertEquals("new", replacement.cache.getIfPresent("t")); + Assertions.assertFalse(root.isClosed()); + Assertions.assertEquals(1, root.allEntries().size()); + Assertions.assertThrows(IllegalStateException.class, () -> first.context.createSiblingConnector( + "sibling", Collections.emptyMap())); + root.invalidateScope(ScopePath.table("db", "t").in(ScopePath.Coordinates.REMOTE)); + Assertions.assertNull(replacement.cache.getIfPresent("t")); + } + } + + @Test + void failedFactoryConstructionRollsBackOnlyItsInstanceDomain() throws Exception { + providers(); + try (CatalogMetaCache root = CatalogMetaCache.unmanaged(); + DefaultConnectorContext context = new DefaultConnectorContext("c", 7L)) { + context.bindCacheRoot(() -> root); + CachedConnector live = (CachedConnector) ConnectorFactory.createConnector( + "primary", Collections.emptyMap(), context); + live.cache.put("t", "live"); + Assertions.assertThrows(IllegalArgumentException.class, () -> ConnectorFactory.createConnector( + "primary", Collections.singletonMap("fail", "true"), context)); + Assertions.assertEquals("live", live.cache.getIfPresent("t")); + Assertions.assertEquals(1, root.allEntries().size()); + Assertions.assertFalse(root.isClosed()); + } + } + + @Test + void contextRetirementDoesNotWaitForTheCatalogMonitorAndRejectsLateDomains() throws Exception { + CountDownLatch entered = new CountDownLatch(1); + CountDownLatch finish = new CountDownLatch(1); + ExecutorService workers = Executors.newFixedThreadPool(2); + try (CatalogMetaCache root = CatalogMetaCache.unmanaged(); + DefaultConnectorContext context = new DefaultConnectorContext("c", 7L)) { + context.bindCacheRoot(() -> { + entered.countDown(); + try { + Assertions.assertTrue(finish.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + return root; + }); + Future construction = workers.submit(() -> Assertions.assertThrows(IllegalStateException.class, + () -> context.createCacheDomain("primary", Collections.emptyMap()))); + Assertions.assertTrue(entered.await(10, TimeUnit.SECONDS)); + workers.submit(() -> { + context.close(); + return null; + }).get(10, TimeUnit.SECONDS); + finish.countDown(); + construction.get(10, TimeUnit.SECONDS); + Assertions.assertFalse(root.isClosed()); + Assertions.assertTrue(root.allEntries().isEmpty()); + } finally { + finish.countDown(); + workers.shutdownNow(); + } + } + + private static final class CachedConnector implements Connector { + private final ConnectorContext context; + private final CatalogMetaCache domain; + private final MetaCache cache; + + private CachedConnector(ConnectorContext context, String engine, Map properties) { + this.context = context; + domain = context.createCacheDomain(engine, properties); + cache = domain.create(MetaCacheDefinition.builder("table", CacheSpec.of(true, -1, 10), + key -> ScopePath.table("db", key)).build()); + } + + @Override + public ConnectorMetadata getMetadata(ConnectorSession session) { + return null; + } + + @Override + public void close() { + domain.close(); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextSiblingTest.java b/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextSiblingTest.java index ac1ac5f2cdffd2..07d6bb5ea7c3bd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextSiblingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/connector/DefaultConnectorContextSiblingTest.java @@ -67,8 +67,9 @@ void createSiblingConnector_buildsViaFactory_passesPropsAndThisContext() { "the sibling must be exactly the connector the matching provider produced"); Assertions.assertSame(props, provider.lastProperties, "the caller-synthesized properties must be forwarded to the sibling provider unchanged"); - Assertions.assertSame(ctx, provider.lastContext, - "the gateway's own context (this) must be passed to the sibling so it shares id/auth/storage"); + Assertions.assertEquals(ctx.getCatalogId(), provider.lastContext.getCatalogId()); + Assertions.assertEquals(ctx.getCatalogName(), provider.lastContext.getCatalogName()); + Assertions.assertInstanceOf(ConnectorInstanceContext.class, provider.lastContext); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogDeadlockTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogDeadlockTest.java index c9a46b4c1dd772..0c46f865e6da52 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogDeadlockTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalCatalogDeadlockTest.java @@ -74,6 +74,7 @@ public void testResetToUninitializedShouldNotDeadlockWithCacheLoader() throws Ex Assertions.assertTrue(refreshReady.await(DEADLOCK_DETECTION_TIMEOUT_SECONDS, TimeUnit.SECONDS)); assertNoDeadlock(queryThread, refreshThread, backgroundFailure); + catalog.onClose(); } private static void assertNoDeadlock(Thread queryThread, Thread refreshThread, @@ -125,6 +126,7 @@ private static class DeadlockCatalog extends ExternalCatalog { DeadlockCatalog() { super(1L, "deadlock-catalog", Type.TEST, ""); + catalogProperty = new CatalogProperty(null, java.util.Collections.emptyMap()); initialized = true; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseSessionContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseSessionContextTest.java index ece97c60979410..e3faff8c7bc386 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseSessionContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalDatabaseSessionContextTest.java @@ -284,7 +284,9 @@ private static Map props(int lowerCaseTableNames) { } boolean hasSharedDatabaseCacheState(long dbId) { - return databaseNames != null || databases != null || dbIdNameIndex.getName(dbId) != null; + return (databaseNames != null && databaseNames.getIfPresent("") != null) + || (databases != null && databases.stats().getEstimatedSize() > 0) + || dbIdNameIndex.getName(dbId) != null; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java index c73ac018f444f3..288f0f8545a7d9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java @@ -20,10 +20,12 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Config; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.cache.NereidsSortedPartitionsCacheManager; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.util.Util; +import org.apache.doris.connector.cache.CatalogMetaCache; import org.apache.doris.datasource.metacache.ExternalMetaCache; import org.apache.doris.datasource.test.TestExternalCatalog; import org.apache.doris.datasource.test.TestExternalDatabase; @@ -46,14 +48,103 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; public class ExternalRowCountCacheTest { + private ExternalRowCountCache standalone(ExecutorService executor) { + Map owners = new java.util.concurrent.ConcurrentHashMap<>(); + return new ExternalRowCountCache(executor, id -> owners.computeIfAbsent(id, + ignored -> CatalogMetaCache.unmanaged())); + } + + @Test + public void testAutomaticRefreshKeepsLastSuccessfulCountUntilCompletion() throws Exception { + verifyAutomaticRefresh(false); + } + + @Test + public void testFailedAutomaticRefreshKeepsLastSuccessfulCount() throws Exception { + verifyAutomaticRefresh(true); + } + + private void verifyAutomaticRefresh(boolean failRefresh) throws Exception { + CountDownLatch refreshStarted = new CountDownLatch(1); + ExecutorService executor = new ThreadPoolExecutor(1, 1, 0L, TimeUnit.MILLISECONDS, + new LinkedBlockingQueue<>()) { + @Override + protected void afterExecute(Runnable task, Throwable failure) { + // Also make an incorrectly published pending future observable before the assertion. + refreshStarted.countDown(); + } + }; + CompletableFuture> refresh = new CompletableFuture<>(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged(); + MockedConstruction mocked = + Mockito.mockConstruction(ExternalRowCountCache.RowCountCacheLoader.class, + (loader, context) -> { + Mockito.when(loader.asyncLoad(Mockito.any(), Mockito.any())) + .thenReturn(CompletableFuture.completedFuture(Optional.of(100L))) + .thenReturn(refresh); + Mockito.when(loader.doLoad(Mockito.any())).thenAnswer(call -> { + refreshStarted.countDown(); + return refresh.get(5L, TimeUnit.SECONDS); + }); + })) { + ExternalRowCountCache cache = new ExternalRowCountCache(executor, ignored -> owner); + Assertions.assertEquals(100L, cache.getCachedRowCount(1, 2, 3, false)); + Object delegate = Deencapsulation.getField(owner.entries().get("row_count"), "delegate"); + com.github.benmanes.caffeine.cache.Cache data = Deencapsulation.getField(delegate, "data"); + Object versioned = data.asMap().values().iterator().next(); + Deencapsulation.setField(versioned, "writeTimeNanos", System.nanoTime() + - TimeUnit.MINUTES.toNanos(Config.external_cache_refresh_time_minutes + 1L)); + Assertions.assertEquals(100L, cache.getCachedRowCount(1, 2, 3, false)); + Assertions.assertTrue(refreshStarted.await(3L, TimeUnit.SECONDS)); + // A refresh is pending, but callers must still see the last successfully loaded result. + Assertions.assertEquals(100L, cache.getCachedRowCountIfPresent(1, 2, 3)); + refresh.complete(failRefresh ? null : Optional.of(200L)); + executor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assertions.assertEquals(failRefresh ? 100L : 200L, cache.getCachedRowCountIfPresent(1, 2, 3)); + } finally { + refresh.complete(Optional.of(200L)); + executor.shutdownNow(); + } + } + + @Test + public void rowCountRetentionLimitRemainsGlobalAcrossCatalogRoots() throws Exception { + long original = Config.max_external_table_row_count_cache_num; + Config.max_external_table_row_count_cache_num = 1; + try (CatalogMetaCache first = CatalogMetaCache.unmanaged(); + CatalogMetaCache second = CatalogMetaCache.unmanaged(); + MockedStatic stats = Mockito.mockStatic(StatisticsUtil.class)) { + ExternalTable table = Mockito.mock(ExternalTable.class); + Mockito.when(table.fetchRowCountWithMetaCache(false)).thenReturn(100L); + stats.when(() -> StatisticsUtil.findTable(1, 11, 111)).thenReturn(table); + stats.when(() -> StatisticsUtil.findTable(2, 22, 222)).thenReturn(table); + ExternalRowCountCache cache = new ExternalRowCountCache(MoreExecutors.newDirectExecutorService(), + id -> id == 1 ? first : second); + Assertions.assertEquals(100L, cache.getCachedRowCount(1, 11, 111, false)); + Assertions.assertEquals(100L, cache.getCachedRowCount(2, 22, 222, false)); + com.github.benmanes.caffeine.cache.Cache retention = + Deencapsulation.getField(cache, "retention"); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (first.entries().get("row_count").size() + second.entries().get("row_count").size() > 1 + && System.nanoTime() < deadline) { + retention.cleanUp(); + Thread.sleep(10); + } + Assertions.assertTrue(first.entries().get("row_count").size() + + second.entries().get("row_count").size() <= 1); + } finally { + Config.max_external_table_row_count_cache_num = original; + } + } + @Test public void testRowCountKeyUsesTableIdAsCacheIdentity() { ExternalRowCountCache.RowCountKey key1 = new ExternalRowCountCache.RowCountKey(1, 2, 3); @@ -91,7 +182,7 @@ public void testGetCachedRowCountPassesFillMetaCacheToLoader() { mockedStatisticsUtil.when(() -> StatisticsUtil.findTable(1, 2, 3)).thenReturn(table); mockedStatisticsUtil.when(() -> StatisticsUtil.findTable(1, 2, 4)).thenReturn(table); - ExternalRowCountCache cache = new ExternalRowCountCache(MoreExecutors.newDirectExecutorService()); + ExternalRowCountCache cache = standalone(MoreExecutors.newDirectExecutorService()); Assertions.assertEquals(100L, cache.getCachedRowCount(1, 2, 3, true)); Assertions.assertEquals(200L, cache.getCachedRowCount(1, 2, 4, false)); } @@ -109,7 +200,7 @@ public void testInvalidateTableReloadsRowCount() { try (MockedStatic mockedStatisticsUtil = Mockito.mockStatic(StatisticsUtil.class)) { mockedStatisticsUtil.when(() -> StatisticsUtil.findTable(1, 2, 3)).thenReturn(table); - ExternalRowCountCache cache = new ExternalRowCountCache(MoreExecutors.newDirectExecutorService()); + ExternalRowCountCache cache = standalone(MoreExecutors.newDirectExecutorService()); Assertions.assertEquals(100L, cache.getCachedRowCount(1, 2, 3, false)); rowCount.set(200L); @@ -140,7 +231,7 @@ public void testCatalogAndDbInvalidationKeepUnrelatedRowCounts() { mockedStatisticsUtil.when(() -> StatisticsUtil.findTable(1, 12, 112)).thenReturn(secondTable); mockedStatisticsUtil.when(() -> StatisticsUtil.findTable(2, 11, 211)).thenReturn(otherCatalogTable); - ExternalRowCountCache cache = new ExternalRowCountCache(MoreExecutors.newDirectExecutorService()); + ExternalRowCountCache cache = standalone(MoreExecutors.newDirectExecutorService()); Assertions.assertEquals(100L, cache.getCachedRowCount(1, 11, 111, false)); Assertions.assertEquals(200L, cache.getCachedRowCount(1, 12, 112, false)); Assertions.assertEquals(300L, cache.getCachedRowCount(2, 11, 211, false)); @@ -162,7 +253,7 @@ public void testCatalogAndDbInvalidationKeepUnrelatedRowCounts() { } @Test - public void testTableInvalidationWaitsForFuturePublication() throws Exception { + public void testTableInvalidationFencesPublicationWithoutWaitingForExecutor() throws Exception { ExecutorService loadExecutor = Mockito.mock(ExecutorService.class); CountDownLatch loadSubmitted = new CountDownLatch(1); CountDownLatch allowPublication = new CountDownLatch(1); @@ -178,7 +269,7 @@ public void testTableInvalidationWaitsForFuturePublication() throws Exception { Mockito.withSettings().defaultAnswer(Mockito.CALLS_REAL_METHODS), (loader, context) -> Mockito.doReturn(Optional.of(100L)) .when(loader).doLoad(Mockito.any()))) { - ExternalRowCountCache cache = new ExternalRowCountCache(loadExecutor); + ExternalRowCountCache cache = standalone(loadExecutor); Future load = callers.submit(() -> cache.getCachedRowCount(1, 2, 3, false)); Assertions.assertTrue(loadSubmitted.await(3L, TimeUnit.SECONDS)); @@ -188,8 +279,7 @@ public void testTableInvalidationWaitsForFuturePublication() throws Exception { cache.invalidateTable(1, 2, 3); }); Assertions.assertTrue(invalidationStarted.await(3L, TimeUnit.SECONDS)); - Assertions.assertThrows(TimeoutException.class, - () -> invalidation.get(200L, TimeUnit.MILLISECONDS)); + invalidation.get(3L, TimeUnit.SECONDS); allowPublication.countDown(); Assertions.assertEquals(100L, load.get(3L, TimeUnit.SECONDS)); @@ -214,7 +304,7 @@ public void testTableInvalidationDiscardsPendingLoadCompletion() throws Exceptio return loadedRowCount; }))) { ExternalRowCountCache cache = - new ExternalRowCountCache(MoreExecutors.newDirectExecutorService()); + standalone(MoreExecutors.newDirectExecutorService()); Future load = callers.submit(() -> cache.getCachedRowCount(1, 2, 3, false)); Assertions.assertTrue(loadStarted.await(3L, TimeUnit.SECONDS)); @@ -337,7 +427,7 @@ public void testCatalogRefreshScansRowCountCacheOnceForMultipleHotDatabases() { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); ExternalRowCountCache rowCountCache = Mockito.spy( - new ExternalRowCountCache(MoreExecutors.newDirectExecutorService())); + standalone(MoreExecutors.newDirectExecutorService())); Deencapsulation.setField(metaCacheMgr, "rowCountCache", rowCountCache); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); @@ -364,7 +454,7 @@ public void testInvalidateDbMetadataCacheDoesNotAccessDbForRowCount() { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); ExternalRowCountCache rowCountCache = Mockito.spy( - new ExternalRowCountCache(MoreExecutors.newDirectExecutorService())); + standalone(MoreExecutors.newDirectExecutorService())); Deencapsulation.setField(metaCacheMgr, "rowCountCache", rowCountCache); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); @@ -399,7 +489,7 @@ public void testLoadWithException() throws Exception { }).when(mock).doLoad(Mockito.any()); })) { - ExternalRowCountCache cache = new ExternalRowCountCache(executor); + ExternalRowCountCache cache = standalone(executor); long cachedRowCount = cache.getCachedRowCount(1, 1, 1, false); Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cachedRowCount); for (int i = 0; i < 60; i++) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java new file mode 100644 index 00000000000000..9106511e37a5d1 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/FeCatalogCacheOwnerTest.java @@ -0,0 +1,650 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.catalog.RefreshManager; +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.common.util.Util; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheGovernance; +import org.apache.doris.datasource.log.CatalogLog; +import org.apache.doris.datasource.log.ExternalObjectLog; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; +import org.apache.doris.datasource.metacache.FeMetaCacheEntry; +import org.apache.doris.datasource.test.TestExternalCatalog; +import org.apache.doris.datasource.test.TestExternalDatabase; +import org.apache.doris.datasource.test.TestExternalTable; +import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +public class FeCatalogCacheOwnerTest extends TestWithFeService { + private static final AtomicLong IDS = new AtomicLong(72000); + private CountingCatalog catalog; + private CountingDatabase database; + private Gate namesGate; + private Gate objectsGate; + private Gate schemaGate; + private ExecutorService workers; + + @Override + protected void runBeforeAll() throws Exception { + FeConstants.runningUnitTest = true; + createDefaultCtx().setThreadLocalInfo(); + } + + @BeforeEach + public void prepareCatalog() { + namesGate = new Gate(); + objectsGate = new Gate(); + schemaGate = new Gate(); + Provider.TABLES.clear(); + putTable("a", "old_column"); + putTable("b", "sibling_column"); + catalog = new CountingCatalog(IDS.incrementAndGet()); + CatalogMgr manager = Env.getCurrentEnv().getCatalogMgr(); + manager.getIdToCatalog().put(catalog.getId(), catalog); + Map> byName = Deencapsulation.getField(manager, "nameToCatalog"); + byName.put(catalog.getName(), catalog); + catalog.setInitializedForTest(true); + database = new CountingDatabase(catalog); + database.setInitializedForTest(true); + catalog.addDatabaseForTest(database); + workers = Executors.newFixedThreadPool(2); + } + + @AfterEach + public void releaseCatalog() throws Exception { + namesGate.release.countDown(); + objectsGate.release.countDown(); + schemaGate.release.countDown(); + workers.shutdownNow(); + Assertions.assertTrue(workers.awaitTermination(10, TimeUnit.SECONDS)); + catalog.closeFeMetaCacheOwner(); + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalog(catalog.getId()); + CatalogMgr manager = Env.getCurrentEnv().getCatalogMgr(); + manager.getIdToCatalog().remove(catalog.getId()); + Map> byName = Deencapsulation.getField(manager, "nameToCatalog"); + byName.remove(catalog.getName()); + } + + @Test + public void catalogRenameAndReplayReactivateOnlyTheRegisteredInstance() throws Exception { + database.getTableNullable("a").getFullSchema(); + CatalogMetaCache before = catalog.getFeMetaCacheOwner(); + CatalogMgr manager = Env.getCurrentEnv().getCatalogMgr(); + String oldName = catalog.getName(); + String renamed = oldName + "_renamed"; + manager.alterCatalogName(oldName, renamed); + Assertions.assertNull(manager.getCatalog(oldName)); + Assertions.assertSame(catalog, manager.getCatalog(renamed)); + Assertions.assertTrue(before.isClosed()); + Assertions.assertNotNull(catalog.getDbNullable("db").getTableNullable("a").getFullSchema()); + CatalogMetaCache after = catalog.getFeMetaCacheOwner(); + Assertions.assertNotSame(before, after); + + CatalogLog replay = new CatalogLog(); + replay.setCatalogId(catalog.getId()); + replay.setNewCatalogName(renamed + "_replayed"); + manager.replayAlterCatalogName(replay); + Assertions.assertTrue(after.isClosed()); + Assertions.assertNotNull(catalog.getDbNullable("db").getTableNullable("a").getFullSchema()); + Assertions.assertNotSame(after, catalog.getFeMetaCacheOwner()); + Assertions.assertEquals(1, MetaCacheGovernance.catalogCaches(catalog.getId()).size()); + } + + @Test + public void renameInvalidatesCaseEquivalentDestinationAndDerivedCaches() { + verifyCaseEquivalentRenameDestination(false, false, true); + } + + @Test + public void renameReplayInvalidatesCaseEquivalentDestinationWithColdNames() { + verifyCaseEquivalentRenameDestination(true, false, true); + } + + @Test + public void renameReplayInvalidatesPreviousDestinationAfterNamesRefresh() { + verifyCaseEquivalentRenameDestination(true, true, true); + } + + @Test + public void renameEventRefreshReplayInvalidatesPreviousDestinationAfterNamesRefresh() { + verifyCaseEquivalentRenameDestination(true, true, false); + } + + @SuppressWarnings("unchecked") + private void verifyCaseEquivalentRenameDestination(boolean replay, boolean refreshedNames, boolean rename) { + prepareCaseInsensitiveTables(); + putTable("Dest", "previous_destination"); + TestExternalTable source = database.getTableNullable("a"); + TestExternalTable destination = database.getTableNullable("Dest"); + TestExternalTable sibling = database.getTableNullable("b"); + source.getFullSchema(); + destination.getFullSchema(); + List siblingSchema = sibling.getFullSchema(); + CatalogMetaCache root = catalog.getFeMetaCacheOwner(); + MetaCache schemas = + (MetaCache) root.entries().get("schema"); + SchemaCacheKey schemaKey = new SchemaCacheKey(destination.getOrBuildNameMapping()); + ExternalRowCountCache counts = Env.getCurrentEnv().getExtMetaCacheMgr().getRowCountCache(); + counts.getCachedRowCount(catalog.getId(), database.getId(), destination.getId(), false); + counts.getCachedRowCount(catalog.getId(), database.getId(), sibling.getId(), false); + MetaCache rows = + (MetaCache) root.entries().get("row_count"); + ExternalRowCountCache.RowCountKey rowKey = new ExternalRowCountCache.RowCountKey( + catalog.getId(), database.getId(), destination.getId()); + ExternalRowCountCache.RowCountKey siblingRowKey = new ExternalRowCountCache.RowCountKey( + catalog.getId(), database.getId(), sibling.getId()); + Assertions.assertNotNull(schemas.getIfPresent(schemaKey)); + Assertions.assertNotNull(rows.getIfPresent(rowKey)); + Assertions.assertNotNull(rows.getIfPresent(siblingRowKey)); + Assertions.assertEquals("Dest", database.getCachedTableNameByIdForTest(destination.getId())); + + // The destination was removed outside Doris; rename recreates its case-equivalent remote name. + Provider.TABLES.remove("Dest"); + Provider.TABLES.remove("a"); + putTable("DEST", "renamed_source"); + if (replay) { + database.resetMetaCacheNames(); + } + if (refreshedNames) { + Assertions.assertEquals(Set.of("DEST", "b"), database.getTableNamesWithLock()); + } + int namesBeforeChange = catalog.nameLoads.get(); + ExternalObjectLog change = rename + ? ExternalObjectLog.createForRenameTable(catalog.getId(), "db", "a", "DEST") + .withRemoteNames("db", "a") + : ExternalObjectLog.createForExplicitRefreshTable(catalog.getId(), "db", "DEST", 0) + .withRemoteNames("db", "DEST"); + if (rename) { + change.setRemoteNewTableName("DEST"); + } + if (replay) { + ExternalObjectLog restored = GsonUtils.GSON.fromJson( + GsonUtils.GSON.toJson(change), ExternalObjectLog.class); + new RefreshManager().replayRefreshTable(restored); + } else { + CatalogMetadataChanges.apply(catalog, change); + } + Assertions.assertEquals(namesBeforeChange, catalog.nameLoads.get(), "invalidation must not list remotely"); + Assertions.assertNull(database.getCachedTableNameByIdForTest(destination.getId())); + Assertions.assertNull(database.getTableNullable(destination.getId())); + if (rename) { + Assertions.assertNull(database.getTableNullable(source.getId())); + } else { + Assertions.assertSame(source, database.getTableNullable(source.getId())); + } + Assertions.assertNull(schemas.getIfPresent(schemaKey)); + Assertions.assertNull(rows.getIfPresent(rowKey)); + Assertions.assertNull(rows.getIfPresent(siblingRowKey), "mode 2 identity changes fence numeric state for this DB"); + TestExternalTable renamed = database.getTableNullable("DEST"); + Assertions.assertNotSame(destination, renamed); + Assertions.assertEquals("renamed_source", renamed.getFullSchema().get(0).getName()); + Assertions.assertSame(sibling, database.getTableNullable("b")); + Assertions.assertSame(siblingSchema, sibling.getFullSchema()); + } + + private void prepareCaseInsensitiveTables() { + catalog.modifyCatalogProps(Collections.singletonMap(ExternalCatalog.LOWER_CASE_TABLE_NAMES, "2")); + catalog.setInitializedForTest(true); + database = new CountingDatabase(catalog); + database.setInitializedForTest(true); + catalog.addDatabaseForTest(database); + } + + @Test + public void tableNameModeChangesRetireTheScopePolicy() { + CatalogMetaCache sensitive = catalog.getFeMetaCacheOwner(); + prepareCaseInsensitiveTables(); + CatalogMetaCache insensitive = catalog.getFeMetaCacheOwner(); + Assertions.assertTrue(sensitive.isClosed()); + Assertions.assertNotSame(sensitive, insensitive); + catalog.modifyCatalogProps(Collections.singletonMap(ExternalCatalog.LOWER_CASE_TABLE_NAMES, "0")); + Assertions.assertTrue(insensitive.isClosed()); + Assertions.assertNotSame(insensitive, catalog.getFeMetaCacheOwner()); + } + + @Test + public void renameFencesFirstDestinationLoadAfterNamesRefresh() throws Exception { + prepareCaseInsensitiveTables(); + putTable("Dest", "previous_destination"); + objectsGate.arm(); + Future old = workers.submit(() -> database.getTableNullable("Dest")); + objectsGate.awaitStarted(); + Provider.TABLES.remove("Dest"); + Provider.TABLES.remove("a"); + putTable("DEST", "renamed_source"); + database.resetMetaCacheNames(); + Assertions.assertEquals(Set.of("DEST", "b"), database.getTableNamesWithLock()); + CatalogMetadataChanges.apply(catalog, + ExternalObjectLog.createForRenameTable(catalog.getId(), "db", "a", "DEST") + .withRemoteNames("db", "a")); + objectsGate.release.countDown(); + TestExternalTable previous = old.get(10, TimeUnit.SECONDS); + Assertions.assertNotNull(previous); + Assertions.assertNull(database.getCachedTableNameByIdForTest(previous.getId())); + Assertions.assertNull(database.getTableNullable(previous.getId())); + TestExternalTable renamed = database.getTableNullable("DEST"); + Assertions.assertNotSame(previous, renamed); + Assertions.assertEquals("renamed_source", renamed.getFullSchema().get(0).getName()); + } + + @Test + public void disabledDatabaseCacheSupportsCaseInsensitiveDropAndRenameEvents() { + long originalTtl = Config.external_cache_expire_time_seconds_after_access; + try { + Config.external_cache_expire_time_seconds_after_access = 0; + catalog.modifyCatalogProps(Collections.singletonMap(ExternalCatalog.LOWER_CASE_TABLE_NAMES, "2")); + catalog.closeFeMetaCacheOwner(); + ExternalDatabase transientDb = catalog.getDbNullable("db"); + Assertions.assertNotNull(transientDb); + Assertions.assertFalse(transientDb.isInitialized()); + Assertions.assertDoesNotThrow(() -> transientDb.unregisterTable("MixedTable")); + Assertions.assertDoesNotThrow(() -> transientDb.invalidateTableRename("MixedTable", "RenamedTable")); + Assertions.assertNotSame(transientDb, catalog.getDbNullable("db")); + } finally { + Config.external_cache_expire_time_seconds_after_access = originalTtl; + } + } + + @Test + public void fullRefreshSharesOwnerAndPreservesSibling() throws Exception { + TestExternalTable a = database.getTableNullable("a"); + TestExternalTable b = database.getTableNullable("b"); + a.getFullSchema(); + List siblingSchema = b.getFullSchema(); + Assertions.assertEquals(1, MetaCacheGovernance.catalogCaches(catalog.getId()).size()); + Assertions.assertEquals(1, catalog.nameLoads.get()); + Provider.TABLES.remove("a"); + + new RefreshManager().handleRefreshTable(catalog.getName(), "db", "a", false); + + Assertions.assertEquals(Collections.singleton("b"), database.getTableNamesWithLock()); + Assertions.assertEquals(2, catalog.nameLoads.get()); + Assertions.assertNull(database.getTableNullable("a")); + Assertions.assertNull(database.getCachedTableNameByIdForTest(a.getId())); + Assertions.assertSame(b, database.getTableNullable("b")); + Assertions.assertSame(siblingSchema, b.getFullSchema()); + } + + @Test + public void committedWritesAndTheirReplayDoNotReloadNames() { + verifyCommittedWritesDoNotReloadNames(); + } + + @Test + public void caseInsensitiveCommittedWritesAndReplayDoNotReloadNames() { + prepareCaseInsensitiveTables(); + verifyCommittedWritesDoNotReloadNames(); + } + + private void verifyCommittedWritesDoNotReloadNames() { + TestExternalTable a = database.getTableNullable("a"); + database.getTableNamesWithLock(); + RefreshManager manager = new RefreshManager(); + for (int i = 0; i < 3; i++) { + manager.refreshTableAfterExternalMutation(a); + manager.replayRefreshTable(ExternalObjectLog.createForRefreshTable( + catalog.getId(), "db", "a", i + 1L)); + Assertions.assertSame(a, database.getTableNullable("a")); + Assertions.assertEquals(Set.of("a", "b"), database.getTableNamesWithLock()); + } + Assertions.assertEquals(1, catalog.nameLoads.get()); + } + + @Test + public void explicitReplayInvalidatesNamesWhenTableObjectIsCold() { + database.getTableNamesWithLock(); + Provider.TABLES.remove("a"); + refreshExplicitly(); + Assertions.assertEquals(Collections.singleton("b"), database.getTableNamesWithLock()); + Assertions.assertEquals(2, catalog.nameLoads.get()); + } + + @Test + public void oldNameLoadCannotRefillAfterExplicitRefresh() throws Exception { + database.getTableNullable("a"); + database.resetMetaCacheNames(); + namesGate.arm(); + Future> old = workers.submit(database::getTableNamesWithLock); + namesGate.awaitStarted(); + Provider.TABLES.remove("a"); + refreshExplicitly(); + namesGate.release.countDown(); + old.get(10, TimeUnit.SECONDS); + Assertions.assertEquals(Collections.singleton("b"), database.getTableNamesWithLock()); + } + + @Test + public void oldObjectLoadCannotReplaceSameNameRecreation() throws Exception { + database.getTableNamesWithLock(); + objectsGate.arm(); + Future old = workers.submit(() -> database.getTableNullable("a")); + objectsGate.awaitStarted(); + Provider.TABLES.remove("a"); + refreshExplicitly(); + putTable("a", "new_column"); + TestExternalTable recreated = database.getTableNullable("a"); + Assertions.assertEquals("new_column", recreated.getFullSchema().get(0).getName()); + objectsGate.release.countDown(); + Assertions.assertNotSame(recreated, old.get(10, TimeUnit.SECONDS)); + Assertions.assertSame(recreated, database.getTableNullable("a")); + Assertions.assertEquals("a", database.getCachedTableNameByIdForTest(recreated.getId())); + } + + @Test + public void oldSchemaLoadCannotReplaceSameNameRecreation() throws Exception { + TestExternalTable oldTable = database.getTableNullable("a"); + schemaGate.arm(); + Future> old = workers.submit(() -> { + return oldTable.getFullSchema(); + }); + schemaGate.awaitStarted(); + Provider.TABLES.remove("a"); + refreshExplicitly(); + putTable("a", "new_column"); + TestExternalTable recreated = database.getTableNullable("a"); + List newSchema = recreated.getFullSchema(); + Assertions.assertEquals("new_column", newSchema.get(0).getName()); + schemaGate.release.countDown(); + Assertions.assertEquals("old_column", old.get(10, TimeUnit.SECONDS).get(0).getName()); + Assertions.assertSame(newSchema, recreated.getFullSchema()); + } + + @Test + public void oldDatabaseCleanupDoesNotInvalidateReplacementOrAccumulateEntries() { + database.getTableNullable("a").getFullSchema(); + for (int i = 0; i < 5; i++) { + CountingDatabase previous = database; + database = new CountingDatabase(catalog); + database.setInitializedForTest(true); + catalog.addDatabaseForTest(database); + TestExternalTable replacement = database.getTableNullable("a"); + List schema = replacement.getFullSchema(); + previous.releaseCachedEntries(); + Assertions.assertSame(replacement, database.getTableNullable("a")); + Assertions.assertSame(schema, replacement.getFullSchema()); + Assertions.assertEquals(7, catalog.getFeMetaCacheOwner().entries().size()); + Assertions.assertNotNull(previous.getTableNullable("a")); + Assertions.assertEquals(7, catalog.getFeMetaCacheOwner().entries().size()); + } + } + + @Test + public void catalogCloseReleasesNamingOwnerBeforeAnySchemaWasLoaded() { + database.getTableNamesWithLock(); + CatalogMetaCache owner = catalog.getFeMetaCacheOwner(); + catalog.onClose(); + Assertions.assertTrue(owner.isClosed()); + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalog.getId()).isEmpty()); + Assertions.assertThrows(IllegalStateException.class, catalog::getFeMetaCacheOwner); + } + + @Test + public void ownerRetirementDuringOldSchemaLoadCannotRefillReplacementRuntime() throws Exception { + TestExternalTable oldTable = database.getTableNullable("a"); + schemaGate.arm(); + Future> old = workers.submit(() -> { + return oldTable.getFullSchema(); + }); + schemaGate.awaitStarted(); + CatalogMetaCache oldOwner = catalog.getFeMetaCacheOwner(); + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalog(catalog.getId()); + putTable("a", "new_owner_column"); + ExternalDatabase replacement = catalog.getDbNullable("db"); + List schema = replacement.getTableNullable("a").getFullSchema(); + Assertions.assertTrue(oldOwner.isClosed()); + Assertions.assertNotSame(oldOwner, catalog.getFeMetaCacheOwner()); + schemaGate.release.countDown(); + Assertions.assertEquals("old_column", old.get(10, TimeUnit.SECONDS).get(0).getName()); + Assertions.assertSame(schema, replacement.getTableNullable("a").getFullSchema()); + Assertions.assertEquals("new_owner_column", schema.get(0).getName()); + Assertions.assertEquals(1, MetaCacheGovernance.catalogCaches(catalog.getId()).size()); + } + + @Test + public void perSessionDatabaseBypassDoesNotRegisterSharedChildEntries() { + catalog.bypassDatabaseNames = true; + int entries = catalog.getFeMetaCacheOwner().entries().size(); + for (int i = 0; i < 20; i++) { + ExternalDatabase transientDatabase = catalog.getDbNullable("db"); + Assertions.assertEquals(Set.of("a", "b"), transientDatabase.getTableNamesWithLock()); + Assertions.assertNotNull(transientDatabase.getTableNullable("a")); + } + Assertions.assertEquals(entries, catalog.getFeMetaCacheOwner().entries().size()); + } + + @Test + public void weightedPayloadBudgetPreservesLookupCachingAndIdResolution() { + catalog.modifyCatalogProps(Collections.singletonMap("meta.cache.max-weight", "1MB")); + ExternalDatabase current = catalog.getDbNullable("db"); + ExternalTable table = current.getTableNullable("a"); + for (int i = 0; i < 20; i++) { + Assertions.assertSame(current, catalog.getDbNullable("db")); + Assertions.assertEquals(Set.of("a", "b"), current.getTableNamesWithLock()); + Assertions.assertSame(table, current.getTableNullable("a")); + Assertions.assertEquals("old_column", table.getFullSchema().get(0).getName()); + } + Assertions.assertEquals(1, catalog.nameLoads.get()); + Assertions.assertSame(current, catalog.getDbNullable(current.getId())); + Assertions.assertSame(table, current.getTableNullable(table.getId())); + Assertions.assertEquals(7, catalog.getFeMetaCacheOwner().entries().size()); + Assertions.assertTrue(Env.getCurrentEnv().getExtMetaCacheMgr().getCatalogCacheStats(catalog.getId()).stream() + .filter(stat -> "schema".equals(stat.getEntryName())) + .anyMatch(stat -> stat.getEntryStats().getEstimatedWeight() > 0)); + } + + @Test + public void coldDatabaseRefreshReplayAfterOwnerRetirementDoesNotReinitializeCatalog() { + database.getTableNamesWithLock(); + catalog.modifyCatalogProps(Collections.singletonMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, "600")); + Assertions.assertFalse(catalog.isInitialized()); + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalog.getId()).isEmpty()); + Assertions.assertDoesNotThrow(() -> new RefreshManager().replayRefreshDb( + ExternalObjectLog.createForRefreshDb(catalog.getId(), "db"))); + Assertions.assertFalse(catalog.isInitialized()); + Assertions.assertTrue(MetaCacheGovernance.catalogCaches(catalog.getId()).isEmpty()); + } + + @Test + public void schemaEntryReconfigurationDoesNotRetireTheBorrowedCatalogOwner() { + database.getTableNullable("a").getFullSchema(); + CatalogMetaCache owner = catalog.getFeMetaCacheOwner(); + catalog.modifyCatalogProps(Collections.singletonMap("meta.cache.default.schema.ttl-second", "600")); + Assertions.assertFalse(owner.isClosed()); + Assertions.assertFalse(catalog.isInitialized()); + Assertions.assertDoesNotThrow(() -> new RefreshManager().replayRefreshDb( + ExternalObjectLog.createForRefreshDb(catalog.getId(), "db"))); + Assertions.assertFalse(catalog.isInitialized()); + Assertions.assertSame(owner, catalog.getFeMetaCacheOwner()); + Assertions.assertEquals("old_column", catalog.getDbNullable("db").getTableNullable("a") + .getFullSchema().get(0).getName()); + Assertions.assertSame(owner, catalog.getFeMetaCacheOwner()); + } + + @Test + public void namingOnlyOwnerIsVisibleInStatsWithoutSchemaInitialization() { + database.getTableNamesWithLock(); + Assertions.assertEquals(6, Env.getCurrentEnv().getExtMetaCacheMgr() + .getCatalogCacheStats(catalog.getId()).size()); + database.getTableNullable("a").getFullSchema(); + Assertions.assertEquals(7, Env.getCurrentEnv().getExtMetaCacheMgr() + .getCatalogCacheStats(catalog.getId()).size()); + } + + @Test + public void retiredDatabaseEntriesReadThroughWithoutPublishingIds() { + database.getTableNamesWithLock(); + FeMetaCacheEntry entries = Deencapsulation.getField(database, "tables"); + database.releaseCachedEntries(); + AtomicInteger actions = new AtomicInteger(); + TestExternalTable table = entries.getAndRunIfCurrent("a", (key, value) -> actions.incrementAndGet()); + Assertions.assertNotNull(table); + Assertions.assertEquals(0, actions.get()); + Assertions.assertNull(entries.getIfPresent("a")); + Assertions.assertDoesNotThrow(() -> database.invalidateTableLookup("a")); + Assertions.assertNull(database.getCachedTableNameByIdForTest(table.getId())); + } + + @Test + public void databaseRetirementDoesNotCloseAnAcceptedObjectLoad() throws Exception { + database.getTableNamesWithLock(); + CountingDatabase previous = database; + objectsGate.arm(); + Future old = workers.submit(() -> previous.getTableNullable("a")); + objectsGate.awaitStarted(); + // Catalog eviction/replacement must not wait for the in-flight remote table lookup. + previous.releaseCachedEntries(); + database = new CountingDatabase(catalog); + database.setInitializedForTest(true); + catalog.addDatabaseForTest(database); + TestExternalTable replacement = database.getTableNullable("a"); + objectsGate.release.countDown(); + Assertions.assertNotSame(replacement, old.get(10, TimeUnit.SECONDS)); + Assertions.assertSame(replacement, database.getTableNullable("a")); + Assertions.assertNull(previous.getCachedTableNameByIdForTest(replacement.getId())); + Assertions.assertEquals(6, catalog.getFeMetaCacheOwner().entries().size()); + } + + @Test + public void refreshLogRetainsScopeAndOldRecordsDefaultToMetadataOnly() { + ExternalObjectLog old = GsonUtils.GSON.fromJson( + "{\"catalogId\":1,\"dbName\":\"db\",\"tableName\":\"a\"}", ExternalObjectLog.class); + Assertions.assertFalse(old.isInvalidateTableLookup()); + ExternalObjectLog full = ExternalObjectLog.createForExplicitRefreshTable(1L, "db", "a", 2L); + Assertions.assertTrue(GsonUtils.GSON.fromJson( + GsonUtils.GSON.toJson(full), ExternalObjectLog.class).isInvalidateTableLookup()); + } + + private void refreshExplicitly() { + new RefreshManager().replayRefreshTable(ExternalObjectLog.createForExplicitRefreshTable( + catalog.getId(), "db", "a", System.currentTimeMillis())); + } + + private static void putTable(String name, String column) { + Provider.TABLES.put(name, Collections.singletonList(new Column(column, PrimitiveType.INT))); + } + + public static class Provider implements TestExternalCatalog.TestCatalogProvider { + static final Map> TABLES = new ConcurrentHashMap<>(); + + @Override + public Map>> getMetadata() { + return Collections.singletonMap("db", TABLES); + } + } + + private class CountingCatalog extends TestExternalCatalog { + private final AtomicInteger nameLoads = new AtomicInteger(); + private boolean bypassDatabaseNames; + + CountingCatalog(long id) { + super(id, "owner_test_" + id, "", + new HashMap<>(Collections.singletonMap("catalog_provider.class", Provider.class.getName())), ""); + } + + @Override + protected boolean shouldBypassDbNameCache(SessionContext context) { + return bypassDatabaseNames; + } + + @Override + protected List listTableNamesFromRemote(SessionContext context, String dbName) { + nameLoads.incrementAndGet(); + List snapshot = new ArrayList<>(super.listTableNamesFromRemote(context, dbName)); + namesGate.pause(); + return snapshot; + } + } + + private class CountingDatabase extends TestExternalDatabase { + CountingDatabase(CountingCatalog owner) { + super(owner, Util.genIdByName(owner.getName(), "db"), "db", "db"); + } + + @Override + public TestExternalTable buildTableInternal(String remoteName, String localName, long id, + ExternalCatalog owner, ExternalDatabase db) { + TestExternalTable table = new TestExternalTable( + id, localName, remoteName, (TestExternalCatalog) owner, this) { + @Override + public Optional initSchema() { + List snapshot = + new ArrayList<>(((TestExternalCatalog) catalog).mockedSchema("db", getName())); + schemaGate.pause(); + return Optional.of(new SchemaCacheValue(snapshot)); + } + }; + objectsGate.pause(); + return table; + } + } + + private static class Gate { + private final AtomicInteger armed = new AtomicInteger(); + private final CountDownLatch started = new CountDownLatch(1); + private final CountDownLatch release = new CountDownLatch(1); + + void arm() { + armed.set(1); + } + + void awaitStarted() throws InterruptedException { + Assertions.assertTrue(started.await(10, TimeUnit.SECONDS)); + } + + void pause() { + if (armed.compareAndSet(1, 0)) { + started.countDown(); + try { + Assertions.assertTrue(release.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverClassLoaderTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverClassLoaderTest.java index 759e984d2c14f5..a75a5037518fbb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverClassLoaderTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverClassLoaderTest.java @@ -54,7 +54,7 @@ public void eventSourceAcquisitionUsesConnectorClassLoaderAndRestoresCaller() th connectorLoader, new Class[] {Connector.class}, (proxy, method, args) -> { if ("getEventSource".equals(method.getName())) { acquisitionClassLoader.set(Thread.currentThread().getContextClassLoader()); - return null; + return Collections.emptyList(); } if ("getMetadata".equals(method.getName()) || "close".equals(method.getName())) { return null; @@ -187,12 +187,12 @@ private void assertDescriptorApplicationClassLoader(boolean failInvalidation) th }); Connector connector = (Connector) Proxy.newProxyInstance( connectorLoader, new Class[] {Connector.class}, (proxy, method, args) -> { - if ("invalidateDb".equals(method.getName())) { + if ("cacheDependencies".equals(method.getName())) { invalidationClassLoader.set(Thread.currentThread().getContextClassLoader()); if (failInvalidation) { throw new IllegalStateException("expected invalidation failure"); } - return null; + return Collections.emptyList(); } if ("getMetadata".equals(method.getName()) || "close".equals(method.getName())) { return null; @@ -204,6 +204,8 @@ private void assertDescriptorApplicationClassLoader(boolean failInvalidation) th Mockito.when(catalog.getId()).thenReturn(1L); Mockito.when(catalog.getName()).thenReturn("test_catalog"); Mockito.when(catalog.canonicalLocalDatabaseNameFromRemote("db1")).thenReturn("db1"); + Mockito.when(catalog.resolveDatabaseNameForInvalidation("db1")).thenReturn("db1"); + Mockito.when(catalog.getConnectorIfPresent()).thenReturn(connector); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); Mockito.doAnswer(invocation -> { mutationClassLoader.set(Thread.currentThread().getContextClassLoader()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverMappedNameTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverMappedNameTest.java index 4d68fd102721f4..95b4065aedb7b7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverMappedNameTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverMappedNameTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.RefreshManager; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.event.MetastoreChangeDescriptor; import org.apache.doris.connector.spi.event.MetastoreChangeDescriptor.Op; @@ -47,12 +48,13 @@ public void databaseRenameKeepsRemoteConnectorKeysAndUsesLocalFeNames() throws E apply(catalog, connector, descriptor, env); InOrder inOrder = Mockito.inOrder(connector, catalogMgr); - inOrder.verify(connector).invalidateDb("OldDb"); - inOrder.verify(connector).invalidateDb("NewDb"); + inOrder.verify(connector).cacheDependencies(ScopePath.database("OldDb")); + inOrder.verify(connector).cacheDependencies(ScopePath.database("NewDb")); inOrder.verify(catalogMgr).unregisterExternalDatabaseFromEvent("local_OldDb", "test_catalog"); inOrder.verify(catalogMgr).registerExternalDatabaseFromEvent( "NewDb", "local_NewDb", "test_catalog"); - Mockito.verifyNoMoreInteractions(connector, catalogMgr); + Mockito.verifyNoMoreInteractions(catalogMgr); + Mockito.verify(connector, Mockito.never()).getMetadata(Mockito.any()); } @Test @@ -68,13 +70,14 @@ public void tableRenameKeepsRemoteConnectorKeysAndUsesBothTableIdentities() thro apply(catalog, connector, descriptor, env); InOrder inOrder = Mockito.inOrder(connector, catalogMgr); - inOrder.verify(connector).invalidateTable("OldDb", "OldTable"); - inOrder.verify(connector).invalidateTable("NewDb", "NewTable"); + inOrder.verify(connector).cacheDependencies(ScopePath.table("OldDb", "OldTable")); + inOrder.verify(connector).cacheDependencies(ScopePath.table("NewDb", "NewTable")); inOrder.verify(catalogMgr).unregisterExternalTableFromEvent( "local_OldDb", "local_OldDb_OldTable", "test_catalog"); inOrder.verify(catalogMgr).registerExternalTableFromEvent( "local_NewDb", "NewTable", "local_NewDb_NewTable", "test_catalog", 2L); - Mockito.verifyNoMoreInteractions(connector, catalogMgr); + Mockito.verifyNoMoreInteractions(catalogMgr); + Mockito.verify(connector, Mockito.never()).getMetadata(Mockito.any()); } @Test @@ -122,7 +125,7 @@ private static void apply(PluginDrivenExternalCatalog catalog, Connector connect try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); Deencapsulation.invoke( - new MetastoreEventSyncDriver(), "applyOne", catalog, connector, descriptor); + new MetastoreEventSyncDriver(), "applyOne", catalog, descriptor); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverRenameTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverRenameTest.java index 5d447f4a2a12be..4a47d4b6f7b6bb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverRenameTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetastoreEventSyncDriverRenameTest.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.event.MetastoreChangeDescriptor; import org.apache.doris.connector.spi.event.MetastoreChangeDescriptor.Op; @@ -46,15 +47,16 @@ public void databaseRenameAlwaysCleansOldNameAndRegistersNewName() throws Except try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); Deencapsulation.invoke( - new MetastoreEventSyncDriver(), "applyOne", catalog, connector, descriptor); + new MetastoreEventSyncDriver(), "applyOne", catalog, descriptor); } InOrder inOrder = Mockito.inOrder(connector, catalogMgr); - inOrder.verify(connector).invalidateDb("OldDb"); - inOrder.verify(connector).invalidateDb("NewDb"); + inOrder.verify(connector).cacheDependencies(ScopePath.database("OldDb")); + inOrder.verify(connector).cacheDependencies(ScopePath.database("NewDb")); inOrder.verify(catalogMgr).unregisterExternalDatabaseFromEvent("OldDb", "test_catalog"); inOrder.verify(catalogMgr).registerExternalDatabaseFromEvent("NewDb", "NewDb", "test_catalog"); - Mockito.verifyNoMoreInteractions(connector, catalogMgr); + Mockito.verifyNoMoreInteractions(catalogMgr); + Mockito.verify(connector, Mockito.never()).getMetadata(Mockito.any()); } @Test @@ -70,17 +72,18 @@ public void tableRenameAlwaysCleansOldNameAndRegistersNewName() throws Exception try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); Deencapsulation.invoke( - new MetastoreEventSyncDriver(), "applyOne", catalog, connector, descriptor); + new MetastoreEventSyncDriver(), "applyOne", catalog, descriptor); } InOrder inOrder = Mockito.inOrder(connector, catalogMgr); - inOrder.verify(connector).invalidateTable("OldDb", "OldTable"); - inOrder.verify(connector).invalidateTable("NewDb", "NewTable"); + inOrder.verify(connector).cacheDependencies(ScopePath.table("OldDb", "OldTable")); + inOrder.verify(connector).cacheDependencies(ScopePath.table("NewDb", "NewTable")); inOrder.verify(catalogMgr).unregisterExternalTableFromEvent( "OldDb", "OldTable", "test_catalog"); inOrder.verify(catalogMgr).registerExternalTableFromEvent( "NewDb", "NewTable", "NewTable", "test_catalog", 2L); - Mockito.verifyNoMoreInteractions(connector, catalogMgr); + Mockito.verifyNoMoreInteractions(catalogMgr); + Mockito.verify(connector, Mockito.never()).getMetadata(Mockito.any()); } @Test @@ -96,16 +99,17 @@ public void sameNameViewRecreateInvalidatesConnectorOnceBeforeReplacement() thro try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); Deencapsulation.invoke( - new MetastoreEventSyncDriver(), "applyOne", catalog, connector, descriptor); + new MetastoreEventSyncDriver(), "applyOne", catalog, descriptor); } InOrder inOrder = Mockito.inOrder(connector, catalogMgr); - inOrder.verify(connector).invalidateTable("db1", "view1"); + inOrder.verify(connector).cacheDependencies(ScopePath.table("db1", "view1")); inOrder.verify(catalogMgr).unregisterExternalTableFromEvent( "db1", "view1", "test_catalog"); inOrder.verify(catalogMgr).registerExternalTableFromEvent( "db1", "view1", "view1", "test_catalog", 2L); - Mockito.verifyNoMoreInteractions(connector, catalogMgr); + Mockito.verifyNoMoreInteractions(catalogMgr); + Mockito.verify(connector, Mockito.never()).getMetadata(Mockito.any()); } @Test @@ -123,18 +127,19 @@ public void sameNameTableDropCreateInvalidatesBothIncarnations() throws Exceptio try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); MetastoreEventSyncDriver driver = new MetastoreEventSyncDriver(); - Deencapsulation.invoke(driver, "applyOne", catalog, connector, drop); - Deencapsulation.invoke(driver, "applyOne", catalog, connector, create); + Deencapsulation.invoke(driver, "applyOne", catalog, drop); + Deencapsulation.invoke(driver, "applyOne", catalog, create); } InOrder inOrder = Mockito.inOrder(connector, catalogMgr); - inOrder.verify(connector).invalidateTable("db1", "tbl1"); + inOrder.verify(connector).cacheDependencies(ScopePath.table("db1", "tbl1")); inOrder.verify(catalogMgr).unregisterExternalTableFromEvent( "db1", "tbl1", "test_catalog"); - inOrder.verify(connector).invalidateTable("db1", "tbl1"); + inOrder.verify(connector).cacheDependencies(ScopePath.table("db1", "tbl1")); inOrder.verify(catalogMgr).registerExternalTableFromEvent( "db1", "tbl1", "tbl1", "test_catalog", 3L); - Mockito.verifyNoMoreInteractions(connector, catalogMgr); + Mockito.verifyNoMoreInteractions(catalogMgr); + Mockito.verify(connector, Mockito.never()).getMetadata(Mockito.any()); } @Test @@ -152,16 +157,17 @@ public void sameNameDatabaseDropCreateInvalidatesBothIncarnations() throws Excep try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { envStatic.when(Env::getCurrentEnv).thenReturn(env); MetastoreEventSyncDriver driver = new MetastoreEventSyncDriver(); - Deencapsulation.invoke(driver, "applyOne", catalog, connector, drop); - Deencapsulation.invoke(driver, "applyOne", catalog, connector, create); + Deencapsulation.invoke(driver, "applyOne", catalog, drop); + Deencapsulation.invoke(driver, "applyOne", catalog, create); } InOrder inOrder = Mockito.inOrder(connector, catalogMgr); - inOrder.verify(connector).invalidateDb("db1"); + inOrder.verify(connector).cacheDependencies(ScopePath.database("db1")); inOrder.verify(catalogMgr).unregisterExternalDatabaseFromEvent("db1", "test_catalog"); - inOrder.verify(connector).invalidateDb("db1"); + inOrder.verify(connector).cacheDependencies(ScopePath.database("db1")); inOrder.verify(catalogMgr).registerExternalDatabaseFromEvent("db1", "db1", "test_catalog"); - Mockito.verifyNoMoreInteractions(connector, catalogMgr); + Mockito.verifyNoMoreInteractions(catalogMgr); + Mockito.verify(connector, Mockito.never()).getMetadata(Mockito.any()); } private static class IdentityEventCatalog extends PluginDrivenExternalCatalog { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/PluginDrivenSysTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/PluginDrivenSysTableTest.java index d164ccf3a97b14..1bd10f7c859ddb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/PluginDrivenSysTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/PluginDrivenSysTableTest.java @@ -256,7 +256,7 @@ protected synchronized void makeSureInitialized() { } @Test - public void testSysTableEmptyWhenBaseHandleMissing() { + public void testSysTableFailsWhenBaseHandleMissing() { ConnectorMetadata metadata = Mockito.mock(ConnectorMetadata.class); ConnectorSession session = Mockito.mock(ConnectorSession.class); Mockito.when(session.getStatementScope()).thenReturn(ConnectorStatementScope.NONE); @@ -272,8 +272,8 @@ protected synchronized void makeSureInitialized() { } }; - Assertions.assertFalse(sysTable.initSchema().isPresent(), - "no base handle -> no sys handle -> empty schema (no spurious getSysTableHandle)"); + CacheException failure = Assertions.assertThrows(CacheException.class, sysTable::initSchema); + Assertions.assertTrue(failure.getMessage().contains("REMOTE_DB.REMOTE_TBL$snapshots does not exist")); Mockito.verify(metadata, Mockito.never()) .getSysTableHandle(Mockito.any(), Mockito.any(), Mockito.anyString()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CatalogMetadataChangesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CatalogMetadataChangesTest.java new file mode 100644 index 00000000000000..ef15261f72f911 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CatalogMetadataChangesTest.java @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.util.Util; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.datasource.log.ExternalObjectLog; +import org.apache.doris.datasource.plugin.PluginDrivenExternalCatalog; +import org.apache.doris.persist.CreateDbInfo; +import org.apache.doris.persist.CreateTableInfo; +import org.apache.doris.persist.DropDbInfo; +import org.apache.doris.persist.DropInfo; +import org.apache.doris.persist.TruncateTableInfo; +import org.apache.doris.persist.gson.GsonUtils; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.Optional; + +class CatalogMetadataChangesTest { + private final CatalogMetaCache root = CatalogMetaCache.unmanaged(); + private PluginDrivenExternalCatalog catalog; + private MockedStatic env; + + @BeforeEach + void setup() { + env = Mockito.mockStatic(Env.class); + env.when(Env::getCurrentEnv).thenReturn(Mockito.mock(Env.class)); + catalog = Mockito.mock(PluginDrivenExternalCatalog.class); + Mockito.when(catalog.getName()).thenReturn("c"); + Mockito.when(catalog.getMetaCacheOwnerIfPresent()).thenReturn(root); + Mockito.when(catalog.getDbForReplay("db")).thenReturn(Optional.empty()); + Mockito.when(catalog.resolveDatabaseNameForInvalidation("db")).thenReturn("db"); + } + + @AfterEach + void cleanup() { + root.close(); + env.close(); + } + + private MetaCache cache(String name, ScopePath scope) { + MetaCache cache = root.create(MetaCacheDefinition.builder(name, + CacheSpec.of(true, -1, 100), key -> scope).build()); + cache.put("key", "value"); + return cache; + } + + private ScopePath local(ScopePath scope) { + return scope.in(ScopePath.Coordinates.LOCAL); + } + + private ScopePath remote(ScopePath scope) { + return scope.in(ScopePath.Coordinates.REMOTE); + } + + @Test + void contentChangePreservesNamesAndLookupButInvalidatesEveryContentDependency() { + MetaCache names = cache("names", local(ScopePath.tableCollection("db"))); + MetaCache object = cache("object", local(ScopePath.table("db", "t"))); + MetaCache schema = cache("schema", local(ScopePath.tableMetadata("db", "t"))); + MetaCache hive = cache("hive", remote(ScopePath.table("DB", "T"))); + MetaCache iceberg = cache("iceberg", remote(ScopePath.table("DB", "T"))); + MetaCache other = cache("other", remote(ScopePath.table("DB", "OTHER"))); + MetaCache rows = cache("rows", ScopePath.table( + Long.toString(Util.genIdByName("c", "db")), Long.toString(Util.genIdByName("c", "db", "t"))) + .in(ScopePath.Coordinates.ID)); + CatalogMetadataChanges.apply(catalog, ExternalObjectLog.createForRefreshTable(1, "db", "t", 0) + .withRemoteNames("DB", "T")); + Assertions.assertEquals("value", names.getIfPresent("key")); + Assertions.assertEquals("value", object.getIfPresent("key")); + Assertions.assertNull(schema.getIfPresent("key")); + Assertions.assertNull(hive.getIfPresent("key")); + Assertions.assertNull(iceberg.getIfPresent("key")); + Assertions.assertNull(rows.getIfPresent("key")); + Assertions.assertEquals("value", other.getIfPresent("key")); + Mockito.verify(catalog, Mockito.never()).getConnector(); + Mockito.verify(catalog, Mockito.never()).getFeMetaCacheOwner(); + } + + @Test + void coldRenameUsesPersistedRemoteNamesAndFencesOldAndNewResourcesOnly() { + MetaCache old = cache("old", local(ScopePath.table("db", "old"))); + MetaCache replacement = cache("new", local(ScopePath.table("db", "new"))); + MetaCache remoteOld = cache("remoteOld", remote(ScopePath.table("DB", "OLD"))); + MetaCache remoteNew = cache("remoteNew", remote(ScopePath.table("DB", "NEW"))); + MetaCache sibling = cache("sibling", remote(ScopePath.table("DB", "OTHER"))); + ExternalObjectLog change = ExternalObjectLog.createForRenameTable(1, "db", "old", "new") + .withRemoteNames("DB", "OLD"); + change.setRemoteNewTableName("NEW"); + change = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(change), ExternalObjectLog.class); + CatalogMetadataChanges.apply(catalog, change); + Assertions.assertNull(old.getIfPresent("key")); + Assertions.assertNull(replacement.getIfPresent("key")); + Assertions.assertNull(remoteOld.getIfPresent("key")); + Assertions.assertNull(remoteNew.getIfPresent("key")); + Assertions.assertEquals("value", sibling.getIfPresent("key")); + } + + @Test + void oldColdLogWidensOnlyRemoteCoordinates() { + MetaCache remote = cache("remote", remote(ScopePath.table("ANY", "T"))); + MetaCache names = cache("names", local(ScopePath.tableCollection("db"))); + MetaCache sibling = cache("sibling", local(ScopePath.table("db", "other"))); + CatalogMetadataChanges.apply(catalog, ExternalObjectLog.createForRefreshTable(1, "db", "t", 0)); + Assertions.assertNull(remote.getIfPresent("key")); + Assertions.assertEquals("value", names.getIfPresent("key")); + Assertions.assertEquals("value", sibling.getIfPresent("key")); + } + + @Test + void gatewayDependencyProjectionParticipatesInTheSameChange() { + Connector connector = Mockito.mock(Connector.class); + Mockito.when(catalog.getConnectorIfPresent()).thenReturn(connector); + Mockito.when(connector.cacheDependencies(ScopePath.table("DB", "T"))) + .thenReturn(Collections.singletonList(ScopePath.table("lake", "physical"))); + MetaCache source = cache("source", remote(ScopePath.table("DB", "T"))); + MetaCache lake = cache("lake", remote(ScopePath.table("lake", "physical"))); + CatalogMetadataChanges.apply(catalog, ExternalObjectLog.createForRefreshTable(1, "db", "t", 0) + .withRemoteNames("DB", "T")); + Assertions.assertNull(source.getIfPresent("key")); + Assertions.assertNull(lake.getIfPresent("key")); + Mockito.verify(connector, Mockito.never()).invalidateTable(Mockito.anyString(), Mockito.anyString()); + } + + @Test + void ddlJournalRecordsPreserveTheSameChangeForReplay() { + ExternalObjectLog change = ExternalObjectLog.createForExplicitRefreshTable(1, "db", "t", 0) + .withRemoteNames("DB", "T"); + Object[] records = { + new CreateDbInfo("c", "db", null).withExternalCacheChange(change), + new CreateTableInfo("c", "db", "t").withExternalCacheChange(change), + new DropDbInfo("c", "db").withExternalCacheChange(change), + new DropInfo("c", "db", "t").withExternalCacheChange(change), + new TruncateTableInfo("c", "db", "t", null, 0).withExternalCacheChange(change) + }; + for (Object record : records) { + String json = GsonUtils.GSON.toJson(record); + Object restored = GsonUtils.GSON.fromJson(json, record.getClass()); + Assertions.assertEquals(GsonUtils.GSON.toJsonTree(record).getAsJsonObject().get("externalCacheChange"), + GsonUtils.GSON.toJsonTree(restored).getAsJsonObject().get("externalCacheChange")); + Assertions.assertTrue(json.contains("remoteTableName")); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java index 22bbc80ad788f0..53d4e80cd7987b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/FeMetaCacheEntryTest.java @@ -18,6 +18,9 @@ package org.apache.doris.datasource.metacache; import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.ScopePath; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -64,6 +67,97 @@ public void testLoadMutationAndStatsUseSharedRuntime() { } } + @Test + public void testRetiredReadThroughValuesAreDiscardedWithoutPublishingActions() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + AtomicInteger actions = new AtomicInteger(); + FeMetaCacheEntry entry = FeMetaCacheEntry.sharedLookup(owner, + key -> ScopePath.database(key), ScopePath.catalog(), "objects", + key -> new AtomicBoolean(true), ENABLED, executor, false, 8, + (key, value, cause) -> value.set(false)); + entry.close(); + AtomicBoolean transientValue = entry.getAndRunIfCurrent("db", (key, value) -> actions.incrementAndGet()); + Assertions.assertFalse(transientValue.get()); + Assertions.assertEquals(0, actions.get()); + Assertions.assertNull(entry.getIfPresent("db")); + Assertions.assertNull(entry.computeWithCommitAction("db", (key, value) -> new AtomicBoolean(true), + actions::incrementAndGet)); + Assertions.assertEquals(0, actions.get()); + entry.invalidateKeyAndRun("db", actions::incrementAndGet); + Assertions.assertEquals(1, actions.get()); + Assertions.assertTrue(owner.entries().isEmpty()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testRetirementPinsAuxiliaryStateUntilAcceptedActionFinishes() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + ExecutorService worker = Executors.newSingleThreadExecutor(); + CountDownLatch actionEntered = new CountDownLatch(1); + CountDownLatch finishAction = new CountDownLatch(1); + AtomicInteger cleanups = new AtomicInteger(); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + FeMetaCacheEntry entry = FeMetaCacheEntry.sharedLookup(owner, + ScopePath::database, ScopePath.catalog(), "objects", + String::length, ENABLED, executor, false, 8, (key, value, cause) -> { }); + entry.onRetired(cleanups::incrementAndGet); + Assertions.assertEquals(2, entry.get("db")); + Future read = worker.submit(() -> entry.getAndRunIfCurrent("db", (key, value) -> { + actionEntered.countDown(); + await(finishAction); + Assertions.assertEquals(0, cleanups.get()); + })); + Assertions.assertTrue(actionEntered.await(3L, TimeUnit.SECONDS)); + entry.close(); + Assertions.assertEquals(0, cleanups.get()); + finishAction.countDown(); + Assertions.assertEquals(2, read.get(3L, TimeUnit.SECONDS)); + Assertions.assertEquals(1, cleanups.get()); + Assertions.assertTrue(owner.entries().isEmpty()); + entry.close(); + Assertions.assertEquals(1, cleanups.get()); + } finally { + finishAction.countDown(); + worker.shutdownNow(); + executor.shutdownNow(); + } + } + + @Test + @SuppressWarnings("unchecked") + public void testObservationRejectsLogicalRetirementBeforePhysicalRelease() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch loadStarted = new CountDownLatch(1); + CountDownLatch finishLoad = new CountDownLatch(1); + try (CatalogMetaCache owner = CatalogMetaCache.unmanaged()) { + FeMetaCacheEntry entry = FeMetaCacheEntry.sharedLookup(owner, + ScopePath::database, ScopePath.catalog(), "objects", + key -> new Object(), ENABLED, executor, false, 8, null); + Object original = entry.get("db"); + MetaCache.Observation observation = entry.observe("db", original).get(); + Future load = executor.submit(() -> entry.get("other", key -> { + loadStarted.countDown(); + await(finishLoad); + return new Object(); + })); + Assertions.assertTrue(loadStarted.await(3L, TimeUnit.SECONDS)); + entry.close(); + Assertions.assertFalse(owner.entries().isEmpty(), "accepted load pins the physical registration"); + Assertions.assertFalse(observation.invalidateScopes(List.of( + ScopePath.database("db").in(ScopePath.Coordinates.LOCAL)))); + Assertions.assertSame(original, ((MetaCache) owner.entries().get("objects")).getIfPresent("db")); + finishLoad.countDown(); + load.get(3L, TimeUnit.SECONDS); + Assertions.assertTrue(owner.entries().isEmpty()); + } finally { + finishLoad.countDown(); + executor.shutdownNow(); + } + } + @Test public void testRejectedWeightedMutationReturnsWithoutRetryingForever() throws Exception { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/IdNameIndexTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/IdNameIndexTest.java index 4ea6773e24db34..176f8832817333 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/IdNameIndexTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/IdNameIndexTest.java @@ -17,6 +17,12 @@ package org.apache.doris.datasource.metacache; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; + import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -160,6 +166,27 @@ public void testConcurrentConflictingPublicationKeepsOneCompleteMapping() throws } } + @Test + public void identitySurvivesObjectEvictionButNotSemanticRefresh() { + try (CatalogMetaCache root = CatalogMetaCache.unmanaged()) { + IdNameIndex index = new IdNameIndex("table"); + ScopePath scope = ScopePath.table("db", "t").in(ScopePath.Coordinates.LOCAL); + index.bind(root, "ids", name -> ScopePath.table("db", name)); + MetaCache objects = root.create(MetaCacheDefinition.builder("objects", + CacheSpec.of(true, -1, 1), key -> scope).build()); + objects.put("t", "old"); + index.put(1, "t"); + objects.invalidateKey("t"); + Assertions.assertEquals("t", index.getName(1)); + root.invalidateScope(scope); + Assertions.assertNull(index.getName(1)); + index.put(2, "t"); + Assertions.assertEquals("t", index.getName(2)); + Assertions.assertNull(index.getName(1)); + index.close(root); + } + } + private static boolean putAfterStart( IdNameIndex index, CountDownLatch start, long id, String name) throws InterruptedException { start.await(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogCacheTest.java index 7c61d641747475..f13f4182a16488 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogCacheTest.java @@ -18,64 +18,65 @@ package org.apache.doris.datasource.plugin; import org.apache.doris.catalog.Env; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.spi.Connector; +import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; -import org.mockito.InOrder; import org.mockito.MockedStatic; import org.mockito.Mockito; -import java.util.HashMap; -import java.util.Map; +import java.util.Collections; -/** Verifies connector-owned caches follow catalog refresh semantics. */ +/** Catalog refresh reaches connector domains through the root even before schema caches are initialized. */ public class PluginDrivenExternalCatalogCacheTest { - - private static PluginDrivenExternalCatalog catalogWith(Connector connector) { - Map props = new HashMap<>(); - props.put("type", "iceberg"); - return new PluginDrivenExternalCatalog(1L, "test_ctl", null, props, "", connector); - } - @Test public void refreshCatalogWithInvalidateDropsConnectorCaches() { - Connector connector = Mockito.mock(Connector.class); - PluginDrivenExternalCatalog catalog = catalogWith(connector); - Env env = Mockito.mock(Env.class); - ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); - Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - catalog.onRefreshCache(true); - } - // Connector state must be cleared before the engine invalidates row counts derived from it. - InOrder order = Mockito.inOrder(connector, cacheMgr); - order.verify(connector).invalidateAll(); - order.verify(cacheMgr).invalidateCatalog(1L); + refresh(true); } @Test - public void refreshCatalogWithoutInvalidateDoesNotTouchConnector() { + public void refreshCatalogWithoutInvalidatePreservesConnectorCaches() { + refresh(false); + } + + private void refresh(boolean invalidate) { Connector connector = Mockito.mock(Connector.class); - PluginDrivenExternalCatalog catalog = catalogWith(connector); - Env env = Mockito.mock(Env.class); - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); - catalog.onRefreshCache(false); + PluginDrivenExternalCatalog catalog = new PluginDrivenExternalCatalog(1L, "c", null, + Collections.singletonMap("type", "iceberg"), "", connector); + try (CatalogMetaCache root = catalog.getFeMetaCacheOwner()) { + MetaCache remote = root.openDomain("iceberg", ScopePath.Coordinates.REMOTE, + getClass().getClassLoader()).create(MetaCacheDefinition.builder("table", + CacheSpec.of(true, -1, 10), key -> ScopePath.table("DB", key)).build()); + remote.put("T", "old"); + Env env = Mockito.mock(Env.class); + CatalogMgr catalogs = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogs).getCatalog(1L); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogs); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(new ExternalMetaCacheMgr(true)); + try (MockedStatic current = Mockito.mockStatic(Env.class)) { + current.when(Env::getCurrentEnv).thenReturn(env); + catalog.onRefreshCache(invalidate); + } + Assertions.assertEquals(invalidate ? null : "old", remote.getIfPresent("T")); + Mockito.verify(connector, Mockito.never()).invalidateAll(); } - Mockito.verify(connector, Mockito.never()).invalidateAll(); } @Test public void refreshCatalogWithNullConnectorIsSafe() { - // A reset catalog has no connector to invalidate. - PluginDrivenExternalCatalog catalog = catalogWith(null); + PluginDrivenExternalCatalog catalog = new PluginDrivenExternalCatalog(1L, "c", null, + Collections.singletonMap("type", "iceberg"), "", null); Env env = Mockito.mock(Env.class); Mockito.when(env.getExtMetaCacheMgr()).thenReturn(Mockito.mock(ExternalMetaCacheMgr.class)); - try (MockedStatic envStatic = Mockito.mockStatic(Env.class)) { - envStatic.when(Env::getCurrentEnv).thenReturn(env); + try (MockedStatic current = Mockito.mockStatic(Env.class)) { + current.when(Env::getCurrentEnv).thenReturn(env); Assertions.assertDoesNotThrow(() -> catalog.onRefreshCache(true)); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java index 18a0e80c8998aa..6bfe4cf49719c3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogConcurrencyTest.java @@ -72,6 +72,8 @@ public class PluginDrivenExternalCatalogConcurrencyTest { public void setUp() { Env env = Mockito.mock(Env.class); cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(cacheMgr.commonRefreshExecutor()) + .thenReturn(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService()); AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); Mockito.when(env.getAccessManager()).thenReturn(accessManager); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogDdlRoutingTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogDdlRoutingTest.java index 9605674d694a6e..29fe3023bfdc25 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogDdlRoutingTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalCatalogDdlRoutingTest.java @@ -35,6 +35,8 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.common.util.Util; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.ScopePath; import org.apache.doris.connector.ddl.CreateTableInfoToConnectorRequestConverter; import org.apache.doris.connector.spi.Connector; import org.apache.doris.connector.spi.ConnectorColumn; @@ -97,6 +99,7 @@ public class PluginDrivenExternalCatalogDdlRoutingTest { private ConnectorMetadata metadata; private ConnectorSession session; private TestablePluginCatalog catalog; + private CatalogMetaCache cacheRoot; @BeforeEach public void setUp() { @@ -126,10 +129,18 @@ public void setUp() { Mockito.when(mockEnv.getRefreshManager()).thenReturn(mockRefreshManager); Mockito.when(mockEnv.getConstraintManager()).thenReturn(mockConstraintManager); Mockito.when(mockEnv.getExtMetaCacheMgr()).thenReturn(mockMetaCacheMgr); + Mockito.when(mockMetaCacheMgr.commonRefreshExecutor()) + .thenReturn(com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService()); + cacheRoot = Mockito.spy(CatalogMetaCache.unmanaged()); + Deencapsulation.setField(catalog, "feMetaCacheOwner", cacheRoot); + org.apache.doris.datasource.CatalogMgr catalogs = Mockito.mock(org.apache.doris.datasource.CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogs).getCatalog(1L); + Mockito.when(mockEnv.getCatalogMgr()).thenReturn(catalogs); } @AfterEach public void tearDown() { + cacheRoot.close(); if (mockedEnv != null) { mockedEnv.close(); } @@ -146,11 +157,9 @@ public void testCreateDbRoutesToConnectorAndInvalidatesCache() throws Exception Mockito.verify(metadata).createDatabase(session, "db1", props); Mockito.verify(mockEditLog).logCreateDb(Mockito.any()); - Assertions.assertEquals(1, catalog.resetMetaCacheNamesCount); - InOrder order = Mockito.inOrder(connector, mockMetaCacheMgr); - order.verify(connector).invalidateDb("db1"); - order.verify(mockMetaCacheMgr).invalidateDb( - 1L, Util.genIdByName("test-catalog", "db1"), "db1"); + verifyInvalidation(ScopePath.databaseCollection().in(ScopePath.Coordinates.LOCAL)); + verifyRemoteDatabaseInvalidation("db1"); + verifyDeterministicDatabaseInvalidation("db1"); } @Test @@ -162,9 +171,8 @@ public void testCreateDbPersistsCanonicalLocalName() throws Exception { ArgumentCaptor log = ArgumentCaptor.forClass(CreateDbInfo.class); Mockito.verify(mockEditLog).logCreateDb(log.capture()); Assertions.assertEquals("local_db", log.getValue().getDbName()); - Mockito.verify(connector).invalidateDb("REMOTE_DB"); - Mockito.verify(mockMetaCacheMgr).invalidateDb( - 1L, Util.genIdByName("test-catalog", "local_db"), "local_db"); + verifyRemoteDatabaseInvalidation("REMOTE_DB"); + verifyDeterministicDatabaseInvalidation("local_db"); } @Test @@ -175,10 +183,9 @@ public void testCreateDbIfNotExistsRefreshesCachesWhenDbExists() throws Exceptio Mockito.verify(metadata, Mockito.never()).createDatabase(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.verify(mockEditLog, Mockito.never()).logCreateDb(Mockito.any()); - Mockito.verify(connector).invalidateDb("DB1"); - Assertions.assertEquals(1, catalog.resetMetaCacheNamesCount); - Mockito.verify(mockMetaCacheMgr).invalidateDb( - 1L, Util.genIdByName("test-catalog", "db1"), "db1"); + verifyRemoteDatabaseInvalidation("DB1"); + verifyInvalidation(ScopePath.databaseCollection().in(ScopePath.Coordinates.LOCAL)); + verifyDeterministicDatabaseInvalidation("db1"); } @Test @@ -201,9 +208,8 @@ public void testCreateDbIfNotExistsSkipsWhenRemoteExists() throws Exception { Mockito.verify(metadata).databaseExists(session, "db1"); Mockito.verify(metadata, Mockito.never()).createDatabase(Mockito.any(), Mockito.any(), Mockito.any()); Mockito.verify(mockEditLog, Mockito.never()).logCreateDb(Mockito.any()); - Assertions.assertEquals(1, catalog.resetMetaCacheNamesCount); - Mockito.verify(mockMetaCacheMgr).invalidateDb( - 1L, Util.genIdByName("test-catalog", "db1"), "db1"); + verifyInvalidation(ScopePath.databaseCollection().in(ScopePath.Coordinates.LOCAL)); + verifyDeterministicDatabaseInvalidation("db1"); } @Test @@ -217,7 +223,7 @@ public void testCreateDbIfNotExistsCreatesWhenRemoteAbsent() throws Exception { Mockito.verify(metadata).databaseExists(session, "db1"); Mockito.verify(metadata).createDatabase(session, "db1", props); Mockito.verify(mockEditLog).logCreateDb(Mockito.any()); - Assertions.assertEquals(1, catalog.resetMetaCacheNamesCount); + verifyInvalidation(ScopePath.databaseCollection().in(ScopePath.Coordinates.LOCAL)); } @Test @@ -230,9 +236,8 @@ public void testCreateDbIfNotExistsLosingTheCreateRaceReturnsWithoutEditLog() th Mockito.verify(metadata, Mockito.times(2)).databaseExists(session, "db1"); Mockito.verify(mockEditLog, Mockito.never()).logCreateDb(Mockito.any()); - Mockito.verify(connector).invalidateDb("db1"); - Mockito.verify(mockMetaCacheMgr).invalidateDb( - 1L, Util.genIdByName("test-catalog", "db1"), "db1"); + verifyRemoteDatabaseInvalidation("db1"); + verifyDeterministicDatabaseInvalidation("db1"); } @Test @@ -277,9 +282,7 @@ public void testDropDbRoutesToConnectorAndUnregisters() throws Exception { Mockito.verify(metadata).dropDatabase(session, "db1", false, false); Mockito.verify(mockEditLog).logDropDb(Mockito.any()); - Assertions.assertEquals("db1", catalog.unregisteredDb, - "dropDb must remove the db from the cache (legacy afterDropDb parity)"); - Mockito.verify(mockMetaCacheMgr).invalidateDb(1L, DATABASE_ID, "db1"); + verifyDeterministicDatabaseInvalidation("db1"); } @Test @@ -350,9 +353,8 @@ public void testDropDbResolvesRemoteNameRoutesAndUnregisters() throws Exception Mockito.verify(mockEditLog).logDropDb(dropDbInfo.capture()); Assertions.assertEquals("db1", dropDbInfo.getValue().getDbName(), "edit-log DropDbInfo must carry the LOCAL db name for follower replay"); - Assertions.assertEquals("db1", catalog.unregisteredDb, - "cache invalidation must use the LOCAL db name"); - Mockito.verify(connector).invalidateDb("REMOTE_DB1"); + verifyDeterministicDatabaseInvalidation("db1"); + verifyRemoteDatabaseInvalidation("REMOTE_DB1"); } // ==================== DROP TABLE ==================== @@ -366,6 +368,8 @@ public void testDropTableKeepsExactCleanupAfterDatabaseCacheEviction() throws Ex Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("T1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; ExternalDatabase replayDb = mockExternalDatabase(); catalog.dbForReplayResult = Optional.of(replayDb); @@ -391,9 +395,9 @@ public void testDropTableKeepsExactCleanupAfterDatabaseCacheEviction() throws Ex Assertions.assertEquals("db1", catalog.lastGetDbForReplayArg); Mockito.verify(replayDb, Mockito.never()).unregisterTable(Mockito.anyString()); Mockito.verify(db, Mockito.never()).unregisterTable(Mockito.anyString()); - Mockito.verify(mockMetaCacheMgr).invalidateTable(1L, DATABASE_ID, "db1", TABLE_ID, "t1"); + verifyDeterministicTableInvalidation("db1", "t1"); // Connector caches are keyed by the remote identity. - Mockito.verify(connector).invalidateTable("DB1", "TBL1"); + verifyRemoteTableInvalidation("DB1", "TBL1"); } @Test @@ -438,6 +442,8 @@ public void testDropTemporaryTableNeverDropsPermanentExternalTable() { Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; DdlException ex = Assertions.assertThrows(DdlException.class, @@ -455,6 +461,8 @@ public void testDropTemporaryTableIfExistsIsNoopAndKeepsPermanentExternalTable() ExternalTable table = Mockito.mock(ExternalTable.class); Mockito.when(table.getName()).thenReturn("t1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; catalog.dropTable("db1", "t1", false, false, false, true, true, false); @@ -473,6 +481,8 @@ public void testDropTableHandleAbsentAfterLocalResolveCleansLocalStateWithIfExis Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; Mockito.when(metadata.getTableHandle(session, "DB1", "TBL1")).thenReturn(Optional.empty()); @@ -480,9 +490,9 @@ public void testDropTableHandleAbsentAfterLocalResolveCleansLocalStateWithIfExis Mockito.verify(metadata).getTableHandle(session, "DB1", "TBL1"); Mockito.verify(metadata, Mockito.never()).dropTable(Mockito.any(), Mockito.any()); - Mockito.verify(connector).invalidateTable("DB1", "TBL1"); + verifyRemoteTableInvalidation("DB1", "TBL1"); Mockito.verify(mockEditLog).logDropTable(Mockito.any()); - Mockito.verify(mockMetaCacheMgr).invalidateTable(1L, DATABASE_ID, "db1", TABLE_ID, "t1"); + verifyDeterministicTableInvalidation("db1", "t1"); } @Test @@ -492,6 +502,8 @@ public void testDropTableHandleAbsentAfterLocalResolveThrowsWithoutIfExists() { Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; Mockito.when(metadata.getTableHandle(session, "DB1", "TBL1")).thenReturn(Optional.empty()); @@ -507,6 +519,8 @@ public void testDropTableWrapsConnectorException() { Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class); @@ -528,6 +542,8 @@ public void testDropTableRoutesViewToDropViewAndUnregisters() throws Exception { Mockito.when(view.getRemoteDbName()).thenReturn("DB1"); Mockito.when(view.getRemoteName()).thenReturn("V1"); Mockito.doReturn(view).when(db).getTableNullable("v1"); + Mockito.doReturn(db).when(view).getDb(); + Mockito.when(view.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; ExternalDatabase replayDb = mockExternalDatabase(); catalog.dbForReplayResult = Optional.of(replayDb); @@ -546,8 +562,8 @@ public void testDropTableRoutesViewToDropViewAndUnregisters() throws Exception { Assertions.assertEquals("v1", dropInfo.getValue().getTableName(), "edit-log DropInfo must carry the LOCAL view name for follower replay"); Assertions.assertEquals("db1", catalog.lastGetDbForReplayArg); - Mockito.verify(replayDb).unregisterTable("v1"); - Mockito.verify(connector).invalidateTable("DB1", "V1"); + verifyDeterministicTableInvalidation("db1", "v1"); + verifyRemoteTableInvalidation("DB1", "V1"); } @Test @@ -557,6 +573,8 @@ public void testDropViewWrapsConnectorException() { Mockito.when(view.getRemoteDbName()).thenReturn("DB1"); Mockito.when(view.getRemoteName()).thenReturn("V1"); Mockito.doReturn(view).when(db).getTableNullable("v1"); + Mockito.doReturn(db).when(view).getDb(); + Mockito.when(view.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; Mockito.when(metadata.viewExists(session, "DB1", "V1")).thenReturn(true); Mockito.doThrow(new DorisConnectorException("boom")).when(metadata).dropView(session, "DB1", "V1"); @@ -578,6 +596,8 @@ public void testRenameTableKeepsCanonicalExactCleanupAfterDatabaseCacheEviction( Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); Mockito.when(db.canonicalLocalTableNameFromRemote("REMOTE_T2")).thenReturn("local_t2"); catalog.dbNullableResult = db; ExternalDatabase replayDb = mockExternalDatabase(); @@ -606,11 +626,10 @@ public void testRenameTableKeepsCanonicalExactCleanupAfterDatabaseCacheEviction( Assertions.assertEquals("t1", logCap.getValue().getTableName()); Assertions.assertEquals("local_t2", logCap.getValue().getNewTableName()); // The destination has no pre-rename mapping, so its remote name is the requested new name. - Mockito.verify(connector).invalidateTable("DB1", "TBL1"); - Mockito.verify(connector).invalidateTable("DB1", "REMOTE_T2"); - Mockito.verify(mockMetaCacheMgr).invalidateTableRename( - 1L, DATABASE_ID, "db1", TABLE_ID, "t1", - Util.genIdByName("test-catalog", "db1", "local_t2"), "local_t2"); + verifyRemoteTableInvalidation("DB1", "TBL1"); + verifyRemoteTableInvalidation("DB1", "REMOTE_T2"); + verifyDeterministicTableInvalidation("db1", "t1"); + verifyDeterministicTableInvalidation("db1", "local_t2"); } @Test @@ -620,6 +639,8 @@ public void testRenameResolvesDestinationIdentityBeforeRemoteMutation() throws E Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); Mockito.when(db.canonicalLocalTableNameFromRemote("TBL2")) .thenThrow(new IllegalStateException("name mapping failed")); catalog.dbNullableResult = db; @@ -642,6 +663,8 @@ public void testRenameTableUsesStructuralSourceInvalidationAndDestinationBarrier Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; ExternalDatabase cachedDb = mockExternalDatabase(); catalog.dbForReplayResult = Optional.of(cachedDb); @@ -650,14 +673,13 @@ public void testRenameTableUsesStructuralSourceInvalidationAndDestinationBarrier catalog.renameTable("db1", "t1", "t2"); - long destinationId = Util.genIdByName("test-catalog", "db1", "t2"); InOrder order = Mockito.inOrder(mockEditLog, connector, cachedDb, mockMetaCacheMgr); order.verify(mockEditLog).logRefreshExternalTable(Mockito.any()); - order.verify(connector).invalidateTable("DB1", "TBL1"); - order.verify(connector).invalidateTable("DB1", "t2"); - order.verify(cachedDb).invalidateTableRename("t1", "t2"); - order.verify(mockMetaCacheMgr).invalidateTableRename( - 1L, DATABASE_ID, "db1", TABLE_ID, "t1", destinationId, "t2"); + verifyRemoteTableInvalidation("DB1", "TBL1"); + verifyRemoteTableInvalidation("DB1", "t2"); + verifyDeterministicTableInvalidation("db1", "t1"); + verifyDeterministicTableInvalidation("db1", "t1"); + verifyDeterministicTableInvalidation("db1", "t2"); } @Test @@ -685,6 +707,8 @@ public void testRenameTableWrapsConnectorExceptionAndSkipsBookkeeping() { Mockito.when(table.getRemoteDbName()).thenReturn("DB1"); Mockito.when(table.getRemoteName()).thenReturn("TBL1"); Mockito.doReturn(table).when(db).getTableNullable("t1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; ExternalDatabase replayDb = mockExternalDatabase(); catalog.dbForReplayResult = Optional.of(replayDb); @@ -763,9 +787,8 @@ public void testCreateTableInvalidatesDbCacheUsingLocalNames() throws UserExcept "edit-log CreateTableInfo must carry the LOCAL db name for follower replay"); Assertions.assertEquals("t1", persist.getValue().getTblName(), "edit-log CreateTableInfo must carry the LOCAL table name for follower replay"); - Mockito.verify(mockMetaCacheMgr).invalidateTable( - 1L, DATABASE_ID, "db1", Util.genIdByName("test-catalog", "db1", "t1"), "t1"); - Mockito.verify(connector).invalidateTable("DB1", "t1"); + verifyDeterministicTableInvalidation("db1", "t1"); + verifyRemoteTableInvalidation("DB1", "t1"); } } @@ -793,9 +816,8 @@ public void testCreateTablePersistsCanonicalLocalTableName() throws UserExceptio Mockito.verify(mockEditLog).logCreateTable(log.capture()); Assertions.assertEquals("db1", log.getValue().getDbName()); Assertions.assertEquals("local_t1", log.getValue().getTblName()); - Mockito.verify(connector).invalidateTable("DB1", "REMOTE_T1"); - Mockito.verify(mockMetaCacheMgr).invalidateTable( - 1L, DATABASE_ID, "db1", Util.genIdByName("test-catalog", "db1", "local_t1"), "local_t1"); + verifyRemoteTableInvalidation("DB1", "REMOTE_T1"); + verifyDeterministicTableInvalidation("db1", "local_t1"); } } @@ -831,7 +853,7 @@ public void testCreateTableIfNotExistsEvictsPreexistingRowCount() throws Excepti Assertions.assertTrue(result); Mockito.verify(metadata, Mockito.never()).createTable(Mockito.any(), Mockito.any()); Mockito.verify(mockEditLog, Mockito.never()).logCreateTable(Mockito.any()); - Mockito.verify(connector).invalidateTable("DB1", "t1"); + verifyRemoteTableInvalidation("DB1", "t1"); Assertions.assertEquals(200L, rowCountCache.getCachedRowCount(1L, DATABASE_ID, tableId, false)); } } @@ -860,9 +882,8 @@ public void testCreateTableIfNotExistsLosingTheCreateRaceReturnsTrueWithoutEditL Assertions.assertTrue(res, "losing the create race under IF NOT EXISTS must return true"); Mockito.verify(mockEditLog, Mockito.never()).logCreateTable(Mockito.any()); - Mockito.verify(connector).invalidateTable("DB1", "t1"); - Mockito.verify(mockMetaCacheMgr).invalidateTable( - 1L, DATABASE_ID, "db1", Util.genIdByName("test-catalog", "db1", "t1"), "t1"); + verifyRemoteTableInvalidation("DB1", "t1"); + verifyDeterministicTableInvalidation("db1", "t1"); } @Test @@ -908,8 +929,7 @@ public void testCreateTableIfNotExistsExistingLocalTableReturnsTrue() throws Exc "an existing IF NOT EXISTS target must remain a no-op even if new validation rules exist"); Mockito.verify(metadata, Mockito.never()).createTable(Mockito.any(), Mockito.any()); Mockito.verify(mockEditLog, Mockito.never()).logCreateTable(Mockito.any()); - Mockito.verify(mockMetaCacheMgr).invalidateTable( - 1L, DATABASE_ID, "db1", Util.genIdByName("test-catalog", "db1", "t1"), "t1"); + verifyDeterministicTableInvalidation("db1", "t1"); } @Test @@ -997,10 +1017,13 @@ public void testCreateTableLocalConflictWithoutIfNotExistsRejects() throws Excep public void testTruncateTablePersistsCanonicalLocalNames() throws Exception { ExternalDatabase db = mockExternalDatabase(); Mockito.when(db.getId()).thenReturn(DATABASE_ID); + Mockito.when(db.resolveTableNameForInvalidation(Mockito.anyString())).thenAnswer(inv -> inv.getArgument(0)); ExternalTable table = mockAlterTable(); Mockito.when(table.getId()).thenReturn(TABLE_ID); Mockito.when(db.getFullName()).thenReturn("db1"); Mockito.doReturn(table).when(db).getTableNullable("T1"); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); catalog.dbNullableResult = db; ConnectorTableHandle handle = Mockito.mock(ConnectorTableHandle.class); Mockito.when(metadata.getTableHandle(session, "DB1", "TBL1")).thenReturn(Optional.of(handle)); @@ -1025,8 +1048,10 @@ public void testReplayTruncateUsesNormalRefreshWhenTableObjectIsCached() { catalog.replayTruncateTable(info); - Mockito.verify(mockRefreshManager).refreshTableInternal(db, table, 123L); - Mockito.verifyNoInteractions(mockMetaCacheMgr); + Mockito.verify(table).unsetObjectCreated(); + Mockito.verify(table).setUpdateTime(123L); + verifyDeterministicTableInvalidation("db1", "t1"); + verifyRemoteTableInvalidation("DB1", "TBL1"); } @Test @@ -1038,11 +1063,8 @@ public void testReplayTruncateInvalidatesByCanonicalNameWhenTableObjectIsCold() catalog.replayTruncateTable(new TruncateTableInfo("test-catalog", "db1", "t1", null, 123L)); - InOrder order = Mockito.inOrder(connector, mockMetaCacheMgr); - order.verify(connector).invalidateDb("DB1"); - order.verify(mockMetaCacheMgr).invalidateTable( - 1L, Util.genIdByName("test-catalog", "db1"), "db1", - Util.genIdByName("test-catalog", "db1", "t1"), "t1"); + verifyRemoteDatabaseInvalidation("DB1"); + verifyDeterministicTableInvalidation("db1", "t1"); } @Test @@ -1051,11 +1073,8 @@ public void testReplayTruncateInvalidatesAllConnectorCachesWhenDatabaseObjectIsC catalog.replayTruncateTable(new TruncateTableInfo("test-catalog", "db1", "t1", null, 123L)); - InOrder order = Mockito.inOrder(connector, mockMetaCacheMgr); - order.verify(connector).invalidateAll(); - order.verify(mockMetaCacheMgr).invalidateTable( - 1L, Util.genIdByName("test-catalog", "db1"), "db1", - Util.genIdByName("test-catalog", "db1", "t1"), "t1"); + verifyInvalidation(ScopePath.catalog().in(ScopePath.Coordinates.REMOTE)); + verifyDeterministicTableInvalidation("db1", "t1"); } // ==================== EDIT-LOG REPLAY (follower / observer propagation) ==================== @@ -1071,8 +1090,8 @@ public void testReplayDropTableInvalidatesConnectorOnFollower() { catalog.replayDropTable("db1", "t1"); - Mockito.verify(connector).invalidateTable("DB1", "TBL1"); - Mockito.verify(replayDb).unregisterTable("t1"); + verifyRemoteTableInvalidation("DB1", "TBL1"); + verifyDeterministicTableInvalidation("db1", "t1"); } @Test @@ -1097,8 +1116,8 @@ public void testReplayDropTableInvalidatesDatabaseConnectorCacheWhenTableObjectI catalog.replayDropTable("db1", "t1"); - Mockito.verify(connector).invalidateDb("DB1"); - Mockito.verify(replayDb).unregisterTable("t1"); + verifyRemoteDatabaseInvalidation("DB1"); + verifyDeterministicTableInvalidation("db1", "t1"); } @Test @@ -1107,7 +1126,7 @@ public void testReplayDropTableInvalidatesAllConnectorCachesWhenDatabaseObjectIs catalog.replayDropTable("db1", "t1"); - Mockito.verify(connector).invalidateAll(); + verifyInvalidation(ScopePath.catalog().in(ScopePath.Coordinates.REMOTE)); verifyDeterministicTableInvalidation("db1", "t1"); } @@ -1131,8 +1150,7 @@ public void testReplayDropDbInvalidatesConnectorOnFollower() { catalog.replayDropDb("db1"); - Mockito.verify(connector).invalidateDb("DB1"); - Assertions.assertEquals("db1", catalog.unregisteredDb); + verifyRemoteDatabaseInvalidation("DB1"); verifyDeterministicDatabaseInvalidation("db1"); } @@ -1142,8 +1160,7 @@ public void testReplayDropDbInvalidatesAllWhenDatabaseObjectIsCold() { catalog.replayDropDb("db1"); - Mockito.verify(connector).invalidateAll(); - Assertions.assertEquals("db1", catalog.unregisteredDb); + verifyInvalidation(ScopePath.catalog().in(ScopePath.Coordinates.REMOTE)); verifyDeterministicDatabaseInvalidation("db1"); } @@ -1155,7 +1172,7 @@ public void testReplayCreateTableInvalidatesConnectorOnFollower() { catalog.replayCreateTable("db1", "t1"); - Mockito.verify(connector).invalidateDb("DB1"); + verifyRemoteDatabaseInvalidation("DB1"); verifyDeterministicTableInvalidation("db1", "t1"); } @@ -1165,7 +1182,7 @@ public void testReplayCreateTableInvalidatesAllWhenDatabaseObjectIsCold() { catalog.replayCreateTable("db1", "t1"); - Mockito.verify(connector).invalidateAll(); + verifyInvalidation(ScopePath.catalog().in(ScopePath.Coordinates.REMOTE)); verifyDeterministicTableInvalidation("db1", "t1"); } @@ -1173,8 +1190,8 @@ public void testReplayCreateTableInvalidatesAllWhenDatabaseObjectIsCold() { public void testReplayCreateDbInvalidatesDatabaseCaches() { catalog.replayCreateDb("db1"); - Assertions.assertEquals(1, catalog.resetMetaCacheNamesCount); - Mockito.verify(connector).invalidateAll(); + verifyInvalidation(ScopePath.databaseCollection().in(ScopePath.Coordinates.LOCAL)); + verifyInvalidation(ScopePath.catalog().in(ScopePath.Coordinates.REMOTE)); verifyDeterministicDatabaseInvalidation("db1"); } @@ -1575,6 +1592,7 @@ private ExternalTable mockAlterTable() { Mockito.when(table.getRemoteName()).thenReturn("TBL1"); ExternalDatabase db = mockExternalDatabase(); Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getCatalog()).thenReturn(catalog); return table; } @@ -1586,15 +1604,30 @@ private ConnectorTableHandle stubAlterHandle() { } /** A nullable INT Doris column (iceberg add/modify reject non-nullable adds). */ - private void verifyDeterministicDatabaseInvalidation(String dbName) { - Mockito.verify(mockMetaCacheMgr).invalidateDb( - 1L, Util.genIdByName("test-catalog", dbName), dbName); + private void verifyInvalidation(ScopePath expected) { + boolean invalidated = Mockito.mockingDetails(cacheRoot).getInvocations().stream() + .filter(invocation -> invocation.getMethod().getName().equals("invalidateScopes")) + .map(invocation -> (java.util.Collection) invocation.getArgument(0)) + .flatMap(java.util.Collection::stream) + .map(ScopePath.class::cast) + .anyMatch(scope -> scope.contains(expected)); + Assertions.assertTrue(invalidated, "Missing resource invalidation: " + expected); + } + + private void verifyRemoteTableInvalidation(String database, String table) { + verifyInvalidation(ScopePath.table(database, table).in(ScopePath.Coordinates.REMOTE)); + } + + private void verifyRemoteDatabaseInvalidation(String database) { + verifyInvalidation(ScopePath.database(database).in(ScopePath.Coordinates.REMOTE)); + } + + private void verifyDeterministicDatabaseInvalidation(String database) { + verifyInvalidation(ScopePath.database(database).in(ScopePath.Coordinates.LOCAL)); } - private void verifyDeterministicTableInvalidation(String dbName, String tableName) { - Mockito.verify(mockMetaCacheMgr).invalidateTable( - 1L, Util.genIdByName("test-catalog", dbName), dbName, - Util.genIdByName("test-catalog", dbName, tableName), tableName); + private void verifyDeterministicTableInvalidation(String database, String table) { + verifyInvalidation(ScopePath.tableMetadata(database, table).in(ScopePath.Coordinates.LOCAL)); } private static Column nullableIntColumn(String name) { @@ -1608,6 +1641,7 @@ private ExternalDatabase mockExternalDatabase() { Mockito.when(db.getFullName()).thenReturn("db1"); Mockito.when(db.getRemoteName()).thenReturn("DB1"); Mockito.when(db.getId()).thenReturn(DATABASE_ID); + Mockito.when(db.resolveTableNameForInvalidation(Mockito.anyString())).thenAnswer(inv -> inv.getArgument(0)); Mockito.when(db.isInitialized()).thenReturn(true); Mockito.when(db.canonicalLocalTableNameFromRemote(Mockito.anyString())) .thenAnswer(inv -> inv.getArgument(0)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableEngineTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableEngineTest.java index 67f872d27250c7..745ebb49c7e1f1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableEngineTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableEngineTest.java @@ -31,6 +31,7 @@ import org.apache.doris.connector.spi.ConnectorTableSchema; import org.apache.doris.connector.spi.ConnectorType; import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.SessionContext; @@ -206,13 +207,12 @@ public void testTableTypeIsAlwaysPluginExternalTable() { } @Test - public void testInitSchemaReturnsEmptyWhenTableHandleMissing() { + public void testInitSchemaFailsWhenTableHandleMissing() { Connector connector = createMockConnector(false, false); PluginDrivenExternalTable table = createTableWithCatalogType("jdbc", connector); - // Return an empty schema result when the connector cannot resolve the table handle. - Assertions.assertFalse(table.initSchema().isPresent(), - "Missing connector table handles should produce an empty schema result"); + CacheException failure = Assertions.assertThrows(CacheException.class, table::initSchema); + Assertions.assertTrue(failure.getMessage().contains("test_db.test_table does not exist")); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java index f21ed9f8006890..4605f1f2784b66 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTableTest.java @@ -37,6 +37,7 @@ import org.apache.doris.connector.spi.pushdown.ConnectorExpression; import org.apache.doris.connector.spi.write.ConnectorRowChangeStyle; import org.apache.doris.connector.spi.write.ConnectorWritePlanProvider; +import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.mvcc.MvccSnapshot; @@ -984,6 +985,13 @@ public void initSchemaBuildsViewSchemaFromViewDefinitionColumns() { "a view has no partition columns"); } + @Test + public void missingTableHandlePreservesNotFoundInsteadOfGenericSchemaFailure() { + PluginDrivenExternalTable table = initSchemaViewTable(null, false); + CacheException error = Assertions.assertThrows(CacheException.class, table::initSchema); + Assertions.assertEquals("Table db1.v1 does not exist", error.getMessage()); + } + @Test public void initSchemaUsesTableHandlePathForNonView() { // WHY: the isView() branch must NOT hijack ordinary tables — a non-view must still resolve its schema diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenMissingTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenMissingTableTest.java new file mode 100644 index 00000000000000..479527c84e3db0 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/plugin/PluginDrivenMissingTableTest.java @@ -0,0 +1,472 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.plugin; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.FeConstants; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.common.util.Util; +import org.apache.doris.connector.cache.CacheSpec; +import org.apache.doris.connector.cache.CatalogMetaCache; +import org.apache.doris.connector.cache.MetaCache; +import org.apache.doris.connector.cache.MetaCacheDefinition; +import org.apache.doris.connector.cache.ScopePath; +import org.apache.doris.connector.spi.Connector; +import org.apache.doris.connector.spi.ConnectorColumn; +import org.apache.doris.connector.spi.ConnectorMetadata; +import org.apache.doris.connector.spi.ConnectorSession; +import org.apache.doris.connector.spi.ConnectorTableSchema; +import org.apache.doris.connector.spi.ConnectorType; +import org.apache.doris.connector.spi.handle.ConnectorTableHandle; +import org.apache.doris.datasource.CacheException; +import org.apache.doris.datasource.CatalogIf; +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.SchemaCacheKey; +import org.apache.doris.datasource.SchemaCacheValue; +import org.apache.doris.datasource.SessionContext; +import org.apache.doris.datasource.log.ExternalObjectLog; +import org.apache.doris.datasource.metacache.CatalogMetadataChanges; +import org.apache.doris.datasource.metacache.FeMetaCacheEntry; +import org.apache.doris.datasource.metacache.NameCacheValue; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicLong; + +/** Tests read-observed absence through real FE object/schema caches and a controllable connector. */ +public class PluginDrivenMissingTableTest extends TestWithFeService { + private static final AtomicLong IDS = new AtomicLong(910000); + private final Map remoteTables = new ConcurrentHashMap<>(); + private Connector connector; + private ConnectorMetadata metadata; + private TestCatalog catalog; + private TestDatabase database; + private MetaCache remoteCache; + private ExecutorService worker; + private CountDownLatch missingStarted; + private CountDownLatch releaseMissing; + private AtomicBoolean blockMissing; + private boolean failLookup; + + @Override + protected void runBeforeAll() throws Exception { + FeConstants.runningUnitTest = true; + createDefaultCtx().setThreadLocalInfo(); + } + + @BeforeEach + void setup() { + failLookup = false; + remoteTables.put("a", "old_column"); + remoteTables.put("b", "sibling_column"); + missingStarted = new CountDownLatch(1); + releaseMissing = new CountDownLatch(1); + blockMissing = new AtomicBoolean(); + worker = Executors.newFixedThreadPool(2); + connector = Mockito.mock(Connector.class); + metadata = Mockito.mock(ConnectorMetadata.class); + Mockito.when(connector.getMetadata(Mockito.any())).thenReturn(metadata); + Mockito.when(metadata.getTableHandle(Mockito.any(), Mockito.anyString(), Mockito.anyString())) + .thenAnswer(call -> { + if (failLookup) { + throw new IllegalStateException("metastore timeout"); + } + String name = call.getArgument(2); + String column = remoteTables.get(name); + if (column == null) { + if (blockMissing.compareAndSet(true, false)) { + missingStarted.countDown(); + Assertions.assertTrue(releaseMissing.await(10, TimeUnit.SECONDS)); + } + return Optional.empty(); + } + return Optional.of(new Handle(name, column)); + }); + Mockito.when(metadata.getTableSchema(Mockito.any(), Mockito.any())).thenAnswer(call -> { + Handle handle = call.getArgument(1); + return new ConnectorTableSchema(handle.name, Collections.singletonList(new ConnectorColumn( + handle.column, ConnectorType.of("INT", -1, -1), "", true, null, true)), + null, Collections.emptyMap()); + }); + Mockito.when(metadata.fromRemoteColumnName(Mockito.any(), Mockito.anyString(), + Mockito.anyString(), Mockito.anyString())).thenAnswer(call -> call.getArgument(3)); + Mockito.when(metadata.fromRemoteTableName(Mockito.any(), Mockito.anyString(), Mockito.anyString())) + .thenAnswer(call -> call.getArgument(2)); + Mockito.when(metadata.fromRemoteDatabaseName(Mockito.any(), Mockito.anyString())) + .thenAnswer(call -> call.getArgument(1)); + catalog = new TestCatalog(IDS.incrementAndGet()); + CatalogMgr manager = Env.getCurrentEnv().getCatalogMgr(); + manager.getIdToCatalog().put(catalog.getId(), catalog); + Map> byName = Deencapsulation.getField(manager, "nameToCatalog"); + byName.put(catalog.getName(), catalog); + installDatabase(); + } + + @AfterEach + void cleanup() throws Exception { + releaseMissing.countDown(); + worker.shutdownNow(); + Assertions.assertTrue(worker.awaitTermination(10, TimeUnit.SECONDS)); + catalog.closeFeMetaCacheOwner(); + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalog(catalog.getId()); + CatalogMgr manager = Env.getCurrentEnv().getCatalogMgr(); + manager.getIdToCatalog().remove(catalog.getId()); + Map> byName = Deencapsulation.getField(manager, "nameToCatalog"); + byName.remove(catalog.getName()); + remoteTables.clear(); + } + + private void installDatabase() { + catalog.setInitializedForTest(true); + database = new TestDatabase(); + database.setInitializedForTest(true); + catalog.addDatabaseForTest(database); + remoteCache = catalog.getFeMetaCacheOwner().openDomain("missing-test", ScopePath.Coordinates.REMOTE, + getClass().getClassLoader()).create(MetaCacheDefinition.builder( + "remote", CacheSpec.of(true, -1, 100), key -> ScopePath.table("db", key)).build()); + } + + @SuppressWarnings("unchecked") + private MetaCache schemas() { + return (MetaCache) catalog.getFeMetaCacheOwner().entries().get("schema"); + } + + private PluginDrivenExternalTable warmTarget() { + Assertions.assertEquals(Set.of("a", "b"), database.getTableNamesWithLock()); + PluginDrivenExternalTable table = database.getTableNullable("a"); + table.getFullSchema(); + database.getTableNullable("b").getFullSchema(); + remoteCache.put("a", "old"); + remoteCache.put("b", "sibling"); + // Keep membership/object entries hot while forcing the next schema read to consult the source. + schemas().invalidateKey(new SchemaCacheKey(table.getOrBuildNameMapping())); + return table; + } + + @Test + void missingTableRepairsNamesObjectsIdsAndConnectorMetadata() { + PluginDrivenExternalTable old = warmTarget(); + PluginDrivenExternalTable sibling = database.getTableNullable("b"); + List siblingSchema = sibling.getFullSchema(); + remoteTables.remove("a"); + Assertions.assertThrows(CacheException.class, old::getFullSchema); + Assertions.assertEquals(Set.of("b"), database.getTableNamesWithLock()); + Assertions.assertNull(database.getTableNullable(old.getId())); + Assertions.assertNull(remoteCache.getIfPresent("a")); + Assertions.assertEquals("sibling", remoteCache.getIfPresent("b")); + Assertions.assertSame(sibling, database.getTableNullable("b")); + Assertions.assertSame(siblingSchema, sibling.getFullSchema()); + } + + @Test + void missingHandleRepairsNamesBeforeSchemaLoading() { + PluginDrivenExternalTable table = warmTarget(); + remoteTables.remove("a"); + ConnectorSession session = catalog.buildCrossStatementSession(); + try { + Assertions.assertFalse(table.resolveConnectorTableHandle(session, metadata).isPresent()); + } finally { + session.getStatementScope().closeAll(); + } + Assertions.assertEquals(Set.of("b"), database.getTableNamesWithLock()); + Assertions.assertNull(database.getTableNullable(table.getId())); + Assertions.assertNull(remoteCache.getIfPresent("a")); + } + + @Test + void delayedMissingDoesNotInvalidateRecreatedTable() throws Exception { + verifyDelayedMissing(false); + } + + @Test + void delayedMissingDoesNotInvalidateExactKeyReplacement() throws Exception { + verifyDelayedMissing(true); + } + + private void verifyDelayedMissing(boolean replaceWithoutScopeInvalidation) throws Exception { + PluginDrivenExternalTable old = warmTarget(); + remoteTables.remove("a"); + blockMissing.set(true); + Future result = worker.submit(() -> Assertions.assertThrows(CacheException.class, + old::getFullSchema)); + Assertions.assertTrue(missingStarted.await(10, TimeUnit.SECONDS)); + remoteTables.put("a", "new_column"); + PluginDrivenExternalTable replacement; + if (replaceWithoutScopeInvalidation) { + replacement = database.buildTableInternal("a", "a", old.getId(), catalog, database); + database.addTableForTest(replacement); + schemas().put(new SchemaCacheKey(replacement.getOrBuildNameMapping()), replacement.initSchema().get()); + } else { + CatalogMetadataChanges.apply(catalog, ExternalObjectLog.createForExplicitRefreshTable( + catalog.getId(), "db", "a", 0).withRemoteNames("db", "a")); + replacement = database.getTableNullable("a"); + } + List newSchema = replacement.getFullSchema(); + remoteCache.put("a", "new"); + database.getTableNamesWithLock(); + int nameLoads = catalog.nameLoads.get(); + releaseMissing.countDown(); + Assertions.assertTrue(result.get(10, TimeUnit.SECONDS).getMessage().contains("does not exist")); + Assertions.assertSame(replacement, database.getTableNullable("a")); + Assertions.assertSame(replacement, database.getTableNullable(replacement.getId())); + Assertions.assertSame(newSchema, replacement.getFullSchema()); + Assertions.assertTrue(replacement.isObjectCreated()); + Assertions.assertEquals("new_column", newSchema.get(0).getName()); + Assertions.assertEquals("new", remoteCache.getIfPresent("a")); + Assertions.assertEquals(Set.of("a", "b"), database.getTableNamesWithLock()); + Assertions.assertEquals(nameLoads, catalog.nameLoads.get()); + } + + @Test + void delayedMissingDoesNotInvalidateReplacementCatalogRoot() throws Exception { + PluginDrivenExternalTable old = warmTarget(); + remoteTables.remove("a"); + blockMissing.set(true); + Future result = worker.submit(() -> Assertions.assertThrows(CacheException.class, + old::getFullSchema)); + Assertions.assertTrue(missingStarted.await(10, TimeUnit.SECONDS)); + CatalogMetaCache retired = catalog.getFeMetaCacheOwner(); + catalog.closeFeMetaCacheOwner(); + remoteTables.put("a", "new_root_column"); + installDatabase(); + PluginDrivenExternalTable replacement = database.getTableNullable("a"); + List schema = replacement.getFullSchema(); + remoteCache.put("a", "new_root"); + Assertions.assertEquals(Set.of("a", "b"), database.getTableNamesWithLock()); + int nameLoads = catalog.nameLoads.get(); + releaseMissing.countDown(); + result.get(10, TimeUnit.SECONDS); + Assertions.assertTrue(retired.isClosed()); + Assertions.assertSame(replacement, database.getTableNullable("a")); + Assertions.assertSame(schema, replacement.getFullSchema()); + Assertions.assertTrue(replacement.isObjectCreated()); + Assertions.assertEquals("new_root", remoteCache.getIfPresent("a")); + Assertions.assertEquals(Set.of("a", "b"), database.getTableNamesWithLock()); + Assertions.assertEquals(nameLoads, catalog.nameLoads.get()); + } + + @Test + void retiredDatabaseWithAnActiveLoadCannotClearReplacement() throws Exception { + PluginDrivenExternalTable old = warmTarget(); + remoteTables.remove("a"); + blockMissing.set(true); + Future missing = worker.submit(() -> { + ConnectorSession session = catalog.buildCrossStatementSession(); + try { + Assertions.assertFalse(old.resolveConnectorTableHandle(session, metadata).isPresent()); + } finally { + session.getStatementScope().closeAll(); + } + }); + Assertions.assertTrue(missingStarted.await(10, TimeUnit.SECONDS)); + FeMetaCacheEntry oldTables = Deencapsulation.getField(database, "tables"); + CountDownLatch loadStarted = new CountDownLatch(1); + CountDownLatch releaseLoad = new CountDownLatch(1); + Future activeLoad = worker.submit(() -> oldTables.get("loading", key -> { + loadStarted.countDown(); + try { + Assertions.assertTrue(releaseLoad.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + throw new AssertionError(e); + } + return null; + })); + try { + Assertions.assertTrue(loadStarted.await(10, TimeUnit.SECONDS)); + database.releaseCachedEntries(); + remoteTables.put("a", "replacement_column"); + database = new TestDatabase(); + database.setInitializedForTest(true); + catalog.addDatabaseForTest(database); + PluginDrivenExternalTable replacement = database.getTableNullable("a"); + List schema = replacement.getFullSchema(); + remoteCache.put("a", "replacement"); + database.getTableNamesWithLock(); + int loads = catalog.nameLoads.get(); + releaseMissing.countDown(); + missing.get(10, TimeUnit.SECONDS); + Assertions.assertSame(replacement, database.getTableNullable("a")); + Assertions.assertSame(schema, replacement.getFullSchema()); + Assertions.assertEquals("replacement", remoteCache.getIfPresent("a")); + Assertions.assertEquals(loads, catalog.nameLoads.get()); + } finally { + releaseLoad.countDown(); + activeLoad.get(10, TimeUnit.SECONDS); + } + } + + @Test + void changedCanonicalSpellingFencesBothExistingAndNewOldObjectReads() throws Exception { + catalog.lowerCaseMode = 2; + try { + catalog.closeFeMetaCacheOwner(); + installDatabase(); + remoteTables.remove("a"); + remoteTables.put("Foo", "old_column"); + PluginDrivenExternalTable old = database.getTableNullable("foo"); + old.getFullSchema(); + schemas().invalidateKey(new SchemaCacheKey(old.getOrBuildNameMapping())); + remoteTables.remove("Foo"); + blockMissing.set(true); + Future missing = worker.submit(() -> Assertions.assertThrows(CacheException.class, + old::getFullSchema)); + Assertions.assertTrue(missingStarted.await(10, TimeUnit.SECONDS)); + remoteTables.put("FOO", "new_column"); + FeMetaCacheEntry names = Deencapsulation.getField(database, "tableNames"); + names.invalidateKey(""); + PluginDrivenExternalTable replacement = database.getTableNullable("foo"); + Assertions.assertEquals("FOO", replacement.getName()); + List schema = replacement.getFullSchema(); + remoteCache.put("FOO", "new"); + int loads = catalog.nameLoads.get(); + releaseMissing.countDown(); + missing.get(10, TimeUnit.SECONDS); + Assertions.assertSame(replacement, database.getTableNullable("foo")); + Assertions.assertSame(schema, replacement.getFullSchema()); + Assertions.assertEquals("new", remoteCache.getIfPresent("FOO")); + Assertions.assertEquals(loads, catalog.nameLoads.get()); + // A fresh request holding the old object must not combine it with the new names snapshot. + ConnectorSession session = catalog.buildCrossStatementSession(); + try { + Assertions.assertFalse(old.resolveConnectorTableHandle(session, metadata).isPresent()); + } finally { + session.getStatementScope().closeAll(); + } + Assertions.assertSame(replacement, database.getTableNullable("foo")); + Assertions.assertSame(schema, replacement.getFullSchema()); + Assertions.assertEquals("new", remoteCache.getIfPresent("FOO")); + } finally { + catalog.lowerCaseMode = 0; + } + } + + @Test + void lookupFailureDoesNotRepairNamesOrEvictObjects() { + PluginDrivenExternalTable table = warmTarget(); + remoteTables.remove("a"); + failLookup = true; + Assertions.assertThrows(IllegalStateException.class, table::getFullSchema); + Assertions.assertEquals(Set.of("a", "b"), database.getTableNamesWithLock()); + Assertions.assertSame(table, database.getTableNullable(table.getId())); + Assertions.assertEquals("old", remoteCache.getIfPresent("a")); + Assertions.assertEquals(1, catalog.nameLoads.get()); + } + + @Test + void sessionBypassDoesNotInvalidateSharedNames() { + PluginDrivenExternalTable table = warmTarget(); + remoteTables.remove("a"); + catalog.bypassSchema = true; + Assertions.assertThrows(CacheException.class, table::getFullSchema); + Assertions.assertEquals(Set.of("a", "b"), database.getTableNamesWithLock()); + Assertions.assertSame(table, database.getTableNullable(table.getId())); + Assertions.assertEquals("old", remoteCache.getIfPresent("a")); + } + + private static class Handle implements ConnectorTableHandle { + private final String name; + private final String column; + + Handle(String name, String column) { + this.name = name; + this.column = column; + } + } + + private class TestCatalog extends PluginDrivenExternalCatalog { + private final AtomicInteger nameLoads = new AtomicInteger(); + private boolean bypassSchema; + private int lowerCaseMode; + + TestCatalog(long id) { + super(id, "missing_test_" + id, "", new HashMap<>(), "", connector); + } + + @Override + protected void initLocalObjectsImpl() { + // The fixture injects the connector directly instead of loading a production plugin. + } + + @Override + public int getLowerCaseTableNames() { + return lowerCaseMode; + } + + @Override + public Connector getConnector() { + return connector; + } + + @Override + protected List listDatabaseNames() { + return Collections.singletonList("db"); + } + + @Override + protected List listTableNamesFromRemote(SessionContext context, String dbName) { + nameLoads.incrementAndGet(); + return new ArrayList<>(remoteTables.keySet()); + } + + @Override + public boolean shouldBypassSchemaCache(SessionContext context) { + return bypassSchema; + } + } + + private class TestDatabase extends ExternalDatabase { + TestDatabase() { + super(catalog, Util.genIdByName(catalog.getName(), "db"), "db", "db"); + } + + @Override + protected PluginDrivenExternalTable buildTableInternal(String remoteName, String localName, long id, + ExternalCatalog owner, ExternalDatabase db) { + return new PluginDrivenExternalTable(id, localName, remoteName, owner, db) { + @Override + protected synchronized void makeSureInitialized() { + // The fixture registers this database explicitly; connector schema loading remains real. + objectCreated = true; + } + }; + } + } +} diff --git a/regression-test/data/external_table_p0/hive/test_hive_catalog_owner_refresh.out b/regression-test/data/external_table_p0/hive/test_hive_catalog_owner_refresh.out new file mode 100644 index 00000000000000..7730655d8510a0 --- /dev/null +++ b/regression-test/data/external_table_p0/hive/test_hive_catalog_owner_refresh.out @@ -0,0 +1,35 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !names_before -- +owner_sibling +owner_target + +-- !schema_before -- +before_col int Yes true \N + +-- !sibling_before -- +sibling_col int Yes true \N + +-- !names_after_drop -- +owner_sibling + +-- !sibling_after -- +sibling_col int Yes true \N + +-- !names_after_database_refresh -- +owner_sibling +owner_target + +-- !schema_after_database_refresh -- +after_col text Yes true \N + +-- !schema_after_recreation -- +recreated_col bigint Yes true \N + +-- !observed_names_before -- +owner_observed + +-- !observed_schema_before -- +observed_col int Yes true \N + +-- !observed_names_after -- + diff --git a/regression-test/data/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.out b/regression-test/data/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.out index b4b3deb56ec3c3..016e287e04717b 100644 --- a/regression-test/data/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.out +++ b/regression-test/data/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.out @@ -40,9 +40,9 @@ nested_tbl2 nested_tbl3 -- !sql15 -- +ns1.ns2 -- !sql17 -- -ns1.ns2 -- !sql18 -- diff --git a/regression-test/suites/external_table_p0/hive/test_hive_catalog_owner_refresh.groovy b/regression-test/suites/external_table_p0/hive/test_hive_catalog_owner_refresh.groovy new file mode 100644 index 00000000000000..13cb4cf6db544f --- /dev/null +++ b/regression-test/suites/external_table_p0/hive/test_hive_catalog_owner_refresh.groovy @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_hive_catalog_owner_refresh", "p0,external") { + String enabled = context.config.otherConfigs.get("enableHiveTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable Hive test.") + return + } + + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String hmsPort = context.config.otherConfigs.get("hive3HmsPort") + String hdfsPort = context.config.otherConfigs.get("hive3HdfsPort") + for (String catalog : ["hive_owner_source", "hive_owner_target"]) { + sql "DROP CATALOG IF EXISTS ${catalog}" + sql """CREATE CATALOG ${catalog} PROPERTIES ( + 'type' = 'hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfsPort}' + )""" + } + + // Two independent catalogs model a change made through another engine's cache domain. + sql "DROP DATABASE IF EXISTS hive_owner_source.fe_cache_owner_refresh_db FORCE" + sql "CREATE DATABASE hive_owner_source.fe_cache_owner_refresh_db" + sql "CREATE TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_target (before_col INT)" + sql "CREATE TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_sibling (sibling_col INT)" + order_qt_names_before "SHOW TABLES FROM hive_owner_target.fe_cache_owner_refresh_db" + qt_schema_before "DESC hive_owner_target.fe_cache_owner_refresh_db.owner_target" + qt_sibling_before "DESC hive_owner_target.fe_cache_owner_refresh_db.owner_sibling" + + sql "DROP TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_target" + sql "REFRESH TABLE hive_owner_target.fe_cache_owner_refresh_db.owner_target" + // Listing first proves refresh itself invalidated membership, without a failing SELECT repairing it. + order_qt_names_after_drop "SHOW TABLES FROM hive_owner_target.fe_cache_owner_refresh_db" + test { + sql "SELECT * FROM hive_owner_target.fe_cache_owner_refresh_db.owner_target" + exception "does not exist" + } + test { + sql "EXPLAIN SELECT * FROM hive_owner_target.fe_cache_owner_refresh_db.owner_target" + exception "does not exist" + } + qt_sibling_after "DESC hive_owner_target.fe_cache_owner_refresh_db.owner_sibling" + + sql "CREATE TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_target (after_col STRING)" + sql "REFRESH DATABASE hive_owner_target.fe_cache_owner_refresh_db" + order_qt_names_after_database_refresh "SHOW TABLES FROM hive_owner_target.fe_cache_owner_refresh_db" + qt_schema_after_database_refresh "DESC hive_owner_target.fe_cache_owner_refresh_db.owner_target" + + sql "DROP TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_target" + sql "CREATE TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_target (recreated_col BIGINT)" + sql "REFRESH TABLE hive_owner_target.fe_cache_owner_refresh_db.owner_target" + qt_schema_after_recreation "DESC hive_owner_target.fe_cache_owner_refresh_db.owner_target" + + sql "DROP CATALOG IF EXISTS hive_owner_observed" + sql """CREATE CATALOG hive_owner_observed PROPERTIES ( + 'type' = 'hms', + 'hadoop.username' = 'hadoop', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfsPort}', + 'schema.cache.ttl-second' = '0' + )""" + sql "CREATE TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_observed (observed_col INT)" + // Schema/HMS table reads are live, while the FE table-name and object caches stay enabled. + order_qt_observed_names_before """SHOW TABLES FROM hive_owner_observed.fe_cache_owner_refresh_db + LIKE 'owner_observed'""" + qt_observed_schema_before "DESC hive_owner_observed.fe_cache_owner_refresh_db.owner_observed" + sql "DROP TABLE hive_owner_source.fe_cache_owner_refresh_db.owner_observed" + test { + sql "SELECT * FROM hive_owner_observed.fe_cache_owner_refresh_db.owner_observed" + // MVCC observes absence before a later schema lookup reports the now-missing cached object. + exception "failed to load schema cache value" + } + order_qt_observed_names_after """SHOW TABLES FROM hive_owner_observed.fe_cache_owner_refresh_db + LIKE 'owner_observed'""" + +} diff --git a/regression-test/suites/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.groovy b/regression-test/suites/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.groovy index abf060b85cec7c..02fe9eb464536d 100644 --- a/regression-test/suites/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.groovy +++ b/regression-test/suites/external_table_p0/iceberg/iceberg_and_internal_nested_namespace.groovy @@ -208,14 +208,14 @@ suite("iceberg_and_internal_nested_namespace", "p0,external") { sql """refresh table `ns1.ns2`.nested_tbl2xxx""" exception """Table nested_tbl2xxx does not exist in db ns1.ns2""" } - // drop ns1.ns2 first, we can still see it after refresh, because ns1.ns2.ns3 still exists + // DROP invalidates database membership immediately. The remaining child keeps its parent visible. sql """drop database `ns1.ns2` force""" - qt_sql15 """show databases like "ns1.ns2"""" // empty + qt_sql15 """show databases like "ns1.ns2"""" // 1 sql """refresh catalog ${catalog_name}""" qtWithRefreshRetry("sql16", """show databases like "ns1.ns2"""", "ns1.ns2") // 1 - // then we drop ns1.ns2.ns3, after refresh, ns1.ns2 also disappear + // Removing the last child also removes the inferred parent, without requiring a catalog refresh. sql """drop database `ns1.ns2.ns3` force""" - qt_sql17 """show databases like "ns1.ns2"""" // 1 + qt_sql17 """show databases like "ns1.ns2"""" // empty qt_sql18 """show databases like "ns1.ns2.ns3"""" // empty sql """refresh catalog ${catalog_name}""" qtWithRefreshRetry("sql19", """show databases like "ns1.ns2"""", "") // empty