From 798dedf1680a61540cd3e2645d6afd3d767030dd Mon Sep 17 00:00:00 2001 From: 924060929 Date: Thu, 24 Sep 2026 09:59:27 +0800 Subject: [PATCH 01/16] [fix](fe) Invalidate external row count cache after metadata changes Squashed branch-4.1 pick of the row-count invalidation fix, rebased onto the current branch-4.1. --- .../apache/doris/catalog/RefreshManager.java | 79 ++- .../apache/doris/datasource/CatalogMgr.java | 73 ++- .../doris/datasource/ExternalCatalog.java | 77 ++- .../doris/datasource/ExternalDatabase.java | 49 +- .../datasource/ExternalMetaCacheMgr.java | 190 ++++++- .../datasource/ExternalRowCountCache.java | 205 +++++++- .../datasource/hive/HMSExternalCatalog.java | 33 +- .../datasource/hive/HiveMetadataOps.java | 10 + .../iceberg/IcebergExternalCatalog.java | 15 +- .../iceberg/IcebergMetadataOps.java | 4 + .../datasource/jdbc/JdbcExternalCatalog.java | 15 +- .../maxcompute/MaxComputeMetadataOps.java | 4 + .../paimon/PaimonExternalCatalog.java | 16 +- .../BaseExternalTableInsertExecutor.java | 9 +- .../commands/insert/HiveInsertExecutor.java | 37 +- .../doris/catalog/RefreshManagerTest.java | 296 +++++++++++ .../doris/datasource/CatalogMgrTest.java | 473 ++++++++++++++++++ .../ExternalMetaCacheRouteResolverTest.java | 209 +++++++- .../datasource/ExternalRowCountCacheTest.java | 271 +++++++++- .../datasource/hive/HiveMetadataOpsTest.java | 96 ++++ .../lance/LanceCatalogLifecycleTest.java | 33 ++ .../paimon/PaimonExternalMetaCacheTest.java | 1 + .../insert/HiveInsertExecutorTest.java | 337 +++++++++++++ 23 files changed, 2399 insertions(+), 133 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index 952acc3efb2084..1204d4eb608edc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -120,6 +120,9 @@ public void replayRefreshDb(ExternalObjectLog log) { if (!db.isPresent()) { LOG.warn("failed to find db when replaying refresh db: {}", log.debugForRefreshDb()); + // No canonical identity is available: retire the catalog scope so engine entries and + // row counts cannot survive the committed refresh. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(catalog.getId()); invalidatePaimonCatalogForUnresolvedReplay(catalog); } else { refreshDbInternal(db.get()); @@ -174,6 +177,22 @@ public void handleRefreshTable(String catalogName, String dbName, String tableNa Env.getCurrentEnv().getEditLog().logRefreshExternalTable(log); } + public void refreshTableAfterCommit(ExternalTable table) { + long updateTime = System.currentTimeMillis(); + try { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table); + refreshTableInternal((ExternalDatabase) table.getDatabase(), table, updateTime); + } catch (RuntimeException e) { + // The external transaction is already committed. Still notify follower FEs even if a + // cache layer failed, so peers do not keep serving the pre-commit state. + LOG.warn("Failed to refresh table cache after committing external insert for {}", + table.getNameWithFullQualifiers(), e); + } + ExternalObjectLog log = ExternalObjectLog.createForRefreshTable( + table.getCatalog().getId(), table.getDatabase().getFullName(), table.getName(), updateTime); + Env.getCurrentEnv().getEditLog().logRefreshExternalTable(log); + } + public void replayRefreshTable(ExternalObjectLog log) { replayRefreshSafely("refresh table " + log.getCatalogId(), () -> { ExternalCatalog catalog = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr() @@ -192,6 +211,9 @@ public void replayRefreshTable(ExternalObjectLog log) { // See comment in refreshDbInternal for why db and table may be null. if (!db.isPresent()) { LOG.warn("failed to find db when replaying refresh table: {}", log.debugForRefreshTable()); + // No canonical identity is available: retire the catalog scope so engine entries and + // row counts cannot survive the committed refresh. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(catalog.getId()); invalidatePaimonCatalogForUnresolvedReplay(catalog); return; } @@ -210,6 +232,10 @@ public void replayRefreshTable(ExternalObjectLog log) { && !db.get().hasLocalTableName(log.getTableName())) { db.get().retireAllTableObjectsWithoutEngineInvalidation(); } + // The independent row-count cache can outlive the table object, so fence the + // canonical database scope before acknowledging the committed refresh. + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateRowCountCache(catalog.getId(), db.get().getId()); invalidatePaimonCatalogForUnresolvedReplay(catalog); return; } @@ -223,11 +249,26 @@ public void replayRefreshTable(ExternalObjectLog log) { if (catalog instanceof HMSExternalCatalog && ((modifiedPartNames != null && !modifiedPartNames.isEmpty()) || (newPartNames != null && !newPartNames.isEmpty()))) { - // Partition-level cache invalidation, only for hive catalog - HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() - .hive(catalog.getId()); - cache.refreshAffectedPartitionsCache( - (HMSExternalTable) table.get(), modifiedPartNames, newPartNames); + // Partition-level cache invalidation, only for hive catalog. Fence the held table + // before the hive(...) lookup can lazily initialize the group and throw; otherwise + // a failed acquisition skips the row-count fence and the full-table fallback. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table.get()); + try { + HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .hive(catalog.getId()); + cache.refreshAffectedPartitionsCache((HMSExternalTable) table.get(), modifiedPartNames, + newPartNames); + // Close the admission window the opening fence left open: a load admitted after + // it can publish the pre-insert value from the still-resident file list. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table.get()); + } catch (RuntimeException e) { + // The insert is already committed. A partial or failed selective refresh must not + // leave stale Hive partition/file entries, so fall back to the same conservative + // full-table invalidation the leader uses. + LOG.warn("failed to refresh affected partitions when replaying refresh table for {}", + table.get().getNameWithFullQualifiers(), e); + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(table.get()); + } if (table.get() instanceof HMSExternalTable && log.getLastUpdateTime() > 0) { ((HMSExternalTable) table.get()).setUpdateTime(log.getLastUpdateTime()); } @@ -253,13 +294,22 @@ public void refreshExternalTableFromEvent(String catalogName, String dbName, Str if (!(catalog instanceof ExternalCatalog)) { throw new DdlException("Only support refresh ExternalCatalog Tables"); } + // Whole-table events are already committed remotely. Fence the row count by cached identity + // before any database/table reload can fail and make the not-found path return. + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateRowCountCache(catalog.getId(), dbName, tableName); DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { + // Cold database: widen so independently resident engine entries are retired too. + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName); return; } TableIf table = db.getTableNullable(tableName); if (table == null) { + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName); return; } refreshTableInternal((ExternalDatabase) db, (ExternalTable) table, updateTime); @@ -314,8 +364,14 @@ public void refreshPartitions(String catalogName, String dbName, String tableNam if (!(catalog instanceof ExternalCatalog)) { throw new DdlException("Only support ExternalCatalog"); } + // Partition events are already committed remotely. Fence the row count by cached identity + // before any database/table reload can fail and make the ignored-not-found path return. + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateRowCountCache(catalog.getId(), dbName, tableName); DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName); if (!ignoreIfNotExists) { throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); } @@ -324,6 +380,8 @@ public void refreshPartitions(String catalogName, String dbName, String tableNam TableIf table = db.getTableNullable(tableName); if (table == null) { + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName); if (!ignoreIfNotExists) { throw new DdlException("Table " + tableName + " does not exist in db " + dbName); } @@ -332,8 +390,15 @@ public void refreshPartitions(String catalogName, String dbName, String tableNam ExternalTable externalTable = (ExternalTable) table; HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr().hive(externalTable.getCatalog().getId()); - for (String partitionName : partitionNames) { - cache.invalidatePartitionCache(externalTable, partitionName); + try { + for (String partitionName : partitionNames) { + cache.invalidatePartitionCache(externalTable, partitionName); + } + } finally { + // Close the admission window opened by the pre-fence above, even when the selective + // invalidation fails partway. + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateRowCountCache(catalog.getId(), dbName, tableName); } ((HMSExternalTable) table).setUpdateTime(updateTime); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 227214e5962811..2150849d0c0e3b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -158,8 +158,13 @@ private RemovedCatalog removeCatalog(long catalogId) { } String catalogName = catalog.getName(); Env.getCurrentEnv().getRefreshManager().removeFromRefreshMap(catalogId); + // Remove both lock-free mappings first: a row-count load admitted after the fence below + // could otherwise still resolve this catalog through them and alias a same-name + // replacement, because RowCountKey equality uses only tableId. Fence last, while this + // write lock still excludes a same-name CREATE. idToCatalog.remove(catalogId); nameToCatalog.remove(catalogName); + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(catalogId); return new RemovedCatalog(catalog, catalogName); } @@ -476,7 +481,11 @@ && hasLanceIdentityKeyChange(oldProperties, newProperties) } log.setNewProps(loggedProperties); replayAlterCatalogProps(log, oldProperties, false); - Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_PROPS, log); + if (!(catalog instanceof ExternalCatalog)) { + // External catalogs journal the committed change from the fenced helper once publication + // has happened; non-external catalogs only reach this point on success. + Env.getCurrentEnv().getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_PROPS, log); + } } finally { writeUnlock(); } @@ -931,15 +940,33 @@ private void alterExternalCatalogPropsFenced(ExternalCatalog externalCatalog, Ca Integer[] sec = {metadataRefreshIntervalSec, metadataRefreshIntervalSec}; Env.getCurrentEnv().getRefreshManager().addToRefreshMap(catalogId, sec); } - externalCatalog.modifyCatalogProps(newProps); // The commit reset the catalog's execution context and closed its SDK resources. Cached // base generations and projections are bound to the replaced context; retire them now so // the next statement loads a generation the planning fences accept, instead of retrying - // against an unplannable cached generation until managed refresh. + // against an unplannable cached generation until managed refresh. The properties are + // published before the reset's throwable cleanup, so retirement must run either way. Env currentEnv = Env.getCurrentEnv(); ExternalMetaCacheMgr cacheMgr = currentEnv == null ? null : currentEnv.getExtMetaCacheMgr(); - if (cacheMgr != null) { - cacheMgr.onCatalogOperationalContextChanged(externalCatalog.getId()); + try { + externalCatalog.modifyCatalogProps(newProps); + } catch (RuntimeException e) { + if (!isReplay) { + throw e; + } + // A follower must not terminate because a local connector cleanup failed while applying + // an already-durable ALTER record. The property publication and the derived-state + // transitions above are failure-safe, so the record is considered applied. + LOG.warn("Failed to complete local cleanup while replaying ALTER CATALOG for {}: {}", + externalCatalog.getName(), e.getMessage(), e); + } finally { + if (cacheMgr != null) { + cacheMgr.onCatalogOperationalContextChanged(externalCatalog.getId()); + } + if (!isReplay && currentEnv != null) { + // Publication has happened once modifyCatalogProps is reached; journal it even if the + // reset's own cleanup throws. Pre-publication validation failures never reach here. + currentEnv.getEditLog().logCatalogLog(OperationType.OP_ALTER_CATALOG_PROPS, log); + } } } @@ -1075,8 +1102,14 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl if (!(catalog instanceof ExternalCatalog)) { throw new DdlException("Only support ExternalCatalog"); } + // Partition events are already committed remotely. Fence the row count by cached identity + // before any database/table reload can fail and make the ignored-not-found path return. + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateRowCountCache(catalog.getId(), dbName, tableName); DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName); if (!ignoreIfNotExists) { throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); } @@ -1085,6 +1118,8 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl TableIf table = db.getTableNullable(tableName); if (table == null) { + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName); if (!ignoreIfNotExists) { throw new DdlException("Table " + tableName + " does not exist in db " + dbName); } @@ -1104,7 +1139,14 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl return; } HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr().hive(catalog.getId()); - cache.addPartitionsCache(hmsTable.getOrBuildNameMapping(), partitionNames, partitionColumnTypes); + try { + cache.addPartitionsCache(hmsTable.getOrBuildNameMapping(), partitionNames, partitionColumnTypes); + } finally { + // Close the admission window opened by the pre-fence above, even when the selective + // update fails partway. + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateRowCountCache(catalog.getId(), dbName, tableName); + } hmsTable.setUpdateTime(updateTime); } @@ -1118,8 +1160,14 @@ public void dropExternalPartitions(String catalogName, String dbName, String tab if (!(catalog instanceof ExternalCatalog)) { throw new DdlException("Only support ExternalCatalog"); } + // Partition events are already committed remotely. Fence the row count by cached identity + // before any database/table reload can fail and make the ignored-not-found path return. + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateRowCountCache(catalog.getId(), dbName, tableName); DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName); if (!ignoreIfNotExists) { throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName()); } @@ -1128,6 +1176,8 @@ public void dropExternalPartitions(String catalogName, String dbName, String tab TableIf table = db.getTableNullable(tableName); if (table == null) { + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateTableByNameOrWider(catalog.getId(), dbName, tableName); if (!ignoreIfNotExists) { throw new DdlException("Table " + tableName + " does not exist in db " + dbName); } @@ -1135,8 +1185,15 @@ public void dropExternalPartitions(String catalogName, String dbName, String tab } HMSExternalTable hmsTable = (HMSExternalTable) table; - Env.getCurrentEnv().getExtMetaCacheMgr().hive(catalog.getId()) - .dropPartitionsCache(hmsTable, partitionNames, true); + try { + Env.getCurrentEnv().getExtMetaCacheMgr().hive(catalog.getId()) + .dropPartitionsCache(hmsTable, partitionNames, true); + } finally { + // Close the admission window opened by the pre-fence above, even when the selective + // update fails partway. + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateRowCountCache(catalog.getId(), dbName, tableName); + } hmsTable.setUpdateTime(updateTime); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index fc1a3ad9c6e528..24ec351e351d59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -187,6 +187,7 @@ public abstract class ExternalCatalog protected MetaCache> metaCache; private ThreadLocal invalidateEngineCacheOnDatabaseRemoval = ThreadLocal.withInitial(() -> true); + private volatile boolean invalidatingAllMetaCache; protected ExecutionAuthenticator executionAuthenticator; protected ThreadPoolExecutor threadPoolWithPreAuth; // Map lowercase database names to actual remote database names for case-insensitive lookup @@ -432,8 +433,7 @@ private void buildMetaCache() { localDbName -> Optional.ofNullable( buildDbForInit(null, localDbName, Util.genIdByName(name, localDbName), logType, true)), - (key, value, cause) -> value.ifPresent( - v -> v.resetMetaToUninitialized(invalidateEngineCacheOnDatabaseRemoval.get())), + (key, value, cause) -> handleDatabaseMetaCacheRemoval(value), this::acquireMetadataLoadEpoch, this::isMetadataLoadEpochCurrent); } @@ -730,11 +730,17 @@ private void refreshMetaCacheOnly(boolean invalidCache) { // prevents one full SDK-cache scan per cached database without affecting concurrent // expiry callbacks on other threads. invalidateEngineCacheOnDatabaseRemoval.set(!invalidCache); + invalidatingAllMetaCache = true; try { metaCache.invalidateAll(); } finally { + invalidatingAllMetaCache = false; invalidateEngineCacheOnDatabaseRemoval.remove(); } + if (!invalidCache) { + // No catalog-wide engine invalidation follows, so fence the row counts once here. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(id); + } } } @@ -924,7 +930,7 @@ private void removeAccessController() { * @return */ public Optional> getDbForReplay(long dbId) { - if (!isInitialized()) { + if (!isInitialized() || metaCache == null) { return Optional.empty(); } return metaCache.getMetaObjById(dbId); @@ -941,7 +947,7 @@ public Optional> getDbForReplay(String LOG.debug("getDbForReplay from metacache, db: {}.{}, catalog id: {}, is catalog init: {}", this.name, dbName, this.id, isInitialized()); } - if (!isInitialized()) { + if (!isInitialized() || metaCache == null) { return Optional.empty(); } @@ -1247,10 +1253,36 @@ public void unregisterDatabase(String dbName) { if (LOG.isDebugEnabled()) { LOG.debug("unregister database [{}]", dbName); } - if (isInitialized()) { - metaCache.invalidate(dbName, Util.genIdByName(name, dbName)); + if (!isInitialized()) { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(getId(), dbName); + return; + } + String localDbName = getLocalDatabaseName(dbName, true); + if (localDbName == null) { + // A mode-2 remote-to-local mapping can disappear (for example after a names refresh) + // while the resident database object survives. The canonical key is then unknown, so + // treat the scope as unknown: retire every cached database object and flush the engine + // caches and row counts catalog-wide instead of evicting the wrong local key. + retireAllDatabaseObjectsWithoutEngineInvalidation(); + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(getId()); + return; } - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(getId(), dbName); + metaCache.invalidate(localDbName, Util.genIdByName(name, localDbName)); + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(getId(), localDbName); + } + + boolean shouldInvalidateRowCountOnDatabaseRemoval() { + return invalidateEngineCacheOnDatabaseRemoval.get() && !invalidatingAllMetaCache; + } + + boolean shouldInvalidateRoutedCacheOnDatabaseRemoval() { + return invalidateEngineCacheOnDatabaseRemoval.get(); + } + + void handleDatabaseMetaCacheRemoval(Optional> value) { + value.ifPresent(v -> v.resetMetaToUninitialized( + shouldInvalidateRoutedCacheOnDatabaseRemoval(), + shouldInvalidateRowCountOnDatabaseRemoval())); } /** @@ -1268,6 +1300,26 @@ public void retireAllDatabaseObjectsWithoutEngineInvalidation() { metaCache.invalidateObjects(); } finally { invalidateEngineCacheOnDatabaseRemoval.remove(); + // The removal callbacks above suppress every per-database row-count fence, so publish + // one catalog-wide fence here; otherwise a recreated same-name object can reuse a stale + // count through its deterministic table id. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(getId()); + } + } + + /** + * Best-effort cleanup for a drop event whose canonical database object cannot be resolved + * (for example a mode-2 name mapping was lost). Retires the hidden database-object generation + * and flushes the catalog-wide engine state so a same-name recreation cannot reuse the old + * incarnation, engine entries, or row counts. + */ + public void retireUnresolvedDatabaseGeneration() { + try { + retireAllDatabaseObjectsWithoutEngineInvalidation(); + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(getId()); + } catch (Exception e) { + LOG.warn("Failed to retire unresolved database objects for catalog {}: {}", + getName(), e.getMessage(), e); } } @@ -1489,7 +1541,16 @@ public int hashCode() { @Override public void notifyPropertiesUpdated(Map updatedProps) { - CatalogIf.super.notifyPropertiesUpdated(updatedProps); + try { + CatalogIf.super.notifyPropertiesUpdated(updatedProps); + } finally { + // The committed ALTER already published the properties; the property-specific cache-group + // removal must still run when the reset's own client cleanup throws. + invalidatePropertySpecificCacheGroups(updatedProps); + } + } + + private void invalidatePropertySpecificCacheGroups(Map updatedProps) { String schemaCacheTtl = updatedProps.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null); ExternalMetaCacheMgr extMetaCacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); if (java.util.Objects.nonNull(schemaCacheTtl) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 2e1950971d84c6..58e9459f09697f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -120,10 +120,14 @@ public void setRemoteName(String remoteName) { } public void resetMetaToUninitialized() { - resetMetaToUninitialized(true); + resetMetaToUninitialized(true, true); } public void resetMetaToUninitialized(boolean invalidateEngineCache) { + resetMetaToUninitialized(invalidateEngineCache, invalidateEngineCache); + } + + public void resetMetaToUninitialized(boolean invalidateEngineCache, boolean invalidateRowCountCache) { if (LOG.isDebugEnabled()) { LOG.debug("resetToUninitialized db name {}, id {}, isInitializing: {}, initialized: {}", this.name, this.id, isInitializing, initialized, new Exception()); @@ -146,8 +150,20 @@ public void resetMetaToUninitialized(boolean invalidateEngineCache) { objectInvalidation.run(); } } - if (invalidateEngineCache) { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(this); + try { + if (invalidateEngineCache) { + // Route through the typed overload: connector-specific caches (for example Paimon's + // table loader) are keyed by the database object and are not fully covered by the + // name-based scan in invalidateDb(long, String). + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(this); + } + } finally { + if (invalidateRowCountCache) { + // Independent of the routed invalidation: a connector cache failure (for example + // Paimon's CacheException) must not skip the row-count fence. + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateRowCountCache(extCatalog.getId(), getId()); + } } } @@ -619,15 +635,36 @@ public boolean unregisterTableForReplay(String tableName) { // Check whether the table still exists in the local replay cache. ExternalTable dorisTable = getTableForReplay(tableName).orElse(null); if (dorisTable == null) { + // A mode-2 table-name mapping can disappear while the old object stays resident in + // MetaCache.metaObjCache; getTableForReplay then misses only because the mapping is + // absent. Retire the hidden table-object generation so a same-name recreation cannot + // reuse the prior incarnation, then widen the engine and row-count fence. + if (!hasLocalTableName(tableName)) { + retireAllTableObjectsWithoutEngineInvalidation(); + } + // The table object cache is much smaller than the row-count and Hive engine caches. A drop + // or rename must still retire stale entries when the table object was evicted, and the + // event carries the caller spelling, so widen to the canonical scope. + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateTableByNameOrWider(extCatalog.getId(), getFullName(), tableName); return false; } - // clear the cache related to this table. + // Fence the held table's row count before releasing its local name/id slot: a query does not + // take this database's write lock, so a same-name replacement admitted right after the removal + // could otherwise observe the retired row-count entry through the deterministic table id. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(dorisTable); + // Local retirement is unconditional: a routed engine failure must not leave the table visible. if (isInitialized()) { metaCache.invalidate(dorisTable.getName(), Util.genIdByName(extCatalog.getName(), name, dorisTable.getName())); } - - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(dorisTable); + // Routed engine invalidation plus completion fencing, isolated from the local retirement. + try { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(dorisTable); + } catch (RuntimeException e) { + LOG.warn("failed to invalidate engine caches after unregistering table {}.{}", name, + dorisTable.getName(), e); + } return true; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index a7980d578716b0..ec2235019d770f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -51,6 +51,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.ExecutorService; import java.util.concurrent.locks.Lock; @@ -380,10 +381,14 @@ private Map runtimeEffectiveCacheProperties( } public void invalidateCatalog(long catalogId) { - invalidateLanceTableAccess(catalogId); - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "invalidateCatalog", - () -> cache.invalidateCatalogEntries(catalogId))); + try { + invalidateLanceTableAccess(catalogId); + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "invalidateCatalog", + () -> cache.invalidateCatalogEntries(catalogId))); + } finally { + rowCountCache.invalidateCatalog(catalogId); + } } public void invalidateCatalogByEngine(long catalogId, String engine) { @@ -433,9 +438,16 @@ public void removeCatalog(long catalogId) { * retained runtime, its group) and lets the next statement load a coherent generation. */ public void onCatalogOperationalContextChanged(long catalogId) { - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "onCatalogOperationalContextChanged", - () -> cache.invalidateCatalogEntries(catalogId))); + try { + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "onCatalogOperationalContextChanged", + () -> cache.invalidateCatalogEntries(catalogId))); + } finally { + // The committed property change reset the catalog against a new target. Database objects + // already evicted from the smaller metadata cache never run removal callbacks, so publish + // one catalog-wide row-count fence to cover those cold scopes as well. + rowCountCache.invalidateCatalog(catalogId); + } } public void removeCatalogPermanently(long catalogId) { @@ -464,11 +476,19 @@ public void rollbackCatalogProperties(ExternalCatalog catalog, Map safeInvalidate( - cache, catalogId, "rollbackCatalogProperties", - () -> cache.invalidateCatalog(catalogId))); + try { + catalog.rollBackCatalogProps(oldProperties); + catalog.resetToUninitialized(false); + } finally { + // Retire any group initialized from the rejected candidate even when the reset's + // own cleanup (JDBC/Trino/Lance close) throws; otherwise an already-initialized + // group keeps taking the fast path against the old target. + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "rollbackCatalogProperties", + () -> cache.invalidateCatalog(catalogId))); + } } finally { + rowCountCache.invalidateCatalog(catalogId); lifecycleLock.unlock(); } } @@ -486,8 +506,37 @@ public void removeCatalogByEngine(long catalogId, String engine) { } public void invalidateDb(long catalogId, String dbName) { - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "invalidateDb", () -> cache.invalidateDb(catalogId, dbName))); + OptionalLong dbId = getCachedDbId(catalogId, dbName); + invalidateDb(catalogId, dbName, dbId); + } + + public void invalidateDb(long catalogId, long dbId, String dbName) { + invalidateDb(catalogId, dbName, OptionalLong.of(dbId), true); + } + + private void invalidateDb(long catalogId, String dbName, OptionalLong dbId) { + invalidateDb(catalogId, dbName, dbId, true); + } + + void invalidateDb(long catalogId, long dbId, String dbName, boolean invalidateRowCountCache) { + invalidateDb(catalogId, dbName, OptionalLong.of(dbId), invalidateRowCountCache); + } + + private void invalidateDb(long catalogId, String dbName, OptionalLong dbId, boolean invalidateRowCountCache) { + try { + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "invalidateDb", () -> cache.invalidateDb(catalogId, dbName))); + } finally { + if (invalidateRowCountCache) { + if (dbId.isPresent()) { + rowCountCache.invalidateDb(catalogId, dbId.getAsLong()); + } else { + // The database object cache is smaller than the row-count cache. If the object has + // already been evicted, retire the catalog scope rather than hashing caller spelling. + rowCountCache.invalidateCatalog(catalogId); + } + } + } } public void invalidateDb(ExternalDatabase database) { @@ -497,16 +546,28 @@ public void invalidateDb(ExternalDatabase database) { } public void invalidateTable(long catalogId, String dbName, String tableName) { - invalidateLanceTableAccess(catalogId); - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "invalidateTable", - () -> cache.invalidateTable(catalogId, dbName, tableName))); + Optional> db = Optional.empty(); + try { + invalidateLanceTableAccess(catalogId); + db = getCachedDb(catalogId, dbName); + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "invalidateTable", + () -> cache.invalidateTable(catalogId, dbName, tableName))); + } finally { + invalidateTableRowCount(catalogId, db, tableName); + } } public void invalidateTableByEngine(long catalogId, String engine, String dbName, String tableName) { - routeSpecifiedEngine(engine, cache -> safeInvalidate( - cache, catalogId, "invalidateTableByEngine", - () -> cache.invalidateTable(catalogId, dbName, tableName))); + Optional> db = Optional.empty(); + try { + db = getCachedDb(catalogId, dbName); + routeSpecifiedEngine(engine, cache -> safeInvalidate( + cache, catalogId, "invalidateTableByEngine", + () -> cache.invalidateTable(catalogId, dbName, tableName))); + } finally { + invalidateTableRowCount(catalogId, db, tableName); + } } private void invalidateLanceTableAccess(long catalogId) { @@ -520,9 +581,42 @@ private void invalidateLanceTableAccess(long catalogId) { public void invalidatePartitions(long catalogId, String dbName, String tableName, List partitions) { - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "invalidatePartitions", - () -> cache.invalidatePartitions(catalogId, dbName, tableName, partitions))); + Optional> db = Optional.empty(); + try { + db = getCachedDb(catalogId, dbName); + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "invalidatePartitions", + () -> cache.invalidatePartitions(catalogId, dbName, tableName, partitions))); + } finally { + invalidateTableRowCount(catalogId, db, tableName); + } + } + + private void invalidateTableRowCount(long catalogId, + Optional> db, String tableName) { + if (db.isPresent()) { + Optional table = db.get().getTableForReplay(tableName); + if (table.isPresent()) { + invalidateRowCountCache(table.get()); + } else { + rowCountCache.invalidateDb(catalogId, db.get().getId()); + } + } else { + rowCountCache.invalidateCatalog(catalogId); + } + } + + private OptionalLong getCachedDbId(long catalogId, String dbName) { + Optional> db = getCachedDb(catalogId, dbName); + return db.isPresent() ? OptionalLong.of(db.get().getId()) : OptionalLong.empty(); + } + + private Optional> getCachedDb(long catalogId, String dbName) { + CatalogIf catalog = getCatalog(catalogId); + if (!(catalog instanceof ExternalCatalog)) { + return Optional.empty(); + } + return ((ExternalCatalog) catalog).getDbForReplay(dbName); } public List getCatalogCacheStats(long catalogId) { @@ -691,15 +785,55 @@ public void invalidateTableCache(ExternalTable dorisTable) { long catalogId = dorisTable.getCatalog().getId(); // Typed table invalidation bypasses the name-based invalidateTable() entry point, so the // Lance access-cache retirement that used to happen there has to be repeated here. - invalidateLanceTableAccess(catalogId); - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "invalidateTable", () -> cache.invalidateTable(dorisTable))); + try { + invalidateLanceTableAccess(catalogId); + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "invalidateTable", () -> cache.invalidateTable(dorisTable))); + } finally { + invalidateRowCountCache(dorisTable); + } if (LOG.isDebugEnabled()) { LOG.debug("invalid table cache for {}.{} in catalog {}", dorisTable.getRemoteDbName(), dorisTable.getRemoteName(), dorisTable.getCatalog().getName()); } } + /** + * Best-effort invalidation for a metadata event that carries the caller's DB/table spelling. + * Resolves canonical local identity, then fences engine caches and row counts at the narrowest + * scope that still covers the event; widens to the canonical database or catalog scope when the + * cached object has already been evicted, so caller spelling can never miss a canonical key. + */ + public void invalidateTableByNameOrWider(long catalogId, String dbName, String tableName) { + Optional> db = getCachedDb(catalogId, dbName); + if (!db.isPresent()) { + invalidateCatalog(catalogId); + return; + } + Optional table = db.get().getTableForReplay(tableName); + if (table.isPresent()) { + invalidateTableCache(table.get()); + } else { + invalidateDb(catalogId, db.get().getId(), db.get().getFullName()); + } + } + + public void invalidateRowCountCache(ExternalTable table) { + rowCountCache.invalidateTable(table.getCatalog().getId(), table.getDb().getId(), table.getId()); + } + + public void invalidateRowCountCache(long catalogId) { + rowCountCache.invalidateCatalog(catalogId); + } + + public void invalidateRowCountCache(long catalogId, long dbId) { + rowCountCache.invalidateDb(catalogId, dbId); + } + + public void invalidateRowCountCache(long catalogId, String dbName, String tableName) { + invalidateTableRowCount(catalogId, getCachedDb(catalogId, dbName), tableName); + } + public LegacyMetaCacheFactory legacyMetaCacheFactory() { return legacyMetaCacheFactory; } @@ -720,6 +854,10 @@ void replaceEngineCachesForTest(List caches) { bindCatalogPreparers(); } + void replaceRowCountCacheForTest(ExternalRowCountCache cache) { + rowCountCache = cache; + } + /** * Fallback implementation of {@link AbstractExternalMetaCache} for engines that do not * provide dedicated cache entries. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java index f32ba5ae20cbc4..8fdce28ba213c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java @@ -24,22 +24,42 @@ import org.apache.doris.statistics.BasicAsyncCacheLoader; import org.apache.doris.statistics.util.StatisticsUtil; +import com.github.benmanes.caffeine.cache.AsyncCacheLoader; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.github.benmanes.caffeine.cache.Ticker; import lombok.Getter; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; +import java.util.Set; import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Supplier; public class ExternalRowCountCache { private static final Logger LOG = LogManager.getLogger(ExternalRowCountCache.class); private final AsyncLoadingCache> rowCountCache; + private final ConcurrentHashMap> inFlightLoads = new ConcurrentHashMap<>(); + // Serialize future publication with explicit invalidation. Invalidation marks matching in-flight loads + // before removing their cache entries, so a refresh that finishes later cannot republish stale data. + private final ReentrantReadWriteLock publicationLock = new ReentrantReadWriteLock(); public ExternalRowCountCache(ExecutorService executor) { + this(executor, null); + } + + ExternalRowCountCache(ExecutorService executor, Ticker ticker) { + this(executor, ticker, new RowCountCacheLoader()); + } + + ExternalRowCountCache(ExecutorService executor, Ticker ticker, RowCountCacheLoader loader) { // 1. set expireAfterWrite to 1 day, avoid too many entries // 2. set refreshAfterWrite to 10min(default), so that the cache will be refreshed after 10min CacheFactory rowCountCacheFactory = new CacheFactory( @@ -47,8 +67,9 @@ public ExternalRowCountCache(ExecutorService executor) { OptionalLong.of(Config.external_cache_refresh_time_minutes * 60), Config.max_external_table_row_count_cache_num, false, - null); - rowCountCache = rowCountCacheFactory.buildAsyncCache(new RowCountCacheLoader(), executor); + ticker); + rowCountCache = rowCountCacheFactory.buildAsyncCache( + new InvalidationAwareLoader(loader), executor); } @Getter @@ -87,6 +108,120 @@ protected Optional doLoad(RowCountKey rowCountKey) { } } + private final class InvalidationAwareLoader implements AsyncCacheLoader> { + private final RowCountCacheLoader delegate; + + private InvalidationAwareLoader(RowCountCacheLoader delegate) { + this.delegate = delegate; + } + + @Override + public CompletableFuture> asyncLoad(RowCountKey key, Executor executor) { + return loadWithInvalidationFence(key, executor, () -> delegate.doLoad(key)); + } + } + + private static final class LoadFence { + private boolean invalidated; + } + + // RowCountKey intentionally uses tableId as the Caffeine cache identity. In-flight loads need + // the complete scope so catalog/database invalidation can fence a same-tableId replacement. + private static final class LoadKey { + private final long catalogId; + private final long dbId; + private final long tableId; + + private LoadKey(RowCountKey key) { + catalogId = key.catalogId; + dbId = key.dbId; + tableId = key.tableId; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof LoadKey)) { + return false; + } + LoadKey other = (LoadKey) obj; + return catalogId == other.catalogId && dbId == other.dbId && tableId == other.tableId; + } + + @Override + public int hashCode() { + return Objects.hash(catalogId, dbId, tableId); + } + } + + private CompletableFuture> loadWithInvalidationFence( + RowCountKey key, Executor executor, Supplier> loader) { + LoadFence fence = new LoadFence(); + LoadKey loadKey = new LoadKey(key); + publicationLock.readLock().lock(); + try { + inFlightLoads.compute(loadKey, (ignored, fences) -> { + Set currentFences = fences == null ? ConcurrentHashMap.newKeySet() : fences; + currentFences.add(fence); + return currentFences; + }); + } finally { + publicationLock.readLock().unlock(); + } + + CompletableFuture> publishedFuture = new CompletableFuture<>(); + CompletableFuture> loadFuture; + try { + loadFuture = CompletableFuture.supplyAsync(loader, executor); + } catch (RuntimeException e) { + publicationLock.readLock().lock(); + try { + removeInFlightLoad(loadKey, fence); + } finally { + publicationLock.readLock().unlock(); + } + throw e; + } + loadFuture.whenComplete((value, throwable) -> { + publicationLock.readLock().lock(); + try { + if (throwable != null) { + publishedFuture.completeExceptionally(throwable); + } else if (fence.invalidated) { + publishedFuture.complete(null); + } else { + publishedFuture.complete(value); + } + } finally { + removeInFlightLoad(loadKey, fence); + publicationLock.readLock().unlock(); + } + }); + return publishedFuture; + } + + private void removeInFlightLoad(LoadKey key, LoadFence fence) { + inFlightLoads.computeIfPresent(key, (ignored, fences) -> { + fences.remove(fence); + return fences.isEmpty() ? null : fences; + }); + } + + int getInFlightLoadCountForTest() { + publicationLock.readLock().lock(); + try { + return inFlightLoads.values().stream().mapToInt(Set::size).sum(); + } finally { + publicationLock.readLock().unlock(); + } + } + + void refreshForTest(long catalogId, long dbId, long tableId) { + rowCountCache.synchronous().refresh(new RowCountKey(catalogId, dbId, tableId)); + } + static Optional loadRowCount(RowCountKey rowCountKey, boolean fillMetaCache) { try { ExternalTable table = (ExternalTable) StatisticsUtil.findTable( @@ -122,10 +257,16 @@ static Optional loadRowCount(RowCountKey rowCountKey, boolean fillMetaCach public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean fillMetaCache) { RowCountKey key = new RowCountKey(catalogId, dbId, tableId); try { - CompletableFuture> f = fillMetaCache - ? rowCountCache.get(key, (rowCountKey, executor) -> CompletableFuture.supplyAsync( - () -> loadRowCount(rowCountKey, true), executor)) - : rowCountCache.get(key); + CompletableFuture> f; + publicationLock.readLock().lock(); + try { + f = fillMetaCache + ? rowCountCache.get(key, (rowCountKey, executor) -> loadWithInvalidationFence( + rowCountKey, executor, () -> loadRowCount(rowCountKey, true))) + : rowCountCache.get(key); + } finally { + publicationLock.readLock().unlock(); + } // Get row count synchronously by default. if (ConnectContext.get() == null || ConnectContext.get().getSessionVariable().fetchHiveRowCountSync) { @@ -150,7 +291,13 @@ public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean f public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) { RowCountKey key = new RowCountKey(catalogId, dbId, tableId); try { - CompletableFuture> f = rowCountCache.getIfPresent(key); + CompletableFuture> f; + publicationLock.readLock().lock(); + try { + f = rowCountCache.getIfPresent(key); + } finally { + publicationLock.readLock().unlock(); + } if (f == null) { return -1; } else if (f.isDone()) { @@ -162,4 +309,48 @@ public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) return -1; } + // Catalog/db invalidation is O(N): row-count keys are numeric ids, and Caffeine + // does not support prefix invalidation by catalog or database id. + void invalidateCatalog(long catalogId) { + publicationLock.writeLock().lock(); + try { + inFlightLoads.forEach((key, fences) -> { + if (key.catalogId == catalogId) { + fences.forEach(fence -> fence.invalidated = true); + } + }); + rowCountCache.asMap().keySet().removeIf(key -> key.catalogId == catalogId); + } finally { + publicationLock.writeLock().unlock(); + } + } + + void invalidateDb(long catalogId, long dbId) { + publicationLock.writeLock().lock(); + try { + inFlightLoads.forEach((key, fences) -> { + if (key.catalogId == catalogId && key.dbId == dbId) { + fences.forEach(fence -> fence.invalidated = true); + } + }); + rowCountCache.asMap().keySet().removeIf(key -> key.catalogId == catalogId && key.dbId == dbId); + } finally { + publicationLock.writeLock().unlock(); + } + } + + void invalidateTable(long catalogId, long dbId, long tableId) { + publicationLock.writeLock().lock(); + try { + RowCountKey key = new RowCountKey(catalogId, dbId, tableId); + Set fences = inFlightLoads.get(new LoadKey(key)); + if (fences != null) { + fences.forEach(fence -> fence.invalidated = true); + } + rowCountCache.synchronous().invalidate(key); + } finally { + publicationLock.writeLock().unlock(); + } + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 68bfeb2034e7f8..756687f7fb3ab7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -322,19 +322,26 @@ public boolean registerDatabaseFromEvent(long dbId, String dbName, long metadata @Override public void notifyPropertiesUpdated(Map updatedProps) { - super.notifyPropertiesUpdated(updatedProps); - String fileMetaCacheTtl = updatedProps.getOrDefault(FILE_META_CACHE_TTL_SECOND, null); - String partitionCacheTtl = updatedProps.getOrDefault(PARTITION_CACHE_TTL_SECOND, null); - if (Objects.nonNull(fileMetaCacheTtl) || Objects.nonNull(partitionCacheTtl)) { - Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HiveExternalMetaCache.ENGINE); - } - if (updatedProps.keySet().stream() - .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, HudiExternalMetaCache.ENGINE))) { - Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HudiExternalMetaCache.ENGINE); - } - if (updatedProps.keySet().stream() - .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, IcebergExternalMetaCache.ENGINE))) { - Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE); + try { + super.notifyPropertiesUpdated(updatedProps); + } finally { + // The committed ALTER already published the properties; the engine-specific groups below + // must still be retired when the generic reset cleanup throws. + String fileMetaCacheTtl = updatedProps.getOrDefault(FILE_META_CACHE_TTL_SECOND, null); + String partitionCacheTtl = updatedProps.getOrDefault(PARTITION_CACHE_TTL_SECOND, null); + if (Objects.nonNull(fileMetaCacheTtl) || Objects.nonNull(partitionCacheTtl)) { + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HiveExternalMetaCache.ENGINE); + } + if (updatedProps.keySet().stream() + .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, HudiExternalMetaCache.ENGINE))) { + Env.getCurrentEnv().getExtMetaCacheMgr() + .removeCatalogByEngine(getId(), HudiExternalMetaCache.ENGINE); + } + if (updatedProps.keySet().stream() + .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, IcebergExternalMetaCache.ENGINE))) { + Env.getCurrentEnv().getExtMetaCacheMgr() + .removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index bb607fb0b1d704..458a216000cab8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -323,6 +323,10 @@ public void afterDropTable(String dbName, String tblName) { Optional> db = catalog.getDbForReplay(dbName); if (db.isPresent()) { db.get().unregisterTable(tblName); + } else { + // A mode-2 mapping can disappear while the canonical database object stays resident, so + // an empty lookup may still hide stale database/table objects, engine entries, and counts. + catalog.retireUnresolvedDatabaseGeneration(); } LOG.info("after drop table {}.{}.{}, is db exists: {}", getCatalog().getName(), dbName, tblName, db.isPresent()); @@ -348,8 +352,14 @@ public void afterTruncateTable(String dbName, String tblName, long updateTime) { if (tbl.isPresent()) { Env.getCurrentEnv().getRefreshManager() .refreshTableInternal(db.get(), (ExternalTable) tbl.get(), updateTime); + return; } } + // The table or database object is cold. The event carries the caller's spelling, which may + // not match canonical local cache keys under lower_case_*_names=2, so widen to whatever + // scope still covers the event instead of routing the raw name. + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateTableByNameOrWider(catalog.getId(), dbName, tblName); } catch (Exception e) { LOG.warn("exception when calling afterTruncateTable for db: {}, table: {}, error: {}", dbName, tblName, e.getMessage(), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java index 03ef1b899eace7..618102e2d05d86 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java @@ -159,11 +159,16 @@ protected void checkProperties(CatalogProperty property) throws DdlException { @Override public void notifyPropertiesUpdated(Map updatedProps) { - super.notifyPropertiesUpdated(updatedProps); - if (updatedProps.keySet().stream() - .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, IcebergExternalMetaCache.ENGINE))) { - Env.getCurrentEnv().getExtMetaCacheMgr() - .removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE); + try { + super.notifyPropertiesUpdated(updatedProps); + } finally { + // The committed ALTER already published the properties; retire the engine group even + // when the generic reset cleanup throws. + if (updatedProps.keySet().stream() + .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, IcebergExternalMetaCache.ENGINE))) { + Env.getCurrentEnv().getExtMetaCacheMgr() + .removeCatalogByEngine(getId(), IcebergExternalMetaCache.ENGINE); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index dca41bb9b4c425..e1db2dd8c03270 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -449,6 +449,10 @@ public void afterDropTable(String dbName, String tblName) { Optional> db = dorisCatalog.getDbForReplay(dbName); if (db.isPresent()) { db.get().unregisterTable(tblName); + } else { + // A mode-2 mapping can disappear while the canonical database object stays resident, so + // an empty lookup may still hide stale database/table objects, engine entries, and counts. + dorisCatalog.retireUnresolvedDatabaseGeneration(); } LOG.info("after drop table {}.{}.{}. is db exists: {}", dorisCatalog.getName(), dbName, tblName, db.isPresent()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index fe48adffff78d7..e6afa99be16689 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -149,11 +149,16 @@ public void setDefaultPropsIfMissing(boolean isReplay) { @Override public void resetToUninitialized(boolean invalidCache) { - super.resetToUninitialized(invalidCache); - this.identifierMapping = new JdbcIdentifierMapping( - (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()), - Boolean.parseBoolean(getLowerCaseMetaNames()), - getMetaNamesMapping()); + try { + super.resetToUninitialized(invalidCache); + } finally { + // The committed ALTER has already published the new properties; rebuild the derived name + // mapping even when the reset's client cleanup (closeClient) throws. + this.identifierMapping = new JdbcIdentifierMapping( + (Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase()), + Boolean.parseBoolean(getLowerCaseMetaNames()), + getMetaNamesMapping()); + } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java index a7c5ba2c7f2c22..21e83cb4e40803 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java @@ -293,6 +293,10 @@ public void afterDropTable(String dbName, String tblName) { Optional> db = dorisCatalog.getDbForReplay(dbName); if (db.isPresent()) { db.get().unregisterTable(tblName); + } else { + // A mode-2 mapping can disappear while the canonical database object stays resident, so + // an empty lookup may still hide stale database/table objects, engine entries, and counts. + dorisCatalog.retireUnresolvedDatabaseGeneration(); } LOG.info("after drop table {}.{}.{}, is db exists: {}", dorisCatalog.getName(), dbName, tblName, db.isPresent()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index 80a9dfc0e2c033..238c695fb3c31b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -469,11 +469,17 @@ private void checkProperties(CatalogProperty property, Map stric @Override public void notifyPropertiesUpdated(Map updatedProps) { - super.notifyPropertiesUpdated(updatedProps); - if (updatedProps.keySet().stream() - .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, PaimonExternalMetaCache.ENGINE) - || AbstractPaimonProperties.isTableOptionProperty(key))) { - Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), PaimonExternalMetaCache.ENGINE); + try { + super.notifyPropertiesUpdated(updatedProps); + } finally { + // The committed ALTER already published the properties; retire the engine group even + // when the generic reset cleanup throws. + if (updatedProps.keySet().stream() + .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, PaimonExternalMetaCache.ENGINE) + || AbstractPaimonProperties.isTableOptionProperty(key))) { + Env.getCurrentEnv().getExtMetaCacheMgr() + .removeCatalogByEngine(getId(), PaimonExternalMetaCache.ENGINE); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java index e597732871b2e1..288bcb32f5fc46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/BaseExternalTableInsertExecutor.java @@ -136,12 +136,9 @@ protected void onComplete() throws UserException { * Default: full table refresh. */ protected void doAfterCommit() throws DdlException { - // Default: full table refresh - Env.getCurrentEnv().getRefreshManager().handleRefreshTable( - catalogName, - table.getDatabase().getFullName(), - table.getName(), - true); + // The transaction is already committed and this executor still owns the exact table + // identity. Do not re-resolve it through an evictable metadata cache before fencing it. + Env.getCurrentEnv().getRefreshManager().refreshTableAfterCommit((ExternalTable) table); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java index 760a2c0551d5a0..e0bdfb50dfdc5e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java @@ -83,17 +83,40 @@ protected void doBeforeCommit() throws UserException { protected void doAfterCommit() throws DdlException { HMSExternalTable hmsTable = (HMSExternalTable) table; + // The transaction is already committed. Fence the row-count cache by the held table + // identity before any fallible cache work (including isPartitionedTable reinitialization), + // so an evicted or partially reloaded table cannot retain the pre-insert count. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(hmsTable); + // For partitioned tables, do selective partition refresh // For non-partitioned tables, do full table cache invalidation List modifiedPartNames = Lists.newArrayList(); List newPartNames = Lists.newArrayList(); - if (hmsTable.isPartitionedTable() && partitionUpdates != null && !partitionUpdates.isEmpty()) { - HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() - .hive(hmsTable.getCatalog().getId()); - cache.refreshAffectedPartitions(hmsTable, partitionUpdates, modifiedPartNames, newPartNames); - } else { - // Non-partitioned table or no partition updates, do full table refresh - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(hmsTable); + try { + if (hmsTable.isPartitionedTable() && partitionUpdates != null && !partitionUpdates.isEmpty()) { + HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .hive(hmsTable.getCatalog().getId()); + cache.refreshAffectedPartitions(hmsTable, partitionUpdates, modifiedPartNames, newPartNames); + // Close the admission window opened by the fence above: a load admitted after it can + // compute the pre-insert value from the still-resident file list and publish it. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(hmsTable); + } else { + // Non-partitioned table or no partition updates, do full table refresh + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(hmsTable); + } + } catch (RuntimeException e) { + // The transaction is already committed. Fall back to a full invalidation and a + // full-refresh edit log instead of reporting a failed insert or skipping peers. + LOG.warn("Failed to refresh caches after committing Hive insert for {}", + hmsTable.getNameWithFullQualifiers(), e); + modifiedPartNames.clear(); + newPartNames.clear(); + try { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(hmsTable); + } catch (RuntimeException fallbackException) { + LOG.warn("Failed to invalidate table cache after committing Hive insert for {}", + hmsTable.getNameWithFullQualifiers(), fallbackException); + } } // Write edit log to notify other FEs diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java new file mode 100644 index 00000000000000..aff6393afd252c --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java @@ -0,0 +1,296 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.datasource.CatalogMgr; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalMetaCacheMgr; +import org.apache.doris.datasource.ExternalObjectLog; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.datasource.hive.HiveExternalMetaCache; +import org.apache.doris.persist.EditLog; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.InOrder; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Optional; + +public class RefreshManagerTest { + + @Test + void testColdDatabaseReplayInvalidatesCatalogRowCount() { + long catalogId = 51L; + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.empty()); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + new RefreshManager().replayRefreshDb(ExternalObjectLog.createForRefreshDb(catalogId, "db1")); + } + + Mockito.verify(cacheMgr).invalidateCatalog(catalogId); + } + + @Test + void testPartitionReplayInvalidatesRowCountBeforeCacheFailure() { + long catalogId = 52L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.of(db)); + Mockito.doReturn(Optional.of(table)).when(db).getTableForReplay("tbl1"); + + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + HiveExternalMetaCache hiveCache = Mockito.mock(HiveExternalMetaCache.class); + Mockito.when(cacheMgr.hive(catalogId)).thenReturn(hiveCache); + Mockito.doThrow(new IllegalStateException("partition cache failure")) + .when(hiveCache).refreshAffectedPartitionsCache( + Mockito.eq(table), Mockito.anyList(), Mockito.anyList()); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + ExternalObjectLog log = ExternalObjectLog.createForRefreshPartitions( + catalogId, "db1", "tbl1", + java.util.Collections.singletonList("p=1"), java.util.Collections.emptyList(), 1L); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + // Replay cleanup failures are logged and swallowed so a committed refresh does not take + // the FE down; the row-count fence must still have been published first. + Assertions.assertDoesNotThrow(() -> new RefreshManager().replayRefreshTable(log)); + } + + Mockito.verify(cacheMgr).invalidateRowCountCache(table); + } + + @Test + void testAlterPartitionInvalidatesRowCountBeforeCacheFailure() throws Exception { + long catalogId = 53L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.doReturn(db).when(catalog).getDbNullable("db1"); + Mockito.doReturn(table).when(db).getTableNullable("tbl1"); + Mockito.when(table.getCatalog()).thenReturn(catalog); + + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog("hms"); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(cacheMgr.hive(catalogId)).thenThrow(new IllegalStateException("partition cache failure")); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Assertions.assertThrows(IllegalStateException.class, + () -> new RefreshManager().refreshPartitions( + "hms", "db1", "tbl1", java.util.Collections.singletonList("p=1"), 1L, true)); + } + + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, "db1", "tbl1"); + } + + @Test + void testColdAlterPartitionEventFencesRowCountBeforeIgnoredTableMiss() throws Exception { + long catalogId = 54L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.doReturn(db).when(catalog).getDbNullable("db1"); + Mockito.doReturn(null).when(db).getTableNullable("tbl1"); + + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog("hms"); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + new RefreshManager().refreshPartitions( + "hms", "db1", "tbl1", java.util.Collections.singletonList("p=1"), 1L, true); + } + + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, "db1", "tbl1"); + Mockito.verify(cacheMgr).invalidateTableByNameOrWider(catalogId, "db1", "tbl1"); + Mockito.verify(cacheMgr, Mockito.never()).hive(catalogId); + } + + @Test + void testSuccessfulPartitionRefreshClosesRowCountAdmissionWindow() throws Exception { + long catalogId = 69L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.doReturn(db).when(catalog).getDbNullable("db1"); + Mockito.doReturn(table).when(db).getTableNullable("tbl1"); + Mockito.when(table.getCatalog()).thenReturn(catalog); + + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog("hms"); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + HiveExternalMetaCache hiveCache = Mockito.mock(HiveExternalMetaCache.class); + Mockito.when(cacheMgr.hive(catalogId)).thenReturn(hiveCache); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + new RefreshManager().refreshPartitions( + "hms", "db1", "tbl1", java.util.Collections.singletonList("p=1"), 1L, true); + } + + // One fence before the selective invalidation, one after it closes the admission window. + Mockito.verify(cacheMgr, Mockito.times(2)) + .invalidateRowCountCache(catalogId, "db1", "tbl1"); + } + + @Test + void testCommittedRefreshUsesHeldTableAndStillLogsAfterCacheFailure() { + long catalogId = 55L; + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + ExternalTable table = Mockito.mock(ExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(db.getFullName()).thenReturn("db1"); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.getDatabase()).thenReturn(db); + Mockito.when(table.getName()).thenReturn("tbl1"); + Mockito.when(table.getNameWithFullQualifiers()).thenReturn("ctl.db1.tbl1"); + + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.doThrow(new IllegalStateException("table cache failure")) + .when(cacheMgr).invalidateTableCache(table); + EditLog editLog = Mockito.mock(EditLog.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(env.getEditLog()).thenReturn(editLog); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Assertions.assertDoesNotThrow(() -> new RefreshManager().refreshTableAfterCommit(table)); + } + + InOrder order = Mockito.inOrder(cacheMgr, editLog); + order.verify(cacheMgr).invalidateRowCountCache(table); + order.verify(cacheMgr).invalidateTableCache(table); + order.verify(editLog).logRefreshExternalTable(Mockito.argThat(log -> + log.getCatalogId() == catalogId + && "db1".equals(log.getDbName()) + && "tbl1".equals(log.getTableName()))); + } + + @Test + void testWholeTableEventFencesRowCountBeforeIgnoredTableMiss() throws Exception { + long catalogId = 56L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.doReturn(db).when(catalog).getDbNullable("db1"); + Mockito.doReturn(null).when(db).getTableNullable("tbl1"); + + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog("hms"); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + new RefreshManager().refreshExternalTableFromEvent("hms", "db1", "tbl1", 1L); + } + + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, "db1", "tbl1"); + Mockito.verify(cacheMgr).invalidateTableByNameOrWider(catalogId, "db1", "tbl1"); + Mockito.verify(cacheMgr, Mockito.never()).hive(catalogId); + } + + @Test + @SuppressWarnings("unchecked") + void testColdRefreshTableReplayWidensToCanonicalDatabaseScope() { + long catalogId = 57L; + long dbId = 58L; + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getDbForReplay("db1")).thenReturn((Optional) Optional.of(db)); + Mockito.when(db.getId()).thenReturn(dbId); + Mockito.when(db.getFullName()).thenReturn("MixedDb"); + Mockito.doReturn(Optional.empty()).when(db).getTableForReplay("tbl1"); + + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + ExternalObjectLog log = ExternalObjectLog.createForRefreshTable(catalogId, "db1", "tbl1", 1L); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + new RefreshManager().replayRefreshTable(log); + } + + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, dbId); + } + + @Test + void testColdRefreshDbReplayWidensToCatalogScope() { + long catalogId = 59L; + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.empty()); + + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + ExternalObjectLog log = ExternalObjectLog.createForRefreshTable(catalogId, "db1", "tbl1", 1L); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + new RefreshManager().replayRefreshTable(log); + } + + Mockito.verify(cacheMgr).invalidateCatalog(catalogId); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index a35716aa95d2e1..36a3a2f5aaad7a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -19,14 +19,24 @@ import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.RefreshManager; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.DdlException; +import org.apache.doris.common.util.Util; +import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.datasource.hive.HiveExternalMetaCache; +import org.apache.doris.datasource.metacache.MetaCache; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.datasource.property.metastore.AbstractPaimonProperties; +import org.apache.doris.nereids.exceptions.NotSupportedException; +import org.apache.doris.persist.EditLog; +import org.apache.doris.statistics.query.QueryStats; import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.InOrder; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -35,6 +45,8 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -54,6 +66,17 @@ private static void addCatalog(CatalogMgr catalogMgr, ExternalCatalog catalog) t idToCatalog.put(catalog.getId(), catalog); } + private static void addNamedCatalog(CatalogMgr catalogMgr, ExternalCatalog catalog) throws Exception { + addCatalog(catalogMgr, catalog); + Field nameToCatalogField = CatalogMgr.class.getDeclaredField("nameToCatalog"); + nameToCatalogField.setAccessible(true); + @SuppressWarnings("unchecked") + ConcurrentMap>> nameToCatalog = + (ConcurrentMap>>) + nameToCatalogField.get(catalogMgr); + nameToCatalog.put(catalog.getName(), catalog); + } + @Test void testAlterCatalogRollsBackUncheckedValidationFailure() throws Exception { CatalogMgr catalogMgr = new CatalogMgr(); @@ -138,6 +161,7 @@ void testCommittedAlterRetiresTheOperationalContextButFailedAlterDoesNot() throw Env env = Mockito.mock(Env.class); ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class)); Mockito.when(cacheMgr.withCatalogLifecycleLock(Mockito.eq(catalogId), Mockito.any())) .thenAnswer(invocation -> { java.util.function.Supplier action = invocation.getArgument(1); @@ -223,6 +247,425 @@ void testReplayPublishesPropertiesOnlyThroughTheFencedCommit() throws Exception Mockito.verify(cacheMgr).onCatalogOperationalContextChanged(catalogId); } + @Test + void testUnsupportedAddPartitionEventStillInvalidatesRowCount() throws Exception { + CatalogMgr catalogMgr = new CatalogMgr(); + long catalogId = 46L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("hms"); + Mockito.doReturn(db).when(catalog).getDbNullable("db1"); + Mockito.doReturn(table).when(db).getTableNullable("tbl1"); + Mockito.when(table.getPartitionColumnTypes(Mockito.any())) + .thenThrow(new NotSupportedException("unsupported table")); + addNamedCatalog(catalogMgr, catalog); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalogMgr.addExternalPartitions( + "hms", "db1", "tbl1", Collections.singletonList("p=1"), 1L, false); + } + + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, "db1", "tbl1"); + Mockito.verify(cacheMgr, Mockito.never()).invalidateTableByNameOrWider(Mockito.anyLong(), + Mockito.anyString(), Mockito.anyString()); + Mockito.verify(cacheMgr, Mockito.never()).hive(catalogId); + } + + @Test + void testDropPartitionEventInvalidatesRowCountBeforeCacheFailure() throws Exception { + CatalogMgr catalogMgr = new CatalogMgr(); + long catalogId = 49L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("hms"); + Mockito.doReturn(db).when(catalog).getDbNullable("db1"); + Mockito.doReturn(table).when(db).getTableNullable("tbl1"); + addNamedCatalog(catalogMgr, catalog); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + HiveExternalMetaCache hiveCache = Mockito.mock(HiveExternalMetaCache.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(cacheMgr.hive(catalogId)).thenReturn(hiveCache); + Mockito.doThrow(new IllegalStateException("partition cache failure")) + .when(hiveCache).dropPartitionsCache( + Mockito.eq(table), Mockito.anyList(), Mockito.eq(true)); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Assertions.assertThrows(IllegalStateException.class, + () -> catalogMgr.dropExternalPartitions( + "hms", "db1", "tbl1", Collections.singletonList("p=1"), 1L, false)); + } + + // The opening fence and the failure-safe closing fence both run. + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId, "db1", "tbl1"); + } + + @Test + void testSuccessfulPartitionEventClosesRowCountAdmissionWindow() throws Exception { + CatalogMgr catalogMgr = new CatalogMgr(); + long catalogId = 62L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("hms"); + Mockito.doReturn(db).when(catalog).getDbNullable("db1"); + Mockito.doReturn(table).when(db).getTableNullable("tbl1"); + Mockito.when(table.getPartitionColumnTypes(Mockito.any())).thenReturn(Collections.emptyList()); + addNamedCatalog(catalogMgr, catalog); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + HiveExternalMetaCache hiveCache = Mockito.mock(HiveExternalMetaCache.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(cacheMgr.hive(catalogId)).thenReturn(hiveCache); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalogMgr.addExternalPartitions( + "hms", "db1", "tbl1", Collections.singletonList("p=1"), 1L, false); + } + + // One fence before the selective update, one after it closes the admission window. + Mockito.verify(cacheMgr, Mockito.times(2)) + .invalidateRowCountCache(catalogId, "db1", "tbl1"); + } + + @Test + void testColdPartitionEventsFenceRowCountBeforeIgnoredTableMiss() throws Exception { + CatalogMgr catalogMgr = new CatalogMgr(); + long catalogId = 50L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("hms"); + Mockito.doReturn(db).when(catalog).getDbNullable("db1"); + Mockito.doReturn(null).when(db).getTableNullable("tbl1"); + addNamedCatalog(catalogMgr, catalog); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalogMgr.addExternalPartitions( + "hms", "db1", "tbl1", Collections.singletonList("p=1"), 1L, true); + catalogMgr.dropExternalPartitions( + "hms", "db1", "tbl1", Collections.singletonList("p=1"), 1L, true); + } + + Mockito.verify(cacheMgr, Mockito.times(2)) + .invalidateRowCountCache(catalogId, "db1", "tbl1"); + Mockito.verify(cacheMgr, Mockito.times(2)) + .invalidateTableByNameOrWider(catalogId, "db1", "tbl1"); + Mockito.verify(cacheMgr, Mockito.never()).hive(catalogId); + } + + @Test + void testCatalogRefreshUsesSingleRowCountInvalidationOwner() { + long catalogId = 50L; + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId); + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalog.onRefreshCache(true); + } + + Mockito.verify(metaCache).invalidateAll(); + Mockito.verify(cacheMgr).invalidateCatalog(catalogId); + Mockito.verify(cacheMgr, Mockito.never()).getRowCountCache(); + } + + @Test + void testMetadataOnlyRefreshStillInvalidatesRowCount() { + long catalogId = 51L; + long dbId = 52L; + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId); + ExternalDatabase db = new ExternalDatabase( + catalog, dbId, "db1", "db1", InitDatabaseLog.Type.TEST) { + @Override + protected ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog externalCatalog, ExternalDatabase externalDatabase) { + return null; + } + }; + ExecutorService removalExecutor = Executors.newSingleThreadExecutor(); + MetaCache> metaCache = new MetaCache<>( + "databaseCache", + removalExecutor, + OptionalLong.empty(), + OptionalLong.empty(), + 10, + key -> Collections.emptyList(), + key -> Optional.empty(), + (key, value, cause) -> catalog.handleDatabaseMetaCacheRemoval(value)); + metaCache.updateCache("db1", "db1", db, dbId); + catalog.installMetaCache(metaCache); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try { + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalog.onRefreshCache(false); + } + } finally { + removalExecutor.shutdownNow(); + } + + Mockito.verify(cacheMgr).invalidateDb(db); + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId); + Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(Mockito.anyLong()); + } + + @Test + void testUnregisterDatabaseRemovesLocalEntryWhenEngineInvalidationFails() { + long catalogId = 47L; + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId); + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.doThrow(new IllegalStateException("engine invalidation failed")) + .when(cacheMgr).invalidateDb(catalogId, "CanonicalDb"); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Assertions.assertThrows(IllegalStateException.class, + () -> catalog.unregisterDatabase("CanonicalDb")); + } + + // The local metadata entry is removed before the engine invalidation runs, so a failing + // engine flush cannot leave the dropped database resident. + Mockito.verify(metaCache).invalidate("CanonicalDb", + Util.genIdByName("testing_catalog", "CanonicalDb")); + } + + @Test + void testResidentDatabaseRemovalInvalidatesLocalAndRoutedCache() { + long catalogId = 53L; + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId); + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalog.unregisterDatabase("CanonicalDb"); + } + + Mockito.verify(metaCache).invalidate("CanonicalDb", + Util.genIdByName("testing_catalog", "CanonicalDb")); + Mockito.verify(cacheMgr).invalidateDb(catalogId, "CanonicalDb"); + } + + @Test + void testUnresolvedModeTwoDatabaseRemovalRetiresCatalogScope() throws Exception { + long catalogId = 60L; + Map properties = ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2"); + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId, properties); + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + // The mode-2 mapping is gone, so the canonical local key cannot be resolved. + catalog.unregisterDatabase("dropped_db"); + } + + // Unknown scope: retire every cached database object and flush engine caches/row counts + // catalog-wide instead of evicting a lowercased local key that may not exist. + Mockito.verify(metaCache).invalidateObjects(); + Mockito.verify(cacheMgr).invalidateCatalog(catalogId); + Mockito.verify(cacheMgr, Mockito.never()).invalidateDb(Mockito.anyLong(), Mockito.anyString()); + } + + @Test + void testUnresolvedDatabaseDropRetiresGenerationAndFencesCatalog() { + long catalogId = 72L; + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId); + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalog.retireUnresolvedDatabaseGeneration(); + } + + // Retire the hidden database generation and fence the catalog scope (engine + row counts), + // even though the per-database removal callbacks suppress their own fences. + Mockito.verify(metaCache).invalidateObjects(); + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId); + Mockito.verify(cacheMgr).invalidateCatalog(catalogId); + } + + @Test + void testUnresolvedTableNameRetiresHiddenTableGeneration() throws Exception { + long catalogId = 73L; + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getLowerCaseTableNames()).thenReturn(2); + ExternalDatabase db = new ExternalDatabase( + catalog, 74L, "db1", "db1", InitDatabaseLog.Type.TEST) { + @Override + protected ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog externalCatalog, ExternalDatabase externalDatabase) { + return null; + } + }; + @SuppressWarnings("unchecked") + MetaCache tableCache = Mockito.mock(MetaCache.class); + Field metaCacheField = ExternalDatabase.class.getDeclaredField("metaCache"); + metaCacheField.setAccessible(true); + metaCacheField.set(db, tableCache); + + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Assertions.assertFalse(db.unregisterTableForReplay("Tbl")); + } + + // The mode-2 mapping is gone: retire the hidden table generation and widen the fence. + Mockito.verify(tableCache).invalidateObjects(); + Mockito.verify(cacheMgr).invalidateTableByNameOrWider(catalogId, "db1", "Tbl"); + } + + @Test + void testCommittedAlterStillRetiresAndLogsWhenResetCleanupThrows() throws Exception { + long catalogId = 58L; + CatalogMgr catalogMgr = new CatalogMgr(); + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("catalog_name"); + Map oldProperties = ImmutableMap.of("s3.access_key", "old", "warehouse", "s3://bucket"); + Map committedProperties = ImmutableMap.of("s3.access_key", "new", "warehouse", "s3://bucket"); + Mockito.when(catalog.getProperties()).thenReturn(oldProperties, committedProperties); + Mockito.doThrow(new RuntimeException("close failed")) + .when(catalog).modifyCatalogProps(Mockito.any()); + addNamedCatalog(catalogMgr, catalog); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + EditLog editLog = Mockito.mock(EditLog.class); + RefreshManager refreshManager = Mockito.mock(RefreshManager.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(env.getEditLog()).thenReturn(editLog); + Mockito.when(env.getRefreshManager()).thenReturn(refreshManager); + Mockito.when(cacheMgr.withCatalogLifecycleLock(Mockito.eq(catalogId), Mockito.any())) + .thenAnswer(invocation -> { + java.util.function.Supplier action = invocation.getArgument(1); + return action.get(); + }); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Assertions.assertThrows(RuntimeException.class, + () -> catalogMgr.alterCatalogProps("catalog_name", ImmutableMap.of("s3.access_key", "new"))); + } + + // The reset published the new properties before its throwable cleanup: retirement and + // durability must still happen. + Mockito.verify(cacheMgr).onCatalogOperationalContextChanged(catalogId); + Mockito.verify(editLog).logCatalogLog(Mockito.anyShort(), Mockito.any()); + } + + @Test + void testReplayAlterCatalogPropsSurvivesResetCleanupFailure() throws Exception { + long catalogId = 61L; + CatalogMgr catalogMgr = new CatalogMgr(); + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("catalog_name"); + Mockito.doThrow(new RuntimeException("close failed during replay")) + .when(catalog).modifyCatalogProps(Mockito.any()); + addCatalog(catalogMgr, catalog); + + CatalogLog log = new CatalogLog(); + log.setCatalogId(catalogId); + log.setNewProps(ImmutableMap.of("s3.access_key", "new")); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(env.getEditLog()).thenReturn(editLog); + Mockito.when(cacheMgr.withCatalogLifecycleLock(Mockito.eq(catalogId), Mockito.any())) + .thenAnswer(invocation -> { + java.util.function.Supplier action = invocation.getArgument(1); + return action.get(); + }); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + // A follower must not terminate when local connector cleanup fails while applying an + // already-durable ALTER record. + Assertions.assertDoesNotThrow(() -> catalogMgr.replayAlterCatalogProps(log, null, true)); + } + + Mockito.verify(cacheMgr).onCatalogOperationalContextChanged(catalogId); + // Replay never journals. + Mockito.verify(editLog, Mockito.never()).logCatalogLog(Mockito.anyShort(), Mockito.any()); + } + + @Test + void testDropCatalogFencesRowCountBeforeCleanup() throws Exception { + long catalogId = 57L; + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("dropme"); + CatalogMgr catalogMgr = new CatalogMgr(); + addNamedCatalog(catalogMgr, catalog); + + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + EditLog editLog = Mockito.mock(EditLog.class); + RefreshManager refreshManager = Mockito.mock(RefreshManager.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(env.getEditLog()).thenReturn(editLog); + Mockito.when(env.getRefreshManager()).thenReturn(refreshManager); + Mockito.when(env.getQueryStats()).thenReturn(Mockito.mock(QueryStats.class)); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalogMgr.dropCatalog("dropme", false); + } + + InOrder order = Mockito.inOrder(cacheMgr, catalog); + order.verify(cacheMgr).invalidateRowCountCache(catalogId); + order.verify(catalog).onClose(); + Assertions.assertNull(catalogMgr.getCatalog("dropme")); + } + private static class LatchingValidationCatalog extends ExternalCatalog { private final CountDownLatch validationStarted = new CountDownLatch(1); private final CountDownLatch initializationReadProperties = new CountDownLatch(1); @@ -274,4 +717,34 @@ public void notifyPropertiesUpdated(Map updatedProps) { // This test isolates edit-log property restoration from environment-owned cache services. } } + + private static class TestingUnregisterCatalog extends ExternalCatalog { + TestingUnregisterCatalog(long id) { + this(id, Collections.emptyMap()); + } + + TestingUnregisterCatalog(long id, Map properties) { + super(id, "testing_catalog", InitCatalogLog.Type.TEST, ""); + catalogProperty = new CatalogProperty(null, properties); + } + + void installMetaCache(MetaCache> cache) { + metaCache = cache; + initialized = true; + } + + @Override + protected List listTableNamesFromRemote(SessionContext ctx, String dbName) { + return Collections.emptyList(); + } + + @Override + public boolean tableExist(SessionContext ctx, String dbName, String tblName) { + return false; + } + + @Override + protected void initLocalObjectsImpl() { + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index 111a8173257c3a..f1555285cee269 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -29,6 +29,7 @@ import org.apache.doris.datasource.metacache.MetaCacheEntryStats; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; +import com.google.common.util.concurrent.Uninterruptibles; import mockit.Mock; import mockit.MockUp; import org.junit.Assert; @@ -39,6 +40,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.CountDownLatch; @@ -304,12 +306,40 @@ public void testRollbackRetiresGroupInitializedFromRejectedCandidate() throws Ex mockCurrentCatalog(catalogId, catalog); hive.initializedCatalogIds.add(catalogId); Map oldProperties = Collections.singletonMap("generation", "old"); - - metaCacheMgr.rollbackCatalogProperties(catalog, oldProperties); - - Mockito.verify(catalog).rollBackCatalogProps(oldProperties); - Assert.assertFalse(hive.isCatalogInitialized(catalogId)); - Assert.assertEquals(1, hive.invalidateCatalogCalls); + CountDownLatch loadStarted = new CountDownLatch(1); + CountDownLatch allowLoadToFinish = new CountDownLatch(1); + ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() { + @Override + protected Optional doLoad(ExternalRowCountCache.RowCountKey rowCountKey) { + loadStarted.countDown(); + Assert.assertTrue(Uninterruptibles.awaitUninterruptibly( + allowLoadToFinish, 30, TimeUnit.SECONDS)); + return Optional.of(100L); + } + }; + ExecutorService loaderExecutor = Executors.newSingleThreadExecutor(); + metaCacheMgr.replaceRowCountCacheForTest(new ExternalRowCountCache(loaderExecutor, null, loader)); + ExecutorService caller = Executors.newSingleThreadExecutor(); + try { + Future load = caller.submit( + () -> metaCacheMgr.getRowCountCache().getCachedRowCount(catalogId, 2L, 3L, false)); + Assert.assertTrue(loadStarted.await(30, TimeUnit.SECONDS)); + + metaCacheMgr.rollbackCatalogProperties(catalog, oldProperties); + allowLoadToFinish.countDown(); + + Assert.assertEquals(TableIf.UNKNOWN_ROW_COUNT, (long) load.get(30, TimeUnit.SECONDS)); + Assert.assertEquals(TableIf.UNKNOWN_ROW_COUNT, + metaCacheMgr.getRowCountCache().getCachedRowCountIfPresent(catalogId, 2L, 3L)); + Mockito.verify(catalog).rollBackCatalogProps(oldProperties); + Mockito.verify(catalog).resetToUninitialized(false); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + Assert.assertEquals(1, hive.invalidateCatalogCalls); + } finally { + allowLoadToFinish.countDown(); + caller.shutdownNow(); + loaderExecutor.shutdownNow(); + } } @Test @@ -410,6 +440,173 @@ public void testMissingCatalogLifecycleOnlyTouchesInitializedEngine() throws Exc Assert.assertEquals(0, paimon.invalidateCatalogCalls); } + @Test + public void testEngineSpecificTableInvalidationAlsoFencesRowCount() throws Exception { + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive); + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + metaCacheMgr.replaceRowCountCacheForTest(rowCountCache); + long catalogId = 17L; + long dbId = 18L; + long tableId = 19L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + ExternalTable table = Mockito.mock(ExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.of(db)); + Mockito.when(db.getId()).thenReturn(dbId); + Mockito.doReturn(Optional.of(table)).when(db).getTableForReplay("tbl1"); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.getDb()).thenReturn(db); + Mockito.when(table.getId()).thenReturn(tableId); + mockCurrentCatalog(catalogId, catalog); + hive.initializedCatalogIds.add(catalogId); + + metaCacheMgr.invalidateTableByEngine(catalogId, "hive", "db1", "tbl1"); + + Assert.assertEquals(1, hive.invalidateTableCalls); + Mockito.verify(rowCountCache).invalidateTable(catalogId, dbId, tableId); + } + + @Test + public void testDatabaseInvalidationCanPreserveSingleBulkRowCountOwner() throws Exception { + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache hudi = new RecordingExternalMetaCache( + "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache( + "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive, hudi, iceberg); + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + metaCacheMgr.replaceRowCountCacheForTest(rowCountCache); + long catalogId = 20L; + long dbId = 21L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + mockCurrentCatalog(catalogId, catalog); + hive.initializedCatalogIds.add(catalogId); + hudi.initializedCatalogIds.add(catalogId); + iceberg.initializedCatalogIds.add(catalogId); + + metaCacheMgr.invalidateDb(catalogId, dbId, "db1", false); + + Assert.assertEquals(1, hive.invalidateDbCalls); + Assert.assertEquals(1, hudi.invalidateDbCalls); + Assert.assertEquals(1, iceberg.invalidateDbCalls); + Mockito.verifyNoInteractions(rowCountCache); + } + + @Test + public void testOperationalContextChangeFencesRowCountForEvictedDatabases() { + long catalogId = 24L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + mockCurrentCatalog(catalogId, catalog); + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + metaCacheMgr.replaceRowCountCacheForTest(rowCountCache); + + metaCacheMgr.onCatalogOperationalContextChanged(catalogId); + + Mockito.verify(rowCountCache).invalidateCatalog(catalogId); + } + + @Test + public void testRollbackRetiresEngineCachesWhenResetCleanupThrows() throws Exception { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + metaCacheMgr.replaceRowCountCacheForTest(rowCountCache); + long catalogId = 25L; + HMSExternalCatalog catalog = Mockito.spy( + new HMSExternalCatalog(catalogId, "hms", null, Collections.emptyMap(), "")); + mockCurrentCatalog(catalogId, catalog); + ExternalMetaCache hive = metaCacheMgr.hive(catalogId); + Assert.assertTrue(hive.isCatalogInitialized(catalogId)); + Mockito.doThrow(new RuntimeException("close failed")).when(catalog).resetToUninitialized(false); + + Assert.assertThrows(RuntimeException.class, + () -> metaCacheMgr.rollbackCatalogProperties(catalog, Collections.emptyMap())); + + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + Mockito.verify(rowCountCache).invalidateCatalog(catalogId); + } + + @Test + public void testNameBasedInvalidationWidensToCanonicalDatabaseWhenTableCold() throws Exception { + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache hudi = new RecordingExternalMetaCache( + "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache( + "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive, hudi, iceberg); + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + metaCacheMgr.replaceRowCountCacheForTest(rowCountCache); + long catalogId = 26L; + long dbId = 27L; + HMSExternalCatalog catalog = Mockito.spy( + new HMSExternalCatalog(catalogId, "hms", null, Collections.emptyMap(), "")); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + Mockito.when(db.getId()).thenReturn(dbId); + Mockito.when(db.getFullName()).thenReturn("MixedDb"); + Mockito.doReturn(Optional.empty()).when(db).getTableForReplay("mixedtbl"); + Mockito.doReturn(Optional.of(db)).when(catalog).getDbForReplay("mixeddb"); + mockCurrentCatalog(catalogId, catalog); + hive.initializedCatalogIds.add(catalogId); + hudi.initializedCatalogIds.add(catalogId); + iceberg.initializedCatalogIds.add(catalogId); + + metaCacheMgr.invalidateTableByNameOrWider(catalogId, "mixeddb", "mixedtbl"); + + Assert.assertEquals(1, hive.invalidateDbCalls); + Mockito.verify(rowCountCache).invalidateDb(catalogId, dbId); + } + + @Test + public void testNameBasedInvalidationWidensToCatalogWhenDatabaseCold() throws Exception { + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache hudi = new RecordingExternalMetaCache( + "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache( + "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive, hudi, iceberg); + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + metaCacheMgr.replaceRowCountCacheForTest(rowCountCache); + long catalogId = 28L; + HMSExternalCatalog catalog = Mockito.spy( + new HMSExternalCatalog(catalogId, "hms", null, Collections.emptyMap(), "")); + Mockito.doReturn(Optional.empty()).when(catalog).getDbForReplay("colddb"); + mockCurrentCatalog(catalogId, catalog); + hive.initializedCatalogIds.add(catalogId); + hudi.initializedCatalogIds.add(catalogId); + iceberg.initializedCatalogIds.add(catalogId); + + metaCacheMgr.invalidateTableByNameOrWider(catalogId, "colddb", "tbl"); + + Assert.assertEquals(1, hive.invalidateCatalogEntriesCalls); + Mockito.verify(rowCountCache).invalidateCatalog(catalogId); + } + + @Test + public void testNameBasedRowCountFenceFallsBackToDatabaseWhenTableIsCold() { + long catalogId = 22L; + long dbId = 23L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.of(db)); + Mockito.when(db.getId()).thenReturn(dbId); + Mockito.doReturn(Optional.empty()).when(db).getTableForReplay("tbl1"); + mockCurrentCatalog(catalogId, catalog); + + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + metaCacheMgr.replaceRowCountCacheForTest(rowCountCache); + + metaCacheMgr.invalidateRowCountCache(catalogId, "db1", "tbl1"); + + Mockito.verify(rowCountCache).invalidateDb(catalogId, dbId); + } + @SuppressWarnings("unchecked") private ExternalMetaCacheMgr newManagerWithCaches(RecordingExternalMetaCache... caches) throws Exception { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java index 075806aa82bc49..826b185dbae904 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java @@ -17,11 +17,13 @@ package org.apache.doris.datasource; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.statistics.util.StatisticsUtil; import com.google.common.util.concurrent.MoreExecutors; +import com.google.common.util.concurrent.Uninterruptibles; import mockit.Mock; import mockit.MockUp; import org.junit.jupiter.api.Assertions; @@ -29,10 +31,79 @@ import org.mockito.Mockito; import java.util.Optional; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.RejectedExecutionException; import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; public class ExternalRowCountCacheTest { + @Test + public void testCatalogCacheResetSuppressesPerDatabaseRowCountScan() { + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(1L); + ExternalMetaCacheMgr metaCacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(metaCacheMgr); + new MockUp() { + @Mock + Env getCurrentEnv() { + return env; + } + }; + ExternalDatabase db = new ExternalDatabase( + catalog, 2L, "db", "db", InitDatabaseLog.Type.TEST) { + @Override + protected ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog externalCatalog, ExternalDatabase externalDatabase) { + return null; + } + }; + + // A catalog-wide refresh suppresses the per-database engine flush, so no per-database + // row-count fence may run; the catalog scope is fenced once by the caller instead. + db.resetMetaToUninitialized(true, false); + Mockito.verify(metaCacheMgr).invalidateDb(db); + Mockito.verify(metaCacheMgr, Mockito.never()).invalidateRowCountCache(1L, 2L); + + // An ordinary removal fences both the engine cache and this database's row counts. + db.resetMetaToUninitialized(); + Mockito.verify(metaCacheMgr).invalidateRowCountCache(1L, 2L); + } + + @Test + public void testDatabaseRemovalFencesRowCountWhenEngineInvalidationFails() { + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(1L); + ExternalMetaCacheMgr metaCacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(metaCacheMgr); + new MockUp() { + @Mock + Env getCurrentEnv() { + return env; + } + }; + ExternalDatabase db = new ExternalDatabase( + catalog, 2L, "db", "db", InitDatabaseLog.Type.TEST) { + @Override + protected ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog externalCatalog, ExternalDatabase externalDatabase) { + return null; + } + }; + Mockito.doThrow(new IllegalStateException("engine invalidation failed")) + .when(metaCacheMgr).invalidateDb(db); + + Assertions.assertThrows(IllegalStateException.class, () -> db.resetMetaToUninitialized(true, true)); + + // The independent row-count fence must still run when the routed invalidation throws. + Mockito.verify(metaCacheMgr).invalidateRowCountCache(1L, 2L); + } + @Test public void testRowCountKeyUsesTableIdAsCacheIdentity() { ExternalRowCountCache.RowCountKey key1 = new ExternalRowCountCache.RowCountKey(1, 2, 3); @@ -84,20 +155,191 @@ public TableIf findTable(long catalogId, long dbId, long tblId) { Mockito.verify(table).fetchRowCountWithMetaCache(false); } + @Test + public void testInvalidationScopes() { + ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() { + @Override + protected Optional doLoad(ExternalRowCountCache.RowCountKey rowCountKey) { + return Optional.of(rowCountKey.getTableId()); + } + }; + + ExternalRowCountCache cache = new ExternalRowCountCache( + MoreExecutors.newDirectExecutorService(), null, loader); + Assertions.assertEquals(100L, cache.getCachedRowCount(1, 10, 100, false)); + Assertions.assertEquals(101L, cache.getCachedRowCount(1, 10, 101, false)); + Assertions.assertEquals(102L, cache.getCachedRowCount(1, 11, 102, false)); + Assertions.assertEquals(200L, cache.getCachedRowCount(2, 20, 200, false)); + + cache.invalidateTable(1, 10, 100); + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cache.getCachedRowCountIfPresent(1, 10, 100)); + Assertions.assertEquals(101L, cache.getCachedRowCountIfPresent(1, 10, 101)); + + cache.invalidateDb(1, 10); + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cache.getCachedRowCountIfPresent(1, 10, 101)); + Assertions.assertEquals(102L, cache.getCachedRowCountIfPresent(1, 11, 102)); + + cache.invalidateCatalog(1); + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cache.getCachedRowCountIfPresent(1, 11, 102)); + Assertions.assertEquals(200L, cache.getCachedRowCountIfPresent(2, 20, 200)); + } + + @Test + public void testInvalidateWhileRefreshIsRunningDoesNotRepublishStaleValue() throws Exception { + AtomicInteger loadCount = new AtomicInteger(); + CountDownLatch refreshStarted = new CountDownLatch(1); + CountDownLatch allowRefreshToFinish = new CountDownLatch(1); + ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() { + @Override + protected Optional doLoad(ExternalRowCountCache.RowCountKey rowCountKey) { + int currentLoad = loadCount.incrementAndGet(); + if (currentLoad == 2) { + refreshStarted.countDown(); + Assertions.assertTrue(Uninterruptibles.awaitUninterruptibly( + allowRefreshToFinish, 30, TimeUnit.SECONDS)); + } + return Optional.of(currentLoad * 100L); + } + }; + + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + ExternalRowCountCache cache = new ExternalRowCountCache(executor, null, loader); + Assertions.assertEquals(100L, cache.getCachedRowCount(1, 10, 100, false)); + + cache.refreshForTest(1, 10, 100); + Assertions.assertTrue(refreshStarted.await(30, TimeUnit.SECONDS)); + Assertions.assertEquals(100L, cache.getCachedRowCount(1, 10, 100, false)); + + cache.invalidateTable(1, 10, 100); + allowRefreshToFinish.countDown(); + executor.submit(() -> { }).get(30, TimeUnit.SECONDS); + + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, + cache.getCachedRowCountIfPresent(1, 10, 100)); + Assertions.assertEquals(300L, cache.getCachedRowCount(1, 10, 100, false)); + } finally { + allowRefreshToFinish.countDown(); + executor.shutdownNow(); + } + } + + @Test + public void testCatalogInvalidationUsesFullInFlightLoadIdentity() throws Exception { + CountDownLatch oldLoadStarted = new CountDownLatch(1); + CountDownLatch newLoadStarted = new CountDownLatch(1); + CountDownLatch allowLoadsToFinish = new CountDownLatch(1); + ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() { + @Override + protected Optional doLoad(ExternalRowCountCache.RowCountKey rowCountKey) { + if (rowCountKey.getCatalogId() == 1L) { + oldLoadStarted.countDown(); + } else { + newLoadStarted.countDown(); + } + Assertions.assertTrue(Uninterruptibles.awaitUninterruptibly( + allowLoadsToFinish, 30, TimeUnit.SECONDS)); + return Optional.of(rowCountKey.getCatalogId() * 100L); + } + }; + + ExecutorService loaderExecutor = Executors.newFixedThreadPool(2); + ExecutorService callers = Executors.newFixedThreadPool(2); + try { + ExternalRowCountCache cache = new ExternalRowCountCache(loaderExecutor, null, loader); + Future oldLoad = callers.submit(() -> cache.getCachedRowCount(1, 10, 100, false)); + Assertions.assertTrue(oldLoadStarted.await(30, TimeUnit.SECONDS)); + cache.invalidateCatalog(1L); + + Future newLoad = callers.submit(() -> cache.getCachedRowCount(2, 20, 100, false)); + Assertions.assertTrue(newLoadStarted.await(30, TimeUnit.SECONDS)); + cache.invalidateCatalog(2L); + allowLoadsToFinish.countDown(); + + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, oldLoad.get(30, TimeUnit.SECONDS)); + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, newLoad.get(30, TimeUnit.SECONDS)); + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, + cache.getCachedRowCountIfPresent(2, 20, 100)); + } finally { + allowLoadsToFinish.countDown(); + callers.shutdownNow(); + loaderExecutor.shutdownNow(); + } + } + + @Test + public void testClosingFenceInvalidatesLoadAdmittedBeforeIt() throws Exception { + CountDownLatch loadAdmitted = new CountDownLatch(1); + CountDownLatch allowLoadToFinish = new CountDownLatch(1); + ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() { + @Override + protected Optional doLoad(ExternalRowCountCache.RowCountKey rowCountKey) { + loadAdmitted.countDown(); + Assertions.assertTrue(Uninterruptibles.awaitUninterruptibly( + allowLoadToFinish, 30, TimeUnit.SECONDS)); + return Optional.of(100L); + } + }; + + ExecutorService executor = Executors.newSingleThreadExecutor(); + ExecutorService callers = Executors.newSingleThreadExecutor(); + try { + ExternalRowCountCache cache = new ExternalRowCountCache(executor, null, loader); + // Opening fence, then a load admitted inside the admission window. + cache.invalidateTable(1, 10, 100); + Future staleLoad = callers.submit(() -> cache.getCachedRowCount(1, 10, 100, false)); + Assertions.assertTrue(loadAdmitted.await(30, TimeUnit.SECONDS)); + + // The selective refresh finishes and closes the window before the load can publish. + cache.invalidateTable(1, 10, 100); + allowLoadToFinish.countDown(); + + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, staleLoad.get(30, TimeUnit.SECONDS)); + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, + cache.getCachedRowCountIfPresent(1, 10, 100)); + } finally { + allowLoadToFinish.countDown(); + callers.shutdownNow(); + executor.shutdownNow(); + } + } + + @Test + public void testRejectedSubmissionRemovesInFlightFence() { + ExecutorService rejectingExecutor = Mockito.mock(ExecutorService.class); + Mockito.doThrow(new RejectedExecutionException("rejected")) + .when(rejectingExecutor).execute(Mockito.any(Runnable.class)); + ExternalRowCountCache cache = new ExternalRowCountCache(rejectingExecutor); + + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, + cache.getCachedRowCount(1, 10, 100, false)); + Assertions.assertEquals(0, cache.getInFlightLoadCountForTest()); + } + @Test public void testLoadWithException() throws Exception { ThreadPoolExecutor executor = ThreadPoolManager.newDaemonFixedThreadPool( 1, Integer.MAX_VALUE, "TEST", true); AtomicInteger counter = new AtomicInteger(0); - new MockUp() { - @Mock + ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() { + @Override protected Optional doLoad(ExternalRowCountCache.RowCountKey rowCountKey) { - counter.incrementAndGet(); - return null; + int currentLoad = counter.incrementAndGet(); + if (currentLoad == 1) { + return null; + } + if (currentLoad == 3) { + try { + Thread.sleep(2000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + } + return Optional.of(100L); } }; - ExternalRowCountCache cache = new ExternalRowCountCache(executor); + ExternalRowCountCache cache = new ExternalRowCountCache(executor, null, loader); long cachedRowCount = cache.getCachedRowCount(1, 1, 1, false); Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cachedRowCount); for (int i = 0; i < 60; i++) { @@ -108,13 +350,6 @@ protected Optional doLoad(ExternalRowCountCache.RowCountKey rowCountKey) { } Assertions.assertEquals(1, counter.get()); - new MockUp() { - @Mock - protected Optional doLoad(ExternalRowCountCache.RowCountKey rowCountKey) { - counter.incrementAndGet(); - return Optional.of(100L); - } - }; cache.getCachedRowCount(1, 1, 1, false); for (int i = 0; i < 60; i++) { cachedRowCount = cache.getCachedRowCount(1, 1, 1, false); @@ -128,18 +363,6 @@ protected Optional doLoad(ExternalRowCountCache.RowCountKey rowCountKey) { Assertions.assertEquals(100, cachedRowCount); Assertions.assertEquals(2, counter.get()); - new MockUp() { - @Mock - protected Optional doLoad(ExternalRowCountCache.RowCountKey rowCountKey) { - counter.incrementAndGet(); - try { - Thread.sleep(2000); - } catch (InterruptedException e) { - e.printStackTrace(); - } - return Optional.of(100L); - } - }; cachedRowCount = cache.getCachedRowCount(2, 2, 2, false); Assertions.assertEquals(100, cachedRowCount); Thread.sleep(1000); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java new file mode 100644 index 00000000000000..e5535b424718e1 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java @@ -0,0 +1,96 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.catalog.Env; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalMetaCacheMgr; + +import mockit.Mock; +import mockit.MockUp; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Optional; + +public class HiveMetadataOpsTest { + @Test + @SuppressWarnings("unchecked") + public void testColdTruncateReplayInvalidatesCanonicalDatabaseScope() { + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(1L); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + Mockito.when(db.getId()).thenReturn(2L); + Mockito.when(db.getFullName()).thenReturn("MixedDb"); + Mockito.when(db.getTableForReplay("mixedtbl")).thenReturn(Optional.empty()); + Mockito.when(catalog.getDbForReplay("mixeddb")).thenReturn((Optional) Optional.of(db)); + + ExternalMetaCacheMgr metaCacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(metaCacheMgr); + new MockUp() { + @Mock + Env getCurrentEnv() { + return env; + } + }; + + HiveMetadataOps metadataOps = new HiveMetadataOps(catalog, Mockito.mock(HMSCachedClient.class)); + metadataOps.afterTruncateTable("mixeddb", "mixedtbl", 100L); + + // The event's lower-cased spelling must not be routed as a cache key directly. + Mockito.verify(metaCacheMgr).invalidateTableByNameOrWider(1L, "mixeddb", "mixedtbl"); + } + + @Test + @SuppressWarnings("unchecked") + public void testColdTruncateReplayWithColdDatabaseUsesWideningHelper() { + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(3L); + Mockito.when(catalog.getDbForReplay("db")).thenReturn(Optional.empty()); + + ExternalMetaCacheMgr metaCacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(metaCacheMgr); + new MockUp() { + @Mock + Env getCurrentEnv() { + return env; + } + }; + + HiveMetadataOps metadataOps = new HiveMetadataOps(catalog, Mockito.mock(HMSCachedClient.class)); + metadataOps.afterTruncateTable("db", "tbl", 100L); + + Mockito.verify(metaCacheMgr).invalidateTableByNameOrWider(3L, "db", "tbl"); + } + + @Test + public void testAfterDropTableWithoutResolvedDatabaseRetiresGeneration() { + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(4L); + Mockito.when(catalog.getName()).thenReturn("hive"); + Mockito.when(catalog.getDbForReplay("db")).thenReturn(Optional.empty()); + + HiveMetadataOps metadataOps = new HiveMetadataOps(catalog, Mockito.mock(HMSCachedClient.class)); + metadataOps.afterDropTable("db", "tbl"); + + // An empty lookup can hide a resident canonical database object after a lost mode-2 mapping. + Mockito.verify(catalog).retireUnresolvedDatabaseGeneration(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java index 0b24a89139b9d9..67f2dbcdff2d96 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java @@ -21,6 +21,7 @@ import org.apache.doris.catalog.RefreshManager; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.ExternalObjectLog; import org.apache.doris.datasource.ExternalTable; @@ -217,8 +218,12 @@ public void testMetadataRefreshInvalidatesAccessWithoutClosingSession() throws E // Leader REFRESH TABLE reaches only the typed table route, which must still retire the // Lance access cache even though it bypasses the name-based invalidateTable() entry. + ExternalDatabase lanceDb = Mockito.mock(ExternalDatabase.class); + Mockito.when(lanceDb.getId()).thenReturn(70L); ExternalTable lanceTable = Mockito.mock(ExternalTable.class); Mockito.when(lanceTable.getCatalog()).thenReturn(catalog); + Mockito.doReturn(lanceDb).when(lanceTable).getDb(); + Mockito.when(lanceTable.getId()).thenReturn(71L); caches.invalidateTableCache(lanceTable); Mockito.verify(client, Mockito.times(3)).invalidateTableAccessCache(); Mockito.verify(session, Mockito.never()).close(); @@ -228,6 +233,34 @@ public void testMetadataRefreshInvalidatesAccessWithoutClosingSession() throws E } } + @Test + public void testHeldTableInvalidationResetsLanceAccess() throws Exception { + Session session = Mockito.mock(Session.class); + LanceCatalogClient client = Mockito.spy(client(session)); + LanceExternalCatalog catalog = catalog(client); + Env env = Mockito.mock(Env.class); + CatalogMgr catalogs = Mockito.mock(CatalogMgr.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogs); + long catalogId = catalog.getId(); + Mockito.doReturn(catalog).when(catalogs).getCatalog(catalogId); + ExternalMetaCacheMgr caches = new ExternalMetaCacheMgr(true); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + ExternalTable table = Mockito.mock(ExternalTable.class); + Mockito.when(db.getId()).thenReturn(70L); + Mockito.doReturn(db).when(table).getDb(); + Mockito.when(table.getId()).thenReturn(71L); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.getDbName()).thenReturn("mapped_db"); + Mockito.when(table.getName()).thenReturn("mapped_table"); + try (MockedStatic currentEnv = Mockito.mockStatic(Env.class)) { + currentEnv.when(Env::getCurrentEnv).thenReturn(env); + caches.invalidateTableCache(table); + Mockito.verify(client).invalidateTableAccessCache(); + } finally { + catalog.onClose(); + } + } + @Test public void testIndexJobLocatorBypassesQueryAccessCache() throws Exception { LanceNamespace namespace = Mockito.mock(LanceNamespace.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 48c0297dd61371..b998121e13fd6e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -2217,6 +2217,7 @@ public void testRefreshTableInvalidatesPaimonCatalogCacheAfterExternalDrop() thr Mockito.when(dorisDatabase.getCatalog()).thenReturn(dorisCatalog); Mockito.when(dorisDatabase.getFullName()).thenReturn("db"); Mockito.when(dorisDatabase.getRemoteName()).thenReturn("db"); + Mockito.when(dorisTable.getDb()).thenReturn(dorisDatabase); // TVFs and table-existence checks can populate only Paimon's CachingCatalog. A // REFRESH TABLE still has the resolved ExternalTable and must invalidate that direct diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java new file mode 100644 index 00000000000000..a6452c09103e15 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java @@ -0,0 +1,337 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands.insert; + +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.EnvFactory; +import org.apache.doris.catalog.RefreshManager; +import org.apache.doris.common.DdlException; +import org.apache.doris.common.UserException; +import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalMetaCacheMgr; +import org.apache.doris.datasource.ExternalObjectLog; +import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.hive.HMSExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalTable; +import org.apache.doris.datasource.hive.HiveExternalMetaCache; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.persist.EditLog; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.Coordinator; +import org.apache.doris.thrift.THivePartitionUpdate; +import org.apache.doris.transaction.TransactionManager; +import org.apache.doris.transaction.TransactionType; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.InOrder; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.lang.reflect.Field; +import java.util.Collections; +import java.util.Optional; + +class HiveInsertExecutorTest { + + @AfterEach + void tearDown() { + ConnectContext.remove(); + } + + @Test + void testCommittedInsertPublishesRowCountFenceWhenPartitionRefreshFails() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setThreadLocalInfo(); + + long catalogId = 61L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("hms"); + Mockito.when(catalog.getTransactionManager()).thenReturn(Mockito.mock(TransactionManager.class)); + Mockito.when(db.getId()).thenReturn(62L); + Mockito.when(db.getFullName()).thenReturn("db1"); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.getDatabase()).thenReturn(db); + Mockito.when(table.getName()).thenReturn("tbl1"); + Mockito.when(table.getNameWithFullQualifiers()).thenReturn("hms.db1.tbl1"); + Mockito.when(table.isPartitionedTable()).thenReturn(true); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + HiveExternalMetaCache hiveCache = Mockito.mock(HiveExternalMetaCache.class); + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(env.getEditLog()).thenReturn(editLog); + Mockito.when(cacheMgr.hive(catalogId)).thenReturn(hiveCache); + Mockito.doThrow(new IllegalStateException("partition cache failure")) + .when(hiveCache).refreshAffectedPartitions( + Mockito.eq(table), Mockito.anyList(), Mockito.anyList(), Mockito.anyList()); + + EnvFactory envFactory = Mockito.mock(EnvFactory.class); + Mockito.when(envFactory.createCoordinator(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong())) + .thenReturn(Mockito.mock(Coordinator.class)); + try (MockedStatic mockedEnvFactory = Mockito.mockStatic(EnvFactory.class); + MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnvFactory.when(EnvFactory::getInstance).thenReturn(envFactory); + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + + TestingHiveInsertExecutor executor = new TestingHiveInsertExecutor( + ctx, table, Mockito.mock(NereidsPlanner.class)); + executor.setPartitionUpdates(Collections.singletonList(new THivePartitionUpdate())); + Assertions.assertDoesNotThrow(executor::runAfterCommit); + } + + InOrder order = Mockito.inOrder(cacheMgr, hiveCache); + order.verify(cacheMgr).invalidateRowCountCache(table); + order.verify(cacheMgr).hive(catalogId); + order.verify(hiveCache).refreshAffectedPartitions( + Mockito.eq(table), Mockito.anyList(), Mockito.anyList(), Mockito.anyList()); + Mockito.verify(cacheMgr).invalidateTableCache(table); + + ArgumentCaptor logCaptor = ArgumentCaptor.forClass(ExternalObjectLog.class); + Mockito.verify(editLog).logRefreshExternalTable(logCaptor.capture()); + ExternalObjectLog log = logCaptor.getValue(); + Assertions.assertEquals(catalogId, log.getCatalogId()); + Assertions.assertEquals("db1", log.getDbName()); + Assertions.assertEquals("tbl1", log.getTableName()); + Assertions.assertNull(log.getPartitionNames()); + Assertions.assertNull(log.getNewPartitionNames()); + } + + @Test + void testDefaultPostCommitRefreshUsesHeldTableIdentity() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setThreadLocalInfo(); + + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + ExternalTable table = Mockito.mock(ExternalTable.class); + Mockito.when(catalog.getName()).thenReturn("iceberg"); + Mockito.when(catalog.getTransactionManager()).thenReturn(Mockito.mock(TransactionManager.class)); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.getDatabase()).thenReturn(db); + + Env env = Mockito.mock(Env.class); + RefreshManager refreshManager = Mockito.mock(RefreshManager.class); + Mockito.when(env.getRefreshManager()).thenReturn(refreshManager); + EnvFactory envFactory = Mockito.mock(EnvFactory.class); + Mockito.when(envFactory.createCoordinator(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong())) + .thenReturn(Mockito.mock(Coordinator.class)); + try (MockedStatic mockedEnvFactory = Mockito.mockStatic(EnvFactory.class); + MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnvFactory.when(EnvFactory::getInstance).thenReturn(envFactory); + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + + TestingExternalInsertExecutor executor = new TestingExternalInsertExecutor( + ctx, table, Mockito.mock(NereidsPlanner.class)); + executor.runAfterCommit(); + } + + Mockito.verify(refreshManager).refreshTableAfterCommit(table); + } + + @Test + void testCommittedInsertFencesRowCountWhenPartitionedPredicateThrows() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setThreadLocalInfo(); + + long catalogId = 63L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("hms"); + Mockito.when(catalog.getTransactionManager()).thenReturn(Mockito.mock(TransactionManager.class)); + Mockito.when(db.getId()).thenReturn(64L); + Mockito.when(db.getFullName()).thenReturn("db1"); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.getDatabase()).thenReturn(db); + Mockito.when(table.getName()).thenReturn("tbl1"); + Mockito.when(table.getNameWithFullQualifiers()).thenReturn("hms.db1.tbl1"); + Mockito.when(table.isPartitionedTable()).thenThrow(new IllegalStateException("reinit failed")); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(env.getEditLog()).thenReturn(editLog); + + EnvFactory envFactory = Mockito.mock(EnvFactory.class); + Mockito.when(envFactory.createCoordinator(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong())) + .thenReturn(Mockito.mock(Coordinator.class)); + try (MockedStatic mockedEnvFactory = Mockito.mockStatic(EnvFactory.class); + MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnvFactory.when(EnvFactory::getInstance).thenReturn(envFactory); + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + + TestingHiveInsertExecutor executor = new TestingHiveInsertExecutor( + ctx, table, Mockito.mock(NereidsPlanner.class)); + executor.setPartitionUpdates(Collections.singletonList(new THivePartitionUpdate())); + Assertions.assertDoesNotThrow(executor::runAfterCommit); + } + + InOrder order = Mockito.inOrder(cacheMgr, editLog); + order.verify(cacheMgr).invalidateRowCountCache(table); + order.verify(cacheMgr).invalidateTableCache(table); + ArgumentCaptor logCaptor = ArgumentCaptor.forClass(ExternalObjectLog.class); + order.verify(editLog).logRefreshExternalTable(logCaptor.capture()); + Assertions.assertNull(logCaptor.getValue().getPartitionNames()); + Assertions.assertNull(logCaptor.getValue().getNewPartitionNames()); + } + + @Test + void testSuccessfulSelectiveRefreshClosesRowCountAdmissionWindow() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setThreadLocalInfo(); + + long catalogId = 67L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("hms"); + Mockito.when(catalog.getTransactionManager()).thenReturn(Mockito.mock(TransactionManager.class)); + Mockito.when(db.getId()).thenReturn(68L); + Mockito.when(db.getFullName()).thenReturn("db1"); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.getDatabase()).thenReturn(db); + Mockito.when(table.getName()).thenReturn("tbl1"); + Mockito.when(table.getNameWithFullQualifiers()).thenReturn("hms.db1.tbl1"); + Mockito.when(table.isPartitionedTable()).thenReturn(true); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + HiveExternalMetaCache hiveCache = Mockito.mock(HiveExternalMetaCache.class); + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(env.getEditLog()).thenReturn(editLog); + Mockito.when(cacheMgr.hive(catalogId)).thenReturn(hiveCache); + + EnvFactory envFactory = Mockito.mock(EnvFactory.class); + Mockito.when(envFactory.createCoordinator(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong())) + .thenReturn(Mockito.mock(Coordinator.class)); + try (MockedStatic mockedEnvFactory = Mockito.mockStatic(EnvFactory.class); + MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnvFactory.when(EnvFactory::getInstance).thenReturn(envFactory); + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + + TestingHiveInsertExecutor executor = new TestingHiveInsertExecutor( + ctx, table, Mockito.mock(NereidsPlanner.class)); + executor.setPartitionUpdates(Collections.singletonList(new THivePartitionUpdate())); + executor.runAfterCommit(); + } + + // One fence before the selective refresh, one after it closes the admission window. + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(table); + } + + @Test + void testCommittedFullInvalidationFailureStillLogsFullRefresh() throws Exception { + ConnectContext ctx = new ConnectContext(); + ctx.setThreadLocalInfo(); + + long catalogId = 65L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("hms"); + Mockito.when(catalog.getTransactionManager()).thenReturn(Mockito.mock(TransactionManager.class)); + Mockito.when(db.getId()).thenReturn(66L); + Mockito.when(db.getFullName()).thenReturn("db1"); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.getDatabase()).thenReturn(db); + Mockito.when(table.getName()).thenReturn("tbl1"); + Mockito.when(table.getNameWithFullQualifiers()).thenReturn("hms.db1.tbl1"); + Mockito.when(table.isPartitionedTable()).thenReturn(false); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(env.getEditLog()).thenReturn(editLog); + Mockito.doThrow(new IllegalStateException("table cache failure")) + .when(cacheMgr).invalidateTableCache(table); + + EnvFactory envFactory = Mockito.mock(EnvFactory.class); + Mockito.when(envFactory.createCoordinator(Mockito.any(), Mockito.any(), Mockito.any(), Mockito.anyLong())) + .thenReturn(Mockito.mock(Coordinator.class)); + try (MockedStatic mockedEnvFactory = Mockito.mockStatic(EnvFactory.class); + MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnvFactory.when(EnvFactory::getInstance).thenReturn(envFactory); + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + + TestingHiveInsertExecutor executor = new TestingHiveInsertExecutor( + ctx, table, Mockito.mock(NereidsPlanner.class)); + executor.setPartitionUpdates(Collections.emptyList()); + Assertions.assertDoesNotThrow(executor::runAfterCommit); + } + + Mockito.verify(cacheMgr).invalidateRowCountCache(table); + Mockito.verify(cacheMgr, Mockito.atLeastOnce()).invalidateTableCache(table); + ArgumentCaptor logCaptor = ArgumentCaptor.forClass(ExternalObjectLog.class); + Mockito.verify(editLog).logRefreshExternalTable(logCaptor.capture()); + Assertions.assertNull(logCaptor.getValue().getPartitionNames()); + Assertions.assertNull(logCaptor.getValue().getNewPartitionNames()); + } + + private static class TestingHiveInsertExecutor extends HiveInsertExecutor { + TestingHiveInsertExecutor(ConnectContext ctx, HMSExternalTable table, NereidsPlanner planner) { + super(ctx, table, "label", planner, Optional.empty(), false, 0L); + } + + void setPartitionUpdates(java.util.List updates) throws Exception { + Field field = HiveInsertExecutor.class.getDeclaredField("partitionUpdates"); + field.setAccessible(true); + field.set(this, updates); + } + + void runAfterCommit() throws DdlException { + doAfterCommit(); + } + } + + private static class TestingExternalInsertExecutor extends BaseExternalTableInsertExecutor { + TestingExternalInsertExecutor(ConnectContext ctx, ExternalTable table, NereidsPlanner planner) { + super(ctx, table, "label", planner, Optional.empty(), false, 0L); + } + + @Override + protected void beforeExec() throws UserException { + } + + @Override + protected void doBeforeCommit() throws UserException { + } + + @Override + protected TransactionType transactionType() { + return TransactionType.ICEBERG; + } + + void runAfterCommit() throws DdlException { + doAfterCommit(); + } + } +} From dbc3f2b5eebe918dc4d22f4112ddd44cf4e84480 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Mon, 28 Sep 2026 08:58:56 +0800 Subject: [PATCH 02/16] [fix](fe) Fence external row counts across cache refreshes ### What problem does this PR solve? Related PR: #68196 Problem Summary: Automatic database-cache eviction could trigger a catalog-wide row-count scan despite no metadata change. A catalog refresh could admit an old asynchronous count before replacing its database generation. Selective Hive insert refresh kept the held HMS table parameters, allowing the new count load to read pre-insert numRows or totalSize. Suppress semantic count invalidation for capacity/expiry eviction, fence both sides of catalog refresh, and retire held HMS metadata before the closing count fence on leader and replay. ### Release note None ### Check List (For Author) - Test: Unit Test (CatalogMgrTest, RefreshManagerTest, HiveInsertExecutorTest); ./build.sh --fe -j4 - Behavior changed: Yes (row-count cache now follows metadata changes without extra eviction scans) - Does this need documentation: No --- .../apache/doris/catalog/RefreshManager.java | 2 ++ .../doris/datasource/ExternalCatalog.java | 30 ++++++++++------ .../doris/datasource/ExternalTable.java | 2 +- .../commands/insert/HiveInsertExecutor.java | 3 ++ .../doris/catalog/RefreshManagerTest.java | 34 +++++++++++++++++++ .../doris/datasource/CatalogMgrTest.java | 24 ++++++++++--- .../insert/HiveInsertExecutorTest.java | 5 +++ 7 files changed, 84 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index 1204d4eb608edc..e4774941330d1e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -258,6 +258,8 @@ public void replayRefreshTable(ExternalObjectLog log) { .hive(catalog.getId()); cache.refreshAffectedPartitionsCache((HMSExternalTable) table.get(), modifiedPartNames, newPartNames); + // The held HMS table also caches the pre-insert table parameters. + table.get().unsetObjectCreated(); // Close the admission window the opening fence left open: a load admitted after // it can publish the pre-insert value from the still-resident file list. Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table.get()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 24ec351e351d59..b58601b5c321fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -72,6 +72,7 @@ import org.apache.doris.qe.GlobalVariable; import org.apache.doris.transaction.TransactionManager; +import com.github.benmanes.caffeine.cache.RemovalCause; import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.base.Strings; @@ -433,7 +434,7 @@ private void buildMetaCache() { localDbName -> Optional.ofNullable( buildDbForInit(null, localDbName, Util.genIdByName(name, localDbName), logType, true)), - (key, value, cause) -> handleDatabaseMetaCacheRemoval(value), + (key, value, cause) -> handleDatabaseMetaCacheRemoval(value, cause), this::acquireMetadataLoadEpoch, this::isMetadataLoadEpochCurrent); } @@ -714,17 +715,23 @@ public void resetToUninitialized(boolean invalidCache) { */ public void onRefreshCache(boolean invalidCache) { setLastUpdateTime(System.currentTimeMillis()); - refreshMetaCacheOnly(invalidCache); - if (invalidCache) { - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(id); + try { + refreshMetaCacheOnly(invalidCache); + } finally { + if (invalidCache) { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(id); + } } } /** * Refresh meta cache only (database level cache), without invalidating catalog level cache. */ - private void refreshMetaCacheOnly(boolean invalidCache) { + private synchronized void refreshMetaCacheOnly(boolean invalidCache) { if (metaCache != null) { + // A concurrent row-count load can finish while invalidateAll retires database objects. + // Fence before the generation swap, then close the window after engine invalidation. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(id); // A catalog-wide engine invalidation below supersedes every database invalidation. // The legacy cache uses a synchronous removal listener, so this thread-local scope // prevents one full SDK-cache scan per cached database without affecting concurrent @@ -736,10 +743,10 @@ private void refreshMetaCacheOnly(boolean invalidCache) { } finally { invalidatingAllMetaCache = false; invalidateEngineCacheOnDatabaseRemoval.remove(); - } - if (!invalidCache) { - // No catalog-wide engine invalidation follows, so fence the row counts once here. - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(id); + if (!invalidCache) { + // No catalog-wide engine invalidation follows, so close the row-count fence here. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(id); + } } } } @@ -1279,10 +1286,11 @@ boolean shouldInvalidateRoutedCacheOnDatabaseRemoval() { return invalidateEngineCacheOnDatabaseRemoval.get(); } - void handleDatabaseMetaCacheRemoval(Optional> value) { + void handleDatabaseMetaCacheRemoval(Optional> value, + RemovalCause cause) { value.ifPresent(v -> v.resetMetaToUninitialized( shouldInvalidateRoutedCacheOnDatabaseRemoval(), - shouldInvalidateRowCountOnDatabaseRemoval())); + !cause.wasEvicted() && shouldInvalidateRowCountOnDatabaseRemoval())); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java index 3e194a725393bc..2a5f5240e60c78 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java @@ -413,7 +413,7 @@ public Optional initSchema() { throw new NotImplementedException("implement in sub class"); } - public void unsetObjectCreated() { + public synchronized void unsetObjectCreated() { this.objectCreated = false; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java index e0bdfb50dfdc5e..486b90a4c91b67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java @@ -97,6 +97,9 @@ protected void doAfterCommit() throws DdlException { HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() .hive(hmsTable.getCatalog().getId()); cache.refreshAffectedPartitions(hmsTable, partitionUpdates, modifiedPartNames, newPartNames); + // The held HMS table also caches table parameters (including numRows/totalSize). + // Reload those after the selective partition refresh, before closing the count fence. + hmsTable.unsetObjectCreated(); // Close the admission window opened by the fence above: a load admitted after it can // compute the pre-insert value from the still-resident file list and publish it. Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(hmsTable); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java index aff6393afd252c..b6d92a4f2112ea 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java @@ -94,6 +94,40 @@ void testPartitionReplayInvalidatesRowCountBeforeCacheFailure() { Mockito.verify(cacheMgr).invalidateRowCountCache(table); } + @Test + void testPartitionReplayReloadsHeldHiveTableBeforeClosingRowCountFence() { + long catalogId = 80L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.of(db)); + Mockito.doReturn(Optional.of(table)).when(db).getTableForReplay("tbl1"); + + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + HiveExternalMetaCache hiveCache = Mockito.mock(HiveExternalMetaCache.class); + Mockito.when(cacheMgr.hive(catalogId)).thenReturn(hiveCache); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + ExternalObjectLog log = ExternalObjectLog.createForRefreshPartitions( + catalogId, "db1", "tbl1", + java.util.Collections.singletonList("p=1"), java.util.Collections.emptyList(), 1L); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + new RefreshManager().replayRefreshTable(log); + } + + InOrder order = Mockito.inOrder(hiveCache, table, cacheMgr); + order.verify(hiveCache).refreshAffectedPartitionsCache(Mockito.eq(table), Mockito.anyList(), + Mockito.anyList()); + order.verify(table).unsetObjectCreated(); + order.verify(cacheMgr).invalidateRowCountCache(table); + } + @Test void testAlterPartitionInvalidatesRowCountBeforeCacheFailure() throws Exception { long catalogId = 53L; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 36a3a2f5aaad7a..fe3fd70a00e8cb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -33,6 +33,7 @@ import org.apache.doris.persist.EditLog; import org.apache.doris.statistics.query.QueryStats; +import com.github.benmanes.caffeine.cache.RemovalCause; import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -56,6 +57,19 @@ public class CatalogMgrTest { + @Test + void testDatabaseCapacityEvictionDoesNotFenceAllRowCounts() { + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(49L); + @SuppressWarnings("unchecked") + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + + catalog.handleDatabaseMetaCacheRemoval(Optional.of(db), RemovalCause.SIZE); + Mockito.verify(db).resetMetaToUninitialized(true, false); + + catalog.handleDatabaseMetaCacheRemoval(Optional.of(db), RemovalCause.EXPLICIT); + Mockito.verify(db).resetMetaToUninitialized(true, true); + } + private static void addCatalog(CatalogMgr catalogMgr, ExternalCatalog catalog) throws Exception { Field idToCatalogField = CatalogMgr.class.getDeclaredField("idToCatalog"); idToCatalogField.setAccessible(true); @@ -385,8 +399,10 @@ void testCatalogRefreshUsesSingleRowCountInvalidationOwner() { catalog.onRefreshCache(true); } - Mockito.verify(metaCache).invalidateAll(); - Mockito.verify(cacheMgr).invalidateCatalog(catalogId); + InOrder order = Mockito.inOrder(cacheMgr, metaCache); + order.verify(cacheMgr).invalidateRowCountCache(catalogId); + order.verify(metaCache).invalidateAll(); + order.verify(cacheMgr).invalidateCatalog(catalogId); Mockito.verify(cacheMgr, Mockito.never()).getRowCountCache(); } @@ -412,7 +428,7 @@ protected ExternalTable buildTableInternal(String remoteTableName, String localT 10, key -> Collections.emptyList(), key -> Optional.empty(), - (key, value, cause) -> catalog.handleDatabaseMetaCacheRemoval(value)); + (key, value, cause) -> catalog.handleDatabaseMetaCacheRemoval(value, cause)); metaCache.updateCache("db1", "db1", db, dbId); catalog.installMetaCache(metaCache); @@ -429,7 +445,7 @@ protected ExternalTable buildTableInternal(String remoteTableName, String localT } Mockito.verify(cacheMgr).invalidateDb(db); - Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId); + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId); Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(Mockito.anyLong()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java index a6452c09103e15..341928f3d6a740 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java @@ -244,6 +244,11 @@ void testSuccessfulSelectiveRefreshClosesRowCountAdmissionWindow() throws Except // One fence before the selective refresh, one after it closes the admission window. Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(table); + InOrder order = Mockito.inOrder(hiveCache, table, cacheMgr); + order.verify(hiveCache).refreshAffectedPartitions(Mockito.eq(table), Mockito.anyList(), + Mockito.anyList(), Mockito.anyList()); + order.verify(table).unsetObjectCreated(); + order.verify(cacheMgr).invalidateRowCountCache(table); } @Test From fed6afc371b94938896c76b113671e5035142c1b Mon Sep 17 00:00:00 2001 From: 924060929 Date: Mon, 28 Sep 2026 16:38:07 +0800 Subject: [PATCH 03/16] [fix](fe) Scope cold external database replay invalidation ### What problem does this PR solve? Related PR: #68196 Problem Summary: A follower can retain a canonical external database name and ID after its bounded database object cache evicts that object. Refresh and DROP TABLE replay treated this ordinary cold miss like a lost name mapping and invalidated every database in the catalog, discarding unrelated hot engine entries and row counts. Resolve the canonical identity without loading the object, invalidate only that database when known, and reserve catalog-wide retirement for genuinely unresolved mappings. Cover both name-based and ID-only replay identity and cold refresh/drop routing. ### Release note None ### Check List (For Author) - Test: Unit Test (RefreshManagerTest, CatalogMgrTest, HiveMetadataOpsTest, two PaimonExternalMetaCacheTest cases); ./build.sh --fe -j4 - Behavior changed: Yes (cold replay invalidation is database-scoped when identity is known) - Does this need documentation: No --- .../apache/doris/catalog/RefreshManager.java | 31 ++++++++----- .../doris/datasource/ExternalCatalog.java | 30 +++++++++++++ .../datasource/hive/HiveMetadataOps.java | 6 +-- .../iceberg/IcebergMetadataOps.java | 5 +-- .../maxcompute/MaxComputeMetadataOps.java | 5 +-- .../doris/datasource/metacache/MetaCache.java | 5 +++ .../datasource/paimon/PaimonMetadataOps.java | 9 ++-- .../doris/catalog/RefreshManagerTest.java | 35 +++++++++++++++ .../doris/datasource/CatalogMgrTest.java | 43 +++++++++++++++++++ .../datasource/hive/HiveMetadataOpsTest.java | 2 +- 10 files changed, 145 insertions(+), 26 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index e4774941330d1e..ddb912e952ee86 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -18,6 +18,7 @@ package org.apache.doris.catalog; import org.apache.doris.common.DdlException; +import org.apache.doris.common.Pair; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.UserException; import org.apache.doris.datasource.CatalogIf; @@ -119,11 +120,7 @@ public void replayRefreshDb(ExternalObjectLog log) { } if (!db.isPresent()) { - LOG.warn("failed to find db when replaying refresh db: {}", log.debugForRefreshDb()); - // No canonical identity is available: retire the catalog scope so engine entries and - // row counts cannot survive the committed refresh. - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(catalog.getId()); - invalidatePaimonCatalogForUnresolvedReplay(catalog); + invalidateColdReplayDatabase(catalog, log); } else { refreshDbInternal(db.get()); } @@ -210,11 +207,7 @@ public void replayRefreshTable(ExternalObjectLog log) { } // See comment in refreshDbInternal for why db and table may be null. if (!db.isPresent()) { - LOG.warn("failed to find db when replaying refresh table: {}", log.debugForRefreshTable()); - // No canonical identity is available: retire the catalog scope so engine entries and - // row counts cannot survive the committed refresh. - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(catalog.getId()); - invalidatePaimonCatalogForUnresolvedReplay(catalog); + invalidateColdReplayDatabase(catalog, log); return; } Optional table; @@ -352,6 +345,24 @@ private void invalidatePaimonCatalogForUnresolvedReplay(ExternalCatalog catalog) } } + private void invalidateColdReplayDatabase(ExternalCatalog catalog, ExternalObjectLog log) { + Optional> identity = catalog.getDbIdentityForReplay(log.getDbName(), log.getDbId()); + if (identity.isPresent()) { + // The object may be evicted while its canonical name and deterministic ID are still + // known. Retire only that database's engine entries and independent row counts. + LOG.debug("database object is cold when replaying refresh in catalog {}, db {}", + catalog.getName(), identity.get().first); + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateDb(catalog.getId(), identity.get().second, identity.get().first); + } else { + // A lost mode-2 mapping cannot identify the database safely; widen the fence. + LOG.warn("failed to resolve database identity when replaying refresh: {}", + log.debugForRefreshDb()); + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(catalog.getId()); + invalidatePaimonCatalogForUnresolvedReplay(catalog); + } + } + // Refresh partition public void refreshPartitions(String catalogName, String dbName, String tableName, List partitionNames, long updateTime, boolean ignoreIfNotExists) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index b58601b5c321fe..5be0351dd67a26 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -967,6 +967,19 @@ public Optional> getDbForReplay(String return metaCache.tryGetMetaObj(localDbName); } + /** Resolve a replay log's database identity without reloading an evicted database object. */ + public Optional> getDbIdentityForReplay(String dbName, long dbId) { + if (!isInitialized() || metaCache == null) { + return Optional.empty(); + } + if (dbName != null && !dbName.isEmpty()) { + String localName = getLocalDatabaseName(dbName, true); + return localName == null ? Optional.empty() + : Optional.of(Pair.of(localName, Util.genIdByName(name, localName))); + } + return metaCache.getNameByIdIfPresent(dbId).map(localName -> Pair.of(localName, dbId)); + } + /** * Build a database instance. * If checkExists is true, it will check if the database exists in the remote system. @@ -1331,6 +1344,23 @@ public void retireUnresolvedDatabaseGeneration() { } } + /** Retire a cold replay database narrowly when its canonical name survived object eviction. */ + public void invalidateColdDatabaseForReplay(String dbName) { + try { + Optional> identity = getDbIdentityForReplay(dbName, 0L); + if (identity.isPresent()) { + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateDb(getId(), identity.get().second, identity.get().first); + } else { + retireUnresolvedDatabaseGeneration(); + } + } catch (Exception e) { + // The remote DROP is already committed; local cleanup cannot suppress its edit log. + LOG.warn("Failed to invalidate cold database {} in catalog {}: {}", + dbName, getName(), e.getMessage(), e); + } + } + public void registerDatabase(long dbId, String dbName) { throw new NotImplementedException("registerDatabase not implemented"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index 458a216000cab8..6940a6fba4f166 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -324,9 +324,9 @@ public void afterDropTable(String dbName, String tblName) { if (db.isPresent()) { db.get().unregisterTable(tblName); } else { - // A mode-2 mapping can disappear while the canonical database object stays resident, so - // an empty lookup may still hide stale database/table objects, engine entries, and counts. - catalog.retireUnresolvedDatabaseGeneration(); + // A cold object can retain a known canonical name; only a lost mode-2 mapping needs + // catalog-wide retirement of hidden objects and engine entries. + catalog.invalidateColdDatabaseForReplay(dbName); } LOG.info("after drop table {}.{}.{}, is db exists: {}", getCatalog().getName(), dbName, tblName, db.isPresent()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index e1db2dd8c03270..d93cfac896abde 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -450,9 +450,8 @@ public void afterDropTable(String dbName, String tblName) { if (db.isPresent()) { db.get().unregisterTable(tblName); } else { - // A mode-2 mapping can disappear while the canonical database object stays resident, so - // an empty lookup may still hide stale database/table objects, engine entries, and counts. - dorisCatalog.retireUnresolvedDatabaseGeneration(); + // Retire a cold known DB narrowly; a lost mode-2 mapping needs catalog-wide retirement. + dorisCatalog.invalidateColdDatabaseForReplay(dbName); } LOG.info("after drop table {}.{}.{}. is db exists: {}", dorisCatalog.getName(), dbName, tblName, db.isPresent()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java index 21e83cb4e40803..0727d83c6c101b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java @@ -294,9 +294,8 @@ public void afterDropTable(String dbName, String tblName) { if (db.isPresent()) { db.get().unregisterTable(tblName); } else { - // A mode-2 mapping can disappear while the canonical database object stays resident, so - // an empty lookup may still hide stale database/table objects, engine entries, and counts. - dorisCatalog.retireUnresolvedDatabaseGeneration(); + // Retire a cold known DB narrowly; a lost mode-2 mapping needs catalog-wide retirement. + dorisCatalog.invalidateColdDatabaseForReplay(dbName); } LOG.info("after drop table {}.{}.{}, is db exists: {}", dorisCatalog.getName(), dbName, tblName, db.isPresent()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java index 3d82518b24e766..2ab532b99c1bbd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java @@ -553,6 +553,11 @@ public Optional getMetaObjById(long id) { return name == null ? Optional.empty() : getMetaObj(name, id); } + /** Return the retained canonical name without loading an evicted metadata object. */ + public Optional getNameByIdIfPresent(long id) { + return Optional.ofNullable(withMetaObjLifecycleReadLock(() -> idToName.get(id))); + } + public void updateCache(String remoteName, String localName, T obj, long id) { updateCache(remoteName, localName, obj, id, namesLoadEpochSupplier.getAsLong()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java index 718ffa1d992eba..6e12bc7a4af05d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java @@ -408,12 +408,9 @@ public void afterDropTable(String dbName, String tblName) { invalidatePaimonCatalogForUnresolvedReplay(); } } else { - // The database itself could not be resolved (for example a mode-2 mapping was lost - // before replay). Retire any retained legacy database object first so a same-name - // recreation cannot reuse its stale nested table-name cache, then flush the engine - // group; a failure in either is best-effort so the drop log is still written. - dorisCatalog.retireAllDatabaseObjectsWithoutEngineInvalidation(); - invalidatePaimonCatalogForUnresolvedReplay(); + // A cold DB with a retained canonical mapping has a narrow invalidation target. + // Only a genuinely lost mapping requires catalog-wide hidden-object retirement. + dorisCatalog.invalidateColdDatabaseForReplay(dbName); } } catch (Exception e) { // The remote drop is already committed and ExternalCatalog.dropTable still has to diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java index b6d92a4f2112ea..37a4e33046fa51 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java @@ -17,6 +17,7 @@ package org.apache.doris.catalog; +import org.apache.doris.common.Pair; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; @@ -38,6 +39,40 @@ public class RefreshManagerTest { + @Test + @SuppressWarnings("unchecked") + void testColdKnownDatabaseReplayKeepsUnrelatedHotDatabase() { + long catalogId = 81L; + long coldDbId = 82L; + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getDbForReplay("cold_db")).thenReturn(Optional.empty()); + Mockito.when(catalog.getDbIdentityForReplay("cold_db", 0L)) + .thenReturn(Optional.of(Pair.of("CanonicalDb", coldDbId))); + ExternalDatabase hotDb = Mockito.mock(ExternalDatabase.class); + Mockito.when(catalog.getDbForReplay("hot_db")) + .thenReturn((Optional) Optional.of(hotDb)); + + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + new RefreshManager().replayRefreshDb( + ExternalObjectLog.createForRefreshDb(catalogId, "cold_db")); + new RefreshManager().replayRefreshTable( + ExternalObjectLog.createForRefreshTable(catalogId, "cold_db", "tbl", 1L)); + } + + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateDb(catalogId, coldDbId, "CanonicalDb"); + Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(catalogId); + Mockito.verify(hotDb, Mockito.never()).resetMetaToUninitialized(); + } + @Test void testColdDatabaseReplayInvalidatesCatalogRowCount() { long catalogId = 51L; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index fe3fd70a00e8cb..288221b52ccdd7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -22,6 +22,7 @@ import org.apache.doris.catalog.RefreshManager; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.DdlException; +import org.apache.doris.common.Pair; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; @@ -70,6 +71,48 @@ void testDatabaseCapacityEvictionDoesNotFenceAllRowCounts() { Mockito.verify(db).resetMetaToUninitialized(true, true); } + @Test + void testReplayDatabaseIdentitySurvivesObjectEviction() throws Exception { + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(83L, + ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2")); + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + Field mappingField = ExternalCatalog.class.getDeclaredField("lowerCaseToDatabaseName"); + mappingField.setAccessible(true); + @SuppressWarnings("unchecked") + Map mapping = (Map) mappingField.get(catalog); + mapping.put("mixeddb", "MixedDb"); + + Assertions.assertEquals(Pair.of("MixedDb", Util.genIdByName("testing_catalog", "MixedDb")), + catalog.getDbIdentityForReplay("mixeddb", 0L).orElseThrow(AssertionError::new)); + Mockito.verify(metaCache, Mockito.never()).getMetaObj(Mockito.anyString(), Mockito.anyLong()); + Mockito.when(metaCache.getNameByIdIfPresent(84L)).thenReturn(Optional.of("MixedDb")); + Assertions.assertEquals(Pair.of("MixedDb", 84L), + catalog.getDbIdentityForReplay(null, 84L).orElseThrow(AssertionError::new)); + } + + @Test + void testColdKnownDropInvalidatesOnlyItsDatabase() { + long catalogId = 85L; + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId); + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalog.invalidateColdDatabaseForReplay("cold_db"); + } + + Mockito.verify(cacheMgr).invalidateDb(catalogId, + Util.genIdByName("testing_catalog", "cold_db"), "cold_db"); + Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(catalogId); + } + private static void addCatalog(CatalogMgr catalogMgr, ExternalCatalog catalog) throws Exception { Field idToCatalogField = CatalogMgr.class.getDeclaredField("idToCatalog"); idToCatalogField.setAccessible(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java index e5535b424718e1..55bb9ae0e6aa46 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java @@ -91,6 +91,6 @@ public void testAfterDropTableWithoutResolvedDatabaseRetiresGeneration() { metadataOps.afterDropTable("db", "tbl"); // An empty lookup can hide a resident canonical database object after a lost mode-2 mapping. - Mockito.verify(catalog).retireUnresolvedDatabaseGeneration(); + Mockito.verify(catalog).invalidateColdDatabaseForReplay("db"); } } From 315e4082e52964caf4b6245997819070ce29b79e Mon Sep 17 00:00:00 2001 From: 924060929 Date: Mon, 28 Sep 2026 18:09:31 +0800 Subject: [PATCH 04/16] [fix](fe) Fence external metadata invalidation at known scopes ### What problem does this PR solve? Related PR: #68196 Problem Summary: Cold databases retained a canonical name and ID but name-based event invalidation widened to the entire catalog, while database removal discarded the ID before row-count invalidation. Paimon name-based invalidation missed SDK-only table handles. Catalog property changes and database refreshes also retired object generations before fencing row counts. Resolve known cold identities, preserve the database ID through removal, invalidate Paimon SDK state independently, and fence row counts before and after generation changes. ### Release note External metadata refresh and drop events invalidate stale row counts and Paimon handles at the correct scope. ### Check List (For Author) - Test: Unit Test (ExternalMetaCacheRouteResolverTest, CatalogMgrTest, ExternalRowCountCacheTest, PaimonExternalMetaCacheTest) - Behavior changed: Yes (metadata invalidation scope and ordering) - Does this need documentation: No --- .../apache/doris/datasource/CatalogMgr.java | 5 ++ .../doris/datasource/ExternalCatalog.java | 5 +- .../doris/datasource/ExternalDatabase.java | 5 ++ .../datasource/ExternalMetaCacheMgr.java | 44 +++++++++++++----- .../paimon/PaimonExternalMetaCache.java | 5 +- .../doris/datasource/CatalogMgrTest.java | 12 +++-- .../ExternalMetaCacheRouteResolverTest.java | 31 +++++++++++++ .../datasource/ExternalRowCountCacheTest.java | 46 ++++++++++++++++++- .../paimon/PaimonExternalMetaCacheTest.java | 6 +-- 9 files changed, 136 insertions(+), 23 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 2150849d0c0e3b..6454a0e9d54e99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -948,6 +948,11 @@ private void alterExternalCatalogPropsFenced(ExternalCatalog externalCatalog, Ca Env currentEnv = Env.getCurrentEnv(); ExternalMetaCacheMgr cacheMgr = currentEnv == null ? null : currentEnv.getExtMetaCacheMgr(); try { + if (cacheMgr != null) { + // Close the old row-count generation before publishing the new catalog context. + // The completion fence below also retires values loaded during the reset. + cacheMgr.invalidateRowCountCache(externalCatalog.getId()); + } externalCatalog.modifyCatalogProps(newProps); } catch (RuntimeException e) { if (!isReplay) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 5be0351dd67a26..cf6c7eabaeec64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -1287,8 +1287,9 @@ public void unregisterDatabase(String dbName) { Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(getId()); return; } - metaCache.invalidate(localDbName, Util.genIdByName(name, localDbName)); - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(getId(), localDbName); + long dbId = Util.genIdByName(name, localDbName); + metaCache.invalidate(localDbName, dbId); + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(getId(), dbId, localDbName); } boolean shouldInvalidateRowCountOnDatabaseRemoval() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 58e9459f09697f..2c3f82d25595d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -134,6 +134,11 @@ public void resetMetaToUninitialized(boolean invalidateEngineCache, boolean inva } MetaCache cacheToInvalidate = null; Runnable objectInvalidation = null; + if (invalidateRowCountCache) { + // Fence readers before the table-object generation is retired. The final fence below + // also removes values loaded during the reset and routed engine invalidation. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(extCatalog.getId(), getId()); + } try { synchronized (this) { metadataLoadEpoch.updateAndGet(epoch -> epoch.next(epoch.catalogEpoch)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index ec2235019d770f..4b6dff361e6ea8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; +import org.apache.doris.common.Pair; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.datasource.doris.DorisExternalMetaCache; import org.apache.doris.datasource.hive.HiveExternalMetaCache; @@ -506,18 +507,23 @@ public void removeCatalogByEngine(long catalogId, String engine) { } public void invalidateDb(long catalogId, String dbName) { - OptionalLong dbId = getCachedDbId(catalogId, dbName); - invalidateDb(catalogId, dbName, dbId); + Optional> db = getCachedDb(catalogId, dbName); + if (db.isPresent()) { + invalidateDb(catalogId, db.get().getId(), db.get().getFullName()); + return; + } + Optional> identity = getDbIdentityForReplay(catalogId, dbName); + if (identity.isPresent()) { + invalidateDb(catalogId, identity.get().second, identity.get().first); + } else { + invalidateDb(catalogId, dbName, OptionalLong.empty(), true); + } } public void invalidateDb(long catalogId, long dbId, String dbName) { invalidateDb(catalogId, dbName, OptionalLong.of(dbId), true); } - private void invalidateDb(long catalogId, String dbName, OptionalLong dbId) { - invalidateDb(catalogId, dbName, dbId, true); - } - void invalidateDb(long catalogId, long dbId, String dbName, boolean invalidateRowCountCache) { invalidateDb(catalogId, dbName, OptionalLong.of(dbId), invalidateRowCountCache); } @@ -606,9 +612,10 @@ private void invalidateTableRowCount(long catalogId, } } - private OptionalLong getCachedDbId(long catalogId, String dbName) { - Optional> db = getCachedDb(catalogId, dbName); - return db.isPresent() ? OptionalLong.of(db.get().getId()) : OptionalLong.empty(); + private Optional> getDbIdentityForReplay(long catalogId, String dbName) { + CatalogIf catalog = getCatalog(catalogId); + return catalog instanceof ExternalCatalog + ? ((ExternalCatalog) catalog).getDbIdentityForReplay(dbName, 0L) : Optional.empty(); } private Optional> getCachedDb(long catalogId, String dbName) { @@ -807,7 +814,12 @@ public void invalidateTableCache(ExternalTable dorisTable) { public void invalidateTableByNameOrWider(long catalogId, String dbName, String tableName) { Optional> db = getCachedDb(catalogId, dbName); if (!db.isPresent()) { - invalidateCatalog(catalogId); + Optional> identity = getDbIdentityForReplay(catalogId, dbName); + if (identity.isPresent()) { + invalidateDb(catalogId, identity.get().second, identity.get().first); + } else { + invalidateCatalog(catalogId); + } return; } Optional table = db.get().getTableForReplay(tableName); @@ -831,7 +843,17 @@ public void invalidateRowCountCache(long catalogId, long dbId) { } public void invalidateRowCountCache(long catalogId, String dbName, String tableName) { - invalidateTableRowCount(catalogId, getCachedDb(catalogId, dbName), tableName); + Optional> db = getCachedDb(catalogId, dbName); + if (db.isPresent()) { + invalidateTableRowCount(catalogId, db, tableName); + } else { + Optional> identity = getDbIdentityForReplay(catalogId, dbName); + if (identity.isPresent()) { + rowCountCache.invalidateDb(catalogId, identity.get().second); + } else { + rowCountCache.invalidateCatalog(catalogId); + } + } } public LegacyMetaCacheFactory legacyMetaCacheFactory() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index 1f937156dc4c44..13e54f1d8011c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -150,7 +150,10 @@ public void invalidateTable(ExternalTable table) { @Override public void invalidateDb(long catalogId, String dbName) { try { - invalidatePaimonTables(catalogId, nameMapping -> matchDb(nameMapping, dbName)); + // The SDK cache can be populated without any Doris table entry (for example by a + // metadata TVF). A name-only database invalidation cannot recover the remote name + // from an evicted database object, so fence the SDK catalog independently. + tableLoader.invalidateCatalog(catalogId); } finally { super.invalidateDb(catalogId, dbName); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 288221b52ccdd7..07f7064a934e17 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -300,8 +300,10 @@ void testReplayPublishesPropertiesOnlyThroughTheFencedCommit() throws Exception } Mockito.verify(catalog, Mockito.never()).tryModifyCatalogProps(Mockito.any()); - Mockito.verify(catalog).modifyCatalogProps(newProperties); - Mockito.verify(cacheMgr).onCatalogOperationalContextChanged(catalogId); + InOrder order = Mockito.inOrder(cacheMgr, catalog); + order.verify(cacheMgr).invalidateRowCountCache(catalogId); + order.verify(catalog).modifyCatalogProps(newProperties); + order.verify(cacheMgr).onCatalogOperationalContextChanged(catalogId); } @Test @@ -504,7 +506,8 @@ void testUnregisterDatabaseRemovesLocalEntryWhenEngineInvalidationFails() { ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); Mockito.doThrow(new IllegalStateException("engine invalidation failed")) - .when(cacheMgr).invalidateDb(catalogId, "CanonicalDb"); + .when(cacheMgr).invalidateDb(catalogId, + Util.genIdByName("testing_catalog", "CanonicalDb"), "CanonicalDb"); try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { mockedEnv.when(Env::getCurrentEnv).thenReturn(env); Assertions.assertThrows(IllegalStateException.class, @@ -535,7 +538,8 @@ void testResidentDatabaseRemovalInvalidatesLocalAndRoutedCache() { Mockito.verify(metaCache).invalidate("CanonicalDb", Util.genIdByName("testing_catalog", "CanonicalDb")); - Mockito.verify(cacheMgr).invalidateDb(catalogId, "CanonicalDb"); + Mockito.verify(cacheMgr).invalidateDb(catalogId, + Util.genIdByName("testing_catalog", "CanonicalDb"), "CanonicalDb"); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index f1555285cee269..f32df02bb423f8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -587,6 +587,37 @@ public void testNameBasedInvalidationWidensToCatalogWhenDatabaseCold() throws Ex Mockito.verify(rowCountCache).invalidateCatalog(catalogId); } + @Test + public void testNameBasedInvalidationUsesKnownColdDatabaseIdentity() throws Exception { + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache hudi = new RecordingExternalMetaCache( + "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache( + "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive, hudi, iceberg); + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + metaCacheMgr.replaceRowCountCacheForTest(rowCountCache); + long catalogId = 29L; + long dbId = 30L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + Mockito.when(catalog.getDbForReplay("coldDb")).thenReturn(Optional.empty()); + Mockito.when(catalog.getDbIdentityForReplay("coldDb", 0L)) + .thenReturn(Optional.of(org.apache.doris.common.Pair.of("CanonicalDb", dbId))); + mockCurrentCatalog(catalogId, catalog); + hive.initializedCatalogIds.add(catalogId); + hudi.initializedCatalogIds.add(catalogId); + iceberg.initializedCatalogIds.add(catalogId); + + metaCacheMgr.invalidateTableByNameOrWider(catalogId, "coldDb", "tbl"); + metaCacheMgr.invalidateRowCountCache(catalogId, "coldDb", "tbl"); + + Assert.assertEquals(1, hive.invalidateDbCalls); + Assert.assertEquals(0, hive.invalidateCatalogEntriesCalls); + Mockito.verify(rowCountCache, Mockito.times(2)).invalidateDb(catalogId, dbId); + Mockito.verify(rowCountCache, Mockito.never()).invalidateCatalog(catalogId); + } + @Test public void testNameBasedRowCountFenceFallsBackToDatabaseWhenTableIsCold() { long catalogId = 22L; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java index 826b185dbae904..2661d9ea7b3c3e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.ThreadPoolManager; +import org.apache.doris.datasource.metacache.MetaCache; import org.apache.doris.statistics.util.StatisticsUtil; import com.google.common.util.concurrent.MoreExecutors; @@ -30,6 +31,7 @@ import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.lang.reflect.Field; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; @@ -71,7 +73,10 @@ protected ExternalTable buildTableInternal(String remoteTableName, String localT // An ordinary removal fences both the engine cache and this database's row counts. db.resetMetaToUninitialized(); - Mockito.verify(metaCacheMgr).invalidateRowCountCache(1L, 2L); + org.mockito.InOrder order = Mockito.inOrder(metaCacheMgr); + order.verify(metaCacheMgr).invalidateRowCountCache(1L, 2L); + order.verify(metaCacheMgr).invalidateDb(db); + order.verify(metaCacheMgr).invalidateRowCountCache(1L, 2L); } @Test @@ -101,7 +106,44 @@ protected ExternalTable buildTableInternal(String remoteTableName, String localT Assertions.assertThrows(IllegalStateException.class, () -> db.resetMetaToUninitialized(true, true)); // The independent row-count fence must still run when the routed invalidation throws. - Mockito.verify(metaCacheMgr).invalidateRowCountCache(1L, 2L); + Mockito.verify(metaCacheMgr, Mockito.times(2)).invalidateRowCountCache(1L, 2L); + } + + @Test + public void testDatabaseRefreshFencesRowCountBeforeRetiringTableObjects() throws Exception { + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(1L); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + new MockUp() { + @Mock + Env getCurrentEnv() { + return env; + } + }; + ExternalDatabase db = new ExternalDatabase( + catalog, 2L, "db", "db", InitDatabaseLog.Type.TEST) { + @Override + protected ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog externalCatalog, ExternalDatabase externalDatabase) { + return null; + } + }; + @SuppressWarnings("unchecked") + MetaCache tableCache = Mockito.mock(MetaCache.class); + Mockito.when(tableCache.retireObjects()).thenReturn(() -> { }); + Field metaCacheField = ExternalDatabase.class.getDeclaredField("metaCache"); + metaCacheField.setAccessible(true); + metaCacheField.set(db, tableCache); + + db.resetMetaToUninitialized(); + + org.mockito.InOrder order = Mockito.inOrder(cacheMgr, tableCache); + order.verify(cacheMgr).invalidateRowCountCache(1L, 2L); + order.verify(tableCache).retireObjects(); + order.verify(cacheMgr).invalidateDb(db); + order.verify(cacheMgr).invalidateRowCountCache(1L, 2L); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index b998121e13fd6e..f2ef0e0061e302 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -3204,8 +3204,9 @@ public void testInvalidateDbAndStats() throws Exception { cache.invalidateDb(catalogId, "db1"); } - Mockito.verify(dorisCatalog).invalidatePaimonTable(db1Table); - Mockito.verify(dorisCatalog, Mockito.never()).invalidatePaimonTable(db2Table); + // Name-only invalidation must also cover SDK-only tables not present in Doris's + // table entry; the SDK scope is catalog-wide when the remote DB name is unknown. + Mockito.verify(dorisCatalog).invalidatePaimonCatalog(); Assert.assertNull(tableEntry.getIfPresent(db1Table)); Assert.assertNotNull(tableEntry.getIfPresent(db2Table)); Assert.assertNull(schemaEntry.getIfPresent(db1Schema)); @@ -3773,4 +3774,3 @@ public void testReplayRefreshTableColdTargetKeepsCachedSibling() throws Exceptio - From 4301c12af815cf37da4aac69ede51b8494c100b5 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Mon, 28 Sep 2026 18:21:02 +0800 Subject: [PATCH 05/16] [test](fe) Verify row-count fence during database generation swap ### What problem does this PR solve? Related PR: #68196 Problem Summary: The database refresh regression test checked call order but did not pause after retiring the old table-object generation and before the final row-count fence. Use a latch at that transition to verify the opening fence is already visible while routed invalidation is still pending. ### Release note None ### Check List (For Author) - Test: Unit Test (ExternalRowCountCacheTest) - Behavior changed: No - Does this need documentation: No --- .../datasource/ExternalRowCountCacheTest.java | 24 +++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java index 2661d9ea7b3c3e..22fee84d36cd14 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java @@ -132,12 +132,32 @@ protected ExternalTable buildTableInternal(String remoteTableName, String localT }; @SuppressWarnings("unchecked") MetaCache tableCache = Mockito.mock(MetaCache.class); - Mockito.when(tableCache.retireObjects()).thenReturn(() -> { }); + CountDownLatch generationSwap = new CountDownLatch(1); + CountDownLatch completeSwap = new CountDownLatch(1); + Mockito.when(tableCache.retireObjects()).thenReturn(() -> { + generationSwap.countDown(); + Uninterruptibles.awaitUninterruptibly(completeSwap); + }); Field metaCacheField = ExternalDatabase.class.getDeclaredField("metaCache"); metaCacheField.setAccessible(true); metaCacheField.set(db, tableCache); - db.resetMetaToUninitialized(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future reset = executor.submit(() -> { + db.resetMetaToUninitialized(); + }); + Assertions.assertTrue(generationSwap.await(10, TimeUnit.SECONDS)); + // The replacement cache has been published while routed invalidation and the final + // fence have not run yet. The opening fence must already be visible. + Mockito.verify(cacheMgr).invalidateRowCountCache(1L, 2L); + Mockito.verify(cacheMgr, Mockito.never()).invalidateDb(db); + completeSwap.countDown(); + reset.get(10, TimeUnit.SECONDS); + } finally { + completeSwap.countDown(); + executor.shutdownNow(); + } org.mockito.InOrder order = Mockito.inOrder(cacheMgr, tableCache); order.verify(cacheMgr).invalidateRowCountCache(1L, 2L); From bb9889cb9985f9e61f936536f62da54e9dbaec6e Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 29 Sep 2026 09:36:58 +0800 Subject: [PATCH 06/16] [fix](fe) Fence cold metadata retirement and scope Paimon SDK invalidation ### What problem does this PR solve? Related PR: #68196 Problem Summary: Cold DROP DATABASE and unresolved database/table replay could publish replacement metadata objects before their old completed row counts were invalidated. Name-based Paimon database invalidation also cleared every SDK database, including unrelated hot entries, and warm DROP duplicated the typed removal flush. Fence row counts before each name or object generation change and again on completion. Route explicit database removal once, and match Paimon SDK entries by the canonical local database name even when Doris has no table object cached. ### Release note External metadata drops and refreshes no longer expose stale row counts during replacement; Paimon database invalidation preserves unrelated SDK caches. ### Check List (For Author) - Test: Unit Test (CatalogMgrTest, ExternalRowCountCacheTest, PaimonExternalMetaCacheTest, RefreshManagerTest); Manual test (FE build) - Behavior changed: Yes (cache invalidation ordering and scope) - Does this need documentation: No --- .../doris/datasource/ExternalCatalog.java | 17 +++- .../doris/datasource/ExternalDatabase.java | 8 +- .../metacache/paimon/PaimonTableLoader.java | 9 ++ .../paimon/PaimonExternalCatalog.java | 19 ++++ .../paimon/PaimonExternalMetaCache.java | 7 +- .../datasource/paimon/PaimonMetadataOps.java | 8 +- .../doris/datasource/CatalogMgrTest.java | 90 ++++++++++++++++++- .../datasource/ExternalRowCountCacheTest.java | 50 +++++++++++ .../paimon/PaimonExternalMetaCacheTest.java | 13 +-- 9 files changed, 204 insertions(+), 17 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index cf6c7eabaeec64..1a0ea67a30fab9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -1288,8 +1288,18 @@ public void unregisterDatabase(String dbName) { return; } long dbId = Util.genIdByName(name, localDbName); - metaCache.invalidate(localDbName, dbId); - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(getId(), dbId, localDbName); + // A cold database has no removal callback. Fence its deterministic table IDs before + // removing the name slot, so a same-name replacement cannot use an old completed count. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(getId(), dbId); + // The explicit routed invalidation below owns this DROP. Suppress the object removal + // callback's routed work so a warm database does not flush the SDK cache twice. + invalidateEngineCacheOnDatabaseRemoval.set(false); + try { + metaCache.invalidate(localDbName, dbId); + } finally { + invalidateEngineCacheOnDatabaseRemoval.remove(); + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(getId(), dbId, localDbName); + } } boolean shouldInvalidateRowCountOnDatabaseRemoval() { @@ -1317,6 +1327,9 @@ public void retireAllDatabaseObjectsWithoutEngineInvalidation() { if (metaCache == null) { return; } + // A mode-2 mapping loss can hide an old object while its row count remains reachable. + // Close that window before swapping the database-object cache generation. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(getId()); invalidateEngineCacheOnDatabaseRemoval.set(false); try { metaCache.invalidateObjects(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 2c3f82d25595d1..1b642318dd2fb6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -681,7 +681,13 @@ public boolean unregisterTableForReplay(String tableName) { */ public void retireAllTableObjectsWithoutEngineInvalidation() { if (metaCache != null) { - metaCache.invalidateObjects(); + ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + cacheMgr.invalidateRowCountCache(extCatalog.getId(), getId()); + try { + metaCache.invalidateObjects(); + } finally { + cacheMgr.invalidateRowCountCache(extCatalog.getId(), getId()); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java index 902ac2fcf5f36a..c3e9f165f9a49b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java @@ -52,6 +52,15 @@ public void invalidateDatabase(ExternalDatabase database) { } } + public void invalidateDatabase(long catalogId, String localDbName) { + try { + catalog(catalogId).invalidatePaimonDatabaseByLocalName(localDbName); + } catch (Exception e) { + throw new CacheException("failed to invalidate paimon database %s.%s: %s", + e, catalogId, localDbName, e.getMessage()); + } + } + public void invalidateCatalog(long catalogId) { try { catalog(catalogId).invalidatePaimonCatalog(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index 238c695fb3c31b..e0a8286204abbc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -173,6 +173,25 @@ public synchronized void invalidatePaimonDatabase(String remoteDbName) throws Ex })); } + public synchronized void invalidatePaimonDatabaseByLocalName(String localDbName) throws Exception { + if (!isInitialized()) { + return; + } + withSdkCatalogCacheWriteLock(() -> executionAuthenticator.execute(() -> { + boolean caseSensitive = catalog.caseSensitive(); + int nameMode = getLowerCaseDatabaseNames(); + invalidateCachedPaimonTables(identifier -> { + String remoteName = identifier.getDatabaseName(); + String candidateLocalName = nameMode == 2 ? remoteName : fromRemoteDatabaseName(remoteName); + if (nameMode == 1) { + candidateLocalName = candidateLocalName.toLowerCase(Locale.ROOT); + } + return identifierPartEquals(candidateLocalName, localDbName, caseSensitive); + }); + return null; + })); + } + public synchronized void invalidatePaimonCatalog() throws Exception { if (!isInitialized()) { return; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index 13e54f1d8011c6..4f8c525f392617 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -150,10 +150,9 @@ public void invalidateTable(ExternalTable table) { @Override public void invalidateDb(long catalogId, String dbName) { try { - // The SDK cache can be populated without any Doris table entry (for example by a - // metadata TVF). A name-only database invalidation cannot recover the remote name - // from an evicted database object, so fence the SDK catalog independently. - tableLoader.invalidateCatalog(catalogId); + // Match the SDK's own entries independently of Doris tableEntry: direct metadata + // paths can populate the SDK before any Doris table handle exists. + tableLoader.invalidateDatabase(catalogId, dbName); } finally { super.invalidateDb(catalogId, dbName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java index 6e12bc7a4af05d..93371a95e9e257 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java @@ -196,12 +196,14 @@ public void afterDropDb(String dbName) { if (db.isPresent()) { // getDbForReplay normalizes case-insensitive database names (lower_case_database_names // mode 1/2), so an alternate-case DROP DATABASE can resolve the cached database while - // an exact-key eviction with the caller's spelling would miss it. Evict by the resolved - // canonical local key so the removal listener still performs the one typed SDK - // invalidation; do not add a second typed scan under the catalog write fence. + // an exact-key eviction with the caller's spelling would miss it. dorisCatalog.unregisterDatabase(db.get().getFullName()); return; } + if (dorisCatalog.getDbIdentityForReplay(dbName, 0L).isPresent()) { + dorisCatalog.unregisterDatabase(dbName); + return; + } // The cached database could not be resolved (for example a mode-2 case mapping was removed // by a names refresh before replay). Exact-key eviction can miss the canonical local key, // so also retire the remaining legacy database objects; otherwise a same-name recreation diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 07f7064a934e17..919e18a48d8714 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -36,6 +36,8 @@ import com.github.benmanes.caffeine.cache.RemovalCause; import com.google.common.collect.ImmutableMap; +import mockit.Mock; +import mockit.MockUp; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.InOrder; @@ -518,6 +520,8 @@ void testUnregisterDatabaseRemovesLocalEntryWhenEngineInvalidationFails() { // engine flush cannot leave the dropped database resident. Mockito.verify(metaCache).invalidate("CanonicalDb", Util.genIdByName("testing_catalog", "CanonicalDb")); + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, + Util.genIdByName("testing_catalog", "CanonicalDb")); } @Test @@ -538,10 +542,53 @@ void testResidentDatabaseRemovalInvalidatesLocalAndRoutedCache() { Mockito.verify(metaCache).invalidate("CanonicalDb", Util.genIdByName("testing_catalog", "CanonicalDb")); + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, + Util.genIdByName("testing_catalog", "CanonicalDb")); Mockito.verify(cacheMgr).invalidateDb(catalogId, Util.genIdByName("testing_catalog", "CanonicalDb"), "CanonicalDb"); } + @Test + void testColdDatabaseDropFencesBeforeRemovingNameSlot() throws Exception { + long catalogId = 86L; + long dbId = Util.genIdByName("testing_catalog", "cold_db"); + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId); + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + CountDownLatch nameRemovalStarted = new CountDownLatch(1); + CountDownLatch releaseNameRemoval = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + nameRemovalStarted.countDown(); + Assertions.assertTrue(releaseNameRemoval.await(10, TimeUnit.SECONDS)); + return null; + }).when(metaCache).invalidate("cold_db", dbId); + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + new MockUp() { + @Mock + Env getCurrentEnv() { + return env; + } + }; + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future drop = executor.submit(() -> { + catalog.unregisterDatabase("cold_db"); + }); + Assertions.assertTrue(nameRemovalStarted.await(10, TimeUnit.SECONDS)); + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, dbId); + Mockito.verify(cacheMgr, Mockito.never()).invalidateDb(catalogId, dbId, "cold_db"); + releaseNameRemoval.countDown(); + drop.get(10, TimeUnit.SECONDS); + Mockito.verify(cacheMgr).invalidateDb(catalogId, dbId, "cold_db"); + } finally { + releaseNameRemoval.countDown(); + executor.shutdownNow(); + } + } + @Test void testUnresolvedModeTwoDatabaseRemovalRetiresCatalogScope() throws Exception { long catalogId = 60L; @@ -564,6 +611,7 @@ void testUnresolvedModeTwoDatabaseRemovalRetiresCatalogScope() throws Exception // catalog-wide instead of evicting a lowercased local key that may not exist. Mockito.verify(metaCache).invalidateObjects(); Mockito.verify(cacheMgr).invalidateCatalog(catalogId); + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId); Mockito.verify(cacheMgr, Mockito.never()).invalidateDb(Mockito.anyLong(), Mockito.anyString()); } @@ -586,10 +634,49 @@ void testUnresolvedDatabaseDropRetiresGenerationAndFencesCatalog() { // Retire the hidden database generation and fence the catalog scope (engine + row counts), // even though the per-database removal callbacks suppress their own fences. Mockito.verify(metaCache).invalidateObjects(); - Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId); + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId); Mockito.verify(cacheMgr).invalidateCatalog(catalogId); } + @Test + void testUnresolvedDatabaseFencesBeforeObjectGenerationSwap() throws Exception { + long catalogId = 87L; + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId); + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + CountDownLatch swapStarted = new CountDownLatch(1); + CountDownLatch releaseSwap = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + swapStarted.countDown(); + Assertions.assertTrue(releaseSwap.await(10, TimeUnit.SECONDS)); + return null; + }).when(metaCache).invalidateObjects(); + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + new MockUp() { + @Mock + Env getCurrentEnv() { + return env; + } + }; + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future retirement = executor.submit(() -> { + catalog.retireAllDatabaseObjectsWithoutEngineInvalidation(); + }); + Assertions.assertTrue(swapStarted.await(10, TimeUnit.SECONDS)); + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId); + releaseSwap.countDown(); + retirement.get(10, TimeUnit.SECONDS); + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId); + } finally { + releaseSwap.countDown(); + executor.shutdownNow(); + } + } + @Test void testUnresolvedTableNameRetiresHiddenTableGeneration() throws Exception { long catalogId = 73L; @@ -620,6 +707,7 @@ protected ExternalTable buildTableInternal(String remoteTableName, String localT // The mode-2 mapping is gone: retire the hidden table generation and widen the fence. Mockito.verify(tableCache).invalidateObjects(); + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId, 74L); Mockito.verify(cacheMgr).invalidateTableByNameOrWider(catalogId, "db1", "Tbl"); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java index 22fee84d36cd14..d79636439cbe68 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java @@ -166,6 +166,56 @@ protected ExternalTable buildTableInternal(String remoteTableName, String localT order.verify(cacheMgr).invalidateRowCountCache(1L, 2L); } + @Test + public void testUnresolvedTableFencesRowCountBeforeObjectGenerationSwap() throws Exception { + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(1L); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + new MockUp() { + @Mock + Env getCurrentEnv() { + return env; + } + }; + ExternalDatabase db = new ExternalDatabase( + catalog, 2L, "db", "db", InitDatabaseLog.Type.TEST) { + @Override + protected ExternalTable buildTableInternal(String remoteTableName, String localTableName, long tblId, + ExternalCatalog externalCatalog, ExternalDatabase externalDatabase) { + return null; + } + }; + @SuppressWarnings("unchecked") + MetaCache tableCache = Mockito.mock(MetaCache.class); + CountDownLatch swapStarted = new CountDownLatch(1); + CountDownLatch releaseSwap = new CountDownLatch(1); + Mockito.doAnswer(invocation -> { + swapStarted.countDown(); + Assertions.assertTrue(releaseSwap.await(10, TimeUnit.SECONDS)); + return null; + }).when(tableCache).invalidateObjects(); + Field metaCacheField = ExternalDatabase.class.getDeclaredField("metaCache"); + metaCacheField.setAccessible(true); + metaCacheField.set(db, tableCache); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + Future retirement = executor.submit(() -> { + db.retireAllTableObjectsWithoutEngineInvalidation(); + }); + Assertions.assertTrue(swapStarted.await(10, TimeUnit.SECONDS)); + Mockito.verify(cacheMgr).invalidateRowCountCache(1L, 2L); + releaseSwap.countDown(); + retirement.get(10, TimeUnit.SECONDS); + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(1L, 2L); + } finally { + releaseSwap.countDown(); + executor.shutdownNow(); + } + } + @Test public void testRowCountKeyUsesTableIdAsCacheIdentity() { ExternalRowCountCache.RowCountKey key1 = new ExternalRowCountCache.RowCountKey(1, 2, 3); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index f2ef0e0061e302..3543a0338ba0a6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -3206,7 +3206,7 @@ public void testInvalidateDbAndStats() throws Exception { // Name-only invalidation must also cover SDK-only tables not present in Doris's // table entry; the SDK scope is catalog-wide when the remote DB name is unknown. - Mockito.verify(dorisCatalog).invalidatePaimonCatalog(); + Mockito.verify(dorisCatalog).invalidatePaimonDatabaseByLocalName("db1"); Assert.assertNull(tableEntry.getIfPresent(db1Table)); Assert.assertNotNull(tableEntry.getIfPresent(db2Table)); Assert.assertNull(schemaEntry.getIfPresent(db1Schema)); @@ -3282,7 +3282,7 @@ public void testDatabaseInvalidationClearsPartitionCacheInOneBatch() throws Exce } Mockito.clearInvocations(cachingCatalog, delegate); - dorisCatalog.invalidatePaimonDatabase("db1"); + dorisCatalog.invalidatePaimonDatabaseByLocalName("db1"); // The batch path must not re-scan the table cache once per matched identifier. Mockito.verify(cachingCatalog, Mockito.never()).invalidateTable(Mockito.any(Identifier.class)); @@ -3381,9 +3381,11 @@ public void testResolvedDropInvalidatesDatabaseSdkCacheExactlyOnce() throws Exce metadataOps.afterDropDb("db"); - // The cached database's synchronous removal listener performs the typed SDK - // invalidation; the explicit duplicate must not run a second scan under the fence. - Mockito.verify(cacheMgr, Mockito.times(1)).invalidateDb(Mockito.any(ExternalDatabase.class)); + // The explicit DROP owns the one routed SDK invalidation, rather than duplicating + // the removal listener's work after the local object has been retired. + Mockito.verify(cacheMgr, Mockito.never()).invalidateDb(Mockito.any(ExternalDatabase.class)); + Mockito.verify(cacheMgr, Mockito.times(1)).invalidateDb( + Mockito.eq(catalogId), Mockito.anyLong(), Mockito.eq("db")); } finally { dorisCatalog.catalog.close(); } @@ -3773,4 +3775,3 @@ public void testReplayRefreshTableColdTargetKeepsCachedSibling() throws Exceptio - From 1a99da7d7f620603c75339e57d40276dc864bb83 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 29 Sep 2026 10:31:02 +0800 Subject: [PATCH 07/16] [fix](fe) Fence row counts before table invalidation and preserve drop identity ### What problem does this PR solve? Related PR: #68196 Problem Summary: Full table refresh and typed engine invalidation exposed refreshed metadata before retiring the old completed row count. A cold name-based route had the same ordering gap. In case-insensitive mode, a delayed DROP could also follow a rebound name mapping to a replacement database instead of the historical target. Open row-count fences before each metadata or engine transition, retain the completion fences, and resolve a retained historical database ID before using a current mode-2 mapping; widen retirement when the historical identity is unavailable. ### Release note External metadata refreshes and drops no longer reuse stale row counts during replacement. ### Check List (For Author) - Test: Unit Test (RefreshManagerTest, CatalogMgrTest, ExternalMetaCacheRouteResolverTest); Manual test (FE build) - Behavior changed: Yes (row-count invalidation order and conservative mode-2 DROP retirement) - Does this need documentation: No --- .../apache/doris/catalog/RefreshManager.java | 2 + .../doris/datasource/ExternalCatalog.java | 19 ++++++ .../datasource/ExternalMetaCacheMgr.java | 4 ++ .../doris/catalog/RefreshManagerTest.java | 21 ++++++- .../doris/datasource/CatalogMgrTest.java | 39 +++++++++++- .../ExternalMetaCacheRouteResolverTest.java | 60 ++++++++++++++++--- 6 files changed, 136 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index ddb912e952ee86..dcf5ab9b642365 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -311,6 +311,8 @@ public void refreshExternalTableFromEvent(String catalogName, String dbName, Str } public void refreshTableInternal(ExternalDatabase db, ExternalTable table, long updateTime) { + // The table's metadata can become visible to a new load as soon as it is reset. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table); table.unsetObjectCreated(); // Iceberg partition evolution can change partition specs across FEs. // Clear related-table validation cache to avoid stale partitioned/unpartitioned judgment. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 1a0ea67a30fab9..ec0ab3ae2ca849 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -962,6 +962,15 @@ public Optional> getDbForReplay(String String localDbName = getLocalDatabaseName(dbName, true); if (localDbName == null) { localDbName = dbName; // Fallback to original name + } else if (getLowerCaseDatabaseNames() == 2 && !localDbName.equals(dbName)) { + // The current case-insensitive mapping may have rebound after a DROP. A replay + // record names the historical object, not a new same-folded replacement. + long historicalId = Util.genIdByName(name, dbName); + if (metaCache.getNameByIdIfPresent(historicalId).isPresent()) { + localDbName = dbName; + } else { + return Optional.empty(); + } } return metaCache.tryGetMetaObj(localDbName); @@ -974,6 +983,11 @@ public Optional> getDbIdentityForReplay(String dbName, long d } if (dbName != null && !dbName.isEmpty()) { String localName = getLocalDatabaseName(dbName, true); + if (getLowerCaseDatabaseNames() == 2 && localName != null && !localName.equals(dbName)) { + long historicalId = Util.genIdByName(name, dbName); + return metaCache.getNameByIdIfPresent(historicalId) + .map(historicalName -> Pair.of(historicalName, historicalId)); + } return localName == null ? Optional.empty() : Optional.of(Pair.of(localName, Util.genIdByName(name, localName))); } @@ -1278,6 +1292,11 @@ public void unregisterDatabase(String dbName) { return; } String localDbName = getLocalDatabaseName(dbName, true); + if (localDbName != null && getLowerCaseDatabaseNames() == 2 && !localDbName.equals(dbName)) { + // Prefer the logged spelling when its old name slot survives a names refresh. + // Otherwise the current mapping cannot identify the historical DROP target. + localDbName = metaCache.getNameByIdIfPresent(Util.genIdByName(name, dbName)).orElse(null); + } if (localDbName == null) { // A mode-2 remote-to-local mapping can disappear (for example after a names refresh) // while the resident database object survives. The canonical key is then unknown, so diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 4b6dff361e6ea8..506e5a4cddb320 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -790,6 +790,8 @@ public ExternalRowCountCache getRowCountCache() { public void invalidateTableCache(ExternalTable dorisTable) { long catalogId = dorisTable.getCatalog().getId(); + // Engine eviction can expose a new snapshot before the completion fence below. + invalidateRowCountCache(dorisTable); // Typed table invalidation bypasses the name-based invalidateTable() entry point, so the // Lance access-cache retirement that used to happen there has to be repeated here. try { @@ -812,6 +814,8 @@ public void invalidateTableCache(ExternalTable dorisTable) { * cached object has already been evicted, so caller spelling can never miss a canonical key. */ public void invalidateTableByNameOrWider(long catalogId, String dbName, String tableName) { + // A cold object miss may route through engine invalidation before its final fence. + invalidateRowCountCache(catalogId, dbName, tableName); Optional> db = getCachedDb(catalogId, dbName); if (!db.isPresent()) { Optional> identity = getDbIdentityForReplay(catalogId, dbName); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java index 37a4e33046fa51..908709c4381076 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java @@ -277,7 +277,7 @@ void testCommittedRefreshUsesHeldTableAndStillLogsAfterCacheFailure() { } InOrder order = Mockito.inOrder(cacheMgr, editLog); - order.verify(cacheMgr).invalidateRowCountCache(table); + order.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(table); order.verify(cacheMgr).invalidateTableCache(table); order.verify(editLog).logRefreshExternalTable(Mockito.argThat(log -> log.getCatalogId() == catalogId @@ -285,6 +285,25 @@ void testCommittedRefreshUsesHeldTableAndStillLogsAfterCacheFailure() { && "tbl1".equals(log.getTableName()))); } + @Test + void testFullRefreshFencesHeldTableBeforeMetadataReset() { + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + ExternalTable table = Mockito.mock(ExternalTable.class); + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(db.getCatalog()).thenReturn(catalog); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + new RefreshManager().refreshTableInternal(db, table, 0L); + } + InOrder order = Mockito.inOrder(cacheMgr, table); + order.verify(cacheMgr).invalidateRowCountCache(table); + order.verify(table).unsetObjectCreated(); + order.verify(cacheMgr).invalidateTableCache(table); + } + @Test void testWholeTableEventFencesRowCountBeforeIgnoredTableMiss() throws Exception { long catalogId = 56L; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 919e18a48d8714..50aee1695dfa16 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -86,8 +86,15 @@ void testReplayDatabaseIdentitySurvivesObjectEviction() throws Exception { Map mapping = (Map) mappingField.get(catalog); mapping.put("mixeddb", "MixedDb"); - Assertions.assertEquals(Pair.of("MixedDb", Util.genIdByName("testing_catalog", "MixedDb")), + long historicalId = Util.genIdByName("testing_catalog", "mixeddb"); + Mockito.when(metaCache.getNameByIdIfPresent(historicalId)).thenReturn(Optional.of("mixeddb")); + Assertions.assertEquals(Pair.of("mixeddb", historicalId), catalog.getDbIdentityForReplay("mixeddb", 0L).orElseThrow(AssertionError::new)); + Mockito.when(metaCache.getNameByIdIfPresent(historicalId)).thenReturn(Optional.empty()); + Assertions.assertFalse(catalog.getDbIdentityForReplay("mixeddb", 0L).isPresent()); + mapping.put("mixeddb", "MixedDb"); + Assertions.assertEquals(Pair.of("MixedDb", Util.genIdByName("testing_catalog", "MixedDb")), + catalog.getDbIdentityForReplay("MixedDb", 0L).orElseThrow(AssertionError::new)); Mockito.verify(metaCache, Mockito.never()).getMetaObj(Mockito.anyString(), Mockito.anyLong()); Mockito.when(metaCache.getNameByIdIfPresent(84L)).thenReturn(Optional.of("MixedDb")); Assertions.assertEquals(Pair.of("MixedDb", 84L), @@ -615,6 +622,36 @@ void testUnresolvedModeTwoDatabaseRemovalRetiresCatalogScope() throws Exception Mockito.verify(cacheMgr, Mockito.never()).invalidateDb(Mockito.anyLong(), Mockito.anyString()); } + @Test + void testReboundModeTwoDatabaseDropDoesNotRetireReplacement() throws Exception { + long catalogId = 89L; + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId, + ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2")); + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + Field mappingField = ExternalCatalog.class.getDeclaredField("lowerCaseToDatabaseName"); + mappingField.setAccessible(true); + @SuppressWarnings("unchecked") + Map mapping = (Map) mappingField.get(catalog); + mapping.put("foo", "FOO"); + long oldId = Util.genIdByName("testing_catalog", "Foo"); + Mockito.when(metaCache.getNameByIdIfPresent(oldId)).thenReturn(Optional.of("Foo")); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalog.unregisterDatabase("Foo"); + } + + Mockito.verify(metaCache).invalidate("Foo", oldId); + Mockito.verify(metaCache, Mockito.never()).invalidate("FOO", + Util.genIdByName("testing_catalog", "FOO")); + Mockito.verify(cacheMgr).invalidateDb(catalogId, oldId, "Foo"); + } + @Test void testUnresolvedDatabaseDropRetiresGenerationAndFencesCatalog() { long catalogId = 72L; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index f32df02bb423f8..9de801682dd410 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -532,17 +532,24 @@ public void testRollbackRetiresEngineCachesWhenResetCleanupThrows() throws Excep @Test public void testNameBasedInvalidationWidensToCanonicalDatabaseWhenTableCold() throws Exception { + long catalogId = 26L; + long dbId = 27L; + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); RecordingExternalMetaCache hive = new RecordingExternalMetaCache( - "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog); + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog) { + @Override + public void invalidateDb(long id, String name) { + // Observe the opening fence while routed invalidation is still running. + Mockito.verify(rowCountCache).invalidateDb(catalogId, dbId); + super.invalidateDb(id, name); + } + }; RecordingExternalMetaCache hudi = new RecordingExternalMetaCache( "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache( "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive, hudi, iceberg); - ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); metaCacheMgr.replaceRowCountCacheForTest(rowCountCache); - long catalogId = 26L; - long dbId = 27L; HMSExternalCatalog catalog = Mockito.spy( new HMSExternalCatalog(catalogId, "hms", null, Collections.emptyMap(), "")); ExternalDatabase db = Mockito.mock(ExternalDatabase.class); @@ -558,7 +565,46 @@ public void testNameBasedInvalidationWidensToCanonicalDatabaseWhenTableCold() th metaCacheMgr.invalidateTableByNameOrWider(catalogId, "mixeddb", "mixedtbl"); Assert.assertEquals(1, hive.invalidateDbCalls); - Mockito.verify(rowCountCache).invalidateDb(catalogId, dbId); + Mockito.verify(rowCountCache, Mockito.times(2)).invalidateDb(catalogId, dbId); + } + + @Test + public void testTypedTableInvalidationFencesBeforeEngineEviction() throws Exception { + long catalogId = 90L; + long dbId = 91L; + long tableId = 92L; + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog) { + @Override + public void invalidateTable(long id, String dbName, String tableName) { + Mockito.verify(rowCountCache).invalidateTable(catalogId, dbId, tableId); + super.invalidateTable(id, dbName, tableName); + } + }; + RecordingExternalMetaCache hudi = new RecordingExternalMetaCache( + "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache( + "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + ExternalMetaCacheMgr cacheMgr = newManagerWithCaches(hive, hudi, iceberg); + cacheMgr.replaceRowCountCacheForTest(rowCountCache); + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + Mockito.when(db.getId()).thenReturn(dbId); + ExternalTable table = Mockito.mock(ExternalTable.class); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.getDb()).thenReturn(db); + Mockito.when(table.getId()).thenReturn(tableId); + Mockito.when(table.getDbName()).thenReturn("db"); + Mockito.when(table.getName()).thenReturn("tbl"); + mockCurrentCatalog(catalogId, catalog); + hive.initializedCatalogIds.add(catalogId); + + cacheMgr.invalidateTableCache(table); + + Assert.assertEquals(1, hive.invalidateTableCalls); + Mockito.verify(rowCountCache, Mockito.times(2)).invalidateTable(catalogId, dbId, tableId); } @Test @@ -584,7 +630,7 @@ public void testNameBasedInvalidationWidensToCatalogWhenDatabaseCold() throws Ex metaCacheMgr.invalidateTableByNameOrWider(catalogId, "colddb", "tbl"); Assert.assertEquals(1, hive.invalidateCatalogEntriesCalls); - Mockito.verify(rowCountCache).invalidateCatalog(catalogId); + Mockito.verify(rowCountCache, Mockito.times(2)).invalidateCatalog(catalogId); } @Test @@ -614,7 +660,7 @@ public void testNameBasedInvalidationUsesKnownColdDatabaseIdentity() throws Exce Assert.assertEquals(1, hive.invalidateDbCalls); Assert.assertEquals(0, hive.invalidateCatalogEntriesCalls); - Mockito.verify(rowCountCache, Mockito.times(2)).invalidateDb(catalogId, dbId); + Mockito.verify(rowCountCache, Mockito.times(3)).invalidateDb(catalogId, dbId); Mockito.verify(rowCountCache, Mockito.never()).invalidateCatalog(catalogId); } From 1557455a60ef4ed72fc085b2ad2dac0715c50c33 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 29 Sep 2026 11:24:55 +0800 Subject: [PATCH 08/16] [fix](fe) Fence cache invalidation and preserve replay name semantics ### What problem does this PR solve? Related PR: #68196 Problem Summary: Routed catalog, database and table invalidation could expose refreshed engine metadata before stale row counts were fenced. A prior mode-2 replay fix also treated a legitimate alternate-case CREATE alias as a historical DROP, skipping warm table-name refresh. Fence row counts both before and after routed invalidation, confine historical identity lookup to DROP paths, and align Paimon mode-1 SDK cache matching with the catalog locale. Cover the ordering, alias and Turkish-locale cases with focused FE tests. ### Release note External catalog metadata refresh and replay no longer reuse stale row counts or miss case-insensitive database cache updates. ### Check List (For Author) - Test: Unit Test (148 focused FE tests); Manual test (./build.sh --fe -j4) - Behavior changed: Yes (cache invalidation and replay identity handling) - Does this need documentation: No --- .../doris/datasource/ExternalCatalog.java | 50 +++++---- .../datasource/ExternalMetaCacheMgr.java | 27 +++-- .../datasource/hive/HiveMetadataOps.java | 2 +- .../iceberg/IcebergMetadataOps.java | 2 +- .../maxcompute/MaxComputeMetadataOps.java | 2 +- .../paimon/PaimonExternalCatalog.java | 3 +- .../datasource/paimon/PaimonMetadataOps.java | 23 +--- .../doris/datasource/CatalogMgrTest.java | 48 +++++++-- .../ExternalMetaCacheRouteResolverTest.java | 102 +++++++++++++++++- .../datasource/hive/HiveMetadataOpsTest.java | 13 +++ .../paimon/PaimonExternalMetaCacheTest.java | 47 +++++++- 11 files changed, 252 insertions(+), 67 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index ec0ab3ae2ca849..4c24eec7b0d9d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -962,15 +962,6 @@ public Optional> getDbForReplay(String String localDbName = getLocalDatabaseName(dbName, true); if (localDbName == null) { localDbName = dbName; // Fallback to original name - } else if (getLowerCaseDatabaseNames() == 2 && !localDbName.equals(dbName)) { - // The current case-insensitive mapping may have rebound after a DROP. A replay - // record names the historical object, not a new same-folded replacement. - long historicalId = Util.genIdByName(name, dbName); - if (metaCache.getNameByIdIfPresent(historicalId).isPresent()) { - localDbName = dbName; - } else { - return Optional.empty(); - } } return metaCache.tryGetMetaObj(localDbName); @@ -983,17 +974,34 @@ public Optional> getDbIdentityForReplay(String dbName, long d } if (dbName != null && !dbName.isEmpty()) { String localName = getLocalDatabaseName(dbName, true); - if (getLowerCaseDatabaseNames() == 2 && localName != null && !localName.equals(dbName)) { - long historicalId = Util.genIdByName(name, dbName); - return metaCache.getNameByIdIfPresent(historicalId) - .map(historicalName -> Pair.of(historicalName, historicalId)); - } return localName == null ? Optional.empty() : Optional.of(Pair.of(localName, Util.genIdByName(name, localName))); } return metaCache.getNameByIdIfPresent(dbId).map(localName -> Pair.of(localName, dbId)); } + /** A DROP must not follow a mode-2 mapping that has rebound to a case-only replacement. */ + private Optional> getDbIdentityForDrop(String dbName) { + if (!isInitialized() || metaCache == null) { + return Optional.empty(); + } + String localName = getLocalDatabaseName(dbName, true); + if (localName == null) { + return Optional.empty(); + } + if (getLowerCaseDatabaseNames() == 2 && !localName.equals(dbName)) { + long historicalId = Util.genIdByName(name, dbName); + return metaCache.getNameByIdIfPresent(historicalId) + .map(historicalName -> Pair.of(historicalName, historicalId)); + } + return Optional.of(Pair.of(localName, Util.genIdByName(name, localName))); + } + + /** Cache-only DROP lookup; normal replay lookups may follow a current case-insensitive alias. */ + public Optional> getDbForDropReplay(String dbName) { + return getDbIdentityForDrop(dbName).flatMap(identity -> metaCache.tryGetMetaObj(identity.first)); + } + /** * Build a database instance. * If checkExists is true, it will check if the database exists in the remote system. @@ -1291,13 +1299,8 @@ public void unregisterDatabase(String dbName) { Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb(getId(), dbName); return; } - String localDbName = getLocalDatabaseName(dbName, true); - if (localDbName != null && getLowerCaseDatabaseNames() == 2 && !localDbName.equals(dbName)) { - // Prefer the logged spelling when its old name slot survives a names refresh. - // Otherwise the current mapping cannot identify the historical DROP target. - localDbName = metaCache.getNameByIdIfPresent(Util.genIdByName(name, dbName)).orElse(null); - } - if (localDbName == null) { + Optional> identity = getDbIdentityForDrop(dbName); + if (!identity.isPresent()) { // A mode-2 remote-to-local mapping can disappear (for example after a names refresh) // while the resident database object survives. The canonical key is then unknown, so // treat the scope as unknown: retire every cached database object and flush the engine @@ -1306,7 +1309,8 @@ public void unregisterDatabase(String dbName) { Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(getId()); return; } - long dbId = Util.genIdByName(name, localDbName); + String localDbName = identity.get().first; + long dbId = identity.get().second; // A cold database has no removal callback. Fence its deterministic table IDs before // removing the name slot, so a same-name replacement cannot use an old completed count. Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(getId(), dbId); @@ -1380,7 +1384,7 @@ public void retireUnresolvedDatabaseGeneration() { /** Retire a cold replay database narrowly when its canonical name survived object eviction. */ public void invalidateColdDatabaseForReplay(String dbName) { try { - Optional> identity = getDbIdentityForReplay(dbName, 0L); + Optional> identity = getDbIdentityForDrop(dbName); if (identity.isPresent()) { Env.getCurrentEnv().getExtMetaCacheMgr() .invalidateDb(getId(), identity.get().second, identity.get().first); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 506e5a4cddb320..1c23272d5094cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -382,6 +382,7 @@ private Map runtimeEffectiveCacheProperties( } public void invalidateCatalog(long catalogId) { + rowCountCache.invalidateCatalog(catalogId); try { invalidateLanceTableAccess(catalogId); routeCatalogEngines(catalogId, cache -> safeInvalidate( @@ -529,22 +530,29 @@ void invalidateDb(long catalogId, long dbId, String dbName, boolean invalidateRo } private void invalidateDb(long catalogId, String dbName, OptionalLong dbId, boolean invalidateRowCountCache) { + if (invalidateRowCountCache) { + invalidateDatabaseRowCount(catalogId, dbId); + } try { routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidateDb", () -> cache.invalidateDb(catalogId, dbName))); } finally { if (invalidateRowCountCache) { - if (dbId.isPresent()) { - rowCountCache.invalidateDb(catalogId, dbId.getAsLong()); - } else { - // The database object cache is smaller than the row-count cache. If the object has - // already been evicted, retire the catalog scope rather than hashing caller spelling. - rowCountCache.invalidateCatalog(catalogId); - } + invalidateDatabaseRowCount(catalogId, dbId); } } } + private void invalidateDatabaseRowCount(long catalogId, OptionalLong dbId) { + if (dbId.isPresent()) { + rowCountCache.invalidateDb(catalogId, dbId.getAsLong()); + } else { + // The database object cache is smaller than the row-count cache. If the object has + // already been evicted, retire the catalog scope rather than hashing caller spelling. + rowCountCache.invalidateCatalog(catalogId); + } + } + public void invalidateDb(ExternalDatabase database) { long catalogId = database.getCatalog().getId(); routeCatalogEngines(catalogId, cache -> safeInvalidate( @@ -554,8 +562,9 @@ public void invalidateDb(ExternalDatabase database) { public void invalidateTable(long catalogId, String dbName, String tableName) { Optional> db = Optional.empty(); try { - invalidateLanceTableAccess(catalogId); db = getCachedDb(catalogId, dbName); + invalidateTableRowCount(catalogId, db, tableName); + invalidateLanceTableAccess(catalogId); routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidateTable", () -> cache.invalidateTable(catalogId, dbName, tableName))); @@ -568,6 +577,7 @@ public void invalidateTableByEngine(long catalogId, String engine, String dbName Optional> db = Optional.empty(); try { db = getCachedDb(catalogId, dbName); + invalidateTableRowCount(catalogId, db, tableName); routeSpecifiedEngine(engine, cache -> safeInvalidate( cache, catalogId, "invalidateTableByEngine", () -> cache.invalidateTable(catalogId, dbName, tableName))); @@ -590,6 +600,7 @@ public void invalidatePartitions(long catalogId, Optional> db = Optional.empty(); try { db = getCachedDb(catalogId, dbName); + invalidateTableRowCount(catalogId, db, tableName); routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidatePartitions", () -> cache.invalidatePartitions(catalogId, dbName, tableName, partitions))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index 6940a6fba4f166..73dd509c918bf0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -320,7 +320,7 @@ public void dropTableImpl(ExternalTable dorisTable, boolean ifExists) throws Ddl @Override public void afterDropTable(String dbName, String tblName) { - Optional> db = catalog.getDbForReplay(dbName); + Optional> db = catalog.getDbForDropReplay(dbName); if (db.isPresent()) { db.get().unregisterTable(tblName); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index d93cfac896abde..4ce1cd235958ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -446,7 +446,7 @@ public void dropTableImpl(ExternalTable dorisTable, boolean ifExists) throws Ddl @Override public void afterDropTable(String dbName, String tblName) { - Optional> db = dorisCatalog.getDbForReplay(dbName); + Optional> db = dorisCatalog.getDbForDropReplay(dbName); if (db.isPresent()) { db.get().unregisterTable(tblName); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java index 0727d83c6c101b..c9059e023b857f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java @@ -290,7 +290,7 @@ public void dropTableImpl(ExternalTable dorisTable, boolean ifExists) throws Ddl @Override public void afterDropTable(String dbName, String tblName) { - Optional> db = dorisCatalog.getDbForReplay(dbName); + Optional> db = dorisCatalog.getDbForDropReplay(dbName); if (db.isPresent()) { db.get().unregisterTable(tblName); } else { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index e0a8286204abbc..5884885dac225d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -184,7 +184,8 @@ public synchronized void invalidatePaimonDatabaseByLocalName(String localDbName) String remoteName = identifier.getDatabaseName(); String candidateLocalName = nameMode == 2 ? remoteName : fromRemoteDatabaseName(remoteName); if (nameMode == 1) { - candidateLocalName = candidateLocalName.toLowerCase(Locale.ROOT); + // Match ExternalCatalog's mode-1 local database name and deterministic ID. + candidateLocalName = candidateLocalName.toLowerCase(); } return identifierPartEquals(candidateLocalName, localDbName, caseSensitive); }); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java index 93371a95e9e257..adafb34c906127 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java @@ -191,27 +191,10 @@ private boolean performDropDb(String dbName, boolean ifExists, boolean force) th @Override public void afterDropDb(String dbName) { - Optional> db = dorisCatalog.getDbForReplay(dbName); try { - if (db.isPresent()) { - // getDbForReplay normalizes case-insensitive database names (lower_case_database_names - // mode 1/2), so an alternate-case DROP DATABASE can resolve the cached database while - // an exact-key eviction with the caller's spelling would miss it. - dorisCatalog.unregisterDatabase(db.get().getFullName()); - return; - } - if (dorisCatalog.getDbIdentityForReplay(dbName, 0L).isPresent()) { - dorisCatalog.unregisterDatabase(dbName); - return; - } - // The cached database could not be resolved (for example a mode-2 case mapping was removed - // by a names refresh before replay). Exact-key eviction can miss the canonical local key, - // so also retire the remaining legacy database objects; otherwise a same-name recreation - // could reuse the stale object and its nested table-name cache. The catalog-wide engine - // flush below covers the SDK side, so the per-database engine callbacks are suppressed. + // The DROP owns historical-name resolution. Ordinary replay lookup also serves CREATE + // and REFRESH, where a case-insensitive alias must still resolve the current database. dorisCatalog.unregisterDatabase(dbName); - dorisCatalog.retireAllDatabaseObjectsWithoutEngineInvalidation(); - invalidatePaimonCatalogForUnresolvedReplay(); } catch (Exception e) { // The remote drop is already committed and ExternalCatalog.dropDb still has to journal // it; keep the post-drop cache cleanup best-effort so the log and its replay are not @@ -396,7 +379,7 @@ private void performDropTable(String dBName, String tableName, boolean ifExists) @Override public void afterDropTable(String dbName, String tblName) { - Optional> db = dorisCatalog.getDbForReplay(dbName); + Optional> db = dorisCatalog.getDbForDropReplay(dbName); try { if (db.isPresent()) { boolean invalidated = db.get().unregisterTableForReplay(tblName); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 50aee1695dfa16..cdfca6ceac0658 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -86,13 +86,12 @@ void testReplayDatabaseIdentitySurvivesObjectEviction() throws Exception { Map mapping = (Map) mappingField.get(catalog); mapping.put("mixeddb", "MixedDb"); - long historicalId = Util.genIdByName("testing_catalog", "mixeddb"); - Mockito.when(metaCache.getNameByIdIfPresent(historicalId)).thenReturn(Optional.of("mixeddb")); - Assertions.assertEquals(Pair.of("mixeddb", historicalId), + @SuppressWarnings("unchecked") + ExternalDatabase warmDb = Mockito.mock(ExternalDatabase.class); + Mockito.when(metaCache.tryGetMetaObj("MixedDb")).thenReturn(Optional.of(warmDb)); + Assertions.assertSame(warmDb, catalog.getDbForReplay("mixeddb").orElseThrow(AssertionError::new)); + Assertions.assertEquals(Pair.of("MixedDb", Util.genIdByName("testing_catalog", "MixedDb")), catalog.getDbIdentityForReplay("mixeddb", 0L).orElseThrow(AssertionError::new)); - Mockito.when(metaCache.getNameByIdIfPresent(historicalId)).thenReturn(Optional.empty()); - Assertions.assertFalse(catalog.getDbIdentityForReplay("mixeddb", 0L).isPresent()); - mapping.put("mixeddb", "MixedDb"); Assertions.assertEquals(Pair.of("MixedDb", Util.genIdByName("testing_catalog", "MixedDb")), catalog.getDbIdentityForReplay("MixedDb", 0L).orElseThrow(AssertionError::new)); Mockito.verify(metaCache, Mockito.never()).getMetaObj(Mockito.anyString(), Mockito.anyLong()); @@ -637,6 +636,14 @@ void testReboundModeTwoDatabaseDropDoesNotRetireReplacement() throws Exception { mapping.put("foo", "FOO"); long oldId = Util.genIdByName("testing_catalog", "Foo"); Mockito.when(metaCache.getNameByIdIfPresent(oldId)).thenReturn(Optional.of("Foo")); + @SuppressWarnings("unchecked") + ExternalDatabase oldDb = Mockito.mock(ExternalDatabase.class); + @SuppressWarnings("unchecked") + ExternalDatabase replacementDb = Mockito.mock(ExternalDatabase.class); + Mockito.when(metaCache.tryGetMetaObj("Foo")).thenReturn(Optional.of(oldDb)); + Mockito.when(metaCache.tryGetMetaObj("FOO")).thenReturn(Optional.of(replacementDb)); + Assertions.assertSame(replacementDb, catalog.getDbForReplay("Foo").orElseThrow(AssertionError::new)); + Assertions.assertSame(oldDb, catalog.getDbForDropReplay("Foo").orElseThrow(AssertionError::new)); Env env = Mockito.mock(Env.class); ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); @@ -652,6 +659,35 @@ void testReboundModeTwoDatabaseDropDoesNotRetireReplacement() throws Exception { Mockito.verify(cacheMgr).invalidateDb(catalogId, oldId, "Foo"); } + @Test + void testColdReboundModeTwoDropUsesHistoricalIdentity() throws Exception { + long catalogId = 90L; + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId, + ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2")); + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + Field mappingField = ExternalCatalog.class.getDeclaredField("lowerCaseToDatabaseName"); + mappingField.setAccessible(true); + @SuppressWarnings("unchecked") + Map mapping = (Map) mappingField.get(catalog); + mapping.put("foo", "FOO"); + long oldId = Util.genIdByName("testing_catalog", "Foo"); + Mockito.when(metaCache.getNameByIdIfPresent(oldId)).thenReturn(Optional.of("Foo")); + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalog.invalidateColdDatabaseForReplay("Foo"); + } + + Mockito.verify(cacheMgr).invalidateDb(catalogId, oldId, "Foo"); + Mockito.verify(cacheMgr, Mockito.never()).invalidateDb(catalogId, + Util.genIdByName("testing_catalog", "FOO"), "FOO"); + } + @Test void testUnresolvedDatabaseDropRetiresGenerationAndFencesCatalog() { long catalogId = 72L; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index 9de801682dd410..1e7b7d3128a47a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -466,7 +466,7 @@ public void testEngineSpecificTableInvalidationAlsoFencesRowCount() throws Excep metaCacheMgr.invalidateTableByEngine(catalogId, "hive", "db1", "tbl1"); Assert.assertEquals(1, hive.invalidateTableCalls); - Mockito.verify(rowCountCache).invalidateTable(catalogId, dbId, tableId); + Mockito.verify(rowCountCache, Mockito.times(2)).invalidateTable(catalogId, dbId, tableId); } @Test @@ -540,7 +540,7 @@ public void testNameBasedInvalidationWidensToCanonicalDatabaseWhenTableCold() th @Override public void invalidateDb(long id, String name) { // Observe the opening fence while routed invalidation is still running. - Mockito.verify(rowCountCache).invalidateDb(catalogId, dbId); + Mockito.verify(rowCountCache, Mockito.times(2)).invalidateDb(catalogId, dbId); super.invalidateDb(id, name); } }; @@ -565,7 +565,7 @@ public void invalidateDb(long id, String name) { metaCacheMgr.invalidateTableByNameOrWider(catalogId, "mixeddb", "mixedtbl"); Assert.assertEquals(1, hive.invalidateDbCalls); - Mockito.verify(rowCountCache, Mockito.times(2)).invalidateDb(catalogId, dbId); + Mockito.verify(rowCountCache, Mockito.times(3)).invalidateDb(catalogId, dbId); } @Test @@ -607,6 +607,98 @@ public void invalidateTable(long id, String dbName, String tableName) { Mockito.verify(rowCountCache, Mockito.times(2)).invalidateTable(catalogId, dbId, tableId); } + @Test + public void testExplicitDatabaseInvalidationFencesBeforeEngineEviction() throws Exception { + long catalogId = 93L; + long dbId = 94L; + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog) { + @Override + public void invalidateDb(long id, String name) { + Mockito.verify(rowCountCache).invalidateDb(catalogId, dbId); + super.invalidateDb(id, name); + } + }; + RecordingExternalMetaCache hudi = new RecordingExternalMetaCache( + "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache( + "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + ExternalMetaCacheMgr cacheMgr = newManagerWithCaches(hive, hudi, iceberg); + cacheMgr.replaceRowCountCacheForTest(rowCountCache); + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + mockCurrentCatalog(catalogId, catalog); + hive.initializedCatalogIds.add(catalogId); + + cacheMgr.invalidateDb(catalogId, dbId, "db"); + + Assert.assertEquals(1, hive.invalidateDbCalls); + Mockito.verify(rowCountCache, Mockito.times(2)).invalidateDb(catalogId, dbId); + } + + @Test + public void testCatalogInvalidationFencesBeforeEngineEviction() throws Exception { + long catalogId = 98L; + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog) { + @Override + public void invalidateCatalogEntries(long id) { + Mockito.verify(rowCountCache).invalidateCatalog(catalogId); + super.invalidateCatalogEntries(id); + } + }; + RecordingExternalMetaCache hudi = new RecordingExternalMetaCache( + "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache( + "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + ExternalMetaCacheMgr cacheMgr = newManagerWithCaches(hive, hudi, iceberg); + cacheMgr.replaceRowCountCacheForTest(rowCountCache); + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + mockCurrentCatalog(catalogId, catalog); + hive.initializedCatalogIds.add(catalogId); + + cacheMgr.invalidateCatalog(catalogId); + + Assert.assertEquals(1, hive.invalidateCatalogEntriesCalls); + Mockito.verify(rowCountCache, Mockito.times(2)).invalidateCatalog(catalogId); + } + + @Test + public void testEngineSpecificTableInvalidationFencesBeforeEviction() throws Exception { + long catalogId = 95L; + long dbId = 96L; + long tableId = 97L; + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog) { + @Override + public void invalidateTable(long id, String dbName, String tableName) { + Mockito.verify(rowCountCache).invalidateTable(catalogId, dbId, tableId); + super.invalidateTable(id, dbName, tableName); + } + }; + ExternalMetaCacheMgr cacheMgr = newManagerWithCaches(hive); + cacheMgr.replaceRowCountCacheForTest(rowCountCache); + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + ExternalTable table = Mockito.mock(ExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.doReturn(Optional.of(db)).when(catalog).getDbForReplay("db"); + Mockito.doReturn(Optional.of(table)).when(db).getTableForReplay("tbl"); + Mockito.when(table.getCatalog()).thenReturn(catalog); + Mockito.when(table.getDb()).thenReturn(db); + Mockito.when(table.getId()).thenReturn(tableId); + Mockito.when(db.getId()).thenReturn(dbId); + mockCurrentCatalog(catalogId, catalog); + hive.initializedCatalogIds.add(catalogId); + + cacheMgr.invalidateTableByEngine(catalogId, "hive", "db", "tbl"); + + Assert.assertEquals(1, hive.invalidateTableCalls); + Mockito.verify(rowCountCache, Mockito.times(2)).invalidateTable(catalogId, dbId, tableId); + } + @Test public void testNameBasedInvalidationWidensToCatalogWhenDatabaseCold() throws Exception { RecordingExternalMetaCache hive = new RecordingExternalMetaCache( @@ -630,7 +722,7 @@ public void testNameBasedInvalidationWidensToCatalogWhenDatabaseCold() throws Ex metaCacheMgr.invalidateTableByNameOrWider(catalogId, "colddb", "tbl"); Assert.assertEquals(1, hive.invalidateCatalogEntriesCalls); - Mockito.verify(rowCountCache, Mockito.times(2)).invalidateCatalog(catalogId); + Mockito.verify(rowCountCache, Mockito.times(3)).invalidateCatalog(catalogId); } @Test @@ -660,7 +752,7 @@ public void testNameBasedInvalidationUsesKnownColdDatabaseIdentity() throws Exce Assert.assertEquals(1, hive.invalidateDbCalls); Assert.assertEquals(0, hive.invalidateCatalogEntriesCalls); - Mockito.verify(rowCountCache, Mockito.times(3)).invalidateDb(catalogId, dbId); + Mockito.verify(rowCountCache, Mockito.times(4)).invalidateDb(catalogId, dbId); Mockito.verify(rowCountCache, Mockito.never()).invalidateCatalog(catalogId); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java index 55bb9ae0e6aa46..0a4fb1454ff1d0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java @@ -29,6 +29,19 @@ import java.util.Optional; public class HiveMetadataOpsTest { + @Test + @SuppressWarnings("unchecked") + public void testAlternateCaseCreateResetsWarmDatabaseTableNames() { + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + Mockito.when(catalog.getDbForReplay("mixeddb")).thenReturn((Optional) Optional.of(db)); + + new HiveMetadataOps(catalog, Mockito.mock(HMSCachedClient.class)) + .afterCreateTable("mixeddb", "new_table"); + + Mockito.verify(db).resetMetaCacheNames(); + } + @Test @SuppressWarnings("unchecked") public void testColdTruncateReplayInvalidatesCanonicalDatabaseScope() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 3543a0338ba0a6..fc5129f9bc9a86 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -95,6 +95,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.concurrent.Callable; @@ -2624,6 +2625,51 @@ public void testReplayDropInvalidatesSdkOnlyPaimonCache() throws Exception { } } + @Test + public void testModeOneSdkOnlyDatabaseInvalidationUsesCatalogLocale() throws Exception { + Locale previousLocale = Locale.getDefault(); + Locale.setDefault(new Locale("tr", "TR")); + PaimonExternalCatalog dorisCatalog = null; + PaimonExternalCatalog externalCatalog = null; + try { + java.io.File warehouse = temporaryFolder.newFolder("mode_one_turkish_sdk_invalidation"); + Map properties = new HashMap<>(); + properties.put("type", "paimon"); + properties.put(PaimonExternalCatalog.PAIMON_CATALOG_TYPE, + PaimonExternalCatalog.PAIMON_FILESYSTEM); + properties.put("warehouse", warehouse.toURI().toString()); + properties.put(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "1"); + dorisCatalog = new PaimonExternalCatalog(121L, "mode_one_turkish_sdk_test", null, properties, ""); + dorisCatalog.makeSureInitialized(); + dorisCatalog.catalog.createDatabase("I", false); + Identifier identifier = Identifier.create("I", "table"); + Schema schema = Schema.newBuilder().column("id", DataTypes.INT()).build(); + dorisCatalog.catalog.createTable(identifier, schema, false); + dorisCatalog.catalog.getTable(identifier); // Populate only the SDK cache. + + Map uncachedProperties = new HashMap<>(properties); + uncachedProperties.put("paimon.cache-enabled", "false"); + externalCatalog = new PaimonExternalCatalog(122L, "mode_one_turkish_external", null, + uncachedProperties, ""); + externalCatalog.makeSureInitialized(); + externalCatalog.catalog.dropTable(identifier, false); + + // Doris mode 1 stores local "I" as the Turkish dotless "ı" on this FE. + dorisCatalog.invalidatePaimonDatabaseByLocalName("ı"); + PaimonExternalCatalog cachedCatalog = dorisCatalog; + Assert.assertThrows(Catalog.TableNotExistException.class, + () -> cachedCatalog.catalog.getTable(identifier)); + } finally { + if (dorisCatalog != null) { + dorisCatalog.catalog.close(); + } + if (externalCatalog != null) { + externalCatalog.catalog.close(); + } + Locale.setDefault(previousLocale); + } + } + @Test public void testCatalogResetWaitsForInFlightSdkTableLoad() throws Exception { java.io.File warehouse = temporaryFolder.newFolder("reset_fences_inflight_load"); @@ -3774,4 +3820,3 @@ public void testReplayRefreshTableColdTargetKeepsCachedSibling() throws Exceptio - From ef38b5029e03e70994a0ccae612cb3944297b3c6 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 29 Sep 2026 12:50:02 +0800 Subject: [PATCH 09/16] [fix](fe) Preserve resolved external drop targets in replay logs ### What problem does this PR solve? Related PR: #68196 Problem Summary: With lower_case_database_names=2, a caller spelling can resolve to a different case-sensitive database incarnation. Logging or cleaning up with that spelling can retire a historical database instead of the database actually dropped. Cold table invalidation also unnecessarily cleared all catalog row counts when the database ID was retained. Record the resolved database name for live drop hooks and edit-log replay, conservatively retire ambiguous generations for old logs, and fence retained database IDs before and after table metadata invalidation. ### Release note Fix external DROP replay and row-count cache invalidation for case-only database replacement. ### Check List (For Author) - Test: Unit Test (165 focused FE tests); Manual test (./build.sh --fe -j4 and Checkstyle) - Behavior changed: Yes (external DROP uses resolved database identity for cleanup and replay) - Does this need documentation: No --- .../java/org/apache/doris/catalog/Env.java | 2 +- .../doris/datasource/ExternalCatalog.java | 31 ++++++- .../datasource/ExternalMetaCacheMgr.java | 54 ++++++------ .../datasource/hive/HiveMetadataOps.java | 12 ++- .../iceberg/IcebergMetadataOps.java | 16 ++-- .../maxcompute/MaxComputeMetadataOps.java | 12 ++- .../operations/ExternalMetadataOps.java | 25 ++++-- .../datasource/paimon/PaimonMetadataOps.java | 14 ++- .../org/apache/doris/persist/DropDbInfo.java | 12 +++ .../org/apache/doris/persist/DropInfo.java | 12 +++ .../org/apache/doris/persist/EditLog.java | 2 +- .../doris/datasource/CatalogMgrTest.java | 86 +++++++++++++++++++ .../ExternalMetaCacheRouteResolverTest.java | 29 +++++++ .../paimon/PaimonExternalMetaCacheTest.java | 7 +- .../paimon/PaimonMetadataOpsTest.java | 16 ++++ .../doris/persist/DropAndRecoverInfoTest.java | 13 +++ .../apache/doris/persist/DropDbInfoTest.java | 13 +++ 17 files changed, 297 insertions(+), 59 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java index 3d95469223c38c..9eccdb1374b181 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java @@ -3580,7 +3580,7 @@ public void replayDropDb(DropDbInfo info) throws DdlException { } else { ExternalCatalog externalCatalog = (ExternalCatalog) catalogMgr.getCatalog(info.getCtlName()); if (externalCatalog != null) { - externalCatalog.replayDropDb(info.getDbName()); + externalCatalog.replayDropDb(info.getDbName(), info.getResolvedDbName()); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 4c24eec7b0d9d7..ee20a8077c82cb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -1179,14 +1179,15 @@ public void dropDb(String dbName, boolean ifExists, boolean force) throws DdlExc throw new DdlException("Drop database is not supported for catalog: " + getName()); } try { - if (!metadataOps.dropDb(dbName, ifExists, force)) { + Optional resolvedDbName = metadataOps.dropDbWithResolvedName(dbName, ifExists, force); + if (!resolvedDbName.isPresent()) { // No remote drop happened (for example DROP DATABASE IF EXISTS on a missing // database). Do not journal the no-op, otherwise every follower would replay a // post-drop hook that retires caches for a database that was never touched. LOG.info("skip drop database {}.{} because the database does not exist", getName(), dbName); return; } - DropDbInfo info = new DropDbInfo(getName(), dbName); + DropDbInfo info = new DropDbInfo(getName(), resolvedDbName.get(), resolvedDbName.get()); Env.getCurrentEnv().getEditLog().logDropDb(info); } catch (Exception e) { LOG.warn("Failed to drop database {} in catalog {}", dbName, getName(), e); @@ -1200,6 +1201,18 @@ public void replayDropDb(String dbName) { } } + public void replayDropDb(String dbName, String resolvedDbName) { + if (resolvedDbName != null) { + replayDropDb(resolvedDbName); + } else if (getLowerCaseDatabaseNames() == 2) { + // An old name-only record cannot distinguish a historical DROP from an alias that + // targeted a case-only replacement. Retire both possibilities conservatively. + retireUnresolvedDatabaseGeneration(); + } else { + replayDropDb(dbName); + } + } + @Override public boolean createTable(CreateTableInfo createTableInfo) throws UserException { makeSureInitialized(); @@ -1271,7 +1284,7 @@ public void dropTable(String dbName, String tableName, boolean isView, boolean i } try { metadataOps.dropTable(dorisTable, ifExists); - DropInfo info = new DropInfo(getName(), dbName, tableName); + DropInfo info = new DropInfo(getName(), db.getFullName(), dorisTable.getName(), db.getFullName()); Env.getCurrentEnv().getEditLog().logDropTable(info); } catch (Exception e) { LOG.warn("Failed to drop a table", e); @@ -1285,6 +1298,18 @@ public void replayDropTable(String dbName, String tblName) { } } + public void replayDropTable(String dbName, String tblName, String resolvedDbName) { + if (resolvedDbName != null) { + replayDropTable(resolvedDbName, tblName); + } else if (getLowerCaseDatabaseNames() == 2) { + // Old logs carry only caller spelling, which can identify either side of a case-only + // database replacement. Never guess which cached database was actually dropped. + retireUnresolvedDatabaseGeneration(); + } else { + replayDropTable(dbName, tblName); + } + } + /** * Unregisters a database from the catalog. * Internally, remove the database meta from cache diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 1c23272d5094cc..1064c47c31dd97 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -560,29 +560,29 @@ public void invalidateDb(ExternalDatabase database) { } public void invalidateTable(long catalogId, String dbName, String tableName) { - Optional> db = Optional.empty(); + Runnable rowCountFence = () -> rowCountCache.invalidateCatalog(catalogId); try { - db = getCachedDb(catalogId, dbName); - invalidateTableRowCount(catalogId, db, tableName); + rowCountFence = resolveTableRowCountFence(catalogId, dbName, getCachedDb(catalogId, dbName), tableName); + rowCountFence.run(); invalidateLanceTableAccess(catalogId); routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidateTable", () -> cache.invalidateTable(catalogId, dbName, tableName))); } finally { - invalidateTableRowCount(catalogId, db, tableName); + rowCountFence.run(); } } public void invalidateTableByEngine(long catalogId, String engine, String dbName, String tableName) { - Optional> db = Optional.empty(); + Runnable rowCountFence = () -> rowCountCache.invalidateCatalog(catalogId); try { - db = getCachedDb(catalogId, dbName); - invalidateTableRowCount(catalogId, db, tableName); + rowCountFence = resolveTableRowCountFence(catalogId, dbName, getCachedDb(catalogId, dbName), tableName); + rowCountFence.run(); routeSpecifiedEngine(engine, cache -> safeInvalidate( cache, catalogId, "invalidateTableByEngine", () -> cache.invalidateTable(catalogId, dbName, tableName))); } finally { - invalidateTableRowCount(catalogId, db, tableName); + rowCountFence.run(); } } @@ -597,30 +597,37 @@ private void invalidateLanceTableAccess(long catalogId) { public void invalidatePartitions(long catalogId, String dbName, String tableName, List partitions) { - Optional> db = Optional.empty(); + Runnable rowCountFence = () -> rowCountCache.invalidateCatalog(catalogId); try { - db = getCachedDb(catalogId, dbName); - invalidateTableRowCount(catalogId, db, tableName); + rowCountFence = resolveTableRowCountFence(catalogId, dbName, getCachedDb(catalogId, dbName), tableName); + rowCountFence.run(); routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidatePartitions", () -> cache.invalidatePartitions(catalogId, dbName, tableName, partitions))); } finally { - invalidateTableRowCount(catalogId, db, tableName); + rowCountFence.run(); } } - private void invalidateTableRowCount(long catalogId, + private Runnable resolveTableRowCountFence(long catalogId, String dbName, Optional> db, String tableName) { if (db.isPresent()) { Optional table = db.get().getTableForReplay(tableName); if (table.isPresent()) { - invalidateRowCountCache(table.get()); - } else { - rowCountCache.invalidateDb(catalogId, db.get().getId()); + long tableCatalogId = table.get().getCatalog().getId(); + long tableDbId = table.get().getDb().getId(); + long tableId = table.get().getId(); + return () -> rowCountCache.invalidateTable(tableCatalogId, tableDbId, tableId); } - } else { - rowCountCache.invalidateCatalog(catalogId); + long dbId = db.get().getId(); + return () -> rowCountCache.invalidateDb(catalogId, dbId); } + Optional> identity = getDbIdentityForReplay(catalogId, dbName); + if (identity.isPresent()) { + long dbId = identity.get().second; + return () -> rowCountCache.invalidateDb(catalogId, dbId); + } + return () -> rowCountCache.invalidateCatalog(catalogId); } private Optional> getDbIdentityForReplay(long catalogId, String dbName) { @@ -859,16 +866,7 @@ public void invalidateRowCountCache(long catalogId, long dbId) { public void invalidateRowCountCache(long catalogId, String dbName, String tableName) { Optional> db = getCachedDb(catalogId, dbName); - if (db.isPresent()) { - invalidateTableRowCount(catalogId, db, tableName); - } else { - Optional> identity = getDbIdentityForReplay(catalogId, dbName); - if (identity.isPresent()) { - rowCountCache.invalidateDb(catalogId, identity.get().second); - } else { - rowCountCache.invalidateCatalog(catalogId); - } - } + resolveTableRowCountFence(catalogId, dbName, db, tableName).run(); } public LegacyMetaCacheFactory legacyMetaCacheFactory() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index 73dd509c918bf0..fdcd147bee77ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -131,14 +131,20 @@ public void afterCreateDb() { @Override public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws DdlException { + return dropDbImplWithResolvedName(dbName, ifExists, force).isPresent(); + } + + @Override + public Optional dropDbImplWithResolvedName(String dbName, boolean ifExists, boolean force) + throws DdlException { ExternalDatabase dorisDb = catalog.getDbNullable(dbName); if (dorisDb == null) { if (ifExists) { LOG.info("drop database[{}] which does not exist", dbName); - return false; + return Optional.empty(); } else { ErrorReport.reportDdlException(ErrorCode.ERR_DB_DROP_EXISTS, dbName); - return false; + return Optional.empty(); } } try { @@ -161,7 +167,7 @@ public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws } } client.dropDatabase(dorisDb.getRemoteName()); - return true; + return Optional.of(dorisDb.getFullName()); } catch (Exception e) { throw new RuntimeException(e.getMessage(), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index 4ce1cd235958ff..fd3014bfeda451 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -276,6 +276,12 @@ private void validateDatabaseProperties(String catalogType, Map @Override public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws DdlException { + return dropDbImplWithResolvedName(dbName, ifExists, force).isPresent(); + } + + @Override + public Optional dropDbImplWithResolvedName(String dbName, boolean ifExists, boolean force) + throws DdlException { try { return executeCatalogOperation(() -> performDropDb(dbName, ifExists, force)); } catch (Exception e) { @@ -284,15 +290,15 @@ public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws } } - private boolean performDropDb(String dbName, boolean ifExists, boolean force) throws DdlException { + private Optional performDropDb(String dbName, boolean ifExists, boolean force) throws DdlException { ExternalDatabase dorisDb = getDatabaseWithinCatalogGeneration(dbName); if (dorisDb == null) { if (ifExists) { LOG.info("drop database[{}] which does not exist", dbName); - return false; + return Optional.empty(); } else { ErrorReport.reportDdlException(ErrorCode.ERR_DB_DROP_EXISTS, dbName); - return false; + return Optional.empty(); } } if (force) { @@ -317,11 +323,11 @@ private boolean performDropDb(String dbName, boolean ifExists, boolean force) th // The remote namespace is already gone, but the resolved local database object // still has to be unregistered, so treat the drop as handled. LOG.info("drop database[{}] force which does not exist", dbName); - return true; + return Optional.of(dorisDb.getFullName()); } } nsCatalog.dropNamespace(getNamespace(dorisDb.getRemoteName())); - return true; + return Optional.of(dorisDb.getFullName()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java index c9059e023b857f..8c500019d01074 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java @@ -128,14 +128,20 @@ public void afterCreateDb() { @Override public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws DdlException { + return dropDbImplWithResolvedName(dbName, ifExists, force).isPresent(); + } + + @Override + public Optional dropDbImplWithResolvedName(String dbName, boolean ifExists, boolean force) + throws DdlException { ExternalDatabase dorisDb = dorisCatalog.getDbNullable(dbName); if (dorisDb == null) { if (ifExists) { LOG.info("drop database[{}] which does not exist", dbName); - return false; + return Optional.empty(); } else { ErrorReport.reportDdlException(ErrorCode.ERR_DB_DROP_EXISTS, dbName); - return false; + return Optional.empty(); } } if (force) { @@ -153,7 +159,7 @@ public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws } } dorisCatalog.getMcStructureHelper().dropDb(odps, dbName, ifExists); - return true; + return Optional.of(dorisDb.getFullName()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java index acc443ced2051e..1c80c91184df6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java @@ -34,6 +34,7 @@ import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; /** * all external metadata operations use this interface @@ -79,21 +80,33 @@ default void afterCreateDb() { * @throws DdlException */ default boolean dropDb(String dbName, boolean ifExists, boolean force) throws DdlException { - if (!dropDbImpl(dbName, ifExists, force)) { + return dropDbWithResolvedName(dbName, ifExists, force).isPresent(); + } + + default Optional dropDbWithResolvedName(String dbName, boolean ifExists, boolean force) + throws DdlException { + Optional resolvedDbName = dropDbImplWithResolvedName(dbName, ifExists, force); + if (!resolvedDbName.isPresent()) { // No remote mutation happened, so do not run the post-drop hook or journal the // operation. A retained local incarnation may still need cleanup (for example a lost // case-insensitive name mapping), so give the implementation a separate hook. afterDropDbNoOp(dbName); - return false; + return Optional.empty(); } - afterDropDb(dbName); - return true; + afterDropDb(resolvedDbName.get()); + return resolvedDbName; } /** - * @return whether the remote database was dropped. Returns {@code false} when the call was a - * no-op (for example {@code IF EXISTS} on a database that does not exist). + * @return the local name of the database actually dropped, or empty when the remote operation + * was a no-op (for example {@code IF EXISTS} on a missing database). */ + default Optional dropDbImplWithResolvedName(String dbName, boolean ifExists, boolean force) + throws DdlException { + return dropDbImpl(dbName, ifExists, force) ? Optional.of(dbName) : Optional.empty(); + } + + /** Retained for existing connector implementations on release branches. */ boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws DdlException; void afterDropDb(String dbName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java index adafb34c906127..f2f01b526b5ee2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java @@ -147,6 +147,12 @@ public void afterCreateDb() { @Override public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws DdlException { + return dropDbImplWithResolvedName(dbName, ifExists, force).isPresent(); + } + + @Override + public Optional dropDbImplWithResolvedName(String dbName, boolean ifExists, boolean force) + throws DdlException { try { return executionAuthenticator.execute(() -> performDropDb(dbName, ifExists, force)); } catch (Exception e) { @@ -155,17 +161,17 @@ public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) throws } } - private boolean performDropDb(String dbName, boolean ifExists, boolean force) throws DdlException { + private Optional performDropDb(String dbName, boolean ifExists, boolean force) throws DdlException { ExternalDatabase dorisDb = dorisCatalog.getDbNullable(dbName); if (dorisDb == null) { if (ifExists) { LOG.info("drop database[{}] which does not exist", dbName); // Database does not exist and IF EXISTS is specified; treat as no-op. - return false; + return Optional.empty(); } else { ErrorReport.reportDdlException(ErrorCode.ERR_DB_DROP_EXISTS, dbName); // ErrorReport.reportDdlException is expected to throw DdlException. - return false; + return Optional.empty(); } } @@ -186,7 +192,7 @@ private boolean performDropDb(String dbName, boolean ifExists, boolean force) th } catch (DatabaseNotEmptyException e) { throw new RuntimeException("database " + dbName + " is not empty! please check!"); } - return true; + return Optional.of(dorisDb.getFullName()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java index 25e8ab74d1ebef..9ceb4251119777 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropDbInfo.java @@ -34,6 +34,8 @@ public class DropDbInfo implements Writable, GsonPostProcessable { private String ctlName; @SerializedName(value = "dbName") private String dbName; + @SerializedName(value = "resolvedDbName") + private String resolvedDbName; @SerializedName(value = "forceDrop") private boolean forceDrop = false; @SerializedName(value = "recycleTime") @@ -49,6 +51,12 @@ public DropDbInfo(String ctlName, String dbName) { this.dbName = dbName; } + // New external logs retain the database actually resolved by the leader. Older logs lack this field. + public DropDbInfo(String ctlName, String dbName, String resolvedDbName) { + this(ctlName, dbName); + this.resolvedDbName = resolvedDbName; + } + // for internal table public DropDbInfo(String dbName, boolean forceDrop, long recycleTime) { this.dbName = dbName; @@ -64,6 +72,10 @@ public String getDbName() { return dbName; } + public String getResolvedDbName() { + return resolvedDbName; + } + public boolean isForceDrop() { return forceDrop; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java index c2082ecc22610d..872e4c471d2cca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/DropInfo.java @@ -34,6 +34,8 @@ public class DropInfo implements Writable { private String ctl; @SerializedName(value = "db") private String db; + @SerializedName(value = "resolvedDb") + private String resolvedDb; @SerializedName(value = "dbId") private long dbId; @SerializedName(value = "tableId") @@ -61,6 +63,12 @@ public DropInfo(String ctl, String db, String tbl) { this.tableName = tbl; } + // New external logs retain the database actually resolved by the leader. Older logs lack this field. + public DropInfo(String ctl, String db, String tbl, String resolvedDb) { + this(ctl, db, tbl); + this.resolvedDb = resolvedDb; + } + // for internal table public DropInfo(long dbId, long tableId, String tableName, boolean isView, boolean forceDrop, long recycleTime) { @@ -89,6 +97,10 @@ public String getDb() { return db; } + public String getResolvedDb() { + return resolvedDb; + } + public long getDbId() { return this.dbId; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java index d2760bb7179759..ce7ab3219cfac5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java @@ -395,7 +395,7 @@ public static void loadJournal(Env env, Long logId, JournalEntity journal) { ExternalCatalog ctl = (ExternalCatalog) Env.getCurrentEnv().getCatalogMgr() .getCatalog(info.getCtl()); if (ctl != null) { - ctl.replayDropTable(info.getDb(), info.getTableName()); + ctl.replayDropTable(info.getDb(), info.getTableName(), info.getResolvedDb()); } } break; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index cdfca6ceac0658..2682db554dead1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -28,9 +28,12 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveExternalMetaCache; import org.apache.doris.datasource.metacache.MetaCache; +import org.apache.doris.datasource.operations.ExternalMetadataOps; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; import org.apache.doris.datasource.property.metastore.AbstractPaimonProperties; import org.apache.doris.nereids.exceptions.NotSupportedException; +import org.apache.doris.persist.DropDbInfo; +import org.apache.doris.persist.DropInfo; import org.apache.doris.persist.EditLog; import org.apache.doris.statistics.query.QueryStats; @@ -40,6 +43,7 @@ import mockit.MockUp; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; import org.mockito.InOrder; import org.mockito.MockedStatic; import org.mockito.Mockito; @@ -644,6 +648,7 @@ void testReboundModeTwoDatabaseDropDoesNotRetireReplacement() throws Exception { Mockito.when(metaCache.tryGetMetaObj("FOO")).thenReturn(Optional.of(replacementDb)); Assertions.assertSame(replacementDb, catalog.getDbForReplay("Foo").orElseThrow(AssertionError::new)); Assertions.assertSame(oldDb, catalog.getDbForDropReplay("Foo").orElseThrow(AssertionError::new)); + Assertions.assertSame(replacementDb, catalog.getDbForDropReplay("FOO").orElseThrow(AssertionError::new)); Env env = Mockito.mock(Env.class); ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); @@ -688,6 +693,87 @@ void testColdReboundModeTwoDropUsesHistoricalIdentity() throws Exception { Util.genIdByName("testing_catalog", "FOO"), "FOO"); } + @Test + void testModeTwoDropReplayUsesResolvedTargetAndLegacyLogRetiresBothCandidates() throws Exception { + long catalogId = 91L; + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId, + ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2")); + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + ExternalMetadataOps metadataOps = Mockito.mock(ExternalMetadataOps.class); + catalog.metadataOps = metadataOps; + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + // Both histories carry caller spelling Foo. New logs preserve the resolved target. + catalog.replayDropTable("Foo", "t", "FOO"); + catalog.replayDropDb("Foo", "FOO"); + catalog.replayDropTable("Foo", "t", "Foo"); + catalog.replayDropDb("Foo", "Foo"); + // An old name-only log cannot distinguish those histories. + catalog.replayDropTable("Foo", "t", null); + catalog.replayDropDb("Foo", null); + } + + Mockito.verify(metadataOps).afterDropTable("FOO", "t"); + Mockito.verify(metadataOps).afterDropDb("FOO"); + Mockito.verify(metadataOps).afterDropTable("Foo", "t"); + Mockito.verify(metadataOps).afterDropDb("Foo"); + Mockito.verify(metaCache, Mockito.times(2)).invalidateObjects(); + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateCatalog(catalogId); + } + + @Test + void testLiveDropLogsResolvedDatabaseInsteadOfCallerAlias() throws Exception { + ExternalDatabase resolvedDb = Mockito.mock(ExternalDatabase.class); + Mockito.when(resolvedDb.getFullName()).thenReturn("FOO"); + ExternalTable resolvedTable = Mockito.mock(ExternalTable.class); + Mockito.when(resolvedTable.getName()).thenReturn("t"); + Mockito.doReturn(resolvedTable).when(resolvedDb).getTableNullable("t"); + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(92L, + ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2")) { + @Override + public ExternalDatabase getDbNullable(String dbName) { + return resolvedDb; + } + }; + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + Field mappingField = ExternalCatalog.class.getDeclaredField("lowerCaseToDatabaseName"); + mappingField.setAccessible(true); + @SuppressWarnings("unchecked") + Map mapping = (Map) mappingField.get(catalog); + mapping.put("foo", "FOO"); + ExternalMetadataOps metadataOps = Mockito.mock(ExternalMetadataOps.class, Mockito.CALLS_REAL_METHODS); + Mockito.when(metadataOps.dropDbImplWithResolvedName("Foo", false, false)) + .thenReturn(Optional.of("FOO")); + catalog.metadataOps = metadataOps; + Env env = Mockito.mock(Env.class); + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getEditLog()).thenReturn(editLog); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalog.dropDb("Foo", false, false); + catalog.dropTable("Foo", "t", false, false, false, false, false); + } + + Mockito.verify(metadataOps).afterDropDb("FOO"); + ArgumentCaptor log = ArgumentCaptor.forClass(DropDbInfo.class); + Mockito.verify(editLog).logDropDb(log.capture()); + Assertions.assertEquals("FOO", log.getValue().getDbName()); + Assertions.assertEquals("FOO", log.getValue().getResolvedDbName()); + ArgumentCaptor tableLog = ArgumentCaptor.forClass(DropInfo.class); + Mockito.verify(editLog).logDropTable(tableLog.capture()); + Assertions.assertEquals("FOO", tableLog.getValue().getDb()); + Assertions.assertEquals("FOO", tableLog.getValue().getResolvedDb()); + } + @Test void testUnresolvedDatabaseDropRetiresGenerationAndFencesCatalog() { long catalogId = 72L; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index 1e7b7d3128a47a..3190b29cd790d6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -20,6 +20,7 @@ import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.Pair; import org.apache.doris.datasource.doris.RemoteDorisExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.iceberg.IcebergHMSExternalCatalog; @@ -699,6 +700,34 @@ public void invalidateTable(long id, String dbName, String tableName) { Mockito.verify(rowCountCache, Mockito.times(2)).invalidateTable(catalogId, dbId, tableId); } + @Test + public void testColdKnownDatabaseTableRoutesDoNotFenceUnrelatedCounts() throws Exception { + long catalogId = 101L; + long dbId = 102L; + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache hudi = new RecordingExternalMetaCache( + "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache( + "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + ExternalMetaCacheMgr cacheMgr = newManagerWithCaches(hive, hudi, iceberg); + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + cacheMgr.replaceRowCountCacheForTest(rowCountCache); + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + Mockito.doReturn(Optional.empty()).when(catalog).getDbForReplay("cold_db"); + Mockito.when(catalog.getDbIdentityForReplay("cold_db", 0L)) + .thenReturn(Optional.of(Pair.of("ColdDb", dbId))); + mockCurrentCatalog(catalogId, catalog); + hive.initializedCatalogIds.add(catalogId); + + cacheMgr.invalidateTable(catalogId, "cold_db", "tbl"); + cacheMgr.invalidateTableByEngine(catalogId, "hive", "cold_db", "tbl"); + cacheMgr.invalidatePartitions(catalogId, "cold_db", "tbl", Collections.singletonList("p")); + + Mockito.verify(rowCountCache, Mockito.times(6)).invalidateDb(catalogId, dbId); + Mockito.verify(rowCountCache, Mockito.never()).invalidateCatalog(catalogId); + } + @Test public void testNameBasedInvalidationWidensToCatalogWhenDatabaseCold() throws Exception { RecordingExternalMetaCache hive = new RecordingExternalMetaCache( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index fc5129f9bc9a86..94fe573e812863 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -3617,8 +3617,8 @@ public void testMode2UnresolvedLeaderNoOpDropRetiresCanonicalLegacyDatabase() th // still retire it even though the drop is not journaled. PaimonMetadataOps noOpMetadataOps = new PaimonMetadataOps(dorisCatalog, dorisCatalog.catalog) { @Override - public boolean dropDbImpl(String dbName, boolean ifExists, boolean force) { - return false; + public Optional dropDbImplWithResolvedName(String dbName, boolean ifExists, boolean force) { + return Optional.empty(); } }; Assert.assertFalse(noOpMetadataOps.dropDb("DB", true, false)); @@ -3817,6 +3817,3 @@ public void testReplayRefreshTableColdTargetKeepsCachedSibling() throws Exceptio } - - - diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java index fb43b5ccefa62e..ac8b33281c78e6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonMetadataOpsTest.java @@ -493,6 +493,22 @@ public void testDropDB() { } } + @Test + public void testDropDatabaseUsesResolvedNameForLeaderCleanup() throws Exception { + ExternalCatalog dorisCatalog = Mockito.mock(ExternalCatalog.class); + ExternalDatabase resolvedDb = Mockito.mock(ExternalDatabase.class); + Mockito.when(resolvedDb.getFullName()).thenReturn("FOO"); + Mockito.doReturn(resolvedDb).when(dorisCatalog).getDbNullable("Foo"); + Catalog remoteCatalog = Mockito.mock(Catalog.class); + PaimonMetadataOps metadataOps = newMetadataOps(dorisCatalog, remoteCatalog); + + Assert.assertEquals(Optional.of("FOO"), metadataOps.dropDbWithResolvedName("Foo", false, false)); + + Mockito.verify(remoteCatalog).dropDatabase("Foo", false, false); + Mockito.verify(dorisCatalog).unregisterDatabase("FOO"); + Mockito.verify(dorisCatalog, Mockito.never()).unregisterDatabase("Foo"); + } + @Test public void testCreateDatabaseWithPropertiesForSupportedCatalogs() throws Exception { List supportedCatalogTypes = Arrays.asList( diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java index 63afe375548fe0..65dc072dead539 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/DropAndRecoverInfoTest.java @@ -19,6 +19,7 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.meta.MetaContext; +import org.apache.doris.persist.gson.GsonUtils; import org.junit.Assert; import org.junit.Test; @@ -30,6 +31,18 @@ import java.io.FileOutputStream; public class DropAndRecoverInfoTest { + @Test + public void testExternalDropTargetSurvivesSerializationAndLegacyLogHasNoTarget() { + DropInfo current = new DropInfo("catalog", "Foo", "t", "FOO"); + DropInfo decoded = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(current), DropInfo.class); + Assert.assertEquals("Foo", decoded.getDb()); + Assert.assertEquals("FOO", decoded.getResolvedDb()); + + DropInfo legacy = GsonUtils.GSON.fromJson( + GsonUtils.GSON.toJson(new DropInfo("catalog", "Foo", "t")), DropInfo.class); + Assert.assertNull(legacy.getResolvedDb()); + } + @Test public void testDropInfoSerialization() throws Exception { MetaContext metaContext = new MetaContext(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/persist/DropDbInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/persist/DropDbInfoTest.java index aea8530afeb0fd..e57dcbf2f254cf 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/persist/DropDbInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/persist/DropDbInfoTest.java @@ -19,6 +19,7 @@ import org.apache.doris.common.FeMetaVersion; import org.apache.doris.meta.MetaContext; +import org.apache.doris.persist.gson.GsonUtils; import org.junit.Assert; import org.junit.Test; @@ -29,6 +30,18 @@ import java.nio.file.Files; public class DropDbInfoTest { + @Test + public void testExternalDropTargetSurvivesSerializationAndLegacyLogHasNoTarget() { + DropDbInfo current = new DropDbInfo("catalog", "Foo", "FOO"); + DropDbInfo decoded = GsonUtils.GSON.fromJson(GsonUtils.GSON.toJson(current), DropDbInfo.class); + Assert.assertEquals("Foo", decoded.getDbName()); + Assert.assertEquals("FOO", decoded.getResolvedDbName()); + + DropDbInfo legacy = GsonUtils.GSON.fromJson( + GsonUtils.GSON.toJson(new DropDbInfo("catalog", "Foo")), DropDbInfo.class); + Assert.assertNull(legacy.getResolvedDbName()); + } + @Test public void testSerialization() throws Exception { MetaContext metaContext = new MetaContext(); From a9e50249f6fbe54ec926cad9f28ca1594ae0f544 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 29 Sep 2026 14:10:00 +0800 Subject: [PATCH 10/16] [fix](fe) Limit no-op external cache retirement and filtered events ### What problem does this PR solve? Related PR: #68196 Problem Summary: A missing external DROP DATABASE IF EXISTS still invoked post-drop cleanup, which could flush every database when mode-2 name resolution failed. Unfiltered HMS partition events for excluded databases and tables could also widen an ignored miss into unrelated cache invalidation. Legacy mode-2 DROP DATABASE replay retired objects but left a warm database-name snapshot visible. Limit no-op cleanup to matching cached identities, skip excluded partition events before any cache fence, and invalidate names with an opening row-count fence before retiring ambiguous legacy DROP generations. ### Release note Avoid unrelated external metadata cache invalidation and stale database names during DROP replay. ### Check List (For Author) - Test: Unit Test (176 focused FE tests); Manual test (./build.sh --fe -j4 and Checkstyle) - Behavior changed: Yes (no-op drops and filtered HMS events no longer flush unrelated caches; legacy DROP replay refreshes names) - Does this need documentation: No --- .../apache/doris/catalog/RefreshManager.java | 4 + .../apache/doris/datasource/CatalogMgr.java | 8 ++ .../doris/datasource/ExternalCatalog.java | 30 +++++- .../datasource/hive/HMSExternalCatalog.java | 12 +++ .../datasource/hive/HiveMetadataOps.java | 5 + .../iceberg/IcebergMetadataOps.java | 5 + .../maxcompute/MaxComputeMetadataOps.java | 5 + .../doris/datasource/metacache/MetaCache.java | 9 ++ .../operations/ExternalMetadataOps.java | 8 +- .../datasource/paimon/PaimonMetadataOps.java | 8 +- .../doris/datasource/CatalogMgrTest.java | 96 +++++++++++++++++++ .../doris/datasource/MetaCacheTest.java | 9 ++ .../datasource/hive/HiveMetadataOpsTest.java | 11 +++ 13 files changed, 195 insertions(+), 15 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index dcf5ab9b642365..0e7751bf92e88d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -379,6 +379,10 @@ public void refreshPartitions(String catalogName, String dbName, String tableNam if (!(catalog instanceof ExternalCatalog)) { throw new DdlException("Only support ExternalCatalog"); } + if (catalog instanceof HMSExternalCatalog + && ((HMSExternalCatalog) catalog).isPartitionEventTargetExcluded(dbName, tableName)) { + return; + } // Partition events are already committed remotely. Fence the row count by cached identity // before any database/table reload can fail and make the ignored-not-found path return. Env.getCurrentEnv().getExtMetaCacheMgr() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 6454a0e9d54e99..3c160e36caf7a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -1107,6 +1107,10 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl if (!(catalog instanceof ExternalCatalog)) { throw new DdlException("Only support ExternalCatalog"); } + if (catalog instanceof HMSExternalCatalog + && ((HMSExternalCatalog) catalog).isPartitionEventTargetExcluded(dbName, tableName)) { + return; + } // Partition events are already committed remotely. Fence the row count by cached identity // before any database/table reload can fail and make the ignored-not-found path return. Env.getCurrentEnv().getExtMetaCacheMgr() @@ -1165,6 +1169,10 @@ public void dropExternalPartitions(String catalogName, String dbName, String tab if (!(catalog instanceof ExternalCatalog)) { throw new DdlException("Only support ExternalCatalog"); } + if (catalog instanceof HMSExternalCatalog + && ((HMSExternalCatalog) catalog).isPartitionEventTargetExcluded(dbName, tableName)) { + return; + } // Partition events are already committed remotely. Fence the row count by cached identity // before any database/table reload can fail and make the ignored-not-found path return. Env.getCurrentEnv().getExtMetaCacheMgr() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index ee20a8077c82cb..97c114235bf335 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -1206,8 +1206,8 @@ public void replayDropDb(String dbName, String resolvedDbName) { replayDropDb(resolvedDbName); } else if (getLowerCaseDatabaseNames() == 2) { // An old name-only record cannot distinguish a historical DROP from an alias that - // targeted a case-only replacement. Retire both possibilities conservatively. - retireUnresolvedDatabaseGeneration(); + // targeted a case-only replacement. The dropped name must not remain visible either. + retireUnresolvedDatabaseGeneration(true); } else { replayDropDb(dbName); } @@ -1334,8 +1334,21 @@ public void unregisterDatabase(String dbName) { Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(getId()); return; } - String localDbName = identity.get().first; - long dbId = identity.get().second; + unregisterDatabase(identity.get().first, identity.get().second); + } + + /** A no-op remote DROP may retire matching resident history, but never unrelated databases. */ + public void retireCachedDatabaseForNoOp(String dbName) { + if (!isInitialized() || metaCache == null) { + return; + } + for (Pair identity : metaCache.getCachedIdentitiesMatching( + dbName, getLowerCaseDatabaseNames() != 0)) { + unregisterDatabase(identity.first, identity.second); + } + } + + private void unregisterDatabase(String localDbName, long dbId) { // A cold database has no removal callback. Fence its deterministic table IDs before // removing the name slot, so a same-name replacement cannot use an old completed count. Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(getId(), dbId); @@ -1397,7 +1410,16 @@ public void retireAllDatabaseObjectsWithoutEngineInvalidation() { * incarnation, engine entries, or row counts. */ public void retireUnresolvedDatabaseGeneration() { + retireUnresolvedDatabaseGeneration(false); + } + + private void retireUnresolvedDatabaseGeneration(boolean invalidateNames) { try { + if (invalidateNames && metaCache != null) { + // A name refresh can expose a new incarnation before the object generation swaps. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(getId()); + metaCache.invalidateNames(); + } retireAllDatabaseObjectsWithoutEngineInvalidation(); Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(getId()); } catch (Exception e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 756687f7fb3ab7..5c4251cd46bf55 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -320,6 +320,18 @@ public boolean registerDatabaseFromEvent(long dbId, String dbName, long metadata return false; } + /** HMS notifications are unfiltered; excluded targets must not change this catalog's caches. */ + public boolean isPartitionEventTargetExcluded(String dbName, String tableName) { + if (!isDatabaseAllowedByFilterIgnoringCase(dbName)) { + return true; + } + return getIncludeTableMap().entrySet().stream() + .filter(entry -> entry.getKey().equalsIgnoreCase(dbName)) + .findFirst() + .map(entry -> entry.getValue().stream().noneMatch(name -> name.equalsIgnoreCase(tableName))) + .orElse(false); + } + @Override public void notifyPropertiesUpdated(Map updatedProps) { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java index fdcd147bee77ea..b6126db8407b79 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java @@ -178,6 +178,11 @@ public void afterDropDb(String dbName) { catalog.unregisterDatabase(dbName); } + @Override + public void afterDropDbNoOp(String dbName) { + catalog.retireCachedDatabaseForNoOp(dbName); + } + @Override public boolean createTableImpl(CreateTableInfo createTableInfo) throws UserException { String dbName = createTableInfo.getDbName(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index fd3014bfeda451..ccc693f4ad29db 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -335,6 +335,11 @@ public void afterDropDb(String dbName) { dorisCatalog.unregisterDatabase(dbName); } + @Override + public void afterDropDbNoOp(String dbName) { + dorisCatalog.retireCachedDatabaseForNoOp(dbName); + } + @Override public boolean createTableImpl(CreateTableInfo createTableInfo) throws UserException { try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java index 8c500019d01074..3df2e3bee4dd49 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeMetadataOps.java @@ -167,6 +167,11 @@ public void afterDropDb(String dbName) { dorisCatalog.unregisterDatabase(dbName); } + @Override + public void afterDropDbNoOp(String dbName) { + dorisCatalog.retireCachedDatabaseForNoOp(dbName); + } + // ==================== Create Table ==================== @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java index 2ab532b99c1bbd..bfa291bedfbbc5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java @@ -558,6 +558,15 @@ public Optional getNameByIdIfPresent(long id) { return Optional.ofNullable(withMetaObjLifecycleReadLock(() -> idToName.get(id))); } + /** Snapshot resident object identities matching a DROP alias under the catalog's name mode. */ + public List> getCachedIdentitiesMatching(String name, boolean ignoreCase) { + return withMetaObjLifecycleReadLock(() -> idToName.entrySet().stream() + .filter(entry -> ignoreCase ? entry.getValue().equalsIgnoreCase(name) + : entry.getValue().equals(name)) + .map(entry -> Pair.of(entry.getValue(), entry.getKey())) + .collect(Collectors.toList())); + } + public void updateCache(String remoteName, String localName, T obj, long id) { updateCache(remoteName, localName, obj, id, namesLoadEpochSupplier.getAsLong()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java index 1c80c91184df6e..853831c02a8469 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/operations/ExternalMetadataOps.java @@ -113,13 +113,11 @@ default Optional dropDbImplWithResolvedName(String dbName, boolean ifExi /** * Cleanup hook for a drop that did not mutate the remote metastore (for example - * {@code DROP DATABASE IF EXISTS} on a database that does not exist). The default preserves the - * pre-existing local cleanup for connectors whose {@code afterDropDb} only unregisters the - * database, so a retained incarnation is still retired. Paimon overrides it to do targeted - * retirement without its broad engine-cache flush. + * {@code DROP DATABASE IF EXISTS} on a database that does not exist). Remote state did not + * change, so a connector must opt in to any cache-only cleanup. Calling + * {@code afterDropDb} here could turn an absent name into catalog-wide invalidation. */ default void afterDropDbNoOp(String dbName) { - afterDropDb(dbName); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java index f2f01b526b5ee2..dab26f3acc92ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonMetadataOps.java @@ -213,13 +213,9 @@ public void afterDropDb(String dbName) { @Override public void afterDropDbNoOp(String dbName) { try { - // A no-op IF EXISTS drop can still leave a retained canonical database object when a - // case-insensitive (mode 2) name mapping was lost before the statement. Retire the legacy - // database objects so a same-name recreation cannot reuse the stale incarnation. The SDK - // catalog is intentionally not flushed because the remote metastore was not mutated. - dorisCatalog.retireAllDatabaseObjectsWithoutEngineInvalidation(); + dorisCatalog.retireCachedDatabaseForNoOp(dbName); } catch (Exception e) { - LOG.warn("Failed to retire legacy database objects after no-op drop of {}: {}", + LOG.warn("Failed to retire cached database after no-op drop of {}: {}", dbName, e.getMessage(), e); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 2682db554dead1..682559d87ae169 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -61,6 +61,7 @@ import java.util.concurrent.Executors; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicReference; public class CatalogMgrTest { @@ -724,9 +725,104 @@ void testModeTwoDropReplayUsesResolvedTargetAndLegacyLogRetiresBothCandidates() Mockito.verify(metadataOps).afterDropTable("Foo", "t"); Mockito.verify(metadataOps).afterDropDb("Foo"); Mockito.verify(metaCache, Mockito.times(2)).invalidateObjects(); + Mockito.verify(metaCache).invalidateNames(); Mockito.verify(cacheMgr, Mockito.times(2)).invalidateCatalog(catalogId); } + @Test + void testLegacyModeTwoDatabaseDropRefreshesPopulatedNames() { + long catalogId = 94L; + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId, + ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2")); + AtomicReference>> remoteNames = new AtomicReference<>( + Collections.singletonList(Pair.of("Foo", "Foo"))); + ExecutorService executor = Executors.newSingleThreadExecutor(); + MetaCache> metaCache = new MetaCache<>( + "databaseCache", executor, OptionalLong.empty(), OptionalLong.empty(), 10, + ignored -> remoteNames.get(), ignored -> Optional.empty(), (key, value, cause) -> { }); + catalog.installMetaCache(metaCache); + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Assertions.assertEquals(Collections.singletonList("Foo"), catalog.getDbNames()); + remoteNames.set(Collections.emptyList()); + + catalog.replayDropDb("Foo", null); + + Assertions.assertTrue(catalog.getDbNames().isEmpty()); + Mockito.verify(cacheMgr).invalidateCatalog(catalogId); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testNoOpDatabaseDropKeepsUnrelatedCachedDatabase() { + long catalogId = 95L; + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId, + ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2")); + ExecutorService executor = Executors.newSingleThreadExecutor(); + MetaCache> metaCache = new MetaCache<>( + "databaseCache", executor, OptionalLong.empty(), OptionalLong.empty(), 10, + ignored -> Collections.emptyList(), ignored -> Optional.empty(), (key, value, cause) -> { }); + long oldId = Util.genIdByName("testing_catalog", "Foo"); + long unrelatedId = Util.genIdByName("testing_catalog", "sales"); + ExternalDatabase oldDb = Mockito.mock(ExternalDatabase.class); + ExternalDatabase unrelatedDb = Mockito.mock(ExternalDatabase.class); + metaCache.addObjForTest(oldId, "Foo", oldDb); + metaCache.addObjForTest(unrelatedId, "sales", unrelatedDb); + catalog.installMetaCache(metaCache); + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalog.retireCachedDatabaseForNoOp("absent"); + Assertions.assertSame(unrelatedDb, metaCache.tryGetMetaObj("sales").orElseThrow(AssertionError::new)); + Mockito.verifyNoInteractions(cacheMgr); + + // A lost mode-2 mapping still permits narrow retirement of a retained historical ID. + catalog.retireCachedDatabaseForNoOp("foo"); + Assertions.assertFalse(metaCache.tryGetMetaObj("Foo").isPresent()); + Assertions.assertSame(unrelatedDb, metaCache.tryGetMetaObj("sales").orElseThrow(AssertionError::new)); + Mockito.verify(cacheMgr).invalidateDb(catalogId, oldId, "Foo"); + Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(catalogId); + Mockito.verify(cacheMgr, Mockito.never()).invalidateRowCountCache(catalogId, unrelatedId); + } finally { + executor.shutdownNow(); + } + } + + @Test + void testFilteredPartitionEventsLeaveOtherDatabaseCachesUntouched() throws Exception { + long catalogId = 96L; + HMSExternalCatalog catalog = new HMSExternalCatalog(catalogId, "hms", null, + ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2", + ExternalCatalog.EXCLUDE_DATABASE_LIST, "archived", + ExternalCatalog.INCLUDE_TABLE_LIST, "sales.hot"), ""); + CatalogMgr catalogMgr = new CatalogMgr(); + addNamedCatalog(catalogMgr, catalog); + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Assertions.assertFalse(catalog.isPartitionEventTargetExcluded("sales", "hot")); + for (String[] target : new String[][] {{"ARCHIVED", "t"}, {"sales", "cold"}}) { + catalogMgr.addExternalPartitions("hms", target[0], target[1], + Collections.singletonList("p=1"), 1L, true); + catalogMgr.dropExternalPartitions("hms", target[0], target[1], + Collections.singletonList("p=1"), 1L, true); + new RefreshManager().refreshPartitions("hms", target[0], target[1], + Collections.singletonList("p=1"), 1L, true); + } + Mockito.verifyNoInteractions(cacheMgr); + } + } + @Test void testLiveDropLogsResolvedDatabaseInsteadOfCallerAlias() throws Exception { ExternalDatabase resolvedDb = Mockito.mock(ExternalDatabase.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetaCacheTest.java index 8c5b29a21036d7..cf16443e9be4ce 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/MetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/MetaCacheTest.java @@ -88,6 +88,15 @@ public void testListNames() { Assert.assertTrue(names.contains("local2")); } + @Test + public void testCachedIdentityMatchingRespectsDatabaseCaseMode() { + metaCache.addObjForTest(1L, "foo", "foo"); + + Assert.assertTrue(metaCache.getCachedIdentitiesMatching("Foo", false).isEmpty()); + Assert.assertEquals(1, metaCache.getCachedIdentitiesMatching("Foo", true).size()); + Assert.assertEquals("foo", metaCache.getCachedIdentitiesMatching("foo", false).get(0).first); + } + @Test public void testRefreshNamesReloadsCompleteSnapshot() { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java index 0a4fb1454ff1d0..a980cd66425195 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetadataOpsTest.java @@ -106,4 +106,15 @@ public void testAfterDropTableWithoutResolvedDatabaseRetiresGeneration() { // An empty lookup can hide a resident canonical database object after a lost mode-2 mapping. Mockito.verify(catalog).invalidateColdDatabaseForReplay("db"); } + + @Test + public void testNoOpDropDatabaseUsesOnlyCachedIdentityCleanup() { + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + HiveMetadataOps metadataOps = new HiveMetadataOps(catalog, Mockito.mock(HMSCachedClient.class)); + + metadataOps.afterDropDbNoOp("absent"); + + Mockito.verify(catalog).retireCachedDatabaseForNoOp("absent"); + Mockito.verify(catalog, Mockito.never()).unregisterDatabase("absent"); + } } From 0a4e111b71dbdfed7cb183ab94ad8da00c779a18 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 29 Sep 2026 15:16:15 +0800 Subject: [PATCH 11/16] [fix](fe) Preserve external cache identity across HMS events and ALTER ### What problem does this PR solve? Related PR: #68196 Problem Summary: Invalid JDBC name mappings could fail after ALTER had published and journaled them. HMS event names were lowercased before case-sensitive catalog filters, so excluded events could invalidate unrelated caches while visible case-variant targets could be skipped. Cold follower refresh replay left independent engine entries alive, and full Hive insert refresh retained the held table statistics source. Validate mappings before publication, propagate remote HMS name spelling through affected event handlers, scope filtered notifications, retire cold replay engine entries, and reload held Hive table metadata after committed refresh. ### Release note Correct external metadata and row-count cache invalidation after HMS events, catalog ALTER, and Hive inserts. ### Check List (For Author) - Test: Unit Test (focused CatalogMgr, JDBC catalog, refresh replay, and Hive insert tests) - Behavior changed: Yes (reject invalid JDBC mapping ALTER and avoid unrelated HMS cache invalidation) - Does this need documentation: No --- .../apache/doris/catalog/RefreshManager.java | 12 +++++ .../apache/doris/datasource/CatalogMgr.java | 6 ++- .../doris/datasource/ExternalCatalog.java | 5 -- .../datasource/hive/HMSExternalCatalog.java | 38 ++++++++++--- .../hive/event/AddPartitionEvent.java | 3 +- .../hive/event/AlterDatabaseEvent.java | 17 ++++-- .../hive/event/AlterPartitionEvent.java | 6 +-- .../hive/event/CreateDatabaseEvent.java | 3 +- .../hive/event/DropDatabaseEvent.java | 2 +- .../hive/event/DropPartitionEvent.java | 2 +- .../datasource/hive/event/InsertEvent.java | 4 +- .../datasource/jdbc/JdbcExternalCatalog.java | 16 ++++++ .../commands/insert/HiveInsertExecutor.java | 12 ++--- .../catalog/RefreshManagerReplayTest.java | 23 ++++++++ .../doris/datasource/CatalogMgrTest.java | 54 ++++++++++++++++++- .../jdbc/JdbcExternalCatalogTest.java | 10 ++++ .../insert/HiveInsertExecutorTest.java | 3 +- 17 files changed, 182 insertions(+), 34 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index 0e7751bf92e88d..dbb43717292868 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -229,6 +229,14 @@ public void replayRefreshTable(ExternalObjectLog log) { // canonical database scope before acknowledging the committed refresh. Env.getCurrentEnv().getExtMetaCacheMgr() .invalidateRowCountCache(catalog.getId(), db.get().getId()); + if (!Strings.isNullOrEmpty(log.getTableName())) { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableByNameOrWider( + catalog.getId(), db.get().getFullName(), log.getTableName()); + } else { + // Legacy ID-only records cannot recover a cold table's engine key. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb( + catalog.getId(), db.get().getId(), db.get().getFullName()); + } invalidatePaimonCatalogForUnresolvedReplay(catalog); return; } @@ -289,6 +297,10 @@ public void refreshExternalTableFromEvent(String catalogName, String dbName, Str if (!(catalog instanceof ExternalCatalog)) { throw new DdlException("Only support refresh ExternalCatalog Tables"); } + if (catalog instanceof HMSExternalCatalog + && ((HMSExternalCatalog) catalog).isPartitionEventTargetExcluded(dbName, tableName)) { + return; + } // Whole-table events are already committed remotely. Fence the row count by cached identity // before any database/table reload can fail and make the not-found path return. Env.getCurrentEnv().getExtMetaCacheMgr() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 3c160e36caf7a8..57442de5798acf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -1071,7 +1071,11 @@ public void unregisterExternalDatabase(String dbName, String catalogName) if (!(catalog instanceof ExternalCatalog)) { throw new DdlException("Only support drop ExternalCatalog databases"); } - ((HMSExternalCatalog) catalog).unregisterDatabase(dbName); + HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; + if (hmsCatalog.isDatabaseEventTargetExcluded(dbName)) { + return; + } + hmsCatalog.unregisterDatabase(dbName); } public void registerExternalDatabaseFromEvent(String dbName, String catalogName) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 97c114235bf335..b079160d38f04b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -631,10 +631,6 @@ protected boolean isDatabaseAllowedByFilter(String dbName) { return isDatabaseAllowedByFilter(dbName, getIncludeDatabaseMap(), getExcludeDatabaseMap(), false); } - protected boolean isDatabaseAllowedByFilterIgnoringCase(String dbName) { - return isDatabaseAllowedByFilter(dbName, getIncludeDatabaseMap(), getExcludeDatabaseMap(), true); - } - private boolean isDatabaseAllowedByFilter(String dbName, Map includeDatabaseMap, Map excludeDatabaseMap, boolean ignoreCase) { if (dbName.equals(InfoSchemaDb.DATABASE_NAME) || dbName.equals(MysqlDb.DATABASE_NAME)) { @@ -1480,7 +1476,6 @@ protected Map> getIncludeTableMap() { } tbls.add(tbl); } - LOG.info("debug get include table map: {}", includeTableMap); return includeTableMap; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index 5c4251cd46bf55..41fa33a94d3093 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -93,6 +93,9 @@ public class HMSExternalCatalog extends ExternalCatalog { private volatile AbstractHiveProperties hmsProperties; private AtomicLong runtimeGeneration = new AtomicLong(); + private transient volatile String eventIncludeTableList; + private transient volatile Map> eventIncludeTableMap; + public long getRuntimeGeneration() { return runtimeGeneration.get(); } @@ -307,8 +310,8 @@ public boolean registerDatabaseFromEvent(long dbId, String dbName, long metadata LOG.debug("create database [{}]", dbName); } - // HMS notification events normalize database names to lowercase before reaching this boundary. - if (!isDatabaseAllowedByFilterIgnoringCase(dbName)) { + // The event caller passes the original remote spelling used by database-list filters. + if (isDatabaseEventTargetExcluded(dbName)) { return true; } @@ -321,15 +324,34 @@ public boolean registerDatabaseFromEvent(long dbId, String dbName, long metadata } /** HMS notifications are unfiltered; excluded targets must not change this catalog's caches. */ + public boolean isDatabaseEventTargetExcluded(String dbName) { + return !isDatabaseAllowedByFilter(dbName); + } + public boolean isPartitionEventTargetExcluded(String dbName, String tableName) { - if (!isDatabaseAllowedByFilterIgnoringCase(dbName)) { + if (isDatabaseEventTargetExcluded(dbName)) { return true; } - return getIncludeTableMap().entrySet().stream() - .filter(entry -> entry.getKey().equalsIgnoreCase(dbName)) - .findFirst() - .map(entry -> entry.getValue().stream().noneMatch(name -> name.equalsIgnoreCase(tableName))) - .orElse(false); + List includedTables = getEventIncludeTableMap().get(dbName); + return includedTables != null && !includedTables.isEmpty() + && includedTables.stream().noneMatch(name -> name.equalsIgnoreCase(tableName)); + } + + private Map> getEventIncludeTableMap() { + String configuredList = catalogProperty.getOrDefault(INCLUDE_TABLE_LIST, ""); + Map> snapshot = eventIncludeTableMap; + if (snapshot == null || !configuredList.equals(eventIncludeTableList)) { + synchronized (this) { + configuredList = catalogProperty.getOrDefault(INCLUDE_TABLE_LIST, ""); + snapshot = eventIncludeTableMap; + if (snapshot == null || !configuredList.equals(eventIncludeTableList)) { + snapshot = getIncludeTableMap(); + eventIncludeTableMap = snapshot; + eventIncludeTableList = configuredList; + } + } + } + return snapshot; } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java index e582c3d2662bfc..dd5e75e4fedc28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java @@ -106,7 +106,8 @@ protected void process() throws MetastoreNotificationException { return; } Env.getCurrentEnv().getCatalogMgr() - .addExternalPartitions(catalogName, dbName, hmsTbl.getTableName(), partitionNames, eventTime, true); + .addExternalPartitions(catalogName, hmsTbl.getDbName(), hmsTbl.getTableName(), + partitionNames, eventTime, true); } catch (DdlException e) { throw new MetastoreNotificationException( getMsgWithEventInfo("Failed to process event"), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java index 8f2932600585e2..00bc3f3ac39183 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java @@ -22,6 +22,7 @@ import org.apache.doris.common.DdlException; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.hive.HMSExternalCatalog; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; @@ -81,14 +82,24 @@ private void processRename() throws DdlException { if (!(catalog instanceof ExternalCatalog)) { throw new DdlException("Only support ExternalCatalog Databases"); } - if (catalog.getDbNullable(dbAfter.getName()) != null) { + HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; + boolean beforeExcluded = hmsCatalog.isDatabaseEventTargetExcluded(dbBefore.getName()); + boolean afterExcluded = hmsCatalog.isDatabaseEventTargetExcluded(dbAfter.getName()); + if (beforeExcluded && afterExcluded) { + return; + } + if (!afterExcluded && catalog.getDbNullable(dbAfter.getName()) != null) { logInfo("AlterExternalDatabase canceled, because dbAfter has exist, " + "catalogName:[{}],dbName:[{}]", catalogName, dbAfter.getName()); return; } - Env.getCurrentEnv().getCatalogMgr().unregisterExternalDatabase(dbBefore.getName(), catalogName); - Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbAfter.getName(), catalogName); + if (!beforeExcluded) { + Env.getCurrentEnv().getCatalogMgr().unregisterExternalDatabase(dbBefore.getName(), catalogName); + } + if (!afterExcluded) { + Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbAfter.getName(), catalogName); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java index d9898f68d982f7..2a91c6c724bb97 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java @@ -113,14 +113,14 @@ protected void process() throws MetastoreNotificationException { catalogName, dbName, tblName, partitionNameBefore, partitionNameAfter); if (isRename) { Env.getCurrentEnv().getCatalogMgr() - .dropExternalPartitions(catalogName, dbName, tblName, + .dropExternalPartitions(catalogName, hmsTbl.getDbName(), tblName, Lists.newArrayList(partitionNameBefore), eventTime, true); Env.getCurrentEnv().getCatalogMgr() - .addExternalPartitions(catalogName, dbName, tblName, + .addExternalPartitions(catalogName, hmsTbl.getDbName(), tblName, Lists.newArrayList(partitionNameAfter), eventTime, true); } else { Env.getCurrentEnv().getRefreshManager() - .refreshPartitions(catalogName, dbName, hmsTbl.getTableName(), + .refreshPartitions(catalogName, hmsTbl.getDbName(), hmsTbl.getTableName(), Lists.newArrayList(partitionNameAfter), eventTime, true); } } catch (DdlException e) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java index 2d81377f4b6749..2b7474a8c3c11d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java @@ -55,7 +55,8 @@ protected static List getEvents(NotificationEvent event, protected void process() throws MetastoreNotificationException { try { logInfo("catalogName:[{}],dbName:[{}]", catalogName, dbName); - Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent(dbName, catalogName); + Env.getCurrentEnv().getCatalogMgr().registerExternalDatabaseFromEvent( + event == null ? dbName : event.getDbName(), catalogName); } catch (DdlException e) { throw new MetastoreNotificationException( getMsgWithEventInfo("Failed to process event"), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java index 6ab089232b98c1..e48574601f4efc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java @@ -55,7 +55,7 @@ protected void process() throws MetastoreNotificationException { try { logInfo("catalogName:[{}],dbName:[{}]", catalogName, dbName); Env.getCurrentEnv().getCatalogMgr() - .unregisterExternalDatabase(dbName, catalogName); + .unregisterExternalDatabase(event == null ? dbName : event.getDbName(), catalogName); } catch (DdlException e) { throw new MetastoreNotificationException( getMsgWithEventInfo("Failed to process event"), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java index 737ad8f28b9051..9a7e7ab671a9f5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java @@ -105,7 +105,7 @@ protected void process() throws MetastoreNotificationException { return; } Env.getCurrentEnv().getCatalogMgr() - .dropExternalPartitions(catalogName, dbName, hmsTbl.getTableName(), + .dropExternalPartitions(catalogName, hmsTbl.getDbName(), hmsTbl.getTableName(), partitionNames, eventTime, true); } catch (DdlException e) { throw new MetastoreNotificationException( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java index 4b4abd0264d53d..b51282be9770d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java @@ -71,8 +71,8 @@ protected void process() throws MetastoreNotificationException { * the file cache of this table, * but this PR has fixed it. */ - Env.getCurrentEnv().getRefreshManager().refreshExternalTableFromEvent(catalogName, dbName, tblName, - eventTime); + Env.getCurrentEnv().getRefreshManager().refreshExternalTableFromEvent( + catalogName, event == null ? dbName : event.getDbName(), tblName, eventTime); } catch (DdlException e) { throw new MetastoreNotificationException( getMsgWithEventInfo("Failed to process event"), e); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java index e6afa99be16689..a418ec9f0f0f3a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java @@ -120,6 +120,22 @@ public void checkProperties() throws DdlException { ExternalFunctionRules.check(catalogProperty.getProperties().getOrDefault(JdbcResource.FUNCTION_RULES, "")); } + @Override + public boolean validatePropertiesBeforeUpdate( + Map currentProperties, Map updatedProperties) throws DdlException { + // The identifier mapping is rebuilt during reset, after property publication. Reject an + // invalid candidate here so a failed ALTER cannot leave invalid durable properties. + if (updatedProperties.containsKey(META_NAMES_MAPPING) + || updatedProperties.containsKey(LOWER_CASE_META_NAMES)) { + String mapping = updatedProperties.getOrDefault(META_NAMES_MAPPING, getMetaNamesMapping()); + String lowerCase = updatedProperties.getOrDefault(LOWER_CASE_META_NAMES, getLowerCaseMetaNames()); + new JdbcIdentifierMapping( + Env.isTableNamesCaseInsensitive() || Env.isStoredTableNamesLowerCase(), + Boolean.parseBoolean(lowerCase), mapping); + } + return false; + } + /** * Enforce the mandatory driver URL rule on user-facing CREATE and ALTER CATALOG validation. * Catalog replay does not call {@link #checkProperties()}, so existing catalogs remain compatible. diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java index 486b90a4c91b67..46b0dd93618e8c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java @@ -97,12 +97,6 @@ protected void doAfterCommit() throws DdlException { HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() .hive(hmsTable.getCatalog().getId()); cache.refreshAffectedPartitions(hmsTable, partitionUpdates, modifiedPartNames, newPartNames); - // The held HMS table also caches table parameters (including numRows/totalSize). - // Reload those after the selective partition refresh, before closing the count fence. - hmsTable.unsetObjectCreated(); - // Close the admission window opened by the fence above: a load admitted after it can - // compute the pre-insert value from the still-resident file list and publish it. - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(hmsTable); } else { // Non-partitioned table or no partition updates, do full table refresh Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(hmsTable); @@ -122,6 +116,12 @@ protected void doAfterCommit() throws DdlException { } } + // Engine invalidation does not retire the held HMS table parameters. A row-count load + // admitted during either refresh path may still read pre-commit numRows/totalSize, so + // rebuild that source and close the admission window for both selective and full paths. + hmsTable.unsetObjectCreated(); + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(hmsTable); + // Write edit log to notify other FEs long updateTime = System.currentTimeMillis(); hmsTable.setUpdateTime(updateTime); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerReplayTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerReplayTest.java index 85a78719714e20..c69381ac5a364b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerReplayTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerReplayTest.java @@ -87,6 +87,29 @@ public void testReplayRefreshTableContainsInvalidationFailure() { } } + @Test + public void testReplayRefreshColdTableRetiresResidentEngineEntries() { + long catalogId = 4L; + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + @SuppressWarnings("unchecked") + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + Mockito.doReturn(Optional.of(db)).when(catalog).getDbForReplay("db"); + Mockito.doReturn(Optional.empty()).when(db).getTableForReplay("table"); + Mockito.when(db.getFullName()).thenReturn("db"); + Mockito.when(db.getId()).thenReturn(5L); + Mockito.when(db.hasLocalTableName("table")).thenReturn(true); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = mockEnv(catalogId, catalog); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + new RefreshManager().replayRefreshTable( + ExternalObjectLog.createForRefreshTable(catalogId, "db", "table", 0L)); + } + Mockito.verify(cacheMgr).invalidateTableByNameOrWider(catalogId, "db", "table"); + } + private Env mockEnv(long catalogId, ExternalCatalog catalog) { CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 682559d87ae169..590667fdc66eb7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.JdbcResource; import org.apache.doris.catalog.RefreshManager; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.DdlException; @@ -27,6 +28,7 @@ import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveExternalMetaCache; +import org.apache.doris.datasource.jdbc.JdbcExternalCatalog; import org.apache.doris.datasource.metacache.MetaCache; import org.apache.doris.datasource.operations.ExternalMetadataOps; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; @@ -173,6 +175,41 @@ void testAlterCatalogRollsBackUncheckedValidationFailure() throws Exception { Mockito.verify(catalog, Mockito.never()).modifyCatalogProps(newProperties); } + @Test + void testMalformedJdbcMappingAlterDoesNotPublishOrJournal() throws Exception { + Map properties = new HashMap<>(ImmutableMap.of( + "type", "jdbc", + JdbcResource.DRIVER_URL, "driver.jar", + JdbcResource.JDBC_URL, "jdbc:oracle:thin:@127.0.0.1:1521:XE", + JdbcResource.DRIVER_CLASS, "oracle.jdbc.driver.OracleDriver")); + JdbcExternalCatalog catalog = new JdbcExternalCatalog(98L, "jdbc", null, properties, ""); + CatalogMgr catalogMgr = new CatalogMgr(); + addCatalog(catalogMgr, catalog); + Map oldProperties = new HashMap<>(catalog.getProperties()); + CatalogLog log = new CatalogLog(); + log.setCatalogId(catalog.getId()); + log.setNewProps(ImmutableMap.of(ExternalCatalog.META_NAMES_MAPPING, "not-json")); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + EditLog editLog = Mockito.mock(EditLog.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(env.getEditLog()).thenReturn(editLog); + Mockito.when(cacheMgr.withCatalogLifecycleLock(Mockito.eq(catalog.getId()), Mockito.any())) + .thenAnswer(invocation -> { + java.util.function.Supplier action = invocation.getArgument(1); + return action.get(); + }); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Assertions.assertThrows(DdlException.class, + () -> catalogMgr.replayAlterCatalogProps(log, oldProperties, false)); + } + + Assertions.assertEquals(oldProperties, catalog.getProperties()); + Mockito.verify(editLog, Mockito.never()).logCatalogLog(Mockito.anyShort(), Mockito.any()); + } + @Test void testDetachedValidationNeverPublishesCandidateToConcurrentInitialization() throws Exception { CatalogMgr catalogMgr = new CatalogMgr(); @@ -811,18 +848,33 @@ void testFilteredPartitionEventsLeaveOtherDatabaseCachesUntouched() throws Excep try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { mockedEnv.when(Env::getCurrentEnv).thenReturn(env); Assertions.assertFalse(catalog.isPartitionEventTargetExcluded("sales", "hot")); - for (String[] target : new String[][] {{"ARCHIVED", "t"}, {"sales", "cold"}}) { + for (String[] target : new String[][] {{"archived", "t"}, {"sales", "cold"}}) { catalogMgr.addExternalPartitions("hms", target[0], target[1], Collections.singletonList("p=1"), 1L, true); catalogMgr.dropExternalPartitions("hms", target[0], target[1], Collections.singletonList("p=1"), 1L, true); new RefreshManager().refreshPartitions("hms", target[0], target[1], Collections.singletonList("p=1"), 1L, true); + new RefreshManager().refreshExternalTableFromEvent("hms", target[0], target[1], 1L); } + catalogMgr.unregisterExternalDatabase("archived", "hms"); Mockito.verifyNoInteractions(cacheMgr); } } + @Test + void testHmsEventFilterMatchesListedCaseAndRefreshesSnapshot() throws Exception { + HMSExternalCatalog catalog = new HMSExternalCatalog(97L, "hms", null, + new HashMap<>(ImmutableMap.of(ExternalCatalog.EXCLUDE_DATABASE_LIST, "SALES", + ExternalCatalog.INCLUDE_TABLE_LIST, "SALES.cold")), ""); + // Listing compares the remote spelling exactly: neither filter hides sales.hot. + Assertions.assertFalse(catalog.isDatabaseEventTargetExcluded("sales")); + Assertions.assertFalse(catalog.isPartitionEventTargetExcluded("sales", "hot")); + Assertions.assertTrue(catalog.isDatabaseEventTargetExcluded("SALES")); + catalog.getCatalogProperty().addProperty(ExternalCatalog.INCLUDE_TABLE_LIST, "sales.cold"); + Assertions.assertTrue(catalog.isPartitionEventTargetExcluded("sales", "hot")); + } + @Test void testLiveDropLogsResolvedDatabaseInsteadOfCallerAlias() throws Exception { ExternalDatabase resolvedDb = Mockito.mock(ExternalDatabase.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java index a0eb338c4b66b7..9ac97941b5a216 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalogTest.java @@ -72,6 +72,16 @@ public void testProcessCompatibleProperties() { jdbcExternalCatalog.getCatalogProperty().deleteProperty("lower_case_table_names"); } + @Test + public void testMalformedAlterMappingIsRejectedBeforePublication() { + Map original = new HashMap<>(jdbcExternalCatalog.getProperties()); + Map update = new HashMap<>(); + update.put(ExternalCatalog.META_NAMES_MAPPING, "not-json"); + Assert.assertThrows(RuntimeException.class, + () -> jdbcExternalCatalog.validatePropertiesBeforeUpdate(original, update)); + Assert.assertEquals(original, jdbcExternalCatalog.getProperties()); + } + @Test public void replayJdbcCatalogTest() throws DdlException { jdbcExternalCatalog.getCatalogProperty().addProperty(JdbcResource.CONNECTION_POOL_MIN_SIZE, "1"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java index 341928f3d6a740..f7b8affcfe3efc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutorTest.java @@ -293,7 +293,8 @@ void testCommittedFullInvalidationFailureStillLogsFullRefresh() throws Exception Assertions.assertDoesNotThrow(executor::runAfterCommit); } - Mockito.verify(cacheMgr).invalidateRowCountCache(table); + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(table); + Mockito.verify(table).unsetObjectCreated(); Mockito.verify(cacheMgr, Mockito.atLeastOnce()).invalidateTableCache(table); ArgumentCaptor logCaptor = ArgumentCaptor.forClass(ExternalObjectLog.class); Mockito.verify(editLog).logRefreshExternalTable(logCaptor.capture()); From de4f82f96b5cff6963c92c59f9d18591378c88de Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 29 Sep 2026 16:10:10 +0800 Subject: [PATCH 12/16] [fix](fe) Keep external refresh cleanup outside catalog lock ### What problem does this PR solve? Issue Number: close #64160 Related PR: #68196 Problem Summary: A catalog refresh retired cached database objects while holding the catalog initialization monitor. Synchronous removal callbacks can recursively reset all cached databases and tables, delaying metadata planning for the duration of cleanup. A follower replay of a selective HMS partition refresh also left the held table object and row-count cache stale when the selective refresh failed, including when conservative full-cache invalidation failed. Swap metadata cache generations under the monitor, run removal callbacks after releasing it, and always retire the held table state and close the row-count fence on fallback. ### Release note None ### Check List (For Author) - Test: Unit Test (CatalogMgrTest and RefreshManagerTest); build.sh --fe - Behavior changed: Yes (catalog refresh no longer holds catalog monitor during removal callbacks; failed selective replay retires held table state) - Does this need documentation: No --- .../apache/doris/catalog/RefreshManager.java | 7 ++- .../doris/datasource/ExternalCatalog.java | 49 ++++++++++--------- .../doris/catalog/RefreshManagerTest.java | 5 +- .../doris/datasource/CatalogMgrTest.java | 38 +++++++++++++- 4 files changed, 74 insertions(+), 25 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index dbb43717292868..6814c3aebb916b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -270,7 +270,12 @@ public void replayRefreshTable(ExternalObjectLog log) { // full-table invalidation the leader uses. LOG.warn("failed to refresh affected partitions when replaying refresh table for {}", table.get().getNameWithFullQualifiers(), e); - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(table.get()); + try { + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateTableCache(table.get()); + } finally { + table.get().unsetObjectCreated(); + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(table.get()); + } } if (table.get() instanceof HMSExternalTable && log.getLastUpdateTime() > 0) { ((HMSExternalTable) table.get()).setUpdateTime(log.getLastUpdateTime()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index b079160d38f04b..0c010ce5b7f09e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -188,7 +188,7 @@ public abstract class ExternalCatalog protected MetaCache> metaCache; private ThreadLocal invalidateEngineCacheOnDatabaseRemoval = ThreadLocal.withInitial(() -> true); - private volatile boolean invalidatingAllMetaCache; + private transient ThreadLocal invalidatingAllMetaCache = ThreadLocal.withInitial(() -> false); protected ExecutionAuthenticator executionAuthenticator; protected ThreadPoolExecutor threadPoolWithPreAuth; // Map lowercase database names to actual remote database names for case-insensitive lookup @@ -723,26 +723,30 @@ public void onRefreshCache(boolean invalidCache) { /** * Refresh meta cache only (database level cache), without invalidating catalog level cache. */ - private synchronized void refreshMetaCacheOnly(boolean invalidCache) { - if (metaCache != null) { - // A concurrent row-count load can finish while invalidateAll retires database objects. - // Fence before the generation swap, then close the window after engine invalidation. - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(id); - // A catalog-wide engine invalidation below supersedes every database invalidation. - // The legacy cache uses a synchronous removal listener, so this thread-local scope - // prevents one full SDK-cache scan per cached database without affecting concurrent - // expiry callbacks on other threads. - invalidateEngineCacheOnDatabaseRemoval.set(!invalidCache); - invalidatingAllMetaCache = true; - try { - metaCache.invalidateAll(); - } finally { - invalidatingAllMetaCache = false; - invalidateEngineCacheOnDatabaseRemoval.remove(); - if (!invalidCache) { - // No catalog-wide engine invalidation follows, so close the row-count fence here. - Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(id); - } + private void refreshMetaCacheOnly(boolean invalidCache) { + Runnable objectInvalidation; + ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + synchronized (this) { + if (metaCache == null) { + return; + } + // Publish the names/object generation transition together, before allowing another + // catalog initialization. The old cache removal callbacks can be much slower. + cacheMgr.invalidateRowCountCache(id); + metaCache.invalidateNames(); + objectInvalidation = metaCache.retireObjects(); + } + // A catalog-wide engine invalidation below supersedes every database invalidation. + // Removal listeners are synchronous, but run them after releasing the catalog monitor. + invalidateEngineCacheOnDatabaseRemoval.set(!invalidCache); + invalidatingAllMetaCache.set(true); + try { + objectInvalidation.run(); + } finally { + invalidatingAllMetaCache.remove(); + invalidateEngineCacheOnDatabaseRemoval.remove(); + if (!invalidCache) { + cacheMgr.invalidateRowCountCache(id); } } } @@ -1103,6 +1107,7 @@ public void gsonPostProcess() throws IOException { objectCreated = false; metadataLoadEpoch = new AtomicLong(); invalidateEngineCacheOnDatabaseRemoval = ThreadLocal.withInitial(() -> true); + invalidatingAllMetaCache = ThreadLocal.withInitial(() -> false); // TODO: This code is to compatible with older version of metadata. // Could only remove after all users upgrate to the new version. if (logType == null) { @@ -1360,7 +1365,7 @@ private void unregisterDatabase(String localDbName, long dbId) { } boolean shouldInvalidateRowCountOnDatabaseRemoval() { - return invalidateEngineCacheOnDatabaseRemoval.get() && !invalidatingAllMetaCache; + return invalidateEngineCacheOnDatabaseRemoval.get() && !invalidatingAllMetaCache.get(); } boolean shouldInvalidateRoutedCacheOnDatabaseRemoval() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java index 908709c4381076..b82811dc9d24af 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java @@ -112,6 +112,8 @@ void testPartitionReplayInvalidatesRowCountBeforeCacheFailure() { Mockito.doThrow(new IllegalStateException("partition cache failure")) .when(hiveCache).refreshAffectedPartitionsCache( Mockito.eq(table), Mockito.anyList(), Mockito.anyList()); + Mockito.doThrow(new IllegalStateException("full cache failure")) + .when(cacheMgr).invalidateTableCache(table); Env env = Mockito.mock(Env.class); Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); @@ -126,7 +128,8 @@ void testPartitionReplayInvalidatesRowCountBeforeCacheFailure() { Assertions.assertDoesNotThrow(() -> new RefreshManager().replayRefreshTable(log)); } - Mockito.verify(cacheMgr).invalidateRowCountCache(table); + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(table); + Mockito.verify(table).unsetObjectCreated(); } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 590667fdc66eb7..5ebe9f80ce8e6b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -484,6 +484,7 @@ void testCatalogRefreshUsesSingleRowCountInvalidationOwner() { TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId); @SuppressWarnings("unchecked") MetaCache> metaCache = Mockito.mock(MetaCache.class); + Mockito.when(metaCache.retireObjects()).thenReturn(() -> { }); catalog.installMetaCache(metaCache); Env env = Mockito.mock(Env.class); @@ -496,11 +497,46 @@ void testCatalogRefreshUsesSingleRowCountInvalidationOwner() { InOrder order = Mockito.inOrder(cacheMgr, metaCache); order.verify(cacheMgr).invalidateRowCountCache(catalogId); - order.verify(metaCache).invalidateAll(); + order.verify(metaCache).invalidateNames(); + order.verify(metaCache).retireObjects(); order.verify(cacheMgr).invalidateCatalog(catalogId); Mockito.verify(cacheMgr, Mockito.never()).getRowCountCache(); } + @Test + void testCatalogRefreshRunsRemovalCallbacksOutsideCatalogMonitor() throws Exception { + long catalogId = 99L; + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(catalogId); + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + ExecutorService executor = Executors.newSingleThreadExecutor(); + CountDownLatch monitorAcquired = new CountDownLatch(1); + Mockito.when(metaCache.retireObjects()).thenReturn(() -> { + executor.submit(() -> { + synchronized (catalog) { + monitorAcquired.countDown(); + } + }); + try { + Assertions.assertTrue(monitorAcquired.await(5, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + }); + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalog.onRefreshCache(false); + } finally { + executor.shutdownNow(); + } + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId); + } + @Test void testMetadataOnlyRefreshStillInvalidatesRowCount() { long catalogId = 51L; From 0c37f5145e4867838ceac51730bd3b57b14c0dde Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 29 Sep 2026 18:26:00 +0800 Subject: [PATCH 13/16] [fix](fe) Fence external metadata cache transitions ### What problem does this PR solve? Issue Number: None Related PR: #68196 Problem Summary: Metadata refresh and HMS events could expose new external objects while retaining row counts from an older incarnation, and some cold-table invalidations widened into expensive database or catalog scans. Fence row counts before and after fallible metadata transitions, use a constant-time catalog generation for whole-catalog retirement, resolve retained table IDs without loading cold objects, and normalize HMS database event names consistently with discovery and filters. This also fixes the FE UT case-insensitive event filter failure. ### Release note External table row counts and HMS database events now follow refreshed metadata more consistently. ### Check List (For Author) - Test: Unit Test: 15 related FE test classes, 239 tests passed; full FE build passed before the final filter adjustment - Behavior changed: Yes, external metadata refresh and event invalidation retire stale row counts; case-insensitive database filters match normalized HMS event names - Does this need documentation: No --- .../apache/doris/catalog/RefreshManager.java | 57 ++++++++--- .../apache/doris/datasource/CatalogMgr.java | 44 +++++---- .../doris/datasource/ExternalCatalog.java | 29 +++--- .../doris/datasource/ExternalDatabase.java | 7 ++ .../datasource/ExternalMetaCacheMgr.java | 4 + .../datasource/ExternalRowCountCache.java | 63 ++++++++---- .../lance/LanceExternalCatalog.java | 4 + .../doris/catalog/RefreshManagerTest.java | 97 ++++++++++++++++++- .../doris/datasource/CatalogMgrTest.java | 94 +++++++++++++++++- .../ExternalMetaCacheRouteResolverTest.java | 24 ++++- .../datasource/ExternalRowCountCacheTest.java | 37 +++++++ .../lance/LanceCatalogLifecycleTest.java | 30 ++++++ 12 files changed, 419 insertions(+), 71 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index 6814c3aebb916b..c1d671fc5ccd35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -25,6 +25,7 @@ import org.apache.doris.datasource.CatalogLog; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalDatabase; +import org.apache.doris.datasource.ExternalMetaCacheMgr; import org.apache.doris.datasource.ExternalObjectLog; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.hive.HMSExternalCatalog; @@ -96,7 +97,7 @@ public void handleRefreshDb(String catalogName, String dbName) throws DdlExcepti } DatabaseIf db = catalog.getDbOrDdlException(dbName); // Local DB-object eviction also resets metadata; only an explicit refresh retires access. - invalidateLanceTableAccess(catalog); + invalidateLanceTableAccess((ExternalDatabase) db); refreshDbInternal((ExternalDatabase) db); ExternalObjectLog log = ExternalObjectLog.createForRefreshDb(catalog.getId(), db.getFullName()); @@ -111,7 +112,7 @@ public void replayRefreshDb(ExternalObjectLog log) { LOG.warn("failed to find catalog when replaying refresh db: {}", log.debugForRefreshDb()); return; } - invalidateLanceTableAccess(catalog); + invalidateLanceTableAccess(catalog, log); Optional> db; if (!Strings.isNullOrEmpty(log.getDbName())) { db = catalog.getDbForReplay(log.getDbName()); @@ -127,14 +128,34 @@ public void replayRefreshDb(ExternalObjectLog log) { }); } - private void invalidateLanceTableAccess(CatalogIf catalog) { - // Access entries outlive the bounded DB/table object caches. Replay must invalidate by - // catalog identity before its cache-only object lookup can return early, including ID logs. - if (catalog instanceof LanceExternalCatalog) { + private void invalidateLanceTableAccess(ExternalDatabase db) { + if (db.getCatalog() instanceof LanceExternalCatalog) { + ExternalCatalog catalog = (ExternalCatalog) db.getCatalog(); + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(catalog.getId(), db.getId()); ((LanceExternalCatalog) catalog).invalidateTableAccessCache(); } } + private void invalidateLanceTableAccess(ExternalCatalog catalog, ExternalObjectLog log) { + if (!(catalog instanceof LanceExternalCatalog)) { + return; + } + // Access entries outlive the bounded DB/table object caches. Fence the retained identity + // before retiring access, even if the following cache-only object lookup misses. + ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + if (!Strings.isNullOrEmpty(log.getDbName()) && !Strings.isNullOrEmpty(log.getTableName())) { + cacheMgr.invalidateRowCountCache(catalog.getId(), log.getDbName(), log.getTableName()); + } else { + Optional> identity = catalog.getDbIdentityForReplay(log.getDbName(), log.getDbId()); + if (identity.isPresent()) { + cacheMgr.invalidateRowCountCache(catalog.getId(), identity.get().second); + } else { + cacheMgr.invalidateRowCountCache(catalog.getId()); + } + } + ((LanceExternalCatalog) catalog).invalidateTableAccessCache(); + } + private void refreshDbInternal(ExternalDatabase db) { db.resetMetaToUninitialized(); LOG.info("refresh database {} in catalog {}", db.getFullName(), db.getCatalog().getName()); @@ -198,7 +219,7 @@ public void replayRefreshTable(ExternalObjectLog log) { LOG.warn("failed to find catalog when replaying refresh table: {}", log.debugForRefreshTable()); return; } - invalidateLanceTableAccess(catalog); + invalidateLanceTableAccess(catalog, log); Optional> db; if (!Strings.isNullOrEmpty(log.getDbName())) { db = catalog.getDbForReplay(log.getDbName()); @@ -404,6 +425,17 @@ public void refreshPartitions(String catalogName, String dbName, String tableNam // before any database/table reload can fail and make the ignored-not-found path return. Env.getCurrentEnv().getExtMetaCacheMgr() .invalidateRowCountCache(catalog.getId(), dbName, tableName); + try { + refreshPartitionsAfterFence(catalog, dbName, tableName, partitionNames, updateTime, + ignoreIfNotExists); + } finally { + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateRowCountCache(catalog.getId(), dbName, tableName); + } + } + + private void refreshPartitionsAfterFence(CatalogIf catalog, String dbName, String tableName, + List partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException { DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { Env.getCurrentEnv().getExtMetaCacheMgr() @@ -426,15 +458,8 @@ public void refreshPartitions(String catalogName, String dbName, String tableNam ExternalTable externalTable = (ExternalTable) table; HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr().hive(externalTable.getCatalog().getId()); - try { - for (String partitionName : partitionNames) { - cache.invalidatePartitionCache(externalTable, partitionName); - } - } finally { - // Close the admission window opened by the pre-fence above, even when the selective - // invalidation fails partway. - Env.getCurrentEnv().getExtMetaCacheMgr() - .invalidateRowCountCache(catalog.getId(), dbName, tableName); + for (String partitionName : partitionNames) { + cache.invalidatePartitionCache(externalTable, partitionName); } ((HMSExternalTable) table).setUpdateTime(updateTime); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 57442de5798acf..918dac65e40b11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -1090,7 +1090,7 @@ public void registerExternalDatabaseFromEvent(String dbName, String catalogName) HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; long metadataLoadEpoch = hmsCatalog.acquireMetadataLoadEpoch(); - long dbId = Util.genIdByName(catalogName, dbName); + long dbId = Util.genIdByName(catalogName, hmsCatalog.localDatabaseNameFromRemote(dbName)); // -1L means it will be dropped later, ignore if (dbId == ExternalMetaIdMgr.META_ID_FOR_NOT_EXISTS) { return; @@ -1119,6 +1119,17 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl // before any database/table reload can fail and make the ignored-not-found path return. Env.getCurrentEnv().getExtMetaCacheMgr() .invalidateRowCountCache(catalog.getId(), dbName, tableName); + try { + addExternalPartitionsAfterFence(catalog, dbName, tableName, partitionNames, updateTime, + ignoreIfNotExists); + } finally { + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateRowCountCache(catalog.getId(), dbName, tableName); + } + } + + private void addExternalPartitionsAfterFence(CatalogIf catalog, String dbName, String tableName, + List partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException { DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { Env.getCurrentEnv().getExtMetaCacheMgr() @@ -1152,14 +1163,7 @@ public void addExternalPartitions(String catalogName, String dbName, String tabl return; } HiveExternalMetaCache cache = Env.getCurrentEnv().getExtMetaCacheMgr().hive(catalog.getId()); - try { - cache.addPartitionsCache(hmsTable.getOrBuildNameMapping(), partitionNames, partitionColumnTypes); - } finally { - // Close the admission window opened by the pre-fence above, even when the selective - // update fails partway. - Env.getCurrentEnv().getExtMetaCacheMgr() - .invalidateRowCountCache(catalog.getId(), dbName, tableName); - } + cache.addPartitionsCache(hmsTable.getOrBuildNameMapping(), partitionNames, partitionColumnTypes); hmsTable.setUpdateTime(updateTime); } @@ -1181,6 +1185,17 @@ public void dropExternalPartitions(String catalogName, String dbName, String tab // before any database/table reload can fail and make the ignored-not-found path return. Env.getCurrentEnv().getExtMetaCacheMgr() .invalidateRowCountCache(catalog.getId(), dbName, tableName); + try { + dropExternalPartitionsAfterFence(catalog, dbName, tableName, partitionNames, updateTime, + ignoreIfNotExists); + } finally { + Env.getCurrentEnv().getExtMetaCacheMgr() + .invalidateRowCountCache(catalog.getId(), dbName, tableName); + } + } + + private void dropExternalPartitionsAfterFence(CatalogIf catalog, String dbName, String tableName, + List partitionNames, long updateTime, boolean ignoreIfNotExists) throws DdlException { DatabaseIf db = catalog.getDbNullable(dbName); if (db == null) { Env.getCurrentEnv().getExtMetaCacheMgr() @@ -1202,15 +1217,8 @@ public void dropExternalPartitions(String catalogName, String dbName, String tab } HMSExternalTable hmsTable = (HMSExternalTable) table; - try { - Env.getCurrentEnv().getExtMetaCacheMgr().hive(catalog.getId()) - .dropPartitionsCache(hmsTable, partitionNames, true); - } finally { - // Close the admission window opened by the pre-fence above, even when the selective - // update fails partway. - Env.getCurrentEnv().getExtMetaCacheMgr() - .invalidateRowCountCache(catalog.getId(), dbName, tableName); - } + Env.getCurrentEnv().getExtMetaCacheMgr().hive(catalog.getId()) + .dropPartitionsCache(hmsTable, partitionNames, true); hmsTable.setUpdateTime(updateTime); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 0c010ce5b7f09e..5a08cd6952f4b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -581,20 +581,11 @@ private List> getFilteredDatabaseNames() { allDatabases = allDatabases.stream() .filter(dbName -> isDatabaseAllowedByFilter( - dbName, includeDatabaseMap, excludeDatabaseMap, false)) + dbName, includeDatabaseMap, excludeDatabaseMap, getLowerCaseDatabaseNames() != 0)) .collect(Collectors.toList()); for (String remoteDbName : allDatabases) { - String localDbName = fromRemoteDatabaseName(remoteDbName); - // Apply lower_case_database_names mode to local name - int dbNameMode = getLowerCaseDatabaseNames(); - if (dbNameMode == 1) { - localDbName = localDbName.toLowerCase(); - } else if (dbNameMode == 2) { - // Mode 2: preserve original remote case for display - localDbName = remoteDbName; - } - remoteToLocalPairs.add(Pair.of(remoteDbName, localDbName)); + remoteToLocalPairs.add(Pair.of(remoteDbName, localDatabaseNameFromRemote(remoteDbName))); } // Check for conflicts when lower_case_meta_names = true or lower_case_database_names = 2 @@ -627,8 +618,20 @@ private List> getFilteredDatabaseNames() { return remoteToLocalPairs; } + /** Use the same local identity for database discovery and HMS create events. */ + protected final String localDatabaseNameFromRemote(String remoteDbName) { + String localDbName = fromRemoteDatabaseName(remoteDbName); + int mode = getLowerCaseDatabaseNames(); + if (mode == 1) { + return localDbName.toLowerCase(Locale.ROOT); + } + // Mode 2 preserves remote spelling for display and deterministic database IDs. + return mode == 2 ? remoteDbName : localDbName; + } + protected boolean isDatabaseAllowedByFilter(String dbName) { - return isDatabaseAllowedByFilter(dbName, getIncludeDatabaseMap(), getExcludeDatabaseMap(), false); + return isDatabaseAllowedByFilter(dbName, getIncludeDatabaseMap(), getExcludeDatabaseMap(), + getLowerCaseDatabaseNames() != 0); } private boolean isDatabaseAllowedByFilter(String dbName, Map includeDatabaseMap, @@ -1016,7 +1019,7 @@ protected ExternalDatabase buildDbForInit(String remote long dbId, InitCatalogLog.Type logType, boolean checkExists) { // Step 1: Map local database name if not already provided if (localDbName == null && remoteDbName != null) { - localDbName = fromRemoteDatabaseName(remoteDbName); + localDbName = localDatabaseNameFromRemote(remoteDbName); } // Step 2: diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java index 1b642318dd2fb6..d66aff16bf42a4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java @@ -417,6 +417,13 @@ public boolean hasLocalTableName(String tableName) { return getLocalTableName(tableName, true) != null; } + /** Resolve a cold table's deterministic ID without loading its evicted object. */ + public OptionalLong getTableIdForReplay(String tableName) { + String localName = getLocalTableName(tableName, true); + return localName == null ? OptionalLong.empty() + : OptionalLong.of(Util.genIdByName(extCatalog.getName(), name, localName)); + } + @Override public void readLock() { this.rwLock.readLock().lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 1064c47c31dd97..fc76705f9c2438 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -620,6 +620,10 @@ private Runnable resolveTableRowCountFence(long catalogId, String dbName, return () -> rowCountCache.invalidateTable(tableCatalogId, tableDbId, tableId); } long dbId = db.get().getId(); + OptionalLong tableId = db.get().getTableIdForReplay(tableName); + if (tableId.isPresent()) { + return () -> rowCountCache.invalidateTable(catalogId, dbId, tableId.getAsLong()); + } return () -> rowCountCache.invalidateDb(catalogId, dbId); } Optional> identity = getDbIdentityForReplay(catalogId, dbName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java index 8fdce28ba213c1..c6802162999916 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java @@ -39,6 +39,7 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.Executor; import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.locks.ReentrantReadWriteLock; import java.util.function.Supplier; @@ -47,6 +48,12 @@ public class ExternalRowCountCache { private static final Logger LOG = LogManager.getLogger(ExternalRowCountCache.class); private final AsyncLoadingCache> rowCountCache; private final ConcurrentHashMap> inFlightLoads = new ConcurrentHashMap<>(); + // A catalog refresh changes this generation in O(1). Old cache entries remain bounded by + // Caffeine's size/expiry policy but are never addressable by a new metadata generation. + private final ConcurrentHashMap catalogGenerations = new ConcurrentHashMap<>(); + // Generations are globally unique: two catalog IDs with the same table ID must not alias + // when one catalog has retired its old cache generation. + private final AtomicLong nextCatalogGeneration = new AtomicLong(); // Serialize future publication with explicit invalidation. Invalidation marks matching in-flight loads // before removing their cache entries, so a refresh that finishes later cannot republish stale data. private final ReentrantReadWriteLock publicationLock = new ReentrantReadWriteLock(); @@ -77,11 +84,17 @@ public static class RowCountKey { private final long catalogId; private final long dbId; private final long tableId; + private final long catalogGeneration; public RowCountKey(long catalogId, long dbId, long tableId) { + this(catalogId, dbId, tableId, 0); + } + + private RowCountKey(long catalogId, long dbId, long tableId, long catalogGeneration) { this.catalogId = catalogId; this.dbId = dbId; this.tableId = tableId; + this.catalogGeneration = catalogGeneration; } @Override @@ -92,12 +105,13 @@ public boolean equals(Object obj) { if (!(obj instanceof RowCountKey)) { return false; } - return ((RowCountKey) obj).tableId == this.tableId; + RowCountKey other = (RowCountKey) obj; + return other.tableId == tableId && other.catalogGeneration == catalogGeneration; } @Override public int hashCode() { - return (int) tableId; + return 31 * Long.hashCode(tableId) + Long.hashCode(catalogGeneration); } } @@ -125,8 +139,9 @@ private static final class LoadFence { private boolean invalidated; } - // RowCountKey intentionally uses tableId as the Caffeine cache identity. In-flight loads need - // the complete scope so catalog/database invalidation can fence a same-tableId replacement. + // A cache generation keeps the existing table-ID identity within one catalog incarnation. + // In-flight loads need the complete scope so database/table invalidation can fence a + // same-tableId replacement without touching another catalog's load. private static final class LoadKey { private final long catalogId; private final long dbId; @@ -219,7 +234,13 @@ int getInFlightLoadCountForTest() { } void refreshForTest(long catalogId, long dbId, long tableId) { - rowCountCache.synchronous().refresh(new RowCountKey(catalogId, dbId, tableId)); + rowCountCache.synchronous().refresh(new RowCountKey( + catalogId, dbId, tableId, currentCatalogGeneration(catalogId))); + } + + private long currentCatalogGeneration(long catalogId) { + return catalogGenerations.computeIfAbsent(catalogId, + ignored -> new AtomicLong(nextCatalogGeneration.incrementAndGet())).get(); } static Optional loadRowCount(RowCountKey rowCountKey, boolean fillMetaCache) { @@ -255,11 +276,13 @@ static Optional loadRowCount(RowCountKey rowCountKey, boolean fillMetaCach * @return Cached row count or -1 if not exist */ public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean fillMetaCache) { - RowCountKey key = new RowCountKey(catalogId, dbId, tableId); try { CompletableFuture> f; + long generation; publicationLock.readLock().lock(); try { + generation = currentCatalogGeneration(catalogId); + RowCountKey key = new RowCountKey(catalogId, dbId, tableId, generation); f = fillMetaCache ? rowCountCache.get(key, (rowCountKey, executor) -> loadWithInvalidationFence( rowCountKey, executor, () -> loadRowCount(rowCountKey, true))) @@ -270,10 +293,14 @@ public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean f // Get row count synchronously by default. if (ConnectContext.get() == null || ConnectContext.get().getSessionVariable().fetchHiveRowCountSync) { - return f.get().orElse(TableIf.UNKNOWN_ROW_COUNT); + Optional value = f.get(); + return generation == currentCatalogGeneration(catalogId) + ? value.orElse(TableIf.UNKNOWN_ROW_COUNT) : TableIf.UNKNOWN_ROW_COUNT; } else { if (f.isDone()) { - return f.get().orElse(TableIf.UNKNOWN_ROW_COUNT); + Optional value = f.get(); + return generation == currentCatalogGeneration(catalogId) + ? value.orElse(TableIf.UNKNOWN_ROW_COUNT) : TableIf.UNKNOWN_ROW_COUNT; } LOG.info("Row count for table {}.{}.{} is still processing.", catalogId, dbId, tableId); } @@ -289,18 +316,20 @@ public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean f * @return Cached row count or -1 if not exist */ public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) { - RowCountKey key = new RowCountKey(catalogId, dbId, tableId); try { CompletableFuture> f; + long generation; publicationLock.readLock().lock(); try { + generation = currentCatalogGeneration(catalogId); + RowCountKey key = new RowCountKey(catalogId, dbId, tableId, generation); f = rowCountCache.getIfPresent(key); } finally { publicationLock.readLock().unlock(); } if (f == null) { return -1; - } else if (f.isDone()) { + } else if (f.isDone() && generation == currentCatalogGeneration(catalogId)) { return f.get().orElse(-1L); } } catch (Exception e) { @@ -309,17 +338,13 @@ public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) return -1; } - // Catalog/db invalidation is O(N): row-count keys are numeric ids, and Caffeine - // does not support prefix invalidation by catalog or database id. + // Catalog invalidation is a constant-time generation fence. Old entries remain bounded by + // Caffeine and cannot be returned or reused after the generation changes. void invalidateCatalog(long catalogId) { publicationLock.writeLock().lock(); try { - inFlightLoads.forEach((key, fences) -> { - if (key.catalogId == catalogId) { - fences.forEach(fence -> fence.invalidated = true); - } - }); - rowCountCache.asMap().keySet().removeIf(key -> key.catalogId == catalogId); + catalogGenerations.computeIfAbsent(catalogId, ignored -> new AtomicLong()) + .set(nextCatalogGeneration.incrementAndGet()); } finally { publicationLock.writeLock().unlock(); } @@ -342,7 +367,7 @@ void invalidateDb(long catalogId, long dbId) { void invalidateTable(long catalogId, long dbId, long tableId) { publicationLock.writeLock().lock(); try { - RowCountKey key = new RowCountKey(catalogId, dbId, tableId); + RowCountKey key = new RowCountKey(catalogId, dbId, tableId, currentCatalogGeneration(catalogId)); Set fences = inFlightLoads.get(new LoadKey(key)); if (fences != null) { fences.forEach(fence -> fence.invalidated = true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java index 179697d436feca..187d2b19121a23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceExternalCatalog.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.lance; import org.apache.doris.analysis.TableSnapshot; +import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.DdlException; import org.apache.doris.datasource.CatalogProperty; @@ -285,6 +286,7 @@ synchronized LanceCatalogClient.Lease acquireClient() { @Override public void unregisterDatabase(String dbName) { // Dropping a namespace is a semantic change, unlike routine local DB-object eviction. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(getId()); invalidateTableAccessCache(); super.unregisterDatabase(dbName); } @@ -299,6 +301,8 @@ public synchronized void invalidateTableAccessCache() { @Override public void onRefreshCache(boolean invalidCache) { if (invalidCache) { + // A new Session can resolve a replacement dataset before the catalog reset below. + Env.getCurrentEnv().getExtMetaCacheMgr().invalidateRowCountCache(getId()); refreshSessionCache(); } super.onRefreshCache(invalidCache); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java index b82811dc9d24af..776ebe580af694 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java @@ -27,6 +27,7 @@ import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveExternalMetaCache; +import org.apache.doris.datasource.lance.LanceExternalCatalog; import org.apache.doris.persist.EditLog; import org.junit.jupiter.api.Assertions; @@ -94,6 +95,98 @@ void testColdDatabaseReplayInvalidatesCatalogRowCount() { Mockito.verify(cacheMgr).invalidateCatalog(catalogId); } + @Test + void testLanceDatabaseReplayFencesRowCountBeforeRetiringAccess() { + long catalogId = 52L; + long dbId = 53L; + LanceExternalCatalog catalog = Mockito.mock(LanceExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getDbIdentityForReplay("db1", 0L)) + .thenReturn(Optional.of(Pair.of("db1", dbId))); + Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.empty()); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.doAnswer(invocation -> { + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, dbId); + return null; + }).when(catalog).invalidateTableAccessCache(); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + new RefreshManager().replayRefreshDb(ExternalObjectLog.createForRefreshDb(catalogId, "db1")); + } + + Mockito.verify(catalog).invalidateTableAccessCache(); + Mockito.verify(cacheMgr).invalidateDb(catalogId, dbId, "db1"); + } + + @Test + void testLanceLeaderDatabaseRefreshFencesRowCountBeforeRetiringAccess() throws Exception { + long catalogId = 56L; + long dbId = 57L; + LanceExternalCatalog catalog = Mockito.mock(LanceExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("lance"); + Mockito.doReturn(db).when(catalog).getDbOrDdlException("db1"); + Mockito.when(db.getId()).thenReturn(dbId); + Mockito.doReturn(catalog).when(db).getCatalog(); + Mockito.when(db.getFullName()).thenReturn("db1"); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog("lance"); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.doAnswer(invocation -> { + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, dbId); + return null; + }).when(catalog).invalidateTableAccessCache(); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(env.getEditLog()).thenReturn(Mockito.mock(EditLog.class)); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + new RefreshManager().handleRefreshDb("lance", "db1"); + } + + Mockito.verify(catalog).invalidateTableAccessCache(); + Mockito.verify(db).resetMetaToUninitialized(); + } + + @Test + void testLanceTableReplayFencesRowCountBeforeRetiringAccess() { + long catalogId = 54L; + long dbId = 55L; + LanceExternalCatalog catalog = Mockito.mock(LanceExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getDbIdentityForReplay("db1", 0L)) + .thenReturn(Optional.of(Pair.of("db1", dbId))); + Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.empty()); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.doAnswer(invocation -> { + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, "db1", "tbl1"); + return null; + }).when(catalog).invalidateTableAccessCache(); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + new RefreshManager().replayRefreshTable( + ExternalObjectLog.createForRefreshTable(catalogId, "db1", "tbl1", 1L)); + } + + Mockito.verify(catalog).invalidateTableAccessCache(); + Mockito.verify(cacheMgr).invalidateDb(catalogId, dbId, "db1"); + } + @Test void testPartitionReplayInvalidatesRowCountBeforeCacheFailure() { long catalogId = 52L; @@ -192,7 +285,7 @@ void testAlterPartitionInvalidatesRowCountBeforeCacheFailure() throws Exception "hms", "db1", "tbl1", java.util.Collections.singletonList("p=1"), 1L, true)); } - Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, "db1", "tbl1"); + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId, "db1", "tbl1"); } @Test @@ -217,7 +310,7 @@ void testColdAlterPartitionEventFencesRowCountBeforeIgnoredTableMiss() throws Ex "hms", "db1", "tbl1", java.util.Collections.singletonList("p=1"), 1L, true); } - Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, "db1", "tbl1"); + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId, "db1", "tbl1"); Mockito.verify(cacheMgr).invalidateTableByNameOrWider(catalogId, "db1", "tbl1"); Mockito.verify(cacheMgr, Mockito.never()).hive(catalogId); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 5ebe9f80ce8e6b..77da72e5c70290 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -128,6 +128,43 @@ void testColdKnownDropInvalidatesOnlyItsDatabase() { Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(catalogId); } + @Test + void testModeOneDatabaseEventCreateAndRenameUseDiscoveryIdentity() throws Exception { + long catalogId = 86L; + TestingHmsEventCatalog catalog = new TestingHmsEventCatalog(catalogId, + ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "1")); + CatalogMgr catalogMgr = new CatalogMgr(); + addNamedCatalog(catalogMgr, catalog); + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(catalog.metaCache.updateCache(Mockito.anyString(), Mockito.anyString(), + Mockito.any(), Mockito.anyLong(), Mockito.anyLong())).thenReturn(true); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalogMgr.registerExternalDatabaseFromEvent("Foo", "hms"); + catalogMgr.unregisterExternalDatabase("Foo", "hms"); + // ALTER DATABASE rename registers the new remote spelling through the same event route. + catalogMgr.registerExternalDatabaseFromEvent("Bar", "hms"); + catalogMgr.unregisterExternalDatabase("Bar", "hms"); + } + + long fooId = Util.genIdByName("hms", "foo"); + long barId = Util.genIdByName("hms", "bar"); + ArgumentCaptor dbCaptor = ArgumentCaptor.forClass(ExternalDatabase.class); + Mockito.verify(catalog.metaCache).updateCache(Mockito.eq("Foo"), Mockito.eq("foo"), + dbCaptor.capture(), Mockito.eq(fooId), Mockito.anyLong()); + Assertions.assertEquals("Foo", dbCaptor.getValue().getRemoteName()); + Assertions.assertEquals("foo", dbCaptor.getValue().getFullName()); + Mockito.verify(catalog.metaCache).invalidate("foo", fooId); + Mockito.verify(catalog.metaCache).updateCache(Mockito.eq("Bar"), Mockito.eq("bar"), + dbCaptor.capture(), Mockito.eq(barId), Mockito.anyLong()); + Mockito.verify(catalog.metaCache).invalidate("bar", barId); + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, fooId); + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, barId); + } + private static void addCatalog(CatalogMgr catalogMgr, ExternalCatalog catalog) throws Exception { Field idToCatalogField = CatalogMgr.class.getDeclaredField("idToCatalog"); idToCatalogField.setAccessible(true); @@ -380,12 +417,40 @@ void testUnsupportedAddPartitionEventStillInvalidatesRowCount() throws Exception "hms", "db1", "tbl1", Collections.singletonList("p=1"), 1L, false); } - Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, "db1", "tbl1"); + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId, "db1", "tbl1"); Mockito.verify(cacheMgr, Mockito.never()).invalidateTableByNameOrWider(Mockito.anyLong(), Mockito.anyString(), Mockito.anyString()); Mockito.verify(cacheMgr, Mockito.never()).hive(catalogId); } + @Test + void testAddPartitionCacheAcquisitionFailureClosesRowCountFence() throws Exception { + CatalogMgr catalogMgr = new CatalogMgr(); + long catalogId = 48L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + HMSExternalTable table = Mockito.mock(HMSExternalTable.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getName()).thenReturn("hms"); + Mockito.doReturn(db).when(catalog).getDbNullable("db1"); + Mockito.doReturn(table).when(db).getTableNullable("tbl1"); + Mockito.when(table.getPartitionColumnTypes(Mockito.any())).thenReturn(Collections.emptyList()); + addNamedCatalog(catalogMgr, catalog); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(cacheMgr.hive(catalogId)).thenThrow(new IllegalStateException("cache acquisition failed")); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Assertions.assertThrows(IllegalStateException.class, + () -> catalogMgr.addExternalPartitions( + "hms", "db1", "tbl1", Collections.singletonList("p=1"), 1L, false)); + } + + Mockito.verify(cacheMgr, Mockito.times(2)).invalidateRowCountCache(catalogId, "db1", "tbl1"); + } + @Test void testDropPartitionEventInvalidatesRowCountBeforeCacheFailure() throws Exception { CatalogMgr catalogMgr = new CatalogMgr(); @@ -471,7 +536,7 @@ void testColdPartitionEventsFenceRowCountBeforeIgnoredTableMiss() throws Excepti "hms", "db1", "tbl1", Collections.singletonList("p=1"), 1L, true); } - Mockito.verify(cacheMgr, Mockito.times(2)) + Mockito.verify(cacheMgr, Mockito.times(4)) .invalidateRowCountCache(catalogId, "db1", "tbl1"); Mockito.verify(cacheMgr, Mockito.times(2)) .invalidateTableByNameOrWider(catalogId, "db1", "tbl1"); @@ -911,6 +976,18 @@ void testHmsEventFilterMatchesListedCaseAndRefreshesSnapshot() throws Exception Assertions.assertTrue(catalog.isPartitionEventTargetExcluded("sales", "hot")); } + @Test + void testHmsEventFilterMatchesNormalizedDatabaseNames() { + for (String mode : new String[] {"1", "2"}) { + HMSExternalCatalog catalog = new HMSExternalCatalog(97L, "hms", null, + new HashMap<>(ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, mode, + ExternalCatalog.INCLUDE_DATABASE_LIST, "AllowedDb", + ExternalCatalog.EXCLUDE_DATABASE_LIST, "ExcludedDb")), ""); + Assertions.assertFalse(catalog.isDatabaseEventTargetExcluded("alloweddb")); + Assertions.assertTrue(catalog.isDatabaseEventTargetExcluded("excludeddb")); + } + } + @Test void testLiveDropLogsResolvedDatabaseInsteadOfCallerAlias() throws Exception { ExternalDatabase resolvedDb = Mockito.mock(ExternalDatabase.class); @@ -1241,4 +1318,17 @@ public boolean tableExist(SessionContext ctx, String dbName, String tblName) { protected void initLocalObjectsImpl() { } } + + private static class TestingHmsEventCatalog extends HMSExternalCatalog { + @SuppressWarnings("unchecked") + TestingHmsEventCatalog(long id, Map properties) { + super(id, "hms", null, properties, ""); + metaCache = Mockito.mock(MetaCache.class); + initialized = true; + } + + @Override + protected void initLocalObjectsImpl() { + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index 3190b29cd790d6..03b2e9cedd8f1f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -786,7 +786,29 @@ public void testNameBasedInvalidationUsesKnownColdDatabaseIdentity() throws Exce } @Test - public void testNameBasedRowCountFenceFallsBackToDatabaseWhenTableIsCold() { + public void testNameBasedRowCountFenceUsesRetainedColdTableIdentity() { + long catalogId = 22L; + long dbId = 23L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + Mockito.when(catalog.getDbForReplay("db1")).thenReturn(Optional.of(db)); + Mockito.when(db.getId()).thenReturn(dbId); + Mockito.doReturn(Optional.empty()).when(db).getTableForReplay("tbl1"); + Mockito.when(db.getTableIdForReplay("tbl1")).thenReturn(java.util.OptionalLong.of(24L)); + mockCurrentCatalog(catalogId, catalog); + + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + metaCacheMgr.replaceRowCountCacheForTest(rowCountCache); + + metaCacheMgr.invalidateRowCountCache(catalogId, "db1", "tbl1"); + + Mockito.verify(rowCountCache).invalidateTable(catalogId, dbId, 24L); + Mockito.verify(rowCountCache, Mockito.never()).invalidateDb(catalogId, dbId); + } + + @Test + public void testNameBasedRowCountFenceWidensWhenColdTableNameIsLost() { long catalogId = 22L; long dbId = 23L; HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java index d79636439cbe68..5eddcc4314376b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java @@ -296,6 +296,43 @@ protected Optional doLoad(ExternalRowCountCache.RowCountKey rowCountKey) { Assertions.assertEquals(200L, cache.getCachedRowCountIfPresent(2, 20, 200)); } + @Test + public void testCatalogGenerationFenceDoesNotReuseInFlightOrCompletedCount() throws Exception { + AtomicInteger loadCount = new AtomicInteger(); + CountDownLatch oldLoadStarted = new CountDownLatch(1); + CountDownLatch releaseOldLoad = new CountDownLatch(1); + ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() { + @Override + protected Optional doLoad(ExternalRowCountCache.RowCountKey key) { + int load = loadCount.incrementAndGet(); + if (load == 1) { + oldLoadStarted.countDown(); + Uninterruptibles.awaitUninterruptibly(releaseOldLoad); + } + return Optional.of(load * 100L); + } + }; + ExecutorService cacheExecutor = Executors.newFixedThreadPool(2); + ExecutorService readerExecutor = Executors.newSingleThreadExecutor(); + try { + ExternalRowCountCache cache = new ExternalRowCountCache(cacheExecutor, null, loader); + Future oldRead = readerExecutor.submit(() -> cache.getCachedRowCount(1, 10, 100, false)); + Assertions.assertTrue(oldLoadStarted.await(10, TimeUnit.SECONDS)); + + cache.invalidateCatalog(1); + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cache.getCachedRowCountIfPresent(1, 10, 100)); + Assertions.assertEquals(200L, cache.getCachedRowCount(1, 10, 100, false)); + + releaseOldLoad.countDown(); + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, oldRead.get(10, TimeUnit.SECONDS)); + Assertions.assertEquals(200L, cache.getCachedRowCountIfPresent(1, 10, 100)); + } finally { + releaseOldLoad.countDown(); + readerExecutor.shutdownNow(); + cacheExecutor.shutdownNow(); + } + } + @Test public void testInvalidateWhileRefreshIsRunningDoesNotRepublishStaleValue() throws Exception { AtomicInteger loadCount = new AtomicInteger(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java index 67f2dbcdff2d96..8b4dbd73660cb5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/LanceCatalogLifecycleTest.java @@ -233,6 +233,36 @@ public void testMetadataRefreshInvalidatesAccessWithoutClosingSession() throws E } } + @Test + public void testCatalogRefreshAndDatabaseDropFenceCountsBeforeRetiringLanceAccess() { + Map properties = new HashMap<>(); + properties.put("type", "lance"); + properties.put(LanceExternalCatalog.WAREHOUSE, "/unused/lance-warehouse"); + LanceExternalCatalog catalog = Mockito.spy( + new LanceExternalCatalog(903, "lance_fence", null, properties, "")); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.doAnswer(invocation -> { + Mockito.verify(cacheMgr).invalidateRowCountCache(903L); + return null; + }).when(catalog).refreshSessionCache(); + + try (MockedStatic currentEnv = Mockito.mockStatic(Env.class)) { + currentEnv.when(Env::getCurrentEnv).thenReturn(env); + catalog.onRefreshCache(true); + Mockito.verify(cacheMgr).invalidateCatalog(903L); + Mockito.clearInvocations(cacheMgr); + Mockito.doAnswer(invocation -> { + Mockito.verify(cacheMgr).invalidateRowCountCache(903L); + return null; + }).when(catalog).invalidateTableAccessCache(); + catalog.unregisterDatabase("db1"); + } + + Mockito.verify(cacheMgr).invalidateDb(903L, "db1"); + } + @Test public void testHeldTableInvalidationResetsLanceAccess() throws Exception { Session session = Mockito.mock(Session.class); From 04fa4afd6b20b9a436fd10e6e13b2241cd84cf33 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Tue, 29 Sep 2026 21:25:20 +0800 Subject: [PATCH 14/16] [fix](fe) Keep external database cache identities locale independent MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ### What problem does this PR solve? Issue Number: None Related PR: #68196 Problem Summary: Database discovery and HMS CREATE events fold mode-1 names with Locale.ROOT, but lookup and DROP used the process default locale. On a Turkish-locale FE, remote I was cached as i while later lookup and DROP sought dotless ı, leaving the dropped object visible. Use one locale-independent fold for database discovery, event IDs, lookup, routing, and Paimon SDK invalidation. Separately, catalog-wide row-count generation state was retained after permanent DROP even when no count was loaded. Avoid allocating unused generations and release a dropped catalog generation without letting an outstanding old read recreate it. ### Release note External database name lookup and cache invalidation are consistent across FE locale settings. ### Check List (For Author) - Test: Unit Test: four related FE classes, 164 tests passed; full FE build and Checkstyle passed - Behavior changed: Yes, external database case folding is locale independent and permanent catalog drop releases row-count generation state - Does this need documentation: No --- .../doris/datasource/ExternalCatalog.java | 24 ++++---- .../datasource/ExternalMetaCacheMgr.java | 8 ++- .../datasource/ExternalRowCountCache.java | 43 ++++++++++--- .../paimon/PaimonExternalCatalog.java | 7 +-- .../doris/datasource/CatalogMgrTest.java | 60 +++++++++++++++++++ .../ExternalMetaCacheRouteResolverTest.java | 15 +++++ .../datasource/ExternalRowCountCacheTest.java | 37 ++++++++++++ .../paimon/PaimonExternalMetaCacheTest.java | 8 +-- 8 files changed, 173 insertions(+), 29 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 5a08cd6952f4b5..df73f5431846e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -429,8 +429,8 @@ private void buildMetaCache() { Math.max(Config.max_meta_object_cache_num, 1), ignored -> getFilteredDatabaseNames(), this::updateLowerCaseToDatabaseName, - (remoteName, localName) -> lowerCaseToDatabaseName.put(remoteName.toLowerCase(), remoteName), - localName -> lowerCaseToDatabaseName.remove(localName.toLowerCase()), + (remoteName, localName) -> lowerCaseToDatabaseName.put(foldDatabaseName(remoteName), remoteName), + localName -> lowerCaseToDatabaseName.remove(foldDatabaseName(localName)), localDbName -> Optional.ofNullable( buildDbForInit(null, localDbName, Util.genIdByName(name, localDbName), logType, true)), @@ -595,7 +595,7 @@ dbName, includeDatabaseMap, excludeDatabaseMap, getLowerCaseDatabaseNames() != 0 // Collect lowercased local names and their remote counterparts for (Pair pair : remoteToLocalPairs) { - String lowerCaseLocalName = pair.second.toLowerCase(); + String lowerCaseLocalName = foldDatabaseName(pair.second); lowerCaseToRemoteNames.computeIfAbsent(lowerCaseLocalName, k -> Lists.newArrayList()).add(pair.first); } @@ -623,12 +623,16 @@ protected final String localDatabaseNameFromRemote(String remoteDbName) { String localDbName = fromRemoteDatabaseName(remoteDbName); int mode = getLowerCaseDatabaseNames(); if (mode == 1) { - return localDbName.toLowerCase(Locale.ROOT); + return foldDatabaseName(localDbName); } // Mode 2 preserves remote spelling for display and deterministic database IDs. return mode == 2 ? remoteDbName : localDbName; } + private static String foldDatabaseName(String dbName) { + return dbName.toLowerCase(Locale.ROOT); + } + protected boolean isDatabaseAllowedByFilter(String dbName) { return isDatabaseAllowedByFilter(dbName, getIncludeDatabaseMap(), getExcludeDatabaseMap(), getLowerCaseDatabaseNames() != 0); @@ -650,14 +654,14 @@ private boolean containsDatabaseName(Map databaseMap, String db if (!ignoreCase) { return databaseMap.containsKey(dbName); } - String normalizedDbName = dbName.toLowerCase(Locale.ROOT); + String normalizedDbName = foldDatabaseName(dbName); return databaseMap.keySet().stream() - .anyMatch(configuredName -> configuredName.toLowerCase(Locale.ROOT).equals(normalizedDbName)); + .anyMatch(configuredName -> foldDatabaseName(configuredName).equals(normalizedDbName)); } private void updateLowerCaseToDatabaseName(List> names) { Map updated = Maps.newConcurrentMap(); - names.forEach(pair -> updated.put(pair.key().toLowerCase(), pair.key())); + names.forEach(pair -> updated.put(foldDatabaseName(pair.key()), pair.key())); lowerCaseToDatabaseName = updated; } @@ -1542,14 +1546,14 @@ private String getLocalDatabaseName(String dbName, boolean isReplay) { if (mode == 1) { // Mode 1: Store as lowercase - finalName = dbName.toLowerCase(); + finalName = foldDatabaseName(dbName); } else if (mode == 2) { // Mode 2: Case-insensitive comparison - finalName = lowerCaseToDatabaseName.get(dbName.toLowerCase()); + finalName = lowerCaseToDatabaseName.get(foldDatabaseName(dbName)); if (finalName == null && !isReplay) { try { metaCache.refreshNames(); - finalName = lowerCaseToDatabaseName.get(dbName.toLowerCase()); + finalName = lowerCaseToDatabaseName.get(foldDatabaseName(dbName)); } catch (Exception e) { if (Thread.currentThread().isInterrupted() && e instanceof java.util.concurrent.CompletionException diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index fc76705f9c2438..300f2c6c3bf911 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -468,7 +468,13 @@ public void removeCatalogPermanently(long catalogId) { } } } finally { - lifecycleLock.unlock(); + // DROP never reuses the catalog ID. The earlier row-count fence protects readers; + // release its generation after permanent engine cleanup, not on rename. + try { + rowCountCache.releaseCatalog(catalogId); + } finally { + lifecycleLock.unlock(); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java index c6802162999916..b47ceae9ba5174 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java @@ -243,6 +243,15 @@ private long currentCatalogGeneration(long catalogId) { ignored -> new AtomicLong(nextCatalogGeneration.incrementAndGet())).get(); } + private boolean isCatalogGenerationCurrent(long catalogId, long generation) { + AtomicLong current = catalogGenerations.get(catalogId); + return current != null && current.get() == generation; + } + + int getCatalogGenerationCountForTest() { + return catalogGenerations.size(); + } + static Optional loadRowCount(RowCountKey rowCountKey, boolean fillMetaCache) { try { ExternalTable table = (ExternalTable) StatisticsUtil.findTable( @@ -294,12 +303,12 @@ public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean f if (ConnectContext.get() == null || ConnectContext.get().getSessionVariable().fetchHiveRowCountSync) { Optional value = f.get(); - return generation == currentCatalogGeneration(catalogId) + return isCatalogGenerationCurrent(catalogId, generation) ? value.orElse(TableIf.UNKNOWN_ROW_COUNT) : TableIf.UNKNOWN_ROW_COUNT; } else { if (f.isDone()) { Optional value = f.get(); - return generation == currentCatalogGeneration(catalogId) + return isCatalogGenerationCurrent(catalogId, generation) ? value.orElse(TableIf.UNKNOWN_ROW_COUNT) : TableIf.UNKNOWN_ROW_COUNT; } LOG.info("Row count for table {}.{}.{} is still processing.", catalogId, dbId, tableId); @@ -321,7 +330,11 @@ public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) long generation; publicationLock.readLock().lock(); try { - generation = currentCatalogGeneration(catalogId); + AtomicLong current = catalogGenerations.get(catalogId); + if (current == null) { + return -1; + } + generation = current.get(); RowCountKey key = new RowCountKey(catalogId, dbId, tableId, generation); f = rowCountCache.getIfPresent(key); } finally { @@ -329,7 +342,7 @@ public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) } if (f == null) { return -1; - } else if (f.isDone() && generation == currentCatalogGeneration(catalogId)) { + } else if (f.isDone() && isCatalogGenerationCurrent(catalogId, generation)) { return f.get().orElse(-1L); } } catch (Exception e) { @@ -343,8 +356,20 @@ public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) void invalidateCatalog(long catalogId) { publicationLock.writeLock().lock(); try { - catalogGenerations.computeIfAbsent(catalogId, ignored -> new AtomicLong()) - .set(nextCatalogGeneration.incrementAndGet()); + AtomicLong current = catalogGenerations.get(catalogId); + if (current != null) { + current.set(nextCatalogGeneration.incrementAndGet()); + } + } finally { + publicationLock.writeLock().unlock(); + } + } + + /** Catalog IDs are never reused after DROP; old readers still fail the generation check. */ + void releaseCatalog(long catalogId) { + publicationLock.writeLock().lock(); + try { + catalogGenerations.remove(catalogId); } finally { publicationLock.writeLock().unlock(); } @@ -367,7 +392,11 @@ void invalidateDb(long catalogId, long dbId) { void invalidateTable(long catalogId, long dbId, long tableId) { publicationLock.writeLock().lock(); try { - RowCountKey key = new RowCountKey(catalogId, dbId, tableId, currentCatalogGeneration(catalogId)); + AtomicLong current = catalogGenerations.get(catalogId); + if (current == null) { + return; + } + RowCountKey key = new RowCountKey(catalogId, dbId, tableId, current.get()); Set fences = inFlightLoads.get(new LoadKey(key)); if (fences != null) { fences.forEach(fence -> fence.invalidated = true); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index 5884885dac225d..52788a751d04fc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -179,14 +179,9 @@ public synchronized void invalidatePaimonDatabaseByLocalName(String localDbName) } withSdkCatalogCacheWriteLock(() -> executionAuthenticator.execute(() -> { boolean caseSensitive = catalog.caseSensitive(); - int nameMode = getLowerCaseDatabaseNames(); invalidateCachedPaimonTables(identifier -> { String remoteName = identifier.getDatabaseName(); - String candidateLocalName = nameMode == 2 ? remoteName : fromRemoteDatabaseName(remoteName); - if (nameMode == 1) { - // Match ExternalCatalog's mode-1 local database name and deterministic ID. - candidateLocalName = candidateLocalName.toLowerCase(); - } + String candidateLocalName = localDatabaseNameFromRemote(remoteName); return identifierPartEquals(candidateLocalName, localDbName, caseSensitive); }); return null; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 77da72e5c70290..40bb5817ccaefa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -54,6 +54,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.OptionalLong; @@ -165,6 +166,65 @@ void testModeOneDatabaseEventCreateAndRenameUseDiscoveryIdentity() throws Except Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, barId); } + @Test + void testModeOneDatabaseEventLookupAndDropUseRootLocale() throws Exception { + Locale previousLocale = Locale.getDefault(); + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + try { + long catalogId = 88L; + long dbId = Util.genIdByName("hms", "i"); + TestingHmsEventCatalog catalog = new TestingHmsEventCatalog(catalogId, + ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "1")); + CatalogMgr catalogMgr = new CatalogMgr(); + addNamedCatalog(catalogMgr, catalog); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(catalog.metaCache.updateCache(Mockito.anyString(), Mockito.anyString(), + Mockito.any(), Mockito.anyLong(), Mockito.anyLong())).thenReturn(true); + @SuppressWarnings("unchecked") + ExternalDatabase db = Mockito.mock(ExternalDatabase.class); + Mockito.when(catalog.metaCache.getMetaObj("i", dbId)).thenReturn(Optional.of(db)); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalogMgr.registerExternalDatabaseFromEvent("I", "hms"); + Assertions.assertSame(db, catalog.getDbNullable("I")); + catalogMgr.unregisterExternalDatabase("I", "hms"); + } + + Mockito.verify(catalog.metaCache).updateCache(Mockito.eq("I"), Mockito.eq("i"), + Mockito.any(), Mockito.eq(dbId), Mockito.anyLong()); + Mockito.verify(catalog.metaCache).invalidate("i", dbId); + Mockito.verify(cacheMgr).invalidateRowCountCache(catalogId, dbId); + } finally { + Locale.setDefault(previousLocale); + } + } + + @Test + void testModeTwoDatabaseLookupUsesRootLocale() throws Exception { + Locale previousLocale = Locale.getDefault(); + Locale.setDefault(Locale.forLanguageTag("tr-TR")); + try { + TestingUnregisterCatalog catalog = new TestingUnregisterCatalog(89L, + ImmutableMap.of(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2")); + @SuppressWarnings("unchecked") + MetaCache> metaCache = Mockito.mock(MetaCache.class); + catalog.installMetaCache(metaCache); + Field mappingField = ExternalCatalog.class.getDeclaredField("lowerCaseToDatabaseName"); + mappingField.setAccessible(true); + @SuppressWarnings("unchecked") + Map mapping = (Map) mappingField.get(catalog); + mapping.put("i", "I"); + + Assertions.assertEquals(Pair.of("I", Util.genIdByName("testing_catalog", "I")), + catalog.getDbIdentityForReplay("I", 0L).orElseThrow(AssertionError::new)); + } finally { + Locale.setDefault(previousLocale); + } + } + private static void addCatalog(CatalogMgr catalogMgr, ExternalCatalog catalog) throws Exception { Field idToCatalogField = CatalogMgr.class.getDeclaredField("idToCatalog"); idToCatalogField.setAccessible(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index 03b2e9cedd8f1f..598edb8995f1bc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -511,6 +511,21 @@ public void testOperationalContextChangeFencesRowCountForEvictedDatabases() { Mockito.verify(rowCountCache).invalidateCatalog(catalogId); } + @Test + public void testPermanentCatalogRemovalReleasesRowCountGenerationButRenameDoesNot() { + long catalogId = 29L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + mockCurrentCatalog(catalogId, catalog); + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + ExternalRowCountCache rowCountCache = Mockito.mock(ExternalRowCountCache.class); + metaCacheMgr.replaceRowCountCacheForTest(rowCountCache); + + metaCacheMgr.removeCatalog(catalogId); + Mockito.verify(rowCountCache, Mockito.never()).releaseCatalog(catalogId); + metaCacheMgr.removeCatalogPermanently(catalogId); + Mockito.verify(rowCountCache).releaseCatalog(catalogId); + } + @Test public void testRollbackRetiresEngineCachesWhenResetCleanupThrows() throws Exception { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java index 5eddcc4314376b..1050d630224a0e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java @@ -333,6 +333,43 @@ protected Optional doLoad(ExternalRowCountCache.RowCountKey key) { } } + @Test + public void testPermanentCatalogRemovalReleasesGenerationWithoutRevivingOldReader() throws Exception { + CountDownLatch loadStarted = new CountDownLatch(1); + CountDownLatch releaseLoad = new CountDownLatch(1); + ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() { + @Override + protected Optional doLoad(ExternalRowCountCache.RowCountKey key) { + loadStarted.countDown(); + Uninterruptibles.awaitUninterruptibly(releaseLoad); + return Optional.of(100L); + } + }; + ExecutorService cacheExecutor = Executors.newSingleThreadExecutor(); + ExecutorService readerExecutor = Executors.newSingleThreadExecutor(); + try { + ExternalRowCountCache cache = new ExternalRowCountCache(cacheExecutor, null, loader); + cache.invalidateCatalog(2L); + Assertions.assertEquals(0, cache.getCatalogGenerationCountForTest()); + Future oldRead = readerExecutor.submit(() -> cache.getCachedRowCount(1L, 10L, 100L, false)); + Assertions.assertTrue(loadStarted.await(10, TimeUnit.SECONDS)); + Assertions.assertEquals(1, cache.getCatalogGenerationCountForTest()); + + cache.invalidateCatalog(1L); + cache.releaseCatalog(1L); + Assertions.assertEquals(0, cache.getCatalogGenerationCountForTest()); + releaseLoad.countDown(); + + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, oldRead.get(10, TimeUnit.SECONDS)); + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cache.getCachedRowCountIfPresent(1L, 10L, 100L)); + Assertions.assertEquals(0, cache.getCatalogGenerationCountForTest()); + } finally { + releaseLoad.countDown(); + readerExecutor.shutdownNow(); + cacheExecutor.shutdownNow(); + } + } + @Test public void testInvalidateWhileRefreshIsRunningDoesNotRepublishStaleValue() throws Exception { AtomicInteger loadCount = new AtomicInteger(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 94fe573e812863..52854171e3511f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -2626,7 +2626,7 @@ public void testReplayDropInvalidatesSdkOnlyPaimonCache() throws Exception { } @Test - public void testModeOneSdkOnlyDatabaseInvalidationUsesCatalogLocale() throws Exception { + public void testModeOneSdkOnlyDatabaseInvalidationUsesRootLocale() throws Exception { Locale previousLocale = Locale.getDefault(); Locale.setDefault(new Locale("tr", "TR")); PaimonExternalCatalog dorisCatalog = null; @@ -2654,8 +2654,8 @@ public void testModeOneSdkOnlyDatabaseInvalidationUsesCatalogLocale() throws Exc externalCatalog.makeSureInitialized(); externalCatalog.catalog.dropTable(identifier, false); - // Doris mode 1 stores local "I" as the Turkish dotless "ı" on this FE. - dorisCatalog.invalidatePaimonDatabaseByLocalName("ı"); + // Doris mode 1 uses a locale-independent local name on every FE. + dorisCatalog.invalidatePaimonDatabaseByLocalName("i"); PaimonExternalCatalog cachedCatalog = dorisCatalog; Assert.assertThrows(Catalog.TableNotExistException.class, () -> cachedCatalog.catalog.getTable(identifier)); @@ -3815,5 +3815,3 @@ public void testReplayRefreshTableColdTargetKeepsCachedSibling() throws Exceptio } } } - - From d96208503c0454e857375911b07c34842b954837 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Wed, 30 Sep 2026 09:10:11 +0800 Subject: [PATCH 15/16] [fix](fe) Bound external row-count database invalidation ### What problem does this PR solve? Issue Number: None Related PR: #68196 Problem Summary: REFRESH DATABASE fenced external row counts by scanning every cached entry and in-flight load twice under the global publication write lock, delaying unrelated catalog planning. An invalidated async load also completed with null, and the synchronous reader dereferenced it as Optional, emitting an unexpected NPE warning during ordinary refresh/read overlap. Give each database a globally unique, bounded cache generation so invalidation is constant time and old loads cannot publish or return stale counts. Treat a null load result as an expected, non-cacheable UNKNOWN row count. ### Release note External database refresh no longer scans unrelated row-count caches or logs an NPE for an invalidated load. ### Check List (For Author) - Test: Unit Test: ExternalRowCountCacheTest, 17 tests passed; full ./build.sh --fe -j4 and Checkstyle passed - Behavior changed: Yes, database row-count invalidation uses a bounded generation fence and invalidated reads return UNKNOWN without an exception log - Does this need documentation: No --- .../datasource/ExternalRowCountCache.java | 102 ++++++++++++++---- .../datasource/ExternalRowCountCacheTest.java | 75 +++++++++++++ 2 files changed, 156 insertions(+), 21 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java index b47ceae9ba5174..aba16c65547a3a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java @@ -26,6 +26,8 @@ import com.github.benmanes.caffeine.cache.AsyncCacheLoader; import com.github.benmanes.caffeine.cache.AsyncLoadingCache; +import com.github.benmanes.caffeine.cache.Cache; +import com.github.benmanes.caffeine.cache.Caffeine; import com.github.benmanes.caffeine.cache.Ticker; import lombok.Getter; import org.apache.logging.log4j.LogManager; @@ -51,6 +53,10 @@ public class ExternalRowCountCache { // A catalog refresh changes this generation in O(1). Old cache entries remain bounded by // Caffeine's size/expiry policy but are never addressable by a new metadata generation. private final ConcurrentHashMap catalogGenerations = new ConcurrentHashMap<>(); + // Database generations are bounded independently of database CREATE/DROP churn. Eviction + // merely forces a new, globally unique generation and makes the old entries unreachable. + private final Cache dbGenerations = Caffeine.newBuilder() + .maximumSize(Config.max_external_table_row_count_cache_num).build(); // Generations are globally unique: two catalog IDs with the same table ID must not alias // when one catalog has retired its old cache generation. private final AtomicLong nextCatalogGeneration = new AtomicLong(); @@ -85,16 +91,18 @@ public static class RowCountKey { private final long dbId; private final long tableId; private final long catalogGeneration; + private final long dbGeneration; public RowCountKey(long catalogId, long dbId, long tableId) { - this(catalogId, dbId, tableId, 0); + this(catalogId, dbId, tableId, 0, 0); } - private RowCountKey(long catalogId, long dbId, long tableId, long catalogGeneration) { + private RowCountKey(long catalogId, long dbId, long tableId, long catalogGeneration, long dbGeneration) { this.catalogId = catalogId; this.dbId = dbId; this.tableId = tableId; this.catalogGeneration = catalogGeneration; + this.dbGeneration = dbGeneration; } @Override @@ -106,12 +114,14 @@ public boolean equals(Object obj) { return false; } RowCountKey other = (RowCountKey) obj; - return other.tableId == tableId && other.catalogGeneration == catalogGeneration; + return other.tableId == tableId && other.catalogGeneration == catalogGeneration + && other.dbGeneration == dbGeneration; } @Override public int hashCode() { - return 31 * Long.hashCode(tableId) + Long.hashCode(catalogGeneration); + int hash = 31 * Long.hashCode(tableId) + Long.hashCode(catalogGeneration); + return 31 * hash + Long.hashCode(dbGeneration); } } @@ -171,6 +181,33 @@ public int hashCode() { } } + private static final class DatabaseKey { + private final long catalogId; + private final long dbId; + + private DatabaseKey(long catalogId, long dbId) { + this.catalogId = catalogId; + this.dbId = dbId; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (!(obj instanceof DatabaseKey)) { + return false; + } + DatabaseKey other = (DatabaseKey) obj; + return catalogId == other.catalogId && dbId == other.dbId; + } + + @Override + public int hashCode() { + return 31 * Long.hashCode(catalogId) + Long.hashCode(dbId); + } + } + private CompletableFuture> loadWithInvalidationFence( RowCountKey key, Executor executor, Supplier> loader) { LoadFence fence = new LoadFence(); @@ -235,7 +272,7 @@ int getInFlightLoadCountForTest() { void refreshForTest(long catalogId, long dbId, long tableId) { rowCountCache.synchronous().refresh(new RowCountKey( - catalogId, dbId, tableId, currentCatalogGeneration(catalogId))); + catalogId, dbId, tableId, currentCatalogGeneration(catalogId), currentDbGeneration(catalogId, dbId))); } private long currentCatalogGeneration(long catalogId) { @@ -248,6 +285,16 @@ private boolean isCatalogGenerationCurrent(long catalogId, long generation) { return current != null && current.get() == generation; } + private long currentDbGeneration(long catalogId, long dbId) { + return dbGenerations.get(new DatabaseKey(catalogId, dbId), + ignored -> nextCatalogGeneration.incrementAndGet()); + } + + private boolean isDbGenerationCurrent(long catalogId, long dbId, long generation) { + Long current = dbGenerations.getIfPresent(new DatabaseKey(catalogId, dbId)); + return current != null && current == generation; + } + int getCatalogGenerationCountForTest() { return catalogGenerations.size(); } @@ -271,9 +318,7 @@ static Optional loadRowCount(RowCountKey rowCountKey, boolean fillMetaCach // so we can't try to load the row count until the cache expire. // Throw an exception here will cause too much stack log in fe.out. // So we return null when exception happen. - // Null may raise NPE in caller, but that is expected. - // We catch that NPE and return a default value -1 without keep the value in cache, - // so we can trigger the load function to fetch row count again next time in this exception case. + // The caller treats null as an uncached UNKNOWN result, so the next read can retry. return null; } } @@ -288,10 +333,12 @@ public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean f try { CompletableFuture> f; long generation; + long dbGeneration; publicationLock.readLock().lock(); try { generation = currentCatalogGeneration(catalogId); - RowCountKey key = new RowCountKey(catalogId, dbId, tableId, generation); + dbGeneration = currentDbGeneration(catalogId, dbId); + RowCountKey key = new RowCountKey(catalogId, dbId, tableId, generation, dbGeneration); f = fillMetaCache ? rowCountCache.get(key, (rowCountKey, executor) -> loadWithInvalidationFence( rowCountKey, executor, () -> loadRowCount(rowCountKey, true))) @@ -303,12 +350,14 @@ public long getCachedRowCount(long catalogId, long dbId, long tableId, boolean f if (ConnectContext.get() == null || ConnectContext.get().getSessionVariable().fetchHiveRowCountSync) { Optional value = f.get(); - return isCatalogGenerationCurrent(catalogId, generation) + return value != null && isCatalogGenerationCurrent(catalogId, generation) + && isDbGenerationCurrent(catalogId, dbId, dbGeneration) ? value.orElse(TableIf.UNKNOWN_ROW_COUNT) : TableIf.UNKNOWN_ROW_COUNT; } else { if (f.isDone()) { Optional value = f.get(); - return isCatalogGenerationCurrent(catalogId, generation) + return value != null && isCatalogGenerationCurrent(catalogId, generation) + && isDbGenerationCurrent(catalogId, dbId, dbGeneration) ? value.orElse(TableIf.UNKNOWN_ROW_COUNT) : TableIf.UNKNOWN_ROW_COUNT; } LOG.info("Row count for table {}.{}.{} is still processing.", catalogId, dbId, tableId); @@ -328,6 +377,7 @@ public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) try { CompletableFuture> f; long generation; + long dbGeneration; publicationLock.readLock().lock(); try { AtomicLong current = catalogGenerations.get(catalogId); @@ -335,15 +385,22 @@ public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) return -1; } generation = current.get(); - RowCountKey key = new RowCountKey(catalogId, dbId, tableId, generation); + Long currentDb = dbGenerations.getIfPresent(new DatabaseKey(catalogId, dbId)); + if (currentDb == null) { + return -1; + } + dbGeneration = currentDb; + RowCountKey key = new RowCountKey(catalogId, dbId, tableId, generation, dbGeneration); f = rowCountCache.getIfPresent(key); } finally { publicationLock.readLock().unlock(); } if (f == null) { return -1; - } else if (f.isDone() && isCatalogGenerationCurrent(catalogId, generation)) { - return f.get().orElse(-1L); + } else if (f.isDone() && isCatalogGenerationCurrent(catalogId, generation) + && isDbGenerationCurrent(catalogId, dbId, dbGeneration)) { + Optional value = f.get(); + return value == null ? -1L : value.orElse(-1L); } } catch (Exception e) { LOG.warn("Unexpected exception while returning row count if present", e); @@ -378,12 +435,11 @@ void releaseCatalog(long catalogId) { void invalidateDb(long catalogId, long dbId) { publicationLock.writeLock().lock(); try { - inFlightLoads.forEach((key, fences) -> { - if (key.catalogId == catalogId && key.dbId == dbId) { - fences.forEach(fence -> fence.invalidated = true); - } - }); - rowCountCache.asMap().keySet().removeIf(key -> key.catalogId == catalogId && key.dbId == dbId); + DatabaseKey key = new DatabaseKey(catalogId, dbId); + Long current = dbGenerations.getIfPresent(key); + if (current != null) { + dbGenerations.put(key, nextCatalogGeneration.incrementAndGet()); + } } finally { publicationLock.writeLock().unlock(); } @@ -396,7 +452,11 @@ void invalidateTable(long catalogId, long dbId, long tableId) { if (current == null) { return; } - RowCountKey key = new RowCountKey(catalogId, dbId, tableId, current.get()); + Long currentDb = dbGenerations.getIfPresent(new DatabaseKey(catalogId, dbId)); + if (currentDb == null) { + return; + } + RowCountKey key = new RowCountKey(catalogId, dbId, tableId, current.get(), currentDb); Set fences = inFlightLoads.get(new LoadKey(key)); if (fences != null) { fences.forEach(fence -> fence.invalidated = true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java index 1050d630224a0e..ea560602fce3b4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java @@ -333,6 +333,81 @@ protected Optional doLoad(ExternalRowCountCache.RowCountKey key) { } } + @Test + public void testDatabaseGenerationFenceKeepsOtherDatabasesReadable() throws Exception { + AtomicInteger loadCount = new AtomicInteger(); + CountDownLatch oldLoadStarted = new CountDownLatch(1); + CountDownLatch releaseOldLoad = new CountDownLatch(1); + ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() { + @Override + protected Optional doLoad(ExternalRowCountCache.RowCountKey key) { + if (key.getDbId() == 10 && loadCount.incrementAndGet() == 1) { + oldLoadStarted.countDown(); + Uninterruptibles.awaitUninterruptibly(releaseOldLoad); + return Optional.of(100L); + } + return Optional.of(key.getDbId() * 100L + loadCount.get()); + } + }; + ExecutorService cacheExecutor = Executors.newFixedThreadPool(2); + ExecutorService readerExecutor = Executors.newSingleThreadExecutor(); + try { + ExternalRowCountCache cache = new ExternalRowCountCache(cacheExecutor, null, loader); + Assertions.assertEquals(2000L, cache.getCachedRowCount(1, 20, 100, false)); + Future oldRead = readerExecutor.submit(() -> cache.getCachedRowCount(1, 10, 100, false)); + Assertions.assertTrue(oldLoadStarted.await(10, TimeUnit.SECONDS)); + + cache.invalidateDb(1, 10); + Assertions.assertEquals(2000L, cache.getCachedRowCountIfPresent(1, 20, 100)); + Assertions.assertEquals(1002L, cache.getCachedRowCount(1, 10, 100, false)); + releaseOldLoad.countDown(); + + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, oldRead.get(10, TimeUnit.SECONDS)); + Assertions.assertEquals(1002L, cache.getCachedRowCountIfPresent(1, 10, 100)); + Assertions.assertEquals(2000L, cache.getCachedRowCountIfPresent(1, 20, 100)); + } finally { + releaseOldLoad.countDown(); + readerExecutor.shutdownNow(); + cacheExecutor.shutdownNow(); + } + } + + @Test + public void testInvalidatedNullLoadReturnsUnknownWithoutReusingIt() throws Exception { + AtomicInteger loadCount = new AtomicInteger(); + CountDownLatch loadStarted = new CountDownLatch(1); + CountDownLatch releaseLoad = new CountDownLatch(1); + ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() { + @Override + protected Optional doLoad(ExternalRowCountCache.RowCountKey key) { + if (loadCount.incrementAndGet() == 1) { + loadStarted.countDown(); + Uninterruptibles.awaitUninterruptibly(releaseLoad); + } + return Optional.of(100L); + } + }; + ExecutorService cacheExecutor = Executors.newSingleThreadExecutor(); + ExecutorService readerExecutor = Executors.newSingleThreadExecutor(); + try { + ExternalRowCountCache cache = new ExternalRowCountCache(cacheExecutor, null, loader); + Future oldRead = readerExecutor.submit(() -> cache.getCachedRowCount(1, 10, 100, false)); + Assertions.assertTrue(loadStarted.await(10, TimeUnit.SECONDS)); + + cache.invalidateTable(1, 10, 100); + releaseLoad.countDown(); + + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, oldRead.get(10, TimeUnit.SECONDS)); + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, cache.getCachedRowCountIfPresent(1, 10, 100)); + Assertions.assertEquals(100L, cache.getCachedRowCount(1, 10, 100, false)); + Assertions.assertEquals(2, loadCount.get()); + } finally { + releaseLoad.countDown(); + readerExecutor.shutdownNow(); + cacheExecutor.shutdownNow(); + } + } + @Test public void testPermanentCatalogRemovalReleasesGenerationWithoutRevivingOldReader() throws Exception { CountDownLatch loadStarted = new CountDownLatch(1); From 43a27b06fa03afaa1db51197d6f9ee58e69b2159 Mon Sep 17 00:00:00 2001 From: 924060929 Date: Wed, 30 Sep 2026 09:57:27 +0800 Subject: [PATCH 16/16] [fix](fe) Fence cold external refresh identities ### What problem does this PR solve? Issue Number: None Related PR: #68196 Problem Summary: On a mode-2 Paimon follower, a names refresh can rebind a logged Foo database to replacement FOO while a cold REFRESH replay still needs to retire SDK-only Foo handles and old Foo row counts. Resolve this ambiguous replay with an opening and closing catalog row-count fence around Paimon SDK retirement. A held table can also read after permanent DROP and recreate a row-count catalog generation; bound that registry so repeated overlapping drops cannot grow FE memory indefinitely. Finally, no-op DROP matching used equalsIgnoreCase, which equates I with dotless i even though catalog lookup's Locale.ROOT fold distinguishes them; use the same fold for resident identity matching. ### Release note Cold Paimon REFRESH replay and no-op database DROP now preserve the correct external metadata identity; late row-count reads after catalog DROP remain memory bounded. ### Check List (For Author) - Test: Unit Test: four related FE test classes, 153 tests passed; full ./build.sh --fe -j4 and Checkstyle passed - Behavior changed: Yes, ambiguous Paimon replay retires old SDK and row-count state, no-op DROP uses catalog name folding, and catalog generation state is bounded - Does this need documentation: No --- .../apache/doris/catalog/RefreshManager.java | 30 ++++++++-- .../datasource/ExternalRowCountCache.java | 29 ++++++---- .../doris/datasource/metacache/MetaCache.java | 4 +- .../doris/catalog/RefreshManagerTest.java | 31 ++++++++++ .../doris/datasource/CatalogMgrTest.java | 8 +++ .../datasource/ExternalRowCountCacheTest.java | 25 +++++++++ .../paimon/PaimonExternalMetaCacheTest.java | 56 +++++++++++++++++++ 7 files changed, 167 insertions(+), 16 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java index c1d671fc5ccd35..bc8495c78a20cd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java @@ -258,7 +258,7 @@ public void replayRefreshTable(ExternalObjectLog log) { Env.getCurrentEnv().getExtMetaCacheMgr().invalidateDb( catalog.getId(), db.get().getId(), db.get().getFullName()); } - invalidatePaimonCatalogForUnresolvedReplay(catalog); + invalidatePaimonCatalogForReplay(catalog); return; } if (!Strings.isNullOrEmpty(log.getNewTableName())) { @@ -378,7 +378,7 @@ private void replayRefreshSafely(String operation, Runnable action) { } } - private void invalidatePaimonCatalogForUnresolvedReplay(ExternalCatalog catalog) { + private void invalidatePaimonCatalogForReplay(ExternalCatalog catalog) { if (catalog instanceof PaimonExternalCatalog) { Env.getCurrentEnv().getExtMetaCacheMgr() .invalidateCatalogByEngine(catalog.getId(), PaimonExternalMetaCache.ENGINE); @@ -392,14 +392,34 @@ private void invalidateColdReplayDatabase(ExternalCatalog catalog, ExternalObjec // known. Retire only that database's engine entries and independent row counts. LOG.debug("database object is cold when replaying refresh in catalog {}, db {}", catalog.getName(), identity.get().first); - Env.getCurrentEnv().getExtMetaCacheMgr() - .invalidateDb(catalog.getId(), identity.get().second, identity.get().first); + // A mode-2 names refresh can rebind the logged Foo to a new FOO. The current + // identity cannot fence an SDK-only Foo handle or Foo's independent row counts. + boolean reboundPaimonName = catalog instanceof PaimonExternalCatalog + && catalog.getLowerCaseDatabaseNames() == 2 + && !Strings.isNullOrEmpty(log.getDbName()) + && !identity.get().first.equals(log.getDbName()); + ExternalMetaCacheMgr cacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + if (reboundPaimonName) { + cacheMgr.invalidateRowCountCache(catalog.getId()); + } + try { + cacheMgr.invalidateDb(catalog.getId(), identity.get().second, identity.get().first); + } finally { + if (reboundPaimonName) { + try { + invalidatePaimonCatalogForReplay(catalog); + } finally { + // Close the load window opened after the first fence. + cacheMgr.invalidateRowCountCache(catalog.getId()); + } + } + } } else { // A lost mode-2 mapping cannot identify the database safely; widen the fence. LOG.warn("failed to resolve database identity when replaying refresh: {}", log.debugForRefreshDb()); Env.getCurrentEnv().getExtMetaCacheMgr().invalidateCatalog(catalog.getId()); - invalidatePaimonCatalogForUnresolvedReplay(catalog); + invalidatePaimonCatalogForReplay(catalog); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java index aba16c65547a3a..f27543c2f2794a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalRowCountCache.java @@ -52,11 +52,12 @@ public class ExternalRowCountCache { private final ConcurrentHashMap> inFlightLoads = new ConcurrentHashMap<>(); // A catalog refresh changes this generation in O(1). Old cache entries remain bounded by // Caffeine's size/expiry policy but are never addressable by a new metadata generation. - private final ConcurrentHashMap catalogGenerations = new ConcurrentHashMap<>(); + // A held table may issue a late read after permanent DROP releases its catalog generation. + // Keep even those post-DROP admissions bounded; eviction forces a fresh unique generation. + private final Cache catalogGenerations; // Database generations are bounded independently of database CREATE/DROP churn. Eviction // merely forces a new, globally unique generation and makes the old entries unreachable. - private final Cache dbGenerations = Caffeine.newBuilder() - .maximumSize(Config.max_external_table_row_count_cache_num).build(); + private final Cache dbGenerations; // Generations are globally unique: two catalog IDs with the same table ID must not alias // when one catalog has retired its old cache generation. private final AtomicLong nextCatalogGeneration = new AtomicLong(); @@ -73,6 +74,13 @@ public ExternalRowCountCache(ExecutorService executor) { } ExternalRowCountCache(ExecutorService executor, Ticker ticker, RowCountCacheLoader loader) { + this(executor, ticker, loader, Config.max_external_table_row_count_cache_num); + } + + ExternalRowCountCache(ExecutorService executor, Ticker ticker, RowCountCacheLoader loader, + long maxGenerationEntries) { + catalogGenerations = Caffeine.newBuilder().maximumSize(maxGenerationEntries).build(); + dbGenerations = Caffeine.newBuilder().maximumSize(maxGenerationEntries).build(); // 1. set expireAfterWrite to 1 day, avoid too many entries // 2. set refreshAfterWrite to 10min(default), so that the cache will be refreshed after 10min CacheFactory rowCountCacheFactory = new CacheFactory( @@ -276,12 +284,12 @@ void refreshForTest(long catalogId, long dbId, long tableId) { } private long currentCatalogGeneration(long catalogId) { - return catalogGenerations.computeIfAbsent(catalogId, + return catalogGenerations.get(catalogId, ignored -> new AtomicLong(nextCatalogGeneration.incrementAndGet())).get(); } private boolean isCatalogGenerationCurrent(long catalogId, long generation) { - AtomicLong current = catalogGenerations.get(catalogId); + AtomicLong current = catalogGenerations.getIfPresent(catalogId); return current != null && current.get() == generation; } @@ -296,7 +304,8 @@ private boolean isDbGenerationCurrent(long catalogId, long dbId, long generation } int getCatalogGenerationCountForTest() { - return catalogGenerations.size(); + catalogGenerations.cleanUp(); + return catalogGenerations.asMap().size(); } static Optional loadRowCount(RowCountKey rowCountKey, boolean fillMetaCache) { @@ -380,7 +389,7 @@ public long getCachedRowCountIfPresent(long catalogId, long dbId, long tableId) long dbGeneration; publicationLock.readLock().lock(); try { - AtomicLong current = catalogGenerations.get(catalogId); + AtomicLong current = catalogGenerations.getIfPresent(catalogId); if (current == null) { return -1; } @@ -413,7 +422,7 @@ && isDbGenerationCurrent(catalogId, dbId, dbGeneration)) { void invalidateCatalog(long catalogId) { publicationLock.writeLock().lock(); try { - AtomicLong current = catalogGenerations.get(catalogId); + AtomicLong current = catalogGenerations.getIfPresent(catalogId); if (current != null) { current.set(nextCatalogGeneration.incrementAndGet()); } @@ -426,7 +435,7 @@ void invalidateCatalog(long catalogId) { void releaseCatalog(long catalogId) { publicationLock.writeLock().lock(); try { - catalogGenerations.remove(catalogId); + catalogGenerations.invalidate(catalogId); } finally { publicationLock.writeLock().unlock(); } @@ -448,7 +457,7 @@ void invalidateDb(long catalogId, long dbId) { void invalidateTable(long catalogId, long dbId, long tableId) { publicationLock.writeLock().lock(); try { - AtomicLong current = catalogGenerations.get(catalogId); + AtomicLong current = catalogGenerations.getIfPresent(catalogId); if (current == null) { return; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java index bfa291bedfbbc5..cf4ea6b66fbbf9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCache.java @@ -31,6 +31,7 @@ import org.apache.logging.log4j.Logger; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.Optional; @@ -560,8 +561,9 @@ public Optional getNameByIdIfPresent(long id) { /** Snapshot resident object identities matching a DROP alias under the catalog's name mode. */ public List> getCachedIdentitiesMatching(String name, boolean ignoreCase) { + String foldedName = ignoreCase ? name.toLowerCase(Locale.ROOT) : name; return withMetaObjLifecycleReadLock(() -> idToName.entrySet().stream() - .filter(entry -> ignoreCase ? entry.getValue().equalsIgnoreCase(name) + .filter(entry -> ignoreCase ? entry.getValue().toLowerCase(Locale.ROOT).equals(foldedName) : entry.getValue().equals(name)) .map(entry -> Pair.of(entry.getValue(), entry.getKey())) .collect(Collectors.toList())); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java index 776ebe580af694..f19a1a4c4acda7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/RefreshManagerTest.java @@ -28,6 +28,8 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveExternalMetaCache; import org.apache.doris.datasource.lance.LanceExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonExternalMetaCache; import org.apache.doris.persist.EditLog; import org.junit.jupiter.api.Assertions; @@ -74,6 +76,35 @@ void testColdKnownDatabaseReplayKeepsUnrelatedHotDatabase() { Mockito.verify(hotDb, Mockito.never()).resetMetaToUninitialized(); } + @Test + void testColdReboundPaimonReplayFencesOldRowCountsAroundSdkInvalidation() { + long catalogId = 83L; + long replacementDbId = 84L; + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.getLowerCaseDatabaseNames()).thenReturn(2); + Mockito.when(catalog.getDbForReplay("Foo")).thenReturn(Optional.empty()); + Mockito.when(catalog.getDbIdentityForReplay("Foo", 0L)) + .thenReturn(Optional.of(Pair.of("FOO", replacementDbId))); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(catalogId); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + new RefreshManager().replayRefreshDb(ExternalObjectLog.createForRefreshDb(catalogId, "Foo")); + } + + InOrder order = Mockito.inOrder(cacheMgr); + order.verify(cacheMgr).invalidateRowCountCache(catalogId); + order.verify(cacheMgr).invalidateDb(catalogId, replacementDbId, "FOO"); + order.verify(cacheMgr).invalidateCatalogByEngine(catalogId, PaimonExternalMetaCache.ENGINE); + order.verify(cacheMgr).invalidateRowCountCache(catalogId); + } + @Test void testColdDatabaseReplayInvalidatesCatalogRowCount() { long catalogId = 51L; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 40bb5817ccaefa..0fa144a6ef26c9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -967,10 +967,13 @@ void testNoOpDatabaseDropKeepsUnrelatedCachedDatabase() { ignored -> Collections.emptyList(), ignored -> Optional.empty(), (key, value, cause) -> { }); long oldId = Util.genIdByName("testing_catalog", "Foo"); long unrelatedId = Util.genIdByName("testing_catalog", "sales"); + long turkishId = Util.genIdByName("testing_catalog", "I"); ExternalDatabase oldDb = Mockito.mock(ExternalDatabase.class); ExternalDatabase unrelatedDb = Mockito.mock(ExternalDatabase.class); + ExternalDatabase turkishDb = Mockito.mock(ExternalDatabase.class); metaCache.addObjForTest(oldId, "Foo", oldDb); metaCache.addObjForTest(unrelatedId, "sales", unrelatedDb); + metaCache.addObjForTest(turkishId, "I", turkishDb); catalog.installMetaCache(metaCache); Env env = Mockito.mock(Env.class); ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); @@ -979,12 +982,17 @@ void testNoOpDatabaseDropKeepsUnrelatedCachedDatabase() { mockedEnv.when(Env::getCurrentEnv).thenReturn(env); catalog.retireCachedDatabaseForNoOp("absent"); Assertions.assertSame(unrelatedDb, metaCache.tryGetMetaObj("sales").orElseThrow(AssertionError::new)); + // Locale.ROOT distinguishes I (i) from dotless i. A no-op DROP of the latter + // must not retire the resident I database. + catalog.retireCachedDatabaseForNoOp("ı"); + Assertions.assertSame(turkishDb, metaCache.tryGetMetaObj("I").orElseThrow(AssertionError::new)); Mockito.verifyNoInteractions(cacheMgr); // A lost mode-2 mapping still permits narrow retirement of a retained historical ID. catalog.retireCachedDatabaseForNoOp("foo"); Assertions.assertFalse(metaCache.tryGetMetaObj("Foo").isPresent()); Assertions.assertSame(unrelatedDb, metaCache.tryGetMetaObj("sales").orElseThrow(AssertionError::new)); + Assertions.assertSame(turkishDb, metaCache.tryGetMetaObj("I").orElseThrow(AssertionError::new)); Mockito.verify(cacheMgr).invalidateDb(catalogId, oldId, "Foo"); Mockito.verify(cacheMgr, Mockito.never()).invalidateCatalog(catalogId); Mockito.verify(cacheMgr, Mockito.never()).invalidateRowCountCache(catalogId, unrelatedId); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java index ea560602fce3b4..4d7451c64b0b16 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalRowCountCacheTest.java @@ -445,6 +445,31 @@ protected Optional doLoad(ExternalRowCountCache.RowCountKey key) { } } + @Test + public void testLateReadsAfterPermanentRemovalDoNotGrowCatalogGenerations() { + java.util.Set activeCatalogs = new java.util.HashSet<>(); + ExternalRowCountCache.RowCountCacheLoader loader = new ExternalRowCountCache.RowCountCacheLoader() { + @Override + protected Optional doLoad(ExternalRowCountCache.RowCountKey key) { + return activeCatalogs.contains(key.getCatalogId()) ? Optional.of(100L) : null; + } + }; + ExternalRowCountCache cache = new ExternalRowCountCache( + MoreExecutors.newDirectExecutorService(), null, loader, 2); + for (long catalogId = 1; catalogId <= 5; catalogId++) { + activeCatalogs.add(catalogId); + Assertions.assertEquals(100L, cache.getCachedRowCount(catalogId, 10, 100, false)); + activeCatalogs.remove(catalogId); + cache.invalidateCatalog(catalogId); + cache.releaseCatalog(catalogId); + + // A statement retaining an old table can still enter the cache after DROP cleanup. + Assertions.assertEquals(TableIf.UNKNOWN_ROW_COUNT, + cache.getCachedRowCount(catalogId, 10, 100, false)); + Assertions.assertTrue(cache.getCatalogGenerationCountForTest() <= 2); + } + } + @Test public void testInvalidateWhileRefreshIsRunningDoesNotRepublishStaleValue() throws Exception { AtomicInteger loadCount = new AtomicInteger(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 52854171e3511f..4608013b5fbf4d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -2571,6 +2571,62 @@ public void testReplayRefreshInvalidatesSdkOnlyPaimonCache() throws Exception { } } + @Test + public void testMode2ColdReplayRefreshInvalidatesReboundSdkOnlyPaimonHandle() throws Exception { + java.io.File warehouse = temporaryFolder.newFolder("mode2_rebound_refresh_sdk_only"); + Map properties = new HashMap<>(); + properties.put("type", "paimon"); + properties.put(PaimonExternalCatalog.PAIMON_CATALOG_TYPE, + PaimonExternalCatalog.PAIMON_FILESYSTEM); + properties.put("warehouse", warehouse.toURI().toString()); + properties.put(ExternalCatalog.LOWER_CASE_DATABASE_NAMES, "2"); + long catalogId = 120L; + PaimonExternalCatalog dorisCatalog = new PaimonExternalCatalog( + catalogId, "mode2_rebound_refresh_test", null, properties, ""); + dorisCatalog.makeSureInitialized(); + dorisCatalog.catalog.createDatabase("Foo", false); + Identifier oldIdentifier = Identifier.create("Foo", "table"); + Schema schema = Schema.newBuilder().column("id", DataTypes.INT()).build(); + dorisCatalog.catalog.createTable(oldIdentifier, schema, false); + + Map externalProperties = new HashMap<>(properties); + externalProperties.put("paimon.cache-enabled", "false"); + PaimonExternalCatalog externalCatalog = new PaimonExternalCatalog( + 121L, "external_paimon", null, externalProperties, ""); + externalCatalog.makeSureInitialized(); + + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.when(env.getAccessManager()).thenReturn(Mockito.mock(AccessControllerManager.class)); + ExternalMetaCacheMgr cacheMgr = new ExternalMetaCacheMgr(true); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.doReturn(dorisCatalog).when(catalogMgr).getCatalog(catalogId); + Mockito.doReturn(dorisCatalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(catalogId), Mockito.any()); + NameMapping oldMapping = new NameMapping(catalogId, "Foo", "table", "Foo", "table"); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + dorisCatalog.getPaimonTable(oldMapping); + externalCatalog.catalog.dropTable(oldIdentifier, false); + externalCatalog.catalog.dropDatabase("Foo", false, false); + externalCatalog.catalog.createDatabase("FOO", false); + + Field mappingField = ExternalCatalog.class.getDeclaredField("lowerCaseToDatabaseName"); + mappingField.setAccessible(true); + Map reboundNames = new HashMap<>(); + reboundNames.put("foo", "FOO"); + mappingField.set(dorisCatalog, reboundNames); + Assert.assertFalse(dorisCatalog.getDbForReplay("Foo").isPresent()); + + new RefreshManager().replayRefreshDb(ExternalObjectLog.createForRefreshDb(catalogId, "Foo")); + assertPaimonCatalogTableMissing(dorisCatalog, oldMapping); + } finally { + dorisCatalog.catalog.close(); + externalCatalog.catalog.close(); + } + } + @Test public void testReplayDropInvalidatesSdkOnlyPaimonCache() throws Exception { java.io.File warehouse = temporaryFolder.newFolder("replay_drop_sdk_only");