You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2025/05/19 03:36:46 UTC
(doris) branch master updated: [feat](hive) add catalog level schema cache property (#50958)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2eee6d7c8a0 [feat](hive) add catalog level schema cache property (#50958)
2eee6d7c8a0 is described below
commit 2eee6d7c8a0adf9f6d8c154dc2200d1b329151aa
Author: Mingyu Chen (Rayner) <mo...@163.com>
AuthorDate: Mon May 19 11:36:38 2025 +0800
[feat](hive) add catalog level schema cache property (#50958)
### What problem does this PR solve?
Problem Summary:
Just same as #50724, support enable or disable schema cache at Catalog
level for all kinds of external catalogs.
Previously, if user want to disable the schema cache, they can only set
the
`max_external_schema_cache_num=0` in fe.conf and restart FE.
And this config will effect all catalogs.
In this PR, I add a new catalog property `schema.cache.ttl-second`.
If set to 0, the schema cache will be disabled, so if schema is changed
Doris will read the new schema immediately.
---
.../apache/doris/datasource/ExternalCatalog.java | 29 +++++++--
.../doris/datasource/ExternalMetaCacheMgr.java | 35 +++++++----
.../doris/datasource/ExternalSchemaCache.java | 8 ++-
.../doris/datasource/hive/HMSExternalCatalog.java | 5 --
.../doris/datasource/hive/HiveMetaStoreCache.java | 9 +--
.../hive/test_hive_meta_cache.out | Bin 858 -> 1649 bytes
.../hive/test_hive_meta_cache.groovy | 67 +++++++++++++++++++++
7 files changed, 124 insertions(+), 29 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 01114899053..9e9024c038a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -81,6 +81,7 @@ import com.google.gson.annotations.SerializedName;
import lombok.Data;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.math.NumberUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -123,6 +124,11 @@ public abstract class ExternalCatalog
// https://help.aliyun.com/zh/emr/emr-on-ecs/user-guide/use-rootpolicy-to-access-oss-hdfs?spm=a2c4g.11186623.help-menu-search-28066.d_0
public static final String OOS_ROOT_POLICY = "oss.root_policy";
+ public static final String SCHEMA_CACHE_TTL_SECOND = "schema.cache.ttl-second";
+ // -1 means cache with no ttl
+ public static final int CACHE_NO_TTL = -1;
+ // 0 means cache is disabled; >0 means cache with ttl;
+ public static final int CACHE_TTL_DISABLE_CACHE = 0;
// Properties that should not be shown in the `show create catalog` result
public static final Set<String> HIDDEN_PROPERTIES = Sets.newHashSet(
@@ -335,7 +341,7 @@ public abstract class ExternalCatalog
Map<String, String> properties = getCatalogProperty().getProperties();
if (properties.containsKey(CatalogMgr.METADATA_REFRESH_INTERVAL_SEC)) {
try {
- Integer metadataRefreshIntervalSec = Integer.valueOf(
+ int metadataRefreshIntervalSec = Integer.parseInt(
properties.get(CatalogMgr.METADATA_REFRESH_INTERVAL_SEC));
if (metadataRefreshIntervalSec < 0) {
throw new DdlException("Invalid properties: " + CatalogMgr.METADATA_REFRESH_INTERVAL_SEC);
@@ -345,11 +351,13 @@ public abstract class ExternalCatalog
}
}
- // if (properties.getOrDefault(ExternalCatalog.USE_META_CACHE, "true").equals("false")) {
- // LOG.warn("force to set use_meta_cache to true for catalog: {} when creating", name);
- // getCatalogProperty().addProperty(ExternalCatalog.USE_META_CACHE, "true");
- // useMetaCache = Optional.of(true);
- // }
+ // check schema.cache.ttl-second parameter
+ String schemaCacheTtlSecond = catalogProperty.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null);
+ if (java.util.Objects.nonNull(schemaCacheTtlSecond) && NumberUtils.toInt(schemaCacheTtlSecond, CACHE_NO_TTL)
+ < CACHE_TTL_DISABLE_CACHE) {
+ throw new DdlException(
+ "The parameter " + SCHEMA_CACHE_TTL_SECOND + " is wrong, value is " + schemaCacheTtlSecond);
+ }
}
/**
@@ -1250,4 +1258,13 @@ public abstract class ExternalCatalog
public int hashCode() {
return Objects.hashCode(name);
}
+
+ @Override
+ public void notifyPropertiesUpdated(Map<String, String> updatedProps) {
+ CatalogIf.super.notifyPropertiesUpdated(updatedProps);
+ String schemaCacheTtl = updatedProps.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null);
+ if (java.util.Objects.nonNull(schemaCacheTtl)) {
+ Env.getCurrentEnv().getExtMetaCacheMgr().invalidSchemaCache(id);
+ }
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
index 3e0c4d90859..28115303dff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java
@@ -89,7 +89,7 @@ public class ExternalMetaCacheMgr {
// catalog id -> HiveMetaStoreCache
private final Map<Long, HiveMetaStoreCache> cacheMap = Maps.newConcurrentMap();
// catalog id -> table schema cache
- private Map<Long, ExternalSchemaCache> schemaCacheMap = Maps.newHashMap();
+ private final Map<Long, ExternalSchemaCache> schemaCacheMap = Maps.newHashMap();
// hudi partition manager
private final HudiMetadataCacheMgr hudiMetadataCacheMgr;
// all catalogs could share the same fsCache.
@@ -222,8 +222,10 @@ public class ExternalMetaCacheMgr {
if (cacheMap.remove(catalogId) != null) {
LOG.info("remove hive metastore cache for catalog {}", catalogId);
}
- if (schemaCacheMap.remove(catalogId) != null) {
- LOG.info("remove schema cache for catalog {}", catalogId);
+ synchronized (schemaCacheMap) {
+ if (schemaCacheMap.remove(catalogId) != null) {
+ LOG.info("remove schema cache for catalog {}", catalogId);
+ }
}
hudiMetadataCacheMgr.removeCache(catalogId);
icebergMetadataCacheMgr.removeCache(catalogId);
@@ -233,9 +235,11 @@ public class ExternalMetaCacheMgr {
public void invalidateTableCache(long catalogId, String dbName, String tblName) {
dbName = ClusterNamespace.getNameFromFullName(dbName);
- ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId);
- if (schemaCache != null) {
- schemaCache.invalidateTableCache(dbName, tblName);
+ synchronized (schemaCacheMap) {
+ ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId);
+ if (schemaCache != null) {
+ schemaCache.invalidateTableCache(dbName, tblName);
+ }
}
HiveMetaStoreCache metaCache = cacheMap.get(catalogId);
if (metaCache != null) {
@@ -252,9 +256,11 @@ public class ExternalMetaCacheMgr {
public void invalidateDbCache(long catalogId, String dbName) {
dbName = ClusterNamespace.getNameFromFullName(dbName);
- ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId);
- if (schemaCache != null) {
- schemaCache.invalidateDbCache(dbName);
+ synchronized (schemaCacheMap) {
+ ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId);
+ if (schemaCache != null) {
+ schemaCache.invalidateDbCache(dbName);
+ }
}
HiveMetaStoreCache metaCache = cacheMap.get(catalogId);
if (metaCache != null) {
@@ -270,9 +276,8 @@ public class ExternalMetaCacheMgr {
}
public void invalidateCatalogCache(long catalogId) {
- ExternalSchemaCache schemaCache = schemaCacheMap.get(catalogId);
- if (schemaCache != null) {
- schemaCache.invalidateAll();
+ synchronized (schemaCacheMap) {
+ schemaCacheMap.remove(catalogId);
}
HiveMetaStoreCache metaCache = cacheMap.get(catalogId);
if (metaCache != null) {
@@ -287,6 +292,12 @@ public class ExternalMetaCacheMgr {
}
}
+ public void invalidSchemaCache(long catalogId) {
+ synchronized (schemaCacheMap) {
+ schemaCacheMap.remove(catalogId);
+ }
+ }
+
public void addPartitionsCache(long catalogId, HMSExternalTable table, List<String> partitionNames) {
String dbName = ClusterNamespace.getNameFromFullName(table.getDbName());
HiveMetaStoreCache metaCache = cacheMap.get(catalogId);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
index d0673da56fd..da9a25d71be 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalSchemaCache.java
@@ -28,6 +28,7 @@ import org.apache.doris.metric.MetricRepo;
import com.github.benmanes.caffeine.cache.LoadingCache;
import com.google.common.collect.ImmutableList;
import lombok.Data;
+import org.apache.commons.lang3.math.NumberUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -51,13 +52,16 @@ public class ExternalSchemaCache {
}
private void init(ExecutorService executor) {
+ long schemaCacheTtlSecond = NumberUtils.toLong(
+ (catalog.getProperties().get(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND)), ExternalCatalog.CACHE_NO_TTL);
CacheFactory schemaCacheFactory = new CacheFactory(
- OptionalLong.of(86400L),
+ OptionalLong.of(schemaCacheTtlSecond >= ExternalCatalog.CACHE_TTL_DISABLE_CACHE
+ ? schemaCacheTtlSecond : 86400),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60),
Config.max_external_schema_cache_num,
false,
null);
- schemaCache = schemaCacheFactory.buildCache(key -> loadSchema(key), null, executor);
+ schemaCache = schemaCacheFactory.buildCache(this::loadSchema, null, executor);
}
private void initMetrics() {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
index 9ff043d3245..96405bbf1e9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
@@ -76,11 +76,6 @@ public class HMSExternalCatalog extends ExternalCatalog {
// from remoteTable object.
public static final String GET_SCHEMA_FROM_TABLE = "get_schema_from_table";
- // -1 means cache with no ttl
- public static final int CACHE_NO_TTL = -1;
- // 0 means cache is disabled; >0 means cache with ttl;
- public static final int CACHE_TTL_DISABLE_CACHE = 0;
-
private static final int FILE_SYSTEM_EXECUTOR_THREAD_NUM = 16;
private ThreadPoolExecutor fileSystemExecutor;
@Getter
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 1e755843079..f7e5cf2bc5f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -37,6 +37,7 @@ import org.apache.doris.common.util.CacheBulkLoader;
import org.apache.doris.common.util.LocationPath;
import org.apache.doris.common.util.Util;
import org.apache.doris.datasource.CacheException;
+import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalMetaCacheMgr;
import org.apache.doris.datasource.property.PropertyConverter;
import org.apache.doris.fs.DirectoryLister;
@@ -134,10 +135,10 @@ public class HiveMetaStoreCache {
public void init() {
long partitionCacheTtlSecond = NumberUtils.toLong(
(catalog.getProperties().get(HMSExternalCatalog.PARTITION_CACHE_TTL_SECOND)),
- HMSExternalCatalog.CACHE_NO_TTL);
+ ExternalCatalog.CACHE_NO_TTL);
CacheFactory partitionValuesCacheFactory = new CacheFactory(
- OptionalLong.of(partitionCacheTtlSecond >= HMSExternalCatalog.CACHE_TTL_DISABLE_CACHE
+ OptionalLong.of(partitionCacheTtlSecond >= ExternalCatalog.CACHE_TTL_DISABLE_CACHE
? partitionCacheTtlSecond : 28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_hive_partition_table_cache_num,
@@ -176,10 +177,10 @@ public class HiveMetaStoreCache {
// if the file.meta.cache.ttl-second is equal or greater than 0, the cache expired will be set to that value
int fileMetaCacheTtlSecond = NumberUtils.toInt(
(catalog.getProperties().get(HMSExternalCatalog.FILE_META_CACHE_TTL_SECOND)),
- HMSExternalCatalog.CACHE_NO_TTL);
+ ExternalCatalog.CACHE_NO_TTL);
CacheFactory fileCacheFactory = new CacheFactory(
- OptionalLong.of(fileMetaCacheTtlSecond >= HMSExternalCatalog.CACHE_TTL_DISABLE_CACHE
+ OptionalLong.of(fileMetaCacheTtlSecond >= ExternalCatalog.CACHE_TTL_DISABLE_CACHE
? fileMetaCacheTtlSecond : 28800L),
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
Config.max_external_file_cache_num,
diff --git a/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out b/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out
index 7ab9a456bdc..ac5d540cdce 100644
Binary files a/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out and b/regression-test/data/external_table_p0/hive/test_hive_meta_cache.out differ
diff --git a/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy b/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy
index 3b6655f6e39..113f90707e9 100644
--- a/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy
+++ b/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy
@@ -229,6 +229,73 @@ suite("test_hive_meta_cache", "p0,external,hive,external_docker,external_docker_
// select 5 rows
order_qt_sql_5row """select * from test_hive_meta_cache_db.sales"""
sql """drop table test_hive_meta_cache_db.sales"""
+
+ // test schema cache
+ sql """drop catalog if exists ${catalog_name_no_cache};"""
+ // 1. create catalog with default property fisrt
+ sql """
+ create catalog ${catalog_name_no_cache} properties (
+ 'type'='hms',
+ 'hadoop.username' = 'hadoop',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}'
+ );
+ """
+ sql """switch ${catalog_name_no_cache}"""
+ hive_docker """drop database if exists test_hive_meta_cache_db CASCADE"""
+ hive_docker """create database test_hive_meta_cache_db"""
+ hive_docker """
+ CREATE TABLE test_hive_meta_cache_db.sales (
+ id INT,
+ amount DOUBLE
+ )
+ PARTITIONED BY (year INT)
+ STORED AS PARQUET;
+ """
+ // desc table, 3 columns
+ qt_sql_3col "desc test_hive_meta_cache_db.sales";
+ // add a new column in hive
+ hive_docker "alter table test_hive_meta_cache_db.sales add columns(k3 string)"
+ // desc table, still 3 columns
+ qt_sql_3col "desc test_hive_meta_cache_db.sales";
+ // refresh and check
+ sql "refresh table test_hive_meta_cache_db.sales";
+ // desc table, 4 columns
+ qt_sql_4col "desc test_hive_meta_cache_db.sales";
+
+ // create catalog without schema cache
+ sql """drop catalog if exists ${catalog_name_no_cache};"""
+ sql """
+ create catalog ${catalog_name_no_cache} properties (
+ 'type'='hms',
+ 'hadoop.username' = 'hadoop',
+ 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}',
+ 'fs.defaultFS' = 'hdfs://${externalEnvIp}:${hdfs_port}',
+ 'schema.cache.ttl-second' = '0'
+ );
+ """
+ sql """switch ${catalog_name_no_cache}"""
+ // desc table, 4 columns
+ qt_sql_4col "desc test_hive_meta_cache_db.sales";
+ // add a new column in hive
+ hive_docker "alter table test_hive_meta_cache_db.sales add columns(k4 string)"
+ // desc table, 5 columns
+ qt_sql_5col "desc test_hive_meta_cache_db.sales";
+
+ // modify property
+ // alter wrong catalog property
+ test {
+ sql """alter catalog ${catalog_name_no_cache} set properties ("schema.cache.ttl-second" = "-2")"""
+ exception "is wrong"
+ }
+ sql """alter catalog ${catalog_name_no_cache} set properties ("schema.cache.ttl-second" = "0")"""
+ // desc table, 5 columns
+ qt_sql_5col "desc test_hive_meta_cache_db.sales";
+ // add a new column in hive
+ hive_docker "alter table test_hive_meta_cache_db.sales add columns(k5 string)"
+ // desc table, 6 columns
+ qt_sql_6col "desc test_hive_meta_cache_db.sales";
+ sql """drop table test_hive_meta_cache_db.sales"""
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org