You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/04/14 13:19:25 UTC
[doris] 01/04: [Fix](multi-catalog) invalidates the file cache when table is non-partitioned. (#17932)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit c911e688de57a3c3b992bc082ba1bf5df9543f90
Author: Xiangyu Wang <du...@gmail.com>
AuthorDate: Wed Mar 22 23:34:18 2023 +0800
[Fix](multi-catalog) invalidates the file cache when table is non-partitioned. (#17932)
Reference to `org.apache.doris.planner.external.HiveSplitter`, the file cache of `HiveMetaStoreCache`
may be created even the table is a non-partitioned table,
so the `RefreshTableStmt` should consider this scene and handle it.
---
.../org/apache/doris/datasource/hive/HiveMetaStoreCache.java | 12 ++++++++++++
1 file changed, 12 insertions(+)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index 31b8d2f3b4..a2fb14de39 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
+import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.InputFormat;
import org.apache.hadoop.mapred.InputSplit;
@@ -374,6 +375,17 @@ public class HiveMetaStoreCache {
LOG.debug("invalid table cache for {}.{} in catalog {}, cache num: {}, cost: {} ms",
dbName, tblName, catalog.getName(), partitionValues.partitionValuesMap.size(),
(System.currentTimeMillis() - start));
+ } else {
+ /**
+ * A file cache entry can be created reference to
+ * {@link org.apache.doris.planner.external.HiveSplitter#getSplits},
+ * so we need to invalidate it if this is a non-partitioned table.
+ *
+ * */
+ Table table = catalog.getClient().getTable(dbName, tblName);
+ // we just need to assign the `location` filed because the `equals` method of `FileCacheKey`
+ // just compares the value of `location`
+ fileCache.invalidate(new FileCacheKey(table.getSd().getLocation(), null));
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org