You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/06/13 12:33:11 UTC
[doris] branch branch-1.2-lts updated: [Fix](multi-catalog) fix FE abnormal exit when replay OP_REFRESH_EXTERNAL_TABLE (#20760)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-1.2-lts by this push:
new f2c90eff29 [Fix](multi-catalog) fix FE abnormal exit when replay OP_REFRESH_EXTERNAL_TABLE (#20760)
f2c90eff29 is described below
commit f2c90eff2936ea859ee2c4b6577dd0a863191b6a
Author: zhangdong <49...@qq.com>
AuthorDate: Tue Jun 13 20:33:04 2023 +0800
[Fix](multi-catalog) fix FE abnormal exit when replay OP_REFRESH_EXTERNAL_TABLE (#20760)
cherry-pick #19120
---
.../doris/datasource/hive/HiveMetaStoreCache.java | 37 ++++++++++++++++------
.../doris/datasource/hive/HivePartition.java | 18 +++++++++--
.../doris/planner/external/HiveScanProvider.java | 3 +-
3 files changed, 46 insertions(+), 12 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
index b82653016e..8c266168a0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetaStoreCache.java
@@ -53,7 +53,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
-import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputFormat;
@@ -239,7 +238,7 @@ public class HiveMetaStoreCache {
sd.getInputFormat(), sd.getLocation(), key, catalog.getName());
}
// TODO: more info?
- return new HivePartition(sd.getInputFormat(), sd.getLocation(), key.values);
+ return new HivePartition(key.dbName, key.tblName, false, sd.getInputFormat(), sd.getLocation(), key.values);
}
private ImmutableList<InputSplit> loadFiles(FileCacheKey key) {
@@ -348,8 +347,13 @@ public class HiveMetaStoreCache {
public List<InputSplit> getFilesByPartitions(List<HivePartition> partitions) {
long start = System.currentTimeMillis();
List<FileCacheKey> keys = Lists.newArrayListWithExpectedSize(partitions.size());
- partitions.stream().forEach(p -> keys.add(
- new FileCacheKey(p.getPath(), p.getInputFormat(), p.getPartitionValues())));
+ partitions.stream().forEach(p -> {
+ FileCacheKey fileCacheKey = p.isDummyPartition()
+ ? FileCacheKey.createDummyCacheKey(p.getDbName(), p.getTblName(), p.getPath(),
+ p.getInputFormat())
+ : new FileCacheKey(p.getPath(), p.getInputFormat(), p.getPartitionValues());
+ keys.add(fileCacheKey);
+ });
Stream<FileCacheKey> stream;
if (partitions.size() < MIN_BATCH_FETCH_PARTITION_NUM) {
@@ -417,12 +421,13 @@ public class HiveMetaStoreCache {
* A file cache entry can be created reference to
* {@link org.apache.doris.planner.external.HiveSplitter#getSplits},
* so we need to invalidate it if this is a non-partitioned table.
- *
+ * We use {@link org.apache.doris.datasource.hive.HiveMetaStoreCache.FileCacheKey#createDummyCacheKey}
+ * to avoid invocation by Hms Client, because this method may be invoked when salve FE replay journal logs,
+ * and FE will exit if some network problems occur.
* */
- Table table = catalog.getClient().getTable(dbName, tblName);
- // we just need to assign the `location` filed because the `equals` method of `FileCacheKey`
- // just compares the value of `location`
- fileCache.invalidate(new FileCacheKey(table.getSd().getLocation(), null, null));
+ FileCacheKey fileCacheKey = FileCacheKey.createDummyCacheKey(
+ dbName, tblName, null, null);
+ fileCache.invalidate(fileCacheKey);
}
}
@@ -650,6 +655,7 @@ public class HiveMetaStoreCache {
@Data
public static class FileCacheKey {
+ private String dummyKey;
private String location;
// not in key
private String inputFormat;
@@ -673,6 +679,13 @@ public class HiveMetaStoreCache {
}
}
+ public static FileCacheKey createDummyCacheKey(String dbName, String tblName, String location,
+ String inputFormat) {
+ FileCacheKey fileCacheKey = new FileCacheKey(location, inputFormat, null);
+ fileCacheKey.dummyKey = dbName + "." + tblName;
+ return fileCacheKey;
+ }
+
@Override
public boolean equals(Object obj) {
if (this == obj) {
@@ -681,12 +694,18 @@ public class HiveMetaStoreCache {
if (!(obj instanceof FileCacheKey)) {
return false;
}
+ if (dummyKey != null) {
+ return dummyKey.equals(((FileCacheKey) obj).dummyKey);
+ }
return location.equals(((FileCacheKey) obj).location)
&& partitionValues.equals(((FileCacheKey) obj).partitionValues);
}
@Override
public int hashCode() {
+ if (dummyKey != null) {
+ return Objects.hash(dummyKey);
+ }
return Objects.hash(location, partitionValues);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HivePartition.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HivePartition.java
index e5b5178e75..1ee42aba80 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HivePartition.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HivePartition.java
@@ -23,11 +23,18 @@ import java.util.List;
@Data
public class HivePartition {
+ private String dbName;
+ private String tblName;
private String inputFormat;
private String path;
private List<String> partitionValues;
+ private boolean isDummyPartition;
- public HivePartition(String inputFormat, String path, List<String> partitionValues) {
+ public HivePartition(String dbName, String tblName, boolean isDummyPartition,
+ String inputFormat, String path, List<String> partitionValues) {
+ this.dbName = dbName;
+ this.tblName = tblName;
+ this.isDummyPartition = isDummyPartition;
// eg: org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat
this.inputFormat = inputFormat;
// eg: hdfs://hk-dev01:8121/user/doris/parquet/partition_table/nation=cn/city=beijing
@@ -36,10 +43,17 @@ public class HivePartition {
this.partitionValues = partitionValues;
}
+ public boolean isDummyPartition() {
+ return this.isDummyPartition;
+ }
+
@Override
public String toString() {
return "HivePartition{"
- + "inputFormat='" + inputFormat + '\''
+ + "dbName='" + dbName + '\''
+ + ", tblName='" + tblName + '\''
+ + ", isDummyPartition='" + isDummyPartition + '\''
+ + ", inputFormat='" + inputFormat + '\''
+ ", path='" + path + '\''
+ ", partitionValues=" + partitionValues + '}';
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
index cf41de2a86..9aa4ab6dc4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/external/HiveScanProvider.java
@@ -183,7 +183,8 @@ public class HiveScanProvider extends HMSTableScanProvider {
} else {
// unpartitioned table, create a dummy partition to save location and inputformat,
// so that we can unify the interface.
- HivePartition dummyPartition = new HivePartition(hmsTable.getRemoteTable().getSd().getInputFormat(),
+ HivePartition dummyPartition = new HivePartition(hmsTable.getDbName(), hmsTable.getName(), true,
+ hmsTable.getRemoteTable().getSd().getInputFormat(),
hmsTable.getRemoteTable().getSd().getLocation(), null);
getFileSplitByPartitions(cache, Lists.newArrayList(dummyPartition), allFiles);
this.totalPartitionNum = 1;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org