You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2022/06/17 00:49:21 UTC
[incubator-doris] branch master updated: [data lake]Add HMS external data source. (#10088)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2a1d1b951a [data lake]Add HMS external data source. (#10088)
2a1d1b951a is described below
commit 2a1d1b951aeaadad4b0e1a8c265d196d89070012
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Fri Jun 17 08:49:15 2022 +0800
[data lake]Add HMS external data source. (#10088)
---
.../main/java/org/apache/doris/alter/Alter.java | 2 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 2 +-
.../java/org/apache/doris/analysis/Analyzer.java | 2 +-
.../org/apache/doris/analysis/DescribeStmt.java | 2 +-
.../java/org/apache/doris/analysis/SelectStmt.java | 2 +-
.../org/apache/doris/analysis/ShowDataStmt.java | 2 +-
.../org/apache/doris/backup/BackupHandler.java | 2 +-
.../java/org/apache/doris/backup/BackupJob.java | 2 +-
.../org/apache/doris/backup/BackupJobInfo.java | 2 +-
.../java/org/apache/doris/backup/RestoreJob.java | 2 +-
.../java/org/apache/doris/catalog/Catalog.java | 2 +-
.../apache/doris/catalog/CatalogRecycleBin.java | 2 +-
.../java/org/apache/doris/catalog/Database.java | 4 +-
.../java/org/apache/doris/catalog/DatabaseIf.java | 7 +-
.../doris/catalog/HiveMetaStoreClientHelper.java | 81 ++++++++
.../org/apache/doris/catalog/InternalDatabase.java | 91 ---------
.../main/java/org/apache/doris/catalog/Table.java | 14 --
.../java/org/apache/doris/catalog/TableIf.java | 18 +-
.../org/apache/doris/catalog/TabletStatMgr.java | 2 +-
.../doris/catalog/external/ExternalDatabase.java | 104 ++++++----
.../doris/catalog/external/ExternalTable.java | 99 +++++++---
.../catalog/external/HMSExternalDatabase.java | 216 +++++++++++++++++++++
.../doris/catalog/external/HMSExternalTable.java | 134 +++++++++++++
.../doris/common/proc/EsPartitionsProcDir.java | 2 +-
.../apache/doris/common/proc/IndexInfoProcDir.java | 2 +-
.../doris/common/proc/PartitionsProcDir.java | 2 +-
.../doris/common/proc/StatisticProcNode.java | 2 +-
.../org/apache/doris/common/proc/TableProcDir.java | 2 +-
.../apache/doris/common/proc/TablesProcDir.java | 2 +-
.../doris/consistency/ConsistencyChecker.java | 2 +-
.../doris/datasource/EsExternalDataSource.java | 13 --
.../doris/datasource/ExternalDataSource.java | 52 ++---
.../doris/datasource/HMSExternalDataSource.java | 178 ++++++++++++++++-
.../doris/datasource/InternalDataSource.java | 2 +-
.../doris/external/elasticsearch/EsRepository.java | 2 +-
.../org/apache/doris/httpv2/rest/ShowAction.java | 2 +-
.../doris/httpv2/rest/StorageTypeCheckAction.java | 2 +-
.../src/main/java/org/apache/doris/load/Load.java | 2 +-
.../java/org/apache/doris/qe/StmtExecutor.java | 2 +-
.../apache/doris/service/FrontendServiceImpl.java | 2 +-
.../org/apache/doris/catalog/OlapTableTest.java | 2 +-
.../java/org/apache/doris/qe/ShowExecutorTest.java | 2 +-
42 files changed, 809 insertions(+), 260 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
index a22e7e896f..2154ab311c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java
@@ -51,7 +51,7 @@ import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.View;
import org.apache.doris.common.AnalysisException;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index 4f38cdafda..a95bdede1d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -34,7 +34,7 @@ import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Replica.ReplicaState;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletInvertedIndex;
import org.apache.doris.catalog.TabletMeta;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
index e94e3ef290..2a598b77c1 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Analyzer.java
@@ -27,7 +27,7 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Partition.PartitionState;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.View;
import org.apache.doris.cluster.ClusterNamespace;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
index cb13e07492..a2c5139adb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
@@ -26,7 +26,7 @@ import org.apache.doris.catalog.OdbcTable;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
index 067a8882de..9e296499c7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/SelectStmt.java
@@ -27,7 +27,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.FunctionSet;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Type;
import org.apache.doris.catalog.View;
import org.apache.doris.cluster.ClusterNamespace;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
index 22f42448cc..abfce939f3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/ShowDataStmt.java
@@ -25,7 +25,7 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
index 8215e6c1d4..2f0b07e461 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupHandler.java
@@ -37,7 +37,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
index 69b638a3cd..96d89a2707 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJob.java
@@ -32,7 +32,7 @@ import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.Resource;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.View;
import org.apache.doris.common.io.Text;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java
index 2c32005d0a..ec622dbdca 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/BackupJobInfo.java
@@ -29,7 +29,7 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Resource;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.View;
import org.apache.doris.common.FeConstants;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
index df90c9d9dd..25267dbad8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/backup/RestoreJob.java
@@ -46,7 +46,7 @@ import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.catalog.Resource;
import org.apache.doris.catalog.ResourceMgr;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletMeta;
import org.apache.doris.catalog.View;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
index 9cf25f2af2..61419eb4f4 100755
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Catalog.java
@@ -81,7 +81,7 @@ import org.apache.doris.catalog.DistributionInfo.DistributionInfoType;
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
import org.apache.doris.catalog.OlapTable.OlapTableState;
import org.apache.doris.catalog.Replica.ReplicaStatus;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.clone.ColocateTableCheckerAndBalancer;
import org.apache.doris.clone.DynamicPartitionScheduler;
import org.apache.doris.clone.TabletChecker;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
index a9c78d5705..15d25ad153 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/CatalogRecycleBin.java
@@ -18,7 +18,7 @@
package org.apache.doris.catalog;
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.ErrorCode;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 941b005d4f..5607f3f59e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -17,7 +17,7 @@
package org.apache.doris.catalog;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
@@ -71,7 +71,7 @@ import javax.annotation.Nullable;
* if the table has never been loaded * if the table loading failed on the
* previous attempt
*/
-public class Database extends MetaObject implements Writable, InternalDatabase<Table> {
+public class Database extends MetaObject implements Writable, DatabaseIf<Table> {
private static final Logger LOG = LogManager.getLogger(Database.class);
private long id;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
index 840e091c82..caeb3862b5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
@@ -74,6 +74,8 @@ public interface DatabaseIf<T extends TableIf> {
T getTableNullable(String tableName);
+ T getTableNullable(long tableId);
+
Optional<T> getTable(String tableName);
Optional<T> getTable(long tableId);
@@ -87,11 +89,10 @@ public interface DatabaseIf<T extends TableIf> {
T getTableOrMetaException(long tableId) throws MetaNotFoundException;
@SuppressWarnings("unchecked")
- <V extends Table> V getTableOrMetaException(String tableName, Table.TableType tableType)
- throws MetaNotFoundException;
+ T getTableOrMetaException(String tableName, Table.TableType tableType) throws MetaNotFoundException;
@SuppressWarnings("unchecked")
- <V extends Table> V getTableOrMetaException(long tableId, Table.TableType tableType) throws MetaNotFoundException;
+ T getTableOrMetaException(long tableId, Table.TableType tableType) throws MetaNotFoundException;
T getTableOrDdlException(String tableName) throws DdlException;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
index 946657b7e1..1b46d40e75 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HiveMetaStoreClientHelper.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
@@ -71,6 +72,8 @@ import java.util.Date;
import java.util.List;
import java.util.Map;
import java.util.Stack;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
/**
* Helper class for HiveMetaStoreClient
@@ -78,6 +81,8 @@ import java.util.Stack;
public class HiveMetaStoreClientHelper {
private static final Logger LOG = LogManager.getLogger(HiveMetaStoreClientHelper.class);
+ private static final Pattern digitPattern = Pattern.compile("(\\d+)");
+
public enum HiveFileFormat {
TEXT_FILE(0, "text"),
PARQUET(1, "parquet"),
@@ -373,6 +378,26 @@ public class HiveMetaStoreClientHelper {
return table;
}
+ /**
+ * Get table schema.
+ *
+ * @param dbName Database name.
+ * @param tableName Table name.
+ * @param metaStoreUris Hive metastore uri.
+ */
+ public static List<FieldSchema> getSchema(String dbName, String tableName, String metaStoreUris)
+ throws DdlException {
+ HiveMetaStoreClient client = getClient(metaStoreUris);
+ try {
+ return client.getSchema(dbName, tableName);
+ } catch (TException e) {
+ LOG.warn("Hive metastore thrift exception: {}", e.getMessage());
+ throw new DdlException("Connect hive metastore failed. Error: " + e.getMessage());
+ } finally {
+ client.close();
+ }
+ }
+
/**
* Convert Doris expr to Hive expr, only for partition column
* @param dorisExpr
@@ -646,4 +671,60 @@ public class HiveMetaStoreClientHelper {
return this;
}
}
+
+ /**
+ * Convert hive type to doris type.
+ */
+ public static Type hiveTypeToDorisType(String hiveType) {
+ String lowerCaseType = hiveType.toLowerCase();
+ switch (lowerCaseType) {
+ case "boolean":
+ return Type.BOOLEAN;
+ case "tinyint":
+ return Type.TINYINT;
+ case "smallint":
+ return Type.SMALLINT;
+ case "int":
+ return Type.INT;
+ case "bigint":
+ return Type.BIGINT;
+ case "date":
+ return Type.DATE;
+ case "timestamp":
+ return Type.DATETIME;
+ default:
+ break;
+ }
+ if (lowerCaseType.startsWith("char")) {
+ ScalarType type = ScalarType.createType(PrimitiveType.CHAR);
+ Matcher match = digitPattern.matcher(lowerCaseType);
+ if (match.find()) {
+ type.setLength(Integer.parseInt(match.group(1)));
+ }
+ return type;
+ }
+ if (lowerCaseType.startsWith("varchar")) {
+ ScalarType type = ScalarType.createType(PrimitiveType.VARCHAR);
+ Matcher match = digitPattern.matcher(lowerCaseType);
+ if (match.find()) {
+ type.setLength(Integer.parseInt(match.group(1)));
+ }
+ return type;
+ }
+ if (lowerCaseType.startsWith("decimal")) {
+ Matcher match = digitPattern.matcher(lowerCaseType);
+ int precision = ScalarType.DEFAULT_PRECISION;
+ int scale = ScalarType.DEFAULT_SCALE;
+ if (match.find()) {
+ precision = Integer.parseInt(match.group(1));
+ }
+ if (match.find()) {
+ scale = Integer.parseInt(match.group(1));
+ }
+ return ScalarType.createDecimalV2Type(precision, scale);
+ }
+ // TODO: Handle unsupported types.
+ LOG.warn("Hive type {} may not supported yet, will use STRING instead.", hiveType);
+ return Type.STRING;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalDatabase.java
deleted file mode 100644
index 5dab055268..0000000000
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/InternalDatabase.java
+++ /dev/null
@@ -1,91 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.catalog;
-
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.MetaNotFoundException;
-
-import java.util.List;
-import java.util.Optional;
-import java.util.function.Function;
-
-/**
- * InternalDatabase interface for the current Database class.
- *
- * @param <T> current Database table type, subclass of Table.
- */
-public interface InternalDatabase<T extends Table> extends DatabaseIf<T> {
-
- @Override
- public List<T> getTables();
-
- @Override
- public List<T> getTablesOnIdOrder();
-
- @Override
- public List<T> getViews();
-
- @Override
- public List<T> getTablesOnIdOrderIfExist(List<Long> tableIdList);
-
- @Override
- public List<T> getTablesOnIdOrderOrThrowException(List<Long> tableIdList) throws MetaNotFoundException;
-
- @Override
- public T getTableNullable(String tableName);
-
- @Override
- public Optional<T> getTable(String tableName);
-
- @Override
- public Optional<T> getTable(long tableId);
-
- @Override
- public <E extends Exception> T getTableOrException(String tableName, Function<String, E> e) throws E;
-
- @Override
- public <E extends Exception> T getTableOrException(long tableId, Function<Long, E> e) throws E;
-
- @Override
- public T getTableOrMetaException(String tableName) throws MetaNotFoundException;
-
- @Override
- public T getTableOrMetaException(long tableId) throws MetaNotFoundException;
-
- @Override
- public T getTableOrMetaException(String tableName, Table.TableType tableType) throws MetaNotFoundException;
-
- @Override
- public T getTableOrMetaException(long tableId, Table.TableType tableType) throws MetaNotFoundException;
-
- @Override
- public T getTableOrDdlException(String tableName) throws DdlException;
-
- @Override
- public T getTableOrDdlException(long tableId) throws DdlException;
-
- @Override
- public T getTableOrAnalysisException(String tableName) throws AnalysisException;
-
- @Override
- public T getTableOrAnalysisException(long tableId) throws AnalysisException;
-
- @Override
- public OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException;
-}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
index 0ffbadcdc1..4284353faa 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
@@ -58,20 +58,6 @@ public class Table extends MetaObject implements Writable, TableIf {
public volatile boolean isDropped = false;
- public enum TableType {
- MYSQL,
- ODBC,
- OLAP,
- SCHEMA,
- INLINE_VIEW,
- VIEW,
- BROKER,
- ELASTICSEARCH,
- HIVE,
- ICEBERG,
- HUDI
- }
-
protected long id;
protected volatile String name;
protected TableType type;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
index 109fb3f59c..4b4a049253 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
@@ -62,7 +62,7 @@ public interface TableIf {
String getName();
- Table.TableType getType();
+ TableType getType();
List<Column> getFullSchema();
@@ -74,4 +74,20 @@ public interface TableIf {
Column getColumn(String name);
+ /**
+ * Doris table type.
+ */
+ public enum TableType {
+ MYSQL,
+ ODBC,
+ OLAP,
+ SCHEMA,
+ INLINE_VIEW,
+ VIEW,
+ BROKER,
+ ELASTICSEARCH,
+ HIVE,
+ ICEBERG,
+ HUDI
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
index d7a46cb564..09d936d79b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/TabletStatMgr.java
@@ -18,7 +18,7 @@
package org.apache.doris.catalog;
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.ClientPool;
import org.apache.doris.common.Config;
import org.apache.doris.common.util.MasterDaemon;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java
index e193c33dcd..8b604dd142 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java
@@ -17,6 +17,7 @@
package org.apache.doris.catalog.external;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.DatabaseProperty;
import org.apache.doris.catalog.OlapTable;
@@ -46,74 +47,90 @@ public class ExternalDatabase<T extends ExternalTable> implements DatabaseIf<T>
private static final Logger LOG = LogManager.getLogger(ExternalDatabase.class);
- private long id;
- private String name;
- private ReentrantReadWriteLock rwLock;
+ private ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true);
- private ExternalDataSource extDataSource;
-
- public ExternalDatabase() {
+ protected long id;
+ protected String name;
+ protected ExternalDataSource extDataSource;
+ protected DatabaseProperty dbProperties;
+ /**
+ * Create external database.
+ *
+ * @param extDataSource The data source this database belongs to.
+ * @param id Database id.
+ * @param name Database name.
+ */
+ public ExternalDatabase(ExternalDataSource extDataSource, long id, String name) {
+ this.extDataSource = extDataSource;
+ this.id = id;
+ this.name = name;
}
@Override
public void readLock() {
-
+ this.rwLock.readLock().lock();
}
@Override
public void readUnlock() {
-
+ this.rwLock.readLock().unlock();
}
@Override
public void writeLock() {
-
+ this.rwLock.writeLock().lock();
}
@Override
public void writeUnlock() {
-
+ this.rwLock.writeLock().unlock();
}
@Override
public boolean tryWriteLock(long timeout, TimeUnit unit) {
- return true;
+ try {
+ return this.rwLock.writeLock().tryLock(timeout, unit);
+ } catch (InterruptedException e) {
+ LOG.warn("failed to try write lock at external db[" + id + "]", e);
+ return false;
+ }
}
@Override
public boolean isWriteLockHeldByCurrentThread() {
- return false;
+ return this.rwLock.writeLock().isHeldByCurrentThread();
}
@Override
public boolean writeLockIfExist() {
+ writeLock();
return true;
}
@Override
public <E extends Exception> void writeLockOrException(E e) throws E {
-
+ writeLock();
}
@Override
public void writeLockOrDdlException() throws DdlException {
-
+ writeLock();
}
@Override
public long getId() {
- return 0;
+ return id;
}
@Override
public String getFullName() {
- return null;
+ return name;
}
@Override
public DatabaseProperty getDbProperties() {
- return null;
+ return dbProperties;
}
@Override
@@ -123,103 +140,108 @@ public class ExternalDatabase<T extends ExternalTable> implements DatabaseIf<T>
@Override
public List<T> getTables() {
- return null;
+ throw new NotImplementedException();
}
@Override
public List<T> getTablesOnIdOrder() {
- return null;
+ throw new NotImplementedException();
}
@Override
public List<T> getViews() {
- return null;
+ throw new NotImplementedException();
}
@Override
public List<T> getTablesOnIdOrderIfExist(List<Long> tableIdList) {
- return null;
+ throw new NotImplementedException();
}
@Override
public List<T> getTablesOnIdOrderOrThrowException(List<Long> tableIdList) throws MetaNotFoundException {
- return null;
+ throw new NotImplementedException();
}
@Override
public Set<String> getTableNamesWithLock() {
- return null;
+ throw new NotImplementedException();
}
@Override
public T getTableNullable(String tableName) {
- return null;
+ throw new NotImplementedException();
+ }
+
+ @Override
+ public T getTableNullable(long tableId) {
+ throw new NotImplementedException();
}
@Override
public Optional<T> getTable(String tableName) {
- return Optional.empty();
+ throw new NotImplementedException();
}
@Override
public Optional<T> getTable(long tableId) {
- return Optional.empty();
+ throw new NotImplementedException();
}
@Override
public <E extends Exception> T getTableOrException(String tableName, Function<String, E> e) throws E {
- return null;
+ throw new NotImplementedException();
}
@Override
public <E extends Exception> T getTableOrException(long tableId, Function<Long, E> e) throws E {
- return null;
+ throw new NotImplementedException();
}
@Override
public T getTableOrMetaException(String tableName) throws MetaNotFoundException {
- return null;
+ throw new NotImplementedException();
}
@Override
public T getTableOrMetaException(long tableId) throws MetaNotFoundException {
- return null;
+ throw new NotImplementedException();
}
@Override
- public <V extends Table> V getTableOrMetaException(String tableName, Table.TableType tableType)
+ public T getTableOrMetaException(String tableName, Table.TableType tableType)
throws MetaNotFoundException {
- return null;
+ throw new NotImplementedException();
}
@Override
- public <V extends Table> V getTableOrMetaException(long tableId, Table.TableType tableType)
+ public T getTableOrMetaException(long tableId, Table.TableType tableType)
throws MetaNotFoundException {
- return null;
+ throw new NotImplementedException();
}
@Override
public T getTableOrDdlException(String tableName) throws DdlException {
- return null;
+ throw new NotImplementedException();
}
@Override
public T getTableOrDdlException(long tableId) throws DdlException {
- return null;
+ throw new NotImplementedException();
}
@Override
public T getTableOrAnalysisException(String tableName) throws AnalysisException {
- return null;
+ throw new NotImplementedException();
}
@Override
- public OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException {
- return null;
+ public T getTableOrAnalysisException(long tableId) throws AnalysisException {
+ throw new NotImplementedException();
}
@Override
- public T getTableOrAnalysisException(long tableId) throws AnalysisException {
- return null;
+ public OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException {
+ throw new NotImplementedException();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
index f87d955220..7a96dc970e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
@@ -19,13 +19,17 @@ package org.apache.doris.catalog.external;
import org.apache.doris.alter.AlterCancelException;
import org.apache.doris.catalog.Column;
-import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
import java.util.List;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
* External table represent tables that are not self-managed by Doris.
@@ -33,118 +37,167 @@ import java.util.concurrent.TimeUnit;
*/
public class ExternalTable implements TableIf {
+ private static final Logger LOG = LogManager.getLogger(ExternalTable.class);
+
+ protected long id;
+ protected String name;
+ protected ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true);
+ protected TableType type = null;
+ protected List<Column> fullSchema = null;
+
+ /**
+ * Create external table.
+ *
+ * @param id Table id.
+ * @param name Table name.
+ */
+ public ExternalTable(long id, String name) {
+ this.id = id;
+ this.name = name;
+ }
+
+ /**
+ * Create external table.
+ *
+ * @param id Table id.
+ * @param name Table name.
+ * @param type Table type.
+ */
+ public ExternalTable(long id, String name, TableType type) {
+ this.id = id;
+ this.name = name;
+ this.type = type;
+ }
+
@Override
public void readLock() {
-
+ this.rwLock.readLock().lock();
}
@Override
public boolean tryReadLock(long timeout, TimeUnit unit) {
- return false;
+ try {
+ return this.rwLock.readLock().tryLock(timeout, unit);
+ } catch (InterruptedException e) {
+ LOG.warn("failed to try read lock at table[" + name + "]", e);
+ return false;
+ }
}
@Override
public void readUnlock() {
-
+ this.rwLock.readLock().unlock();
}
@Override
public void writeLock() {
-
+ this.rwLock.writeLock().lock();
}
@Override
public boolean writeLockIfExist() {
- return false;
+ writeLock();
+ return true;
}
@Override
public boolean tryWriteLock(long timeout, TimeUnit unit) {
- return false;
+ try {
+ return this.rwLock.writeLock().tryLock(timeout, unit);
+ } catch (InterruptedException e) {
+ LOG.warn("failed to try write lock at table[" + name + "]", e);
+ return false;
+ }
}
@Override
public void writeUnlock() {
-
+ this.rwLock.writeLock().unlock();
}
@Override
public boolean isWriteLockHeldByCurrentThread() {
- return false;
+ return this.rwLock.writeLock().isHeldByCurrentThread();
}
@Override
public <E extends Exception> void writeLockOrException(E e) throws E {
-
+ writeLock();
}
@Override
public void writeLockOrDdlException() throws DdlException {
-
+ writeLockOrException(new DdlException("unknown table, tableName=" + name));
}
@Override
public void writeLockOrMetaException() throws MetaNotFoundException {
-
+ writeLockOrException(new MetaNotFoundException("unknown table, tableName=" + name));
}
@Override
public void writeLockOrAlterCancelException() throws AlterCancelException {
-
+ writeLockOrException(new AlterCancelException("unknown table, tableName=" + name));
}
@Override
public boolean tryWriteLockOrMetaException(long timeout, TimeUnit unit) throws MetaNotFoundException {
- return false;
+ return tryWriteLockOrException(timeout, unit, new MetaNotFoundException("unknown table, tableName=" + name));
}
@Override
public <E extends Exception> boolean tryWriteLockOrException(long timeout, TimeUnit unit, E e) throws E {
+ if (tryWriteLock(timeout, unit)) {
+ return true;
+ }
return false;
}
@Override
public boolean tryWriteLockIfExist(long timeout, TimeUnit unit) {
+ if (tryWriteLock(timeout, unit)) {
+ return true;
+ }
return false;
}
@Override
public long getId() {
- return 0;
+ return id;
}
@Override
public String getName() {
- return null;
+ return name;
}
@Override
- public Table.TableType getType() {
- return null;
+ public TableType getType() {
+ return type;
}
@Override
public List<Column> getFullSchema() {
- return null;
+ throw new NotImplementedException();
}
@Override
public List<Column> getBaseSchema() {
- return null;
+ throw new NotImplementedException();
}
@Override
public List<Column> getBaseSchema(boolean full) {
- return null;
+ throw new NotImplementedException();
}
@Override
public void setNewFullSchema(List<Column> newSchema) {
-
+ this.fullSchema = newSchema;
}
@Override
public Column getColumn(String name) {
- return null;
+ throw new NotImplementedException();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java
new file mode 100644
index 0000000000..64d40a73c6
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java
@@ -0,0 +1,216 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog.external;
+
+import org.apache.doris.catalog.TableIf;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.MetaNotFoundException;
+import org.apache.doris.datasource.ExternalDataSource;
+
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
+import java.util.stream.Collectors;
+
+/**
+ * Hive metastore external database.
+ */
+public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
+
+ private static final Logger LOG = LogManager.getLogger(HMSExternalDatabase.class);
+
+ // Cache of table name to table id.
+ private ConcurrentHashMap<String, Long> tableNameToId = new ConcurrentHashMap<>();
+ private AtomicLong nextId = new AtomicLong(0);
+ private final String hiveMetastoreUri;
+
+ /**
+ * Create HMS external database.
+ *
+ * @param extDataSource External data source this database belongs to.
+ * @param id database id.
+ * @param name database name.
+ * @param uri Hive metastore uri.
+ */
+ public HMSExternalDatabase(ExternalDataSource extDataSource, long id, String name, String uri) {
+ super(extDataSource, id, name);
+ this.hiveMetastoreUri = uri;
+ init();
+ }
+
+ private void init() {
+ List<String> tableNames = extDataSource.listTableNames(null, name);
+ if (tableNames != null) {
+ for (String tableName : tableNames) {
+ tableNameToId.put(tableName, nextId.incrementAndGet());
+ }
+ }
+ }
+
+ @Override
+ public List<HMSExternalTable> getTables() {
+ List<HMSExternalTable> tables = new ArrayList<>();
+ List<String> tableNames = extDataSource.listTableNames(null, name);
+ for (String tableName : tableNames) {
+ tableNameToId.putIfAbsent(tableName, nextId.incrementAndGet());
+ try {
+ tables.add(new HMSExternalTable(tableNameToId.get(tableName), tableName, name, hiveMetastoreUri));
+ } catch (MetaNotFoundException e) {
+ LOG.warn("Table {} in db {} not found in Hive metastore.", tableName, name, e);
+ }
+ }
+ return tables;
+ }
+
+ @Override
+ public List<HMSExternalTable> getTablesOnIdOrder() {
+ // Sort the name instead, because the id may change.
+ return getTables().stream().sorted(Comparator.comparing(TableIf::getName)).collect(Collectors.toList());
+ }
+
+ @Override
+ public Set<String> getTableNamesWithLock() {
+ // Doesn't need to lock because everytime we call the hive metastore api to get table names.
+ return new HashSet<>(extDataSource.listTableNames(null, name));
+ }
+
+ @Override
+ public HMSExternalTable getTableNullable(String tableName) {
+ if (extDataSource.tableExist(null, name, tableName)) {
+ tableNameToId.putIfAbsent(tableName, nextId.incrementAndGet());
+ try {
+ return new HMSExternalTable(tableNameToId.get(tableName), tableName, name, hiveMetastoreUri);
+ } catch (MetaNotFoundException e) {
+ LOG.warn("Table {} in db {} not found in Hive metastore.", tableName, name, e);
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public HMSExternalTable getTableNullable(long tableId) {
+ for (Map.Entry<String, Long> entry : tableNameToId.entrySet()) {
+ if (entry.getValue() == tableId) {
+ if (extDataSource.tableExist(null, name, entry.getKey())) {
+ try {
+ return new HMSExternalTable(tableId, entry.getKey(), name, hiveMetastoreUri);
+ } catch (MetaNotFoundException e) {
+ LOG.warn("Table {} in db {} not found in Hive metastore.", entry.getKey(), name, e);
+ }
+ }
+ return null;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Optional<HMSExternalTable> getTable(String tableName) {
+ return Optional.ofNullable(getTableNullable(tableName));
+ }
+
+ @Override
+ public Optional<HMSExternalTable> getTable(long tableId) {
+ return Optional.ofNullable(getTableNullable(tableId));
+ }
+
+ @Override
+ public <E extends Exception> HMSExternalTable getTableOrException(String tableName, Function<String, E> e)
+ throws E {
+ HMSExternalTable table = getTableNullable(tableName);
+ if (table == null) {
+ throw e.apply(tableName);
+ }
+ return table;
+ }
+
+ @Override
+ public <E extends Exception> HMSExternalTable getTableOrException(long tableId, Function<Long, E> e) throws E {
+ HMSExternalTable table = getTableNullable(tableId);
+ if (table == null) {
+ throw e.apply(tableId);
+ }
+ return table;
+ }
+
+ @Override
+ public HMSExternalTable getTableOrMetaException(String tableName) throws MetaNotFoundException {
+ return getTableOrException(tableName, t -> new MetaNotFoundException("unknown table, tableName=" + t));
+ }
+
+ @Override
+ public HMSExternalTable getTableOrMetaException(long tableId) throws MetaNotFoundException {
+ return getTableOrException(tableId, t -> new MetaNotFoundException("unknown table, tableName=" + t));
+ }
+
+ @Override
+ public HMSExternalTable getTableOrMetaException(String tableName, TableIf.TableType tableType)
+ throws MetaNotFoundException {
+ HMSExternalTable table = getTableOrMetaException(tableName);
+ if (table.getType() != tableType) {
+ throw new MetaNotFoundException("table type is not "
+ + tableType + ", tableName=" + tableName + ", type=" + table.getType());
+ }
+ return table;
+ }
+
+ @Override
+ public HMSExternalTable getTableOrMetaException(long tableId, TableIf.TableType tableType)
+ throws MetaNotFoundException {
+ HMSExternalTable table = getTableOrMetaException(tableId);
+ if (table.getType() != tableType) {
+ throw new MetaNotFoundException("table type is not "
+ + tableType + ", tableId=" + tableId + ", type=" + table.getType());
+ }
+ return table;
+ }
+
+ @Override
+ public HMSExternalTable getTableOrDdlException(String tableName) throws DdlException {
+ return getTableOrException(tableName, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
+ }
+
+ @Override
+ public HMSExternalTable getTableOrDdlException(long tableId) throws DdlException {
+ return getTableOrException(tableId, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
+ }
+
+ @Override
+ public HMSExternalTable getTableOrAnalysisException(String tableName) throws AnalysisException {
+ return getTableOrException(tableName,
+ t -> new AnalysisException(ErrorCode.ERR_UNKNOWN_TABLE.formatErrorMsg(t, name)));
+ }
+
+ @Override
+ public HMSExternalTable getTableOrAnalysisException(long tableId) throws AnalysisException {
+ return getTableOrException(tableId,
+ t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
new file mode 100644
index 0000000000..8c5ea18d06
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
@@ -0,0 +1,134 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog.external;
+
+import org.apache.doris.catalog.Column;
+import org.apache.doris.catalog.HiveMetaStoreClientHelper;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.MetaNotFoundException;
+
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+
+/**
+ * Hive metastore external table.
+ */
+public class HMSExternalTable extends ExternalTable {
+
+ private static final Logger LOG = LogManager.getLogger(HMSExternalTable.class);
+
+ private final String metastoreUri;
+ private final String dbName;
+ private org.apache.hadoop.hive.metastore.api.Table remoteTable = null;
+
+ /**
+ * Create hive metastore external table.
+ *
+ * @param id Table id.
+ * @param name Table name.
+ * @param dbName Database name.
+ * @param uri Hive metastore uri.
+ */
+ public HMSExternalTable(long id, String name, String dbName, String uri) throws MetaNotFoundException {
+ super(id, name);
+ this.dbName = dbName;
+ this.metastoreUri = uri;
+ init();
+ }
+
+ private void init() throws MetaNotFoundException {
+ getRemoteTable();
+ if (remoteTable.getParameters().containsKey("table_type")
+ && remoteTable.getParameters().get("table_type").equalsIgnoreCase("ICEBERG")) {
+ type = TableType.ICEBERG;
+ } else if (remoteTable.getSd().getInputFormat().toLowerCase().contains("hoodie")) {
+ type = TableType.HUDI;
+ } else {
+ type = TableType.HIVE;
+ }
+ }
+
+ /**
+ * Get the related remote hive metastore table.
+ */
+ public org.apache.hadoop.hive.metastore.api.Table getRemoteTable() throws MetaNotFoundException {
+ if (remoteTable == null) {
+ synchronized (this) {
+ if (remoteTable == null) {
+ try {
+ remoteTable = HiveMetaStoreClientHelper.getTable(dbName, name, metastoreUri);
+ } catch (DdlException e) {
+ LOG.warn("Fail to get remote hive table. db {}, table {}, uri {}",
+ dbName, name, metastoreUri);
+ throw new MetaNotFoundException(e);
+ }
+ }
+ }
+ }
+ // TODO: Refresh cached remoteTable
+ return remoteTable;
+ }
+
+ @Override
+ public List<Column> getFullSchema() {
+ if (fullSchema == null) {
+ synchronized (this) {
+ if (fullSchema == null) {
+ try {
+ for (FieldSchema field : HiveMetaStoreClientHelper.getSchema(dbName, name, metastoreUri)) {
+ fullSchema.add(new Column(field.getName(),
+ HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType()),
+ true, null,
+ true, null, field.getComment()));
+ }
+ } catch (DdlException e) {
+ LOG.warn("Fail to get schema of hms table {}", name, e);
+ }
+ }
+ }
+ }
+ // TODO: Refresh cached fullSchema.
+ return fullSchema;
+ }
+
+ @Override
+ public List<Column> getBaseSchema() {
+ return getFullSchema();
+ }
+
+ @Override
+ public List<Column> getBaseSchema(boolean full) {
+ return getFullSchema();
+ }
+
+ @Override
+ public Column getColumn(String name) {
+ if (fullSchema == null) {
+ getFullSchema();
+ }
+ for (Column column : fullSchema) {
+ if (name.equals(column.getName())) {
+ return column;
+ }
+ }
+ return null;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java
index 145905d226..aaee7d7e6f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/EsPartitionsProcDir.java
@@ -22,7 +22,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.EsTable;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.RangePartitionInfo;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.external.elasticsearch.EsShardPartitions;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
index 5857adc629..56c3dc7b26 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
@@ -22,7 +22,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.AnalysisException;
import com.google.common.base.Joiner;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
index cd6c768936..543755b8c5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/PartitionsProcDir.java
@@ -33,7 +33,7 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.PartitionInfo;
import org.apache.doris.catalog.PartitionType;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/StatisticProcNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/StatisticProcNode.java
index dc1efa6243..4ff5efb9b5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/StatisticProcNode.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/StatisticProcNode.java
@@ -23,7 +23,7 @@ import org.apache.doris.catalog.MaterializedIndex;
import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.common.AnalysisException;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TableProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TableProcDir.java
index 3e8d811af9..f8cb5770cd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TableProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TableProcDir.java
@@ -21,7 +21,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.EsTable;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.AnalysisException;
import com.google.common.base.Preconditions;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java
index eec7841502..53d02ea320 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TablesProcDir.java
@@ -23,7 +23,7 @@ import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.PartitionType;
import org.apache.doris.catalog.RangePartitionInfo;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.ListComparator;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java b/fe/fe-core/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java
index c6886ae722..b5e261083e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/consistency/ConsistencyChecker.java
@@ -25,7 +25,7 @@ import org.apache.doris.catalog.MetaObject;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Partition;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.common.Config;
import org.apache.doris.common.MetaNotFoundException;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalDataSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalDataSource.java
index 37e16708e0..58a709e3c6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalDataSource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalDataSource.java
@@ -17,9 +17,6 @@
package org.apache.doris.datasource;
-import org.apache.doris.catalog.Column;
-import org.apache.doris.external.ExternalScanRange;
-
import java.util.List;
/**
@@ -41,14 +38,4 @@ public class EsExternalDataSource extends ExternalDataSource {
public boolean tableExist(SessionContext ctx, String dbName, String tblName) {
return false;
}
-
- @Override
- public List<Column> getSchema(SessionContext ctx, String dbName, String tblName) {
- return null;
- }
-
- @Override
- public List<ExternalScanRange> getExternalScanRanges(SessionContext ctx) {
- return null;
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java
index a4c6c6390f..67b27fbd38 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java
@@ -17,14 +17,13 @@
package org.apache.doris.datasource;
-import org.apache.doris.catalog.Column;
+import org.apache.commons.lang.NotImplementedException;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
-import org.apache.doris.external.ExternalScanRange;
import org.apache.doris.persist.gson.GsonUtils;
import com.google.gson.annotations.SerializedName;
@@ -43,14 +42,14 @@ import java.util.function.Function;
public abstract class ExternalDataSource implements DataSourceIf, Writable {
// Unique id of this data source, will be assigned after data source is loaded.
@SerializedName(value = "id")
- private long id;
+ protected long id;
@SerializedName(value = "name")
- private String name;
+ protected String name;
@SerializedName(value = "type")
- private String type;
+ protected String type;
// save properties of this data source, such as hive meta store url.
@SerializedName(value = "dsProperty")
- private DataSourceProperty dsProperty = new DataSourceProperty();
+ protected DataSourceProperty dsProperty = new DataSourceProperty();
/**
* @return names of database in this data source.
@@ -72,21 +71,6 @@ public abstract class ExternalDataSource implements DataSourceIf, Writable {
*/
public abstract boolean tableExist(SessionContext ctx, String dbName, String tblName);
- /**
- * get schema of the specified table
- *
- * @param dbName
- * @param tblName
- * @return list of columns as table's schema
- */
- public abstract List<Column> getSchema(SessionContext ctx, String dbName, String tblName);
-
- /**
- * @return list of ExternalScanRange
- */
- public abstract List<ExternalScanRange> getExternalScanRanges(SessionContext ctx);
-
-
@Override
public long getId() {
return id;
@@ -104,69 +88,69 @@ public abstract class ExternalDataSource implements DataSourceIf, Writable {
@Override
public List<String> getDbNames() {
- return null;
+ return listDatabaseNames(null);
}
@Nullable
@Override
public DatabaseIf getDbNullable(String dbName) {
- return null;
+ throw new NotImplementedException();
}
@Nullable
@Override
public DatabaseIf getDbNullable(long dbId) {
- return null;
+ throw new NotImplementedException();
}
@Override
public Optional<DatabaseIf> getDb(String dbName) {
- return Optional.empty();
+ throw new NotImplementedException();
}
@Override
public Optional<DatabaseIf> getDb(long dbId) {
- return Optional.empty();
+ throw new NotImplementedException();
}
@Override
public <E extends Exception> DatabaseIf getDbOrException(String dbName, Function<String, E> e) throws E {
- return null;
+ throw new NotImplementedException();
}
@Override
public <E extends Exception> DatabaseIf getDbOrException(long dbId, Function<Long, E> e) throws E {
- return null;
+ throw new NotImplementedException();
}
@Override
public DatabaseIf getDbOrMetaException(String dbName) throws MetaNotFoundException {
- return null;
+ throw new NotImplementedException();
}
@Override
public DatabaseIf getDbOrMetaException(long dbId) throws MetaNotFoundException {
- return null;
+ throw new NotImplementedException();
}
@Override
public DatabaseIf getDbOrDdlException(String dbName) throws DdlException {
- return null;
+ throw new NotImplementedException();
}
@Override
public DatabaseIf getDbOrDdlException(long dbId) throws DdlException {
- return null;
+ throw new NotImplementedException();
}
@Override
public DatabaseIf getDbOrAnalysisException(String dbName) throws AnalysisException {
- return null;
+ throw new NotImplementedException();
}
@Override
public DatabaseIf getDbOrAnalysisException(long dbId) throws AnalysisException {
- return null;
+ throw new NotImplementedException();
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java
index 8b2ada2946..6de84387dd 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java
@@ -17,46 +17,206 @@
package org.apache.doris.datasource;
-import org.apache.doris.catalog.Column;
-import org.apache.doris.external.ExternalScanRange;
+import org.apache.doris.catalog.DatabaseIf;
+import org.apache.doris.catalog.external.HMSExternalDatabase;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
+import org.apache.doris.common.MetaNotFoundException;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.api.MetaException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
+import org.apache.thrift.TException;
+import org.jetbrains.annotations.Nullable;
import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.Function;
/**
* External data source for hive metastore compatible data sources.
*/
public class HMSExternalDataSource extends ExternalDataSource {
private static final Logger LOG = LogManager.getLogger(HMSExternalDataSource.class);
- public static final String HIVE_METASTORE_URIS = "hive.metastore.uris";
- public HMSExternalDataSource() {
+ //Cache of db name to db id.
+ private ConcurrentHashMap<String, Long> dbNameToId = new ConcurrentHashMap();
+ private AtomicLong nextId = new AtomicLong(0);
+
+ protected String hiveMetastoreUris;
+ protected HiveMetaStoreClient client;
+
+ /**
+ * Hive metastore data source implementation.
+ *
+ * @param hiveMetastoreUris e.g. thrift://127.0.0.1:9083
+ */
+ public HMSExternalDataSource(long id, String name, String type, DataSourceProperty dsProperty,
+ String hiveMetastoreUris)
+ throws DdlException {
+ this.id = id;
+ this.name = name;
+ this.type = type;
+ this.dsProperty = dsProperty;
+ this.hiveMetastoreUris = hiveMetastoreUris;
+ init();
+ }
+
+ private void init() throws DdlException {
+ HiveConf hiveConf = new HiveConf();
+ hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, hiveMetastoreUris);
+ try {
+ client = new HiveMetaStoreClient(hiveConf);
+ } catch (MetaException e) {
+ LOG.warn("Failed to create HiveMetaStoreClient: {}", e.getMessage());
+ throw new DdlException("Create HMSExternalDataSource failed.", e);
+ }
+ List<String> allDatabases;
+ try {
+ allDatabases = client.getAllDatabases();
+ } catch (MetaException e) {
+ LOG.warn("Fail to init db name to id map. {}", e.getMessage());
+ return;
+ }
+ // Update the db name to id map.
+ if (allDatabases == null) {
+ return;
+ }
+ for (String db : allDatabases) {
+ dbNameToId.put(db, nextId.incrementAndGet());
+ }
}
@Override
public List<String> listDatabaseNames(SessionContext ctx) {
- return null;
+ try {
+ List<String> allDatabases = client.getAllDatabases();
+ // Update the db name to id map.
+ for (String db : allDatabases) {
+ dbNameToId.putIfAbsent(db, nextId.incrementAndGet());
+ }
+ return allDatabases;
+ } catch (MetaException e) {
+ LOG.warn("List Database Names failed. {}", e.getMessage());
+ }
+ return Lists.newArrayList();
}
@Override
public List<String> listTableNames(SessionContext ctx, String dbName) {
- return null;
+ try {
+ return client.getAllTables(dbName);
+ } catch (MetaException e) {
+ LOG.warn("List Table Names failed. {}", e.getMessage());
+ }
+ return Lists.newArrayList();
}
@Override
public boolean tableExist(SessionContext ctx, String dbName, String tblName) {
+ try {
+ return client.tableExists(dbName, tblName);
+ } catch (TException e) {
+ LOG.warn("Check table exist failed. {}", e.getMessage());
+ }
return false;
}
+ @Nullable
@Override
- public List<Column> getSchema(SessionContext ctx, String dbName, String tblName) {
- return null;
+ public DatabaseIf getDbNullable(String dbName) {
+ try {
+ client.getDatabase(dbName);
+ } catch (TException e) {
+ LOG.warn("External database {} not exist.", dbName);
+ return null;
+ }
+ // The id may change after FE restart since we don't persist it.
+ // Different FEs may have different ids for the same dbName.
+ // May duplicate with internal db id as well.
+ dbNameToId.putIfAbsent(dbName, nextId.incrementAndGet());
+ return new HMSExternalDatabase(this, dbNameToId.get(dbName), dbName, hiveMetastoreUris);
}
+ @Nullable
@Override
- public List<ExternalScanRange> getExternalScanRanges(SessionContext ctx) {
+ public DatabaseIf getDbNullable(long dbId) {
+ for (Map.Entry<String, Long> entry : dbNameToId.entrySet()) {
+ if (entry.getValue() == dbId) {
+ return new HMSExternalDatabase(this, dbId, entry.getKey(), hiveMetastoreUris);
+ }
+ }
return null;
}
+
+ @Override
+ public Optional<DatabaseIf> getDb(String dbName) {
+ return Optional.ofNullable(getDbNullable(dbName));
+ }
+
+ @Override
+ public Optional<DatabaseIf> getDb(long dbId) {
+ return Optional.ofNullable(getDbNullable(dbId));
+ }
+
+ @Override
+ public <E extends Exception> DatabaseIf getDbOrException(String dbName, Function<String, E> e) throws E {
+ DatabaseIf db = getDbNullable(dbName);
+ if (db == null) {
+ throw e.apply(dbName);
+ }
+ return db;
+ }
+
+ @Override
+ public <E extends Exception> DatabaseIf getDbOrException(long dbId, Function<Long, E> e) throws E {
+ DatabaseIf db = getDbNullable(dbId);
+ if (db == null) {
+ throw e.apply(dbId);
+ }
+ return db;
+ }
+
+ @Override
+ public DatabaseIf getDbOrMetaException(String dbName) throws MetaNotFoundException {
+ return getDbOrException(dbName, s -> new MetaNotFoundException("unknown databases, dbName=" + s,
+ ErrorCode.ERR_BAD_DB_ERROR));
+ }
+
+ @Override
+ public DatabaseIf getDbOrMetaException(long dbId) throws MetaNotFoundException {
+ return getDbOrException(dbId, s -> new MetaNotFoundException("unknown databases, dbId=" + s,
+ ErrorCode.ERR_BAD_DB_ERROR));
+ }
+
+ @Override
+ public DatabaseIf getDbOrDdlException(String dbName) throws DdlException {
+ return getDbOrException(dbName, s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s),
+ ErrorCode.ERR_BAD_DB_ERROR));
+ }
+
+ @Override
+ public DatabaseIf getDbOrDdlException(long dbId) throws DdlException {
+ return getDbOrException(dbId, s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s),
+ ErrorCode.ERR_BAD_DB_ERROR));
+ }
+
+ @Override
+ public DatabaseIf getDbOrAnalysisException(String dbName) throws AnalysisException {
+ return getDbOrException(dbName, s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s),
+ ErrorCode.ERR_BAD_DB_ERROR));
+ }
+
+ @Override
+ public DatabaseIf getDbOrAnalysisException(long dbId) throws AnalysisException {
+ return getDbOrException(dbId, s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s),
+ ErrorCode.ERR_BAD_DB_ERROR));
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
index 579715e46c..5e32f2f418 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
@@ -99,7 +99,7 @@ import org.apache.doris.catalog.Replica.ReplicaState;
import org.apache.doris.catalog.ReplicaAllocation;
import org.apache.doris.catalog.SinglePartitionInfo;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.TableIndexes;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletInvertedIndex;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java
index 46108314d3..e92d06ec3f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/external/elasticsearch/EsRepository.java
@@ -22,7 +22,7 @@ import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.EsTable;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.Config;
import org.apache.doris.common.util.MasterDaemon;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
index 6b9b7f6818..d358df47f2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/ShowAction.java
@@ -21,7 +21,7 @@ import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.proc.ProcNodeInterface;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StorageTypeCheckAction.java b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StorageTypeCheckAction.java
index 6cdaf140cb..efcf005a21 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StorageTypeCheckAction.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/httpv2/rest/StorageTypeCheckAction.java
@@ -22,7 +22,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.MaterializedIndexMeta;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.httpv2.entity.ResponseEntityBuilder;
import org.apache.doris.mysql.privilege.PrivPredicate;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
index 53426529c9..df65545ce0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/load/Load.java
@@ -57,7 +57,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.Replica;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletInvertedIndex;
import org.apache.doris.catalog.TabletMeta;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
index e03e2485a1..a3fd1fed5e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/StmtExecutor.java
@@ -57,7 +57,7 @@ import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.ScalarType;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.catalog.Type;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.AuditLog;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
index d4d90782b0..be9f8b463e 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java
@@ -24,7 +24,7 @@ import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.OlapTable;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.AuthenticationException;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
index c60776a488..c557992729 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/OlapTableTest.java
@@ -18,7 +18,7 @@
package org.apache.doris.catalog;
import org.apache.doris.analysis.IndexDef;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.io.FastByteArrayOutputStream;
import org.apache.doris.common.util.UnitTestUtil;
diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java
index 6f1737dc6e..b1eee5ead3 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/qe/ShowExecutorTest.java
@@ -45,7 +45,7 @@ import org.apache.doris.catalog.PrimitiveType;
import org.apache.doris.catalog.RandomDistributionInfo;
import org.apache.doris.catalog.SinglePartitionInfo;
import org.apache.doris.catalog.Table;
-import org.apache.doris.catalog.Table.TableType;
+import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.PatternMatcher;
import org.apache.doris.common.UserException;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org