You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by yi...@apache.org on 2022/07/05 05:40:29 UTC
[doris] branch master updated: [feature-wip](multi-catalog) support "show proc 'catalogs/'" (#10596)
This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new f40ae7c654 [feature-wip](multi-catalog) support "show proc 'catalogs/'" (#10596)
f40ae7c654 is described below
commit f40ae7c654e5c5e49a916fc628846413f7b5577f
Author: Mingyu Chen <mo...@gmail.com>
AuthorDate: Tue Jul 5 13:40:24 2022 +0800
[feature-wip](multi-catalog) support "show proc 'catalogs/'" (#10596)
---
.../org/apache/doris/analysis/DescribeStmt.java | 64 ++-----
.../java/org/apache/doris/catalog/Database.java | 95 +---------
.../java/org/apache/doris/catalog/DatabaseIf.java | 113 +++++++++---
.../doris/catalog/external/ExternalDatabase.java | 72 --------
.../catalog/external/HMSExternalDatabase.java | 167 +++--------------
.../doris/catalog/external/HMSExternalTable.java | 89 +++++----
.../apache/doris/common/proc/CatalogsProcDir.java | 110 +++++++++++
.../org/apache/doris/common/proc/DbsProcDir.java | 23 ++-
.../apache/doris/common/proc/IndexInfoProcDir.java | 2 +-
.../org/apache/doris/common/proc/ProcService.java | 3 +-
.../org/apache/doris/datasource/DataSourceIf.java | 76 ++++++--
.../org/apache/doris/datasource/DataSourceMgr.java | 8 +-
.../doris/datasource/ExternalDataSource.java | 55 ------
.../doris/datasource/HMSExternalDataSource.java | 91 +---------
.../doris/datasource/InternalDataSource.java | 68 +------
.../java/org/apache/doris/qe/ShowExecutor.java | 1 -
.../java/org/apache/doris/alter/AlterTest.java | 202 ++++++++++-----------
.../ShowCreateMaterializedViewStmtTest.java | 97 ----------
.../apache/doris/common/proc/DbsProcDirTest.java | 16 +-
.../load/routineload/RoutineLoadManagerTest.java | 41 ++++-
20 files changed, 524 insertions(+), 869 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
index 48b4e4d664..015f238a31 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DescribeStmt.java
@@ -37,12 +37,12 @@ import org.apache.doris.common.proc.ProcResult;
import org.apache.doris.common.proc.ProcService;
import org.apache.doris.common.proc.TableProcDir;
import org.apache.doris.common.util.Util;
+import org.apache.doris.datasource.DataSourceIf;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSetMetaData;
import com.google.common.base.Preconditions;
-import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import org.apache.commons.lang.StringUtils;
@@ -50,10 +50,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
-import java.util.Locale;
import java.util.Map;
import java.util.Set;
-import java.util.stream.Collectors;
public class DescribeStmt extends ShowStmt {
private static final ShowResultSetMetaData DESC_OLAP_TABLE_ALL_META_DATA =
@@ -79,17 +77,6 @@ public class DescribeStmt extends ShowStmt {
.addColumn(new Column("Table", ScalarType.createVarchar(30)))
.build();
- // The same columns in IndexSchemaProcNode.TITLE_NAMES
- private static final ShowResultSetMetaData HMS_EXTERNAL_TABLE_META_DATA =
- ShowResultSetMetaData.builder()
- .addColumn(new Column("Field", ScalarType.createVarchar(20)))
- .addColumn(new Column("Type", ScalarType.createVarchar(20)))
- .addColumn(new Column("Null", ScalarType.createVarchar(20)))
- .addColumn(new Column("Key", ScalarType.createVarchar(20)))
- .addColumn(new Column("Default", ScalarType.createVarchar(20)))
- .addColumn(new Column("Extra", ScalarType.createVarchar(20)))
- .build();
-
// empty col num equals to DESC_OLAP_TABLE_ALL_META_DATA.size()
private static final List<String> EMPTY_ROW = initEmptyRow();
@@ -114,47 +101,35 @@ public class DescribeStmt extends ShowStmt {
}
@Override
- public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
+ public void analyze(Analyzer analyzer) throws UserException {
dbTableName.analyze(analyzer);
- if (!Catalog.getCurrentCatalog().getAuth().checkTblPriv(
- ConnectContext.get(), dbTableName, PrivPredicate.SHOW)) {
+ if (!Catalog.getCurrentCatalog().getAuth()
+ .checkTblPriv(ConnectContext.get(), dbTableName, PrivPredicate.SHOW)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLEACCESS_DENIED_ERROR, "DESCRIBE",
- ConnectContext.get().getQualifiedUser(),
- ConnectContext.get().getRemoteIP(),
- dbTableName.toString());
+ ConnectContext.get().getQualifiedUser(), ConnectContext.get().getRemoteIP(),
+ dbTableName.toString());
}
- DatabaseIf db = Catalog.getCurrentCatalog().getDataSourceMgr()
- .getCatalogOrAnalysisException(dbTableName.getCtl()).getDbOrAnalysisException(dbTableName.getDb());
+ DataSourceIf ds = Catalog.getCurrentCatalog().getDataSourceMgr().getCatalog(dbTableName.getCtl());
+ DatabaseIf db = ds.getDbOrAnalysisException(dbTableName.getDb());
TableIf table = db.getTableOrAnalysisException(dbTableName.getTbl());
table.readLock();
try {
if (!isAllTables) {
- if (table.getType() == TableType.HMS_EXTERNAL_TABLE) {
- hmsSchema = table.getFullSchema().stream().map(col -> Arrays.asList(
- col.getName(),
- col.getType().toSql().toUpperCase(Locale.ROOT),
- Boolean.toString(col.isAllowNull()),
- Boolean.toString(col.isKey()),
- Strings.nullToEmpty(col.getDefaultValue()),
- "" /* no extra field */))
- .collect(Collectors.toList());
+ // show base table schema only
+ String procString = "/catalogs/" + ds.getId() + "/" + db.getId() + "/" + table.getId() + "/"
+ + TableProcDir.INDEX_SCHEMA + "/";
+ if (table.getType() == TableType.OLAP) {
+ procString += ((OlapTable) table).getBaseIndexId();
} else {
- // show base table schema only
- String procString = "/dbs/" + db.getId() + "/" + table.getId() + "/" + TableProcDir.INDEX_SCHEMA
- + "/";
- if (table.getType() == TableType.OLAP) {
- procString += ((OlapTable) table).getBaseIndexId();
- } else {
- procString += table.getId();
- }
+ procString += table.getId();
+ }
- node = ProcService.getInstance().open(procString);
- if (node == null) {
- throw new AnalysisException("Describe table[" + dbTableName.getTbl() + "] failed");
- }
+ node = ProcService.getInstance().open(procString);
+ if (node == null) {
+ throw new AnalysisException("Describe table[" + dbTableName.getTbl() + "] failed");
}
} else {
Util.prohibitExternalCatalog(dbTableName.getCtl(), this.getClass().getSimpleName() + " ALL");
@@ -272,9 +247,6 @@ public class DescribeStmt extends ShowStmt {
@Override
public ShowResultSetMetaData getMetaData() {
if (!isAllTables) {
- if (hmsSchema != null) {
- return HMS_EXTERNAL_TABLE_META_DATA;
- }
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
ProcResult result = null;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
index 48a2f61194..50e4faa619 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Database.java
@@ -22,7 +22,6 @@ import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
-import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.Pair;
@@ -50,13 +49,11 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Objects;
-import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.stream.Collectors;
-import javax.annotation.Nullable;
/**
* Internal representation of db-related metadata. Owned by Catalog instance.
@@ -457,7 +454,7 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
/**
* This is a thread-safe method when nameToTable is a concurrent hash map
*/
- @Nullable
+ @Override
public Table getTableNullable(String tableName) {
if (Catalog.isStoredTableNamesLowerCase()) {
tableName = tableName.toLowerCase();
@@ -474,99 +471,11 @@ public class Database extends MetaObject implements Writable, DatabaseIf<Table>
/**
* This is a thread-safe method when idToTable is a concurrent hash map
*/
- @Nullable
+ @Override
public Table getTableNullable(long tableId) {
return idToTable.get(tableId);
}
- public Optional<Table> getTable(String tableName) {
- return Optional.ofNullable(getTableNullable(tableName));
- }
-
- public Optional<Table> getTable(long tableId) {
- return Optional.ofNullable(getTableNullable(tableId));
- }
-
- public <E extends Exception> Table getTableOrException(
- String tableName, java.util.function.Function<String, E> e) throws E {
- Table table = getTableNullable(tableName);
- if (table == null) {
- throw e.apply(tableName);
- }
- return table;
- }
-
- public <E extends Exception> Table getTableOrException(
- long tableId, java.util.function.Function<Long, E> e) throws E {
- Table table = getTableNullable(tableId);
- if (table == null) {
- throw e.apply(tableId);
- }
- return table;
- }
-
- public Table getTableOrMetaException(String tableName) throws MetaNotFoundException {
- return getTableOrException(tableName, t -> new MetaNotFoundException("unknown table, tableName=" + t));
- }
-
- public Table getTableOrMetaException(long tableId) throws MetaNotFoundException {
- return getTableOrException(tableId, t -> new MetaNotFoundException("unknown table, tableId=" + t));
- }
-
- @SuppressWarnings("unchecked")
- public Table getTableOrMetaException(String tableName, TableType tableType) throws MetaNotFoundException {
- Table table = getTableOrMetaException(tableName);
- if (table.getType() != tableType) {
- throw new MetaNotFoundException("table type is not "
- + tableType + ", tableName=" + tableName + ", type=" + table.getType());
- }
- return table;
- }
-
- @SuppressWarnings("unchecked")
- public Table getTableOrMetaException(long tableId, TableType tableType) throws MetaNotFoundException {
- Table table = getTableOrMetaException(tableId);
- if (table.getType() != tableType) {
- throw new MetaNotFoundException("table type is not " + tableType
- + ", tableId=" + tableId + ", type=" + table.getType());
- }
- return table;
- }
-
- public Table getTableOrDdlException(String tableName) throws DdlException {
- return getTableOrException(tableName, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
- }
-
- public OlapTable getOlapTableOrDdlException(String tableName) throws DdlException {
- Table table = getTableOrDdlException(tableName);
- if (!(table instanceof OlapTable)) {
- throw new DdlException(ErrorCode.ERR_NOT_OLAP_TABLE.formatErrorMsg(tableName));
- }
- return (OlapTable) table;
- }
-
- public Table getTableOrDdlException(long tableId) throws DdlException {
- return getTableOrException(tableId, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
- }
-
- public Table getTableOrAnalysisException(String tableName) throws AnalysisException {
- return getTableOrException(tableName,
- t -> new AnalysisException(ErrorCode.ERR_UNKNOWN_TABLE.formatErrorMsg(t, fullQualifiedName)));
- }
-
- public OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException {
- Table table = getTableOrAnalysisException(tableName);
- if (!(table instanceof OlapTable)) {
- throw new AnalysisException(ErrorCode.ERR_NOT_OLAP_TABLE.formatErrorMsg(tableName));
- }
- return (OlapTable) table;
- }
-
- public Table getTableOrAnalysisException(long tableId) throws AnalysisException {
- return getTableOrException(tableId,
- t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
- }
-
public int getMaxReplicationNum() {
int ret = 0;
readLock();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
index caeb3862b5..9fad070fe9 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DatabaseIf.java
@@ -19,12 +19,14 @@ package org.apache.doris.catalog;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.MetaNotFoundException;
import java.util.List;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
/**
* Database interface.
@@ -76,31 +78,88 @@ public interface DatabaseIf<T extends TableIf> {
T getTableNullable(long tableId);
- Optional<T> getTable(String tableName);
-
- Optional<T> getTable(long tableId);
-
- <E extends Exception> T getTableOrException(String tableName, java.util.function.Function<String, E> e) throws E;
-
- <E extends Exception> T getTableOrException(long tableId, java.util.function.Function<Long, E> e) throws E;
-
- T getTableOrMetaException(String tableName) throws MetaNotFoundException;
-
- T getTableOrMetaException(long tableId) throws MetaNotFoundException;
-
- @SuppressWarnings("unchecked")
- T getTableOrMetaException(String tableName, Table.TableType tableType) throws MetaNotFoundException;
-
- @SuppressWarnings("unchecked")
- T getTableOrMetaException(long tableId, Table.TableType tableType) throws MetaNotFoundException;
-
- T getTableOrDdlException(String tableName) throws DdlException;
-
- T getTableOrDdlException(long tableId) throws DdlException;
-
- T getTableOrAnalysisException(String tableName) throws AnalysisException;
-
- T getTableOrAnalysisException(long tableId) throws AnalysisException;
-
- OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException;
+ default Optional<T> getTable(String tableName) {
+ return Optional.ofNullable(getTableNullable(tableName));
+ }
+
+ default Optional<T> getTable(long tableId) {
+ return Optional.ofNullable(getTableNullable(tableId));
+ }
+
+ default <E extends Exception> T getTableOrException(String tableName, java.util.function.Function<String, E> e)
+ throws E {
+ T table = getTableNullable(tableName);
+ if (table == null) {
+ throw e.apply(tableName);
+ }
+ return table;
+ }
+
+ default <E extends Exception> T getTableOrException(long tableId, Function<Long, E> e) throws E {
+ T table = getTableNullable(tableId);
+ if (table == null) {
+ throw e.apply(tableId);
+ }
+ return table;
+ }
+
+ default T getTableOrMetaException(String tableName) throws MetaNotFoundException {
+ return getTableOrException(tableName, t -> new MetaNotFoundException("unknown table, tableName=" + t));
+ }
+
+ default T getTableOrMetaException(long tableId) throws MetaNotFoundException {
+ return getTableOrException(tableId, t -> new MetaNotFoundException("unknown table, tableId=" + t));
+ }
+
+ default T getTableOrMetaException(String tableName, TableIf.TableType tableType) throws MetaNotFoundException {
+ T table = getTableOrMetaException(tableName);
+ if (table.getType() != tableType) {
+ throw new MetaNotFoundException(
+ "table type is not " + tableType + ", tableName=" + tableName + ", type=" + table.getType());
+ }
+ return table;
+ }
+
+ default T getTableOrMetaException(long tableId, TableIf.TableType tableType) throws MetaNotFoundException {
+ T table = getTableOrMetaException(tableId);
+ if (table.getType() != tableType) {
+ throw new MetaNotFoundException(
+ "table type is not " + tableType + ", tableId=" + tableId + ", type=" + table.getType());
+ }
+ return table;
+ }
+
+ default T getTableOrDdlException(String tableName) throws DdlException {
+ return getTableOrException(tableName, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
+ }
+
+ default T getTableOrDdlException(long tableId) throws DdlException {
+ return getTableOrException(tableId, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
+ }
+
+ default T getTableOrAnalysisException(String tableName) throws AnalysisException {
+ return getTableOrException(tableName,
+ t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
+ }
+
+ default T getTableOrAnalysisException(long tableId) throws AnalysisException {
+ return getTableOrException(tableId,
+ t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
+ }
+
+ default OlapTable getOlapTableOrDdlException(String tableName) throws DdlException {
+ T table = getTableOrDdlException(tableName);
+ if (!(table instanceof OlapTable)) {
+ throw new DdlException(ErrorCode.ERR_NOT_OLAP_TABLE.formatErrorMsg(tableName));
+ }
+ return (OlapTable) table;
+ }
+
+ default OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException {
+ T table = getTableOrAnalysisException(tableName);
+ if (!(table instanceof OlapTable)) {
+ throw new AnalysisException(ErrorCode.ERR_NOT_OLAP_TABLE.formatErrorMsg(tableName));
+ }
+ return (OlapTable) table;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java
index dd3d78ff9d..e844da2318 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java
@@ -19,9 +19,6 @@ package org.apache.doris.catalog.external;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.catalog.DatabaseProperty;
-import org.apache.doris.catalog.OlapTable;
-import org.apache.doris.catalog.Table;
-import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.datasource.ExternalDataSource;
@@ -32,11 +29,9 @@ import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.util.List;
-import java.util.Optional;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.function.Function;
/**
* Base class of external database.
@@ -177,71 +172,4 @@ public class ExternalDatabase<T extends ExternalTable> implements DatabaseIf<T>
public T getTableNullable(long tableId) {
throw new NotImplementedException();
}
-
- @Override
- public Optional<T> getTable(String tableName) {
- throw new NotImplementedException();
- }
-
- @Override
- public Optional<T> getTable(long tableId) {
- throw new NotImplementedException();
- }
-
- @Override
- public <E extends Exception> T getTableOrException(String tableName, Function<String, E> e) throws E {
- throw new NotImplementedException();
- }
-
- @Override
- public <E extends Exception> T getTableOrException(long tableId, Function<Long, E> e) throws E {
- throw new NotImplementedException();
- }
-
- @Override
- public T getTableOrMetaException(String tableName) throws MetaNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public T getTableOrMetaException(long tableId) throws MetaNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public T getTableOrMetaException(String tableName, Table.TableType tableType)
- throws MetaNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public T getTableOrMetaException(long tableId, Table.TableType tableType)
- throws MetaNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public T getTableOrDdlException(String tableName) throws DdlException {
- throw new NotImplementedException();
- }
-
- @Override
- public T getTableOrDdlException(long tableId) throws DdlException {
- throw new NotImplementedException();
- }
-
- @Override
- public T getTableOrAnalysisException(String tableName) throws AnalysisException {
- throw new NotImplementedException();
- }
-
- @Override
- public T getTableOrAnalysisException(long tableId) throws AnalysisException {
- throw new NotImplementedException();
- }
-
- @Override
- public OlapTable getOlapTableOrAnalysisException(String tableName) throws AnalysisException {
- throw new NotImplementedException();
- }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java
index 8a142e78bd..abe9fad243 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java
@@ -17,40 +17,33 @@
package org.apache.doris.catalog.external;
+import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.TableIf;
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.ErrorCode;
-import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.datasource.ExternalDataSource;
import org.apache.doris.datasource.HMSExternalDataSource;
+import com.clearspring.analytics.util.Lists;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.util.ArrayList;
import java.util.Comparator;
-import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
import java.util.Set;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.function.Function;
import java.util.stream.Collectors;
/**
* Hive metastore external database.
*/
public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
-
private static final Logger LOG = LogManager.getLogger(HMSExternalDatabase.class);
// Cache of table name to table id.
- private ConcurrentHashMap<String, Long> tableNameToId = new ConcurrentHashMap<>();
- private AtomicLong nextId = new AtomicLong(0);
- private final String hiveMetastoreUri;
+ private Map<String, Long> tableNameToId = Maps.newConcurrentMap();
+ private Map<Long, HMSExternalTable> idToTbl = Maps.newHashMap();
+ private boolean initialized = false;
/**
* Create HMS external database.
@@ -58,37 +51,33 @@ public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
* @param extDataSource External data source this database belongs to.
* @param id database id.
* @param name database name.
- * @param uri Hive metastore uri.
*/
- public HMSExternalDatabase(ExternalDataSource extDataSource, long id, String name, String uri) {
+ public HMSExternalDatabase(ExternalDataSource extDataSource, long id, String name) {
super(extDataSource, id, name);
- this.hiveMetastoreUri = uri;
- init();
+ }
+
+ private synchronized void makeSureInitialized() {
+ if (!initialized) {
+ init();
+ initialized = true;
+ }
}
private void init() {
List<String> tableNames = extDataSource.listTableNames(null, name);
if (tableNames != null) {
for (String tableName : tableNames) {
- tableNameToId.put(tableName, nextId.incrementAndGet());
+ long tblId = Catalog.getCurrentCatalog().getNextId();
+ tableNameToId.put(tableName, tblId);
+ idToTbl.put(tblId, new HMSExternalTable(tblId, tableName, name, (HMSExternalDataSource) extDataSource));
}
}
}
@Override
public List<HMSExternalTable> getTables() {
- List<HMSExternalTable> tables = new ArrayList<>();
- List<String> tableNames = extDataSource.listTableNames(null, name);
- for (String tableName : tableNames) {
- tableNameToId.putIfAbsent(tableName, nextId.incrementAndGet());
- try {
- tables.add(new HMSExternalTable(tableNameToId.get(tableName), tableName, name,
- (HMSExternalDataSource) extDataSource));
- } catch (MetaNotFoundException e) {
- LOG.warn("Table {} in db {} not found in Hive metastore.", tableName, name, e);
- }
- }
- return tables;
+ makeSureInitialized();
+ return Lists.newArrayList(idToTbl.values());
}
@Override
@@ -99,122 +88,22 @@ public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
@Override
public Set<String> getTableNamesWithLock() {
- // Doesn't need to lock because everytime we call the hive metastore api to get table names.
- return new HashSet<>(extDataSource.listTableNames(null, name));
+ makeSureInitialized();
+ return Sets.newHashSet(tableNameToId.keySet());
}
@Override
public HMSExternalTable getTableNullable(String tableName) {
- if (extDataSource.tableExist(null, name, tableName)) {
- tableNameToId.putIfAbsent(tableName, nextId.incrementAndGet());
- try {
- return new HMSExternalTable(tableNameToId.get(tableName), tableName, name,
- (HMSExternalDataSource) extDataSource);
- } catch (MetaNotFoundException e) {
- LOG.warn("Table {} in db {} not found in Hive metastore.", tableName, name, e);
- }
+ makeSureInitialized();
+ if (!tableNameToId.containsKey(tableName)) {
+ return null;
}
- return null;
+ return idToTbl.get(tableNameToId.get(tableName));
}
@Override
public HMSExternalTable getTableNullable(long tableId) {
- for (Map.Entry<String, Long> entry : tableNameToId.entrySet()) {
- if (entry.getValue() == tableId) {
- if (extDataSource.tableExist(null, name, entry.getKey())) {
- try {
- return new HMSExternalTable(tableId, entry.getKey(), name,
- (HMSExternalDataSource) extDataSource);
- } catch (MetaNotFoundException e) {
- LOG.warn("Table {} in db {} not found in Hive metastore.", entry.getKey(), name, e);
- }
- }
- return null;
- }
- }
- return null;
- }
-
- @Override
- public Optional<HMSExternalTable> getTable(String tableName) {
- return Optional.ofNullable(getTableNullable(tableName));
- }
-
- @Override
- public Optional<HMSExternalTable> getTable(long tableId) {
- return Optional.ofNullable(getTableNullable(tableId));
- }
-
- @Override
- public <E extends Exception> HMSExternalTable getTableOrException(String tableName, Function<String, E> e)
- throws E {
- HMSExternalTable table = getTableNullable(tableName);
- if (table == null) {
- throw e.apply(tableName);
- }
- return table;
- }
-
- @Override
- public <E extends Exception> HMSExternalTable getTableOrException(long tableId, Function<Long, E> e) throws E {
- HMSExternalTable table = getTableNullable(tableId);
- if (table == null) {
- throw e.apply(tableId);
- }
- return table;
- }
-
- @Override
- public HMSExternalTable getTableOrMetaException(String tableName) throws MetaNotFoundException {
- return getTableOrException(tableName, t -> new MetaNotFoundException("unknown table, tableName=" + t));
- }
-
- @Override
- public HMSExternalTable getTableOrMetaException(long tableId) throws MetaNotFoundException {
- return getTableOrException(tableId, t -> new MetaNotFoundException("unknown table, tableName=" + t));
- }
-
- @Override
- public HMSExternalTable getTableOrMetaException(String tableName, TableIf.TableType tableType)
- throws MetaNotFoundException {
- HMSExternalTable table = getTableOrMetaException(tableName);
- if (table.getType() != tableType) {
- throw new MetaNotFoundException("table type is not "
- + tableType + ", tableName=" + tableName + ", type=" + table.getType());
- }
- return table;
- }
-
- @Override
- public HMSExternalTable getTableOrMetaException(long tableId, TableIf.TableType tableType)
- throws MetaNotFoundException {
- HMSExternalTable table = getTableOrMetaException(tableId);
- if (table.getType() != tableType) {
- throw new MetaNotFoundException("table type is not "
- + tableType + ", tableId=" + tableId + ", type=" + table.getType());
- }
- return table;
- }
-
- @Override
- public HMSExternalTable getTableOrDdlException(String tableName) throws DdlException {
- return getTableOrException(tableName, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
- }
-
- @Override
- public HMSExternalTable getTableOrDdlException(long tableId) throws DdlException {
- return getTableOrException(tableId, t -> new DdlException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
- }
-
- @Override
- public HMSExternalTable getTableOrAnalysisException(String tableName) throws AnalysisException {
- return getTableOrException(tableName,
- t -> new AnalysisException(ErrorCode.ERR_UNKNOWN_TABLE.formatErrorMsg(t, name)));
- }
-
- @Override
- public HMSExternalTable getTableOrAnalysisException(long tableId) throws AnalysisException {
- return getTableOrException(tableId,
- t -> new AnalysisException(ErrorCode.ERR_BAD_TABLE_ERROR.formatErrorMsg(t)));
+ makeSureInitialized();
+ return idToTbl.get(tableId);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
index 993a124285..49e003b904 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
@@ -26,11 +26,11 @@ import org.apache.doris.thrift.THiveTable;
import org.apache.doris.thrift.TTableDescriptor;
import org.apache.doris.thrift.TTableType;
+import com.google.common.collect.Lists;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -45,12 +45,11 @@ public class HMSExternalTable extends ExternalTable {
private final HMSExternalDataSource ds;
private final String dbName;
private org.apache.hadoop.hive.metastore.api.Table remoteTable = null;
- private DLAType dlaType = null;
+ private DLAType dlaType = DLAType.UNKNOWN;
+ private boolean initialized = false;
public enum DLAType {
- HIVE,
- HUDI,
- ICEBERG
+ UNKNOWN, HIVE, HUDI, ICEBERG
}
/**
@@ -61,24 +60,59 @@ public class HMSExternalTable extends ExternalTable {
* @param dbName Database name.
* @param ds HMSExternalDataSource.
*/
- public HMSExternalTable(long id, String name, String dbName, HMSExternalDataSource ds)
- throws MetaNotFoundException {
+ public HMSExternalTable(long id, String name, String dbName, HMSExternalDataSource ds) {
super(id, name);
this.dbName = dbName;
this.ds = ds;
this.type = TableType.HMS_EXTERNAL_TABLE;
- init();
}
- private void init() throws MetaNotFoundException {
- getRemoteTable();
- if (remoteTable.getParameters().containsKey("table_type")
- && remoteTable.getParameters().get("table_type").equalsIgnoreCase("ICEBERG")) {
- dlaType = DLAType.ICEBERG;
- } else if (remoteTable.getSd().getInputFormat().toLowerCase().contains("hoodie")) {
- dlaType = DLAType.HUDI;
+ private synchronized void makeSureInitialized() {
+ if (!initialized) {
+ init();
+ initialized = true;
+ }
+ }
+
+ private void init() {
+ try {
+ getRemoteTable();
+ } catch (MetaNotFoundException e) {
+ // CHECKSTYLE IGNORE THIS LINE
+ }
+ if (remoteTable == null) {
+ dlaType = DLAType.UNKNOWN;
+ fullSchema = Lists.newArrayList();
} else {
- dlaType = DLAType.HIVE;
+ if (remoteTable.getParameters().containsKey("table_type") && remoteTable.getParameters().get("table_type")
+ .equalsIgnoreCase("ICEBERG")) {
+ dlaType = DLAType.ICEBERG;
+ } else if (remoteTable.getSd().getInputFormat().toLowerCase().contains("hoodie")) {
+ dlaType = DLAType.HUDI;
+ } else {
+ dlaType = DLAType.HIVE;
+ }
+ initSchema();
+ }
+ }
+
+ private void initSchema() {
+ if (fullSchema == null) {
+ synchronized (this) {
+ if (fullSchema == null) {
+ fullSchema = Lists.newArrayList();
+ try {
+ for (FieldSchema field : HiveMetaStoreClientHelper.getSchema(dbName, name,
+ ds.getHiveMetastoreUris())) {
+ fullSchema.add(new Column(field.getName(),
+ HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType()), true, null, true,
+ null, field.getComment()));
+ }
+ } catch (DdlException e) {
+ LOG.warn("Fail to get schema of hms table {}", name, e);
+ }
+ }
+ }
}
}
@@ -105,24 +139,7 @@ public class HMSExternalTable extends ExternalTable {
@Override
public List<Column> getFullSchema() {
- if (fullSchema == null) {
- synchronized (this) {
- if (fullSchema == null) {
- fullSchema = new ArrayList<>();
- try {
- for (FieldSchema field : HiveMetaStoreClientHelper.getSchema(dbName, name,
- ds.getHiveMetastoreUris())) {
- fullSchema.add(new Column(field.getName(),
- HiveMetaStoreClientHelper.hiveTypeToDorisType(field.getType()), true, null, true,
- null, field.getComment()));
- }
- } catch (DdlException e) {
- LOG.warn("Fail to get schema of hms table {}", name, e);
- }
- }
- }
- }
- // TODO: Refresh cached fullSchema.
+ makeSureInitialized();
return fullSchema;
}
@@ -138,9 +155,7 @@ public class HMSExternalTable extends ExternalTable {
@Override
public Column getColumn(String name) {
- if (fullSchema == null) {
- getFullSchema();
- }
+ makeSureInitialized();
for (Column column : fullSchema) {
if (name.equals(column.getName())) {
return column;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java
new file mode 100644
index 0000000000..5bb64062a1
--- /dev/null
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/CatalogsProcDir.java
@@ -0,0 +1,110 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.common.proc;
+
+import org.apache.doris.catalog.Catalog;
+import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.util.ListComparator;
+import org.apache.doris.datasource.DataSourceIf;
+
+import com.clearspring.analytics.util.Lists;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Strings;
+import com.google.common.collect.ImmutableList;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+/*
+ * SHOW PROC /catalogs/
+ * show all catalogs' info
+ */
+public class CatalogsProcDir implements ProcDirInterface {
+ public static final ImmutableList<String> TITLE_NAMES = new ImmutableList.Builder<String>()
+ .add("CatalogIds").add("CatalogName").add("DatabaseNum")
+ .build();
+
+ private Catalog catalog;
+
+ public CatalogsProcDir(Catalog catalog) {
+ this.catalog = catalog;
+ }
+
+ @Override
+ public boolean register(String name, ProcNodeInterface node) {
+ return false;
+ }
+
+ @Override
+ public ProcNodeInterface lookup(String catalogIdStr) throws AnalysisException {
+ if (catalog == null || Strings.isNullOrEmpty(catalogIdStr)) {
+ throw new AnalysisException("Catalog id is null");
+ }
+
+ long catalogId = -1L;
+ try {
+ catalogId = Long.valueOf(catalogIdStr);
+ } catch (NumberFormatException e) {
+ throw new AnalysisException("Invalid catalog id format: " + catalogIdStr);
+ }
+
+ DataSourceIf ds = catalog.getDataSourceMgr().getCatalog(catalogId);
+ if (ds == null) {
+ throw new AnalysisException("Catalog " + catalogIdStr + " does not exist");
+ }
+
+ return new DbsProcDir(catalog, ds);
+ }
+
+ @Override
+ public ProcResult fetchResult() throws AnalysisException {
+ Preconditions.checkNotNull(catalog);
+ BaseProcResult result = new BaseProcResult();
+ result.setNames(TITLE_NAMES);
+
+ List<Long> catalogIds = catalog.getDataSourceMgr().getCatalogIds();
+ // get info
+ List<List<Comparable>> catalogInfos = Lists.newArrayList();
+ for (long catalogId : catalogIds) {
+ DataSourceIf ds = catalog.getDataSourceMgr().getCatalog(catalogId);
+ if (ds == null) {
+ continue;
+ }
+ List<Comparable> catalogInfo = Lists.newArrayList();
+ catalogInfo.add(ds.getId());
+ catalogInfo.add(ds.getName());
+ catalogInfo.add(ds.getDbNames().size());
+ catalogInfos.add(catalogInfo);
+ }
+
+ // order by catalogId, asc
+ ListComparator<List<Comparable>> comparator = new ListComparator<List<Comparable>>(0);
+ Collections.sort(catalogInfos, comparator);
+
+ // set result
+ for (List<Comparable> info : catalogInfos) {
+ List<String> row = new ArrayList<String>(info.size());
+ for (Comparable comparable : info) {
+ row.add(comparable.toString());
+ }
+ result.addRow(row);
+ }
+ return result;
+ }
+}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
index 8ab49edfff..50ada69d70 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/DbsProcDir.java
@@ -21,9 +21,11 @@ import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Database;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.common.AnalysisException;
+import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.DebugUtil;
import org.apache.doris.common.util.ListComparator;
import org.apache.doris.common.util.TimeUtils;
+import org.apache.doris.datasource.DataSourceIf;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@@ -44,9 +46,11 @@ public class DbsProcDir implements ProcDirInterface {
.build();
private Catalog catalog;
+ private DataSourceIf ds;
- public DbsProcDir(Catalog catalog) {
+ public DbsProcDir(Catalog catalog, DataSourceIf ds) {
this.catalog = catalog;
+ this.ds = ds;
}
@Override
@@ -67,7 +71,7 @@ public class DbsProcDir implements ProcDirInterface {
throw new AnalysisException("Invalid db id format: " + dbIdStr);
}
- DatabaseIf db = catalog.getInternalDataSource().getDbNullable(dbId);
+ DatabaseIf db = ds.getDbNullable(dbId);
if (db == null) {
throw new AnalysisException("Database " + dbId + " does not exist");
}
@@ -81,7 +85,7 @@ public class DbsProcDir implements ProcDirInterface {
BaseProcResult result = new BaseProcResult();
result.setNames(TITLE_NAMES);
- List<String> dbNames = catalog.getInternalDataSource().getDbNames();
+ List<String> dbNames = ds.getDbNames();
if (dbNames == null || dbNames.isEmpty()) {
// empty
return result;
@@ -90,7 +94,7 @@ public class DbsProcDir implements ProcDirInterface {
// get info
List<List<Comparable>> dbInfos = new ArrayList<>();
for (String dbName : dbNames) {
- DatabaseIf db = catalog.getInternalDataSource().getDbNullable(dbName);
+ DatabaseIf db = ds.getDbNullable(dbName);
if (db == null) {
continue;
}
@@ -102,13 +106,14 @@ public class DbsProcDir implements ProcDirInterface {
dbInfo.add(dbName);
dbInfo.add(tableNum);
- long usedDataQuota = ((Database) db).getUsedDataQuotaWithLock();
- long dataQuota = ((Database) db).getDataQuota();
+ long usedDataQuota = (db instanceof Database) ? ((Database) db).getUsedDataQuotaWithLock() : 0;
+ long dataQuota = (db instanceof Database) ? ((Database) db).getDataQuota() : 0;
String readableUsedQuota = DebugUtil.printByteWithUnit(usedDataQuota);
String readableQuota = DebugUtil.printByteWithUnit(dataQuota);
- String lastCheckTime = TimeUtils.longToTimeString(((Database) db).getLastCheckTime());
- long replicaCount = ((Database) db).getReplicaCountWithLock();
- long replicaQuota = ((Database) db).getReplicaQuota();
+ String lastCheckTime = (db instanceof Database) ? TimeUtils.longToTimeString(
+ ((Database) db).getLastCheckTime()) : FeConstants.null_string;
+ long replicaCount = (db instanceof Database) ? ((Database) db).getReplicaCountWithLock() : 0;
+ long replicaQuota = (db instanceof Database) ? ((Database) db).getReplicaQuota() : 0;
dbInfo.add(readableUsedQuota);
dbInfo.add(readableQuota);
dbInfo.add(lastCheckTime);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
index 6ab28a094b..4775ba23ef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/IndexInfoProcDir.java
@@ -91,7 +91,7 @@ public class IndexInfoProcDir implements ProcDirInterface {
builder.toString()));
}
} else {
- result.addRow(Lists.newArrayList("-1", table.getName(), "", "", "", "", ""));
+ result.addRow(Lists.newArrayList(String.valueOf(table.getId()), table.getName(), "", "", "", "", ""));
}
return result;
diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java
index 59a354265b..8c21287aea 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/ProcService.java
@@ -37,7 +37,8 @@ public final class ProcService {
root = new BaseProcDir();
root.register("auth", new AuthProcDir(Catalog.getCurrentCatalog().getAuth()));
root.register("backends", new BackendsProcDir(Catalog.getCurrentSystemInfo()));
- root.register("dbs", new DbsProcDir(Catalog.getCurrentCatalog()));
+ root.register("catalogs", new CatalogsProcDir(Catalog.getCurrentCatalog()));
+ root.register("dbs", new DbsProcDir(Catalog.getCurrentCatalog(), Catalog.getCurrentInternalCatalog()));
root.register("jobs", new JobsDbProcDir(Catalog.getCurrentCatalog()));
root.register("statistic", new StatisticProcNode(Catalog.getCurrentCatalog()));
root.register("tasks", new TasksProcDir());
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceIf.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceIf.java
index 93260e2746..7f0e01c2cf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceIf.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceIf.java
@@ -20,11 +20,13 @@ package org.apache.doris.datasource;
import org.apache.doris.catalog.DatabaseIf;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.MetaNotFoundException;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.function.Function;
import javax.annotation.Nullable;
/**
@@ -50,29 +52,63 @@ public interface DataSourceIf<T extends DatabaseIf> {
@Nullable
T getDbNullable(long dbId);
- Optional<T> getDb(String dbName);
-
- Optional<T> getDb(long dbId);
-
- <E extends Exception> T getDbOrException(String dbName, java.util.function.Function<String, E> e) throws E;
-
- <E extends Exception> T getDbOrException(long dbId, java.util.function.Function<Long, E> e) throws E;
-
- T getDbOrMetaException(String dbName) throws MetaNotFoundException;
-
- T getDbOrMetaException(long dbId) throws MetaNotFoundException;
-
- T getDbOrDdlException(String dbName) throws DdlException;
-
- T getDbOrDdlException(long dbId) throws DdlException;
-
- T getDbOrAnalysisException(String dbName) throws AnalysisException;
-
- T getDbOrAnalysisException(long dbId) throws AnalysisException;
-
Map<String, String> getProperties();
void modifyDatasourceName(String name);
void modifyDatasourceProps(Map<String, String> props);
+
+ default Optional<T> getDb(String dbName) {
+ return Optional.ofNullable(getDbNullable(dbName));
+ }
+
+ default Optional<T> getDb(long dbId) {
+ return Optional.ofNullable(getDbNullable(dbId));
+ }
+
+ default <E extends Exception> T getDbOrException(String dbName, Function<String, E> e) throws E {
+ T db = getDbNullable(dbName);
+ if (db == null) {
+ throw e.apply(dbName);
+ }
+ return db;
+ }
+
+ default <E extends Exception> T getDbOrException(long dbId, Function<Long, E> e) throws E {
+ T db = getDbNullable(dbId);
+ if (db == null) {
+ throw e.apply(dbId);
+ }
+ return db;
+ }
+
+ default T getDbOrMetaException(String dbName) throws MetaNotFoundException {
+ return getDbOrException(dbName,
+ s -> new MetaNotFoundException("unknown databases, dbName=" + s, ErrorCode.ERR_BAD_DB_ERROR));
+ }
+
+ default T getDbOrMetaException(long dbId) throws MetaNotFoundException {
+ return getDbOrException(dbId,
+ s -> new MetaNotFoundException("unknown databases, dbId=" + s, ErrorCode.ERR_BAD_DB_ERROR));
+ }
+
+ default T getDbOrDdlException(String dbName) throws DdlException {
+ return getDbOrException(dbName,
+ s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
+ }
+
+ default T getDbOrDdlException(long dbId) throws DdlException {
+ return getDbOrException(dbId,
+ s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
+ }
+
+ default T getDbOrAnalysisException(String dbName) throws AnalysisException {
+ return getDbOrException(dbName,
+ s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
+ }
+
+ default T getDbOrAnalysisException(long dbId) throws AnalysisException {
+ return getDbOrException(dbId,
+ s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceMgr.java
index 45e8b00a1c..8fbe946f47 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/DataSourceMgr.java
@@ -116,8 +116,12 @@ public class DataSourceMgr implements Writable, GsonPostProcessable {
}
public DataSourceIf getCatalogOrAnalysisException(String name) throws AnalysisException {
- return getCatalogOrException(name, ds -> new AnalysisException(
- ErrorCode.ERR_UNKNOWN_CATALOG.formatErrorMsg(ds), ErrorCode.ERR_UNKNOWN_CATALOG));
+ return getCatalogOrException(name, ds -> new AnalysisException(ErrorCode.ERR_UNKNOWN_CATALOG.formatErrorMsg(ds),
+ ErrorCode.ERR_UNKNOWN_CATALOG));
+ }
+
+ public List<Long> getCatalogIds() {
+ return Lists.newArrayList(idToCatalog.keySet());
}
public DatabaseIf getDbNullable(long dbId) {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java
index ca08ff2e67..79223498f8 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDataSource.java
@@ -19,9 +19,6 @@ package org.apache.doris.datasource;
import org.apache.doris.catalog.external.ExternalDatabase;
import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.persist.gson.GsonUtils;
@@ -36,8 +33,6 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
-import java.util.function.Function;
/**
* The abstract class for all types of external data sources.
@@ -107,56 +102,6 @@ public abstract class ExternalDataSource implements DataSourceIf<ExternalDatabas
throw new NotImplementedException();
}
- @Override
- public Optional<ExternalDatabase> getDb(String dbName) {
- throw new NotImplementedException();
- }
-
- @Override
- public Optional<ExternalDatabase> getDb(long dbId) {
- throw new NotImplementedException();
- }
-
- @Override
- public <E extends Exception> ExternalDatabase getDbOrException(String dbName, Function<String, E> e) throws E {
- throw new NotImplementedException();
- }
-
- @Override
- public <E extends Exception> ExternalDatabase getDbOrException(long dbId, Function<Long, E> e) throws E {
- throw new NotImplementedException();
- }
-
- @Override
- public ExternalDatabase getDbOrMetaException(String dbName) throws MetaNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public ExternalDatabase getDbOrMetaException(long dbId) throws MetaNotFoundException {
- throw new NotImplementedException();
- }
-
- @Override
- public ExternalDatabase getDbOrDdlException(String dbName) throws DdlException {
- throw new NotImplementedException();
- }
-
- @Override
- public ExternalDatabase getDbOrDdlException(long dbId) throws DdlException {
- throw new NotImplementedException();
- }
-
- @Override
- public ExternalDatabase getDbOrAnalysisException(String dbName) throws AnalysisException {
- throw new NotImplementedException();
- }
-
- @Override
- public ExternalDatabase getDbOrAnalysisException(long dbId) throws AnalysisException {
- throw new NotImplementedException();
- }
-
@Override
public Map<String, String> getProperties() {
return dsProperty.getProperties();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java
index 22c5f09303..8abc4f30ba 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalDataSource.java
@@ -21,10 +21,6 @@ import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.external.ExternalDatabase;
import org.apache.doris.catalog.external.HMSExternalDatabase;
import org.apache.doris.cluster.ClusterNamespace;
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.DdlException;
-import org.apache.doris.common.ErrorCode;
-import org.apache.doris.common.MetaNotFoundException;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
@@ -38,8 +34,6 @@ import org.jetbrains.annotations.Nullable;
import java.util.List;
import java.util.Map;
-import java.util.Optional;
-import java.util.function.Function;
/**
* External data source for hive metastore compatible data sources.
@@ -48,7 +42,8 @@ public class HMSExternalDataSource extends ExternalDataSource {
private static final Logger LOG = LogManager.getLogger(HMSExternalDataSource.class);
// Cache of db name to db id.
- private Map<String, Long> dbNameToId;
+ private Map<String, Long> dbNameToId = Maps.newConcurrentMap();
+ private Map<Long, HMSExternalDatabase> idToDb = Maps.newConcurrentMap();
private boolean initialized = false;
protected HiveMetaStoreClient client;
@@ -68,7 +63,6 @@ public class HMSExternalDataSource extends ExternalDataSource {
}
private void init() {
- dbNameToId = Maps.newConcurrentMap();
HiveConf hiveConf = new HiveConf();
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, getHiveMetastoreUris());
try {
@@ -87,8 +81,10 @@ public class HMSExternalDataSource extends ExternalDataSource {
if (allDatabases == null) {
return;
}
- for (String db : allDatabases) {
- dbNameToId.put(db, Catalog.getCurrentCatalog().getNextId());
+ for (String dbName : allDatabases) {
+ long dbId = Catalog.getCurrentCatalog().getNextId();
+ dbNameToId.put(dbName, dbId);
+ idToDb.put(dbId, new HMSExternalDatabase(this, dbId, dbName));
}
}
@@ -111,7 +107,6 @@ public class HMSExternalDataSource extends ExternalDataSource {
@Override
public List<String> listTableNames(SessionContext ctx, String dbName) {
- makeSureInitialized();
try {
return client.getAllTables(getRealTableName(dbName));
} catch (MetaException e) {
@@ -122,7 +117,6 @@ public class HMSExternalDataSource extends ExternalDataSource {
@Override
public boolean tableExist(SessionContext ctx, String dbName, String tblName) {
- makeSureInitialized();
try {
return client.tableExists(getRealTableName(dbName), tblName);
} catch (TException e) {
@@ -139,83 +133,14 @@ public class HMSExternalDataSource extends ExternalDataSource {
if (!dbNameToId.containsKey(realDbName)) {
return null;
}
- return new HMSExternalDatabase(this, dbNameToId.get(realDbName), realDbName, getHiveMetastoreUris());
+ return idToDb.get(dbNameToId.get(realDbName));
}
@Nullable
@Override
public ExternalDatabase getDbNullable(long dbId) {
makeSureInitialized();
- for (Map.Entry<String, Long> entry : dbNameToId.entrySet()) {
- if (entry.getValue() == dbId) {
- return new HMSExternalDatabase(this, dbId, entry.getKey(), getHiveMetastoreUris());
- }
- }
- return null;
- }
-
- @Override
- public Optional<ExternalDatabase> getDb(String dbName) {
- return Optional.ofNullable(getDbNullable(dbName));
- }
-
- @Override
- public Optional<ExternalDatabase> getDb(long dbId) {
- return Optional.ofNullable(getDbNullable(dbId));
- }
-
- @Override
- public <E extends Exception> ExternalDatabase getDbOrException(String dbName, Function<String, E> e) throws E {
- ExternalDatabase db = getDbNullable(dbName);
- if (db == null) {
- throw e.apply(dbName);
- }
- return db;
- }
-
- @Override
- public <E extends Exception> ExternalDatabase getDbOrException(long dbId, Function<Long, E> e) throws E {
- ExternalDatabase db = getDbNullable(dbId);
- if (db == null) {
- throw e.apply(dbId);
- }
- return db;
- }
-
- @Override
- public ExternalDatabase getDbOrMetaException(String dbName) throws MetaNotFoundException {
- return getDbOrException(dbName,
- s -> new MetaNotFoundException("unknown databases, dbName=" + s, ErrorCode.ERR_BAD_DB_ERROR));
- }
-
- @Override
- public ExternalDatabase getDbOrMetaException(long dbId) throws MetaNotFoundException {
- return getDbOrException(dbId,
- s -> new MetaNotFoundException("unknown databases, dbId=" + s, ErrorCode.ERR_BAD_DB_ERROR));
- }
-
- @Override
- public ExternalDatabase getDbOrDdlException(String dbName) throws DdlException {
- return getDbOrException(dbName,
- s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
- }
-
- @Override
- public ExternalDatabase getDbOrDdlException(long dbId) throws DdlException {
- return getDbOrException(dbId,
- s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
- }
-
- @Override
- public ExternalDatabase getDbOrAnalysisException(String dbName) throws AnalysisException {
- return getDbOrException(dbName,
- s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
- }
-
- @Override
- public ExternalDatabase getDbOrAnalysisException(long dbId) throws AnalysisException {
- return getDbOrException(dbId,
- s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
+ return idToDb.get(dbId);
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
index 1e355c9e8b..90797a0d66 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/InternalDataSource.java
@@ -187,11 +187,9 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
-import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.TimeUnit;
-import java.util.function.Function;
/**
* The Internal data source will manage all self-managed meta object in a Doris cluster.
@@ -263,70 +261,6 @@ public class InternalDataSource implements DataSourceIf<Database> {
return idToDb.get(dbId);
}
- @Override
- public Optional<Database> getDb(String dbName) {
- return Optional.ofNullable(getDbNullable(dbName));
- }
-
- @Override
- public Optional<Database> getDb(long dbId) {
- return Optional.ofNullable(getDbNullable(dbId));
- }
-
- @Override
- public <E extends Exception> Database getDbOrException(String dbName, Function<String, E> e) throws E {
- Database db = getDbNullable(dbName);
- if (db == null) {
- throw e.apply(dbName);
- }
- return db;
- }
-
- @Override
- public <E extends Exception> Database getDbOrException(long dbId, Function<Long, E> e) throws E {
- Database db = getDbNullable(dbId);
- if (db == null) {
- throw e.apply(dbId);
- }
- return db;
- }
-
- @Override
- public Database getDbOrMetaException(String dbName) throws MetaNotFoundException {
- return getDbOrException(dbName,
- s -> new MetaNotFoundException("unknown databases, dbName=" + s, ErrorCode.ERR_BAD_DB_ERROR));
- }
-
- @Override
- public Database getDbOrMetaException(long dbId) throws MetaNotFoundException {
- return getDbOrException(dbId,
- s -> new MetaNotFoundException("unknown databases, dbId=" + s, ErrorCode.ERR_BAD_DB_ERROR));
- }
-
- @Override
- public Database getDbOrDdlException(String dbName) throws DdlException {
- return getDbOrException(dbName,
- s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
- }
-
- @Override
- public Database getDbOrDdlException(long dbId) throws DdlException {
- return getDbOrException(dbId,
- s -> new DdlException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
- }
-
- @Override
- public Database getDbOrAnalysisException(String dbName) throws AnalysisException {
- return getDbOrException(dbName,
- s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
- }
-
- @Override
- public Database getDbOrAnalysisException(long dbId) throws AnalysisException {
- return getDbOrException(dbId,
- s -> new AnalysisException(ErrorCode.ERR_BAD_DB_ERROR.formatErrorMsg(s), ErrorCode.ERR_BAD_DB_ERROR));
- }
-
@Override
public Map<String, String> getProperties() {
return Maps.newHashMap();
@@ -734,7 +668,7 @@ public class InternalDataSource implements DataSourceIf<Database> {
String dbName = recoverStmt.getDbName();
String tableName = recoverStmt.getTableName();
- Database db = (Database) getDbOrDdlException(dbName);
+ Database db = getDbOrDdlException(dbName);
OlapTable olapTable = db.getOlapTableOrDdlException(tableName);
olapTable.writeLockOrDdlException();
try {
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
index 006d27008e..7f702e83af 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
@@ -909,7 +909,6 @@ public class ShowExecutor {
ShowIndexStmt showStmt = (ShowIndexStmt) stmt;
List<List<String>> rows = Lists.newArrayList();
DatabaseIf db = ctx.getCurrentDataSource().getDbOrAnalysisException(showStmt.getDbName());
-
OlapTable table = db.getOlapTableOrAnalysisException(showStmt.getTableName().getTbl());
table.readLock();
try {
diff --git a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
index dadaa4eeeb..3f056f5d9b 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/alter/AlterTest.java
@@ -19,11 +19,13 @@ package org.apache.doris.alter;
import org.apache.doris.analysis.AlterTableStmt;
import org.apache.doris.analysis.CreateDbStmt;
+import org.apache.doris.analysis.CreateMaterializedViewStmt;
import org.apache.doris.analysis.CreatePolicyStmt;
import org.apache.doris.analysis.CreateResourceStmt;
import org.apache.doris.analysis.CreateTableStmt;
import org.apache.doris.analysis.DateLiteral;
import org.apache.doris.analysis.DropResourceStmt;
+import org.apache.doris.analysis.ShowCreateMaterializedViewStmt;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Column;
import org.apache.doris.catalog.DataProperty;
@@ -38,11 +40,14 @@ import org.apache.doris.catalog.Table;
import org.apache.doris.catalog.Tablet;
import org.apache.doris.catalog.TabletInvertedIndex;
import org.apache.doris.catalog.Type;
+import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
+import org.apache.doris.common.ExceptionChecker;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.util.TimeUtils;
import org.apache.doris.qe.ConnectContext;
+import org.apache.doris.qe.ShowExecutor;
import org.apache.doris.thrift.TStorageMedium;
import org.apache.doris.utframe.UtFrameUtils;
@@ -138,118 +143,63 @@ public class AlterTest {
+ " 'storage_cooldown_time' = '2999-12-31 00:00:00'\n"
+ ");");
- createTable("CREATE TABLE test.tbl5\n"
- + "(\n"
- + " k1 date,\n"
- + " k2 int,\n"
- + " v1 int \n"
- + ") ENGINE=OLAP\n"
- + "UNIQUE KEY (k1,k2)\n"
- + "PARTITION BY RANGE(k1)\n"
- + "(\n"
+ createTable("CREATE TABLE test.tbl5\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int \n"
+ + ") ENGINE=OLAP\n" + "UNIQUE KEY (k1,k2)\n" + "PARTITION BY RANGE(k1)\n" + "(\n"
+ " PARTITION p1 values less than('2020-02-01'),\n"
- + " PARTITION p2 values less than('2020-03-01')\n"
- + ")\n"
- + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n"
- + "PROPERTIES('replication_num' = '1');");
-
- createTable("CREATE TABLE test.tbl6\n"
- + "(\n"
- + " k1 datetime(3),\n"
- + " k2 time(3),\n"
- + " v1 int \n,"
- + " v2 datetime(3)\n"
- + ") ENGINE=OLAP\n"
- + "UNIQUE KEY (k1,k2)\n"
- + "PARTITION BY RANGE(k1)\n"
- + "(\n"
- + " PARTITION p1 values less than('2020-02-01 00:00:00'),\n"
- + " PARTITION p2 values less than('2020-03-01 00:00:00')\n"
- + ")\n"
- + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n"
+ + " PARTITION p2 values less than('2020-03-01')\n" + ")\n" + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n"
+ "PROPERTIES('replication_num' = '1');");
- createTable("create external table test.odbc_table\n"
- + "( `k1` bigint(20) COMMENT \"\",\n"
- + " `k2` datetime COMMENT \"\",\n"
- + " `k3` varchar(20) COMMENT \"\",\n"
- + " `k4` varchar(100) COMMENT \"\",\n"
- + " `k5` float COMMENT \"\"\n"
- + ")ENGINE=ODBC\n"
- + "PROPERTIES (\n"
- + "\"host\" = \"127.0.0.1\",\n"
- + "\"port\" = \"3306\",\n"
- + "\"user\" = \"root\",\n"
- + "\"password\" = \"123\",\n"
- + "\"database\" = \"db1\",\n"
- + "\"table\" = \"tbl1\",\n"
- + "\"driver\" = \"Oracle Driver\",\n"
- + "\"odbc_type\" = \"oracle\"\n"
- + ");");
+ createTable(
+ "CREATE TABLE test.tbl6\n" + "(\n" + " k1 datetime(3),\n" + " k2 time(3),\n" + " v1 int \n,"
+ + " v2 datetime(3)\n" + ") ENGINE=OLAP\n" + "UNIQUE KEY (k1,k2)\n"
+ + "PARTITION BY RANGE(k1)\n" + "(\n"
+ + " PARTITION p1 values less than('2020-02-01 00:00:00'),\n"
+ + " PARTITION p2 values less than('2020-03-01 00:00:00')\n" + ")\n"
+ + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n" + "PROPERTIES('replication_num' = '1');");
+
+ createTable("create external table test.odbc_table\n" + "( `k1` bigint(20) COMMENT \"\",\n"
+ + " `k2` datetime COMMENT \"\",\n" + " `k3` varchar(20) COMMENT \"\",\n"
+ + " `k4` varchar(100) COMMENT \"\",\n" + " `k5` float COMMENT \"\"\n" + ")ENGINE=ODBC\n"
+ + "PROPERTIES (\n" + "\"host\" = \"127.0.0.1\",\n" + "\"port\" = \"3306\",\n" + "\"user\" = \"root\",\n"
+ + "\"password\" = \"123\",\n" + "\"database\" = \"db1\",\n" + "\"table\" = \"tbl1\",\n"
+ + "\"driver\" = \"Oracle Driver\",\n" + "\"odbc_type\" = \"oracle\"\n" + ");");
// s3 resource
- createRemoteStorageResource("create resource \"remote_s3\"\n"
- + "properties\n"
- + "(\n"
- + " \"type\" = \"s3\", \n"
- + " \"s3_endpoint\" = \"bj\",\n"
- + " \"s3_region\" = \"bj\",\n"
- + " \"s3_root_path\" = \"/path/to/root\",\n"
- + " \"s3_access_key\" = \"bbb\",\n"
- + " \"s3_secret_key\" = \"aaaa\",\n"
- + " \"s3_max_connections\" = \"50\",\n"
- + " \"s3_request_timeout_ms\" = \"3000\",\n"
- + " \"s3_connection_timeout_ms\" = \"1000\"\n"
- + ");");
-
- createRemoteStorageResource("create resource \"remote_s3_1\"\n"
- + "properties\n"
- + "(\n"
- + " \"type\" = \"s3\", \n"
- + " \"s3_endpoint\" = \"bj\",\n"
- + " \"s3_region\" = \"bj\",\n"
- + " \"s3_root_path\" = \"/path/to/root\",\n"
- + " \"s3_access_key\" = \"bbb\",\n"
- + " \"s3_secret_key\" = \"aaaa\",\n"
- + " \"s3_max_connections\" = \"50\",\n"
- + " \"s3_request_timeout_ms\" = \"3000\",\n"
- + " \"s3_connection_timeout_ms\" = \"1000\"\n"
- + ");");
-
- createRemoteStoragePolicy("CREATE STORAGE POLICY testPolicy\n"
- + "PROPERTIES(\n"
- + " \"storage_resource\" = \"remote_s3\",\n"
- + " \"cooldown_datetime\" = \"2100-05-10 00:00:00\"\n"
- + ");");
-
- createRemoteStoragePolicy("CREATE STORAGE POLICY testPolicy2\n"
- + "PROPERTIES(\n"
- + " \"storage_resource\" = \"remote_s3\",\n"
- + " \"cooldown_ttl\" = \"1d\"\n"
- + ");");
-
- createTable("CREATE TABLE test.tbl_remote\n"
- + "(\n"
- + " k1 date,\n"
- + " k2 int,\n"
- + " v1 int sum\n"
- + ")\n"
- + "PARTITION BY RANGE(k1)\n"
- + "(\n"
- + " PARTITION p1 values less than('2020-02-01'),\n"
+ createRemoteStorageResource(
+ "create resource \"remote_s3\"\n" + "properties\n" + "(\n" + " \"type\" = \"s3\", \n"
+ + " \"s3_endpoint\" = \"bj\",\n" + " \"s3_region\" = \"bj\",\n"
+ + " \"s3_root_path\" = \"/path/to/root\",\n" + " \"s3_access_key\" = \"bbb\",\n"
+ + " \"s3_secret_key\" = \"aaaa\",\n" + " \"s3_max_connections\" = \"50\",\n"
+ + " \"s3_request_timeout_ms\" = \"3000\",\n" + " \"s3_connection_timeout_ms\" = \"1000\"\n"
+ + ");");
+
+ createRemoteStorageResource(
+ "create resource \"remote_s3_1\"\n" + "properties\n" + "(\n" + " \"type\" = \"s3\", \n"
+ + " \"s3_endpoint\" = \"bj\",\n" + " \"s3_region\" = \"bj\",\n"
+ + " \"s3_root_path\" = \"/path/to/root\",\n" + " \"s3_access_key\" = \"bbb\",\n"
+ + " \"s3_secret_key\" = \"aaaa\",\n" + " \"s3_max_connections\" = \"50\",\n"
+ + " \"s3_request_timeout_ms\" = \"3000\",\n" + " \"s3_connection_timeout_ms\" = \"1000\"\n"
+ + ");");
+
+ createRemoteStoragePolicy(
+ "CREATE STORAGE POLICY testPolicy\n" + "PROPERTIES(\n" + " \"storage_resource\" = \"remote_s3\",\n"
+ + " \"cooldown_datetime\" = \"2100-05-10 00:00:00\"\n" + ");");
+
+ createRemoteStoragePolicy(
+ "CREATE STORAGE POLICY testPolicy2\n" + "PROPERTIES(\n" + " \"storage_resource\" = \"remote_s3\",\n"
+ + " \"cooldown_ttl\" = \"1d\"\n" + ");");
+
+ createTable("CREATE TABLE test.tbl_remote\n" + "(\n" + " k1 date,\n" + " k2 int,\n" + " v1 int sum\n"
+ + ")\n" + "PARTITION BY RANGE(k1)\n" + "(\n" + " PARTITION p1 values less than('2020-02-01'),\n"
+ " PARTITION p2 values less than('2020-03-01'),\n"
+ " PARTITION p3 values less than('2020-04-01'),\n"
- + " PARTITION p4 values less than('2020-05-01')\n"
- + ")\n"
- + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n"
- + "PROPERTIES"
- + "("
- + " 'replication_num' = '1',\n"
- + " 'in_memory' = 'false',\n"
- + " 'storage_medium' = 'SSD',\n"
- + " 'storage_cooldown_time' = '2100-05-09 00:00:00',\n"
- + " 'remote_storage_policy' = 'testPolicy'\n"
- + ");");
+ + " PARTITION p4 values less than('2020-05-01')\n" + ")\n" + "DISTRIBUTED BY HASH(k2) BUCKETS 3\n"
+ + "PROPERTIES" + "(" + " 'replication_num' = '1',\n" + " 'in_memory' = 'false',\n"
+ + " 'storage_medium' = 'SSD',\n" + " 'storage_cooldown_time' = '2100-05-09 00:00:00',\n"
+ + " 'remote_storage_policy' = 'testPolicy'\n" + ");");
+
+ createTable("create table test.show_test (k1 int, k2 int) distributed by hash(k1) "
+ + "buckets 1 properties(\"replication_num\" = \"1\");");
}
@AfterClass
@@ -273,7 +223,7 @@ public class AlterTest {
Catalog.getCurrentCatalog().getPolicyMgr().createPolicy(stmt);
}
- private static void alterTable(String sql, boolean expectedException) throws Exception {
+ private static void alterTable(String sql, boolean expectedException) {
try {
AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
Catalog.getCurrentCatalog().alterTable(alterTableStmt);
@@ -288,6 +238,22 @@ public class AlterTest {
}
}
+ private static void createMV(String sql, boolean expectedException) {
+ try {
+ CreateMaterializedViewStmt createMaterializedViewStmt
+ = (CreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
+ Catalog.getCurrentCatalog().createMaterializedView(createMaterializedViewStmt);
+ if (expectedException) {
+ Assert.fail();
+ }
+ } catch (Exception e) {
+ e.printStackTrace();
+ if (!expectedException) {
+ Assert.fail();
+ }
+ }
+ }
+
private static void alterTableWithExceptionMsg(String sql, String msg) throws Exception {
AlterTableStmt alterTableStmt = (AlterTableStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
try {
@@ -1171,4 +1137,28 @@ public class AlterTest {
DropResourceStmt stmt = (DropResourceStmt) UtFrameUtils.parseAndAnalyzeStmt(sql, connectContext);
Catalog.getCurrentCatalog().getResourceMgr().dropResource(stmt);
}
+
+ @Test
+ public void testShowMV() throws Exception {
+ createMV("CREATE MATERIALIZED VIEW test_mv as select k1 from test.show_test;", false);
+ waitSchemaChangeJobDone(true);
+
+ String showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv on test.show_test;";
+ ShowCreateMaterializedViewStmt showStmt = (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(
+ showMvSql, connectContext);
+ ShowExecutor executor = new ShowExecutor(connectContext, showStmt);
+ Assert.assertEquals(executor.execute().getResultRows().get(0).get(2),
+ "CREATE MATERIALIZED VIEW test_mv as select k1 from test.show_test;");
+
+ showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv_empty on test.show_test;";
+ showStmt = (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext);
+ executor = new ShowExecutor(connectContext, showStmt);
+ Assert.assertTrue(executor.execute().getResultRows().isEmpty());
+
+ showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv on test.table1_error;";
+ showStmt = (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext);
+ executor = new ShowExecutor(connectContext, showStmt);
+ ExceptionChecker.expectThrowsWithMsg(AnalysisException.class, "Unknown table 'table1_error'",
+ executor::execute);
+ }
}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateMaterializedViewStmtTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateMaterializedViewStmtTest.java
deleted file mode 100644
index 501b50b418..0000000000
--- a/fe/fe-core/src/test/java/org/apache/doris/analysis/ShowCreateMaterializedViewStmtTest.java
+++ /dev/null
@@ -1,97 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.doris.analysis;
-
-import org.apache.doris.common.AnalysisException;
-import org.apache.doris.common.ExceptionChecker;
-import org.apache.doris.qe.ConnectContext;
-import org.apache.doris.qe.ShowExecutor;
-import org.apache.doris.utframe.DorisAssert;
-import org.apache.doris.utframe.UtFrameUtils;
-
-import org.junit.AfterClass;
-import org.junit.Assert;
-import org.junit.BeforeClass;
-import org.junit.Test;
-
-import java.io.File;
-import java.util.UUID;
-
-/**
- * test for ShowCreateMaterializedViewStmt.
- **/
-public class ShowCreateMaterializedViewStmtTest {
-
- private static String runningDir = "fe/mocked/ShowCreateMaterializedViewStmtTest/" + UUID.randomUUID() + "/";
-
- private static ConnectContext connectContext;
-
- private static DorisAssert dorisAssert;
-
- /**
- * init.
- **/
- @BeforeClass
- public static void beforeClass() throws Exception {
- UtFrameUtils.createDorisCluster(runningDir);
-
- // create connect context
- connectContext = UtFrameUtils.createDefaultCtx();
- dorisAssert = new DorisAssert(connectContext);
- dorisAssert.withDatabase("test")
- .withTable("create table test.table1 (k1 int, k2 int) distributed by hash(k1) "
- + "buckets 1 properties(\"replication_num\" = \"1\");")
- .withMaterializedView("CREATE MATERIALIZED VIEW test_mv as select k1 from test.table1;");
- }
-
- @AfterClass
- public static void tearDown() {
- File file = new File(runningDir);
- file.delete();
- }
-
- @Test
- public void testNormal() throws Exception {
- String showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv on test.table1;";
- ShowCreateMaterializedViewStmt showStmt =
- (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext);
- ShowExecutor executor = new ShowExecutor(connectContext, showStmt);
- Assert.assertEquals(executor.execute().getResultRows().get(0).get(2),
- "CREATE MATERIALIZED VIEW test_mv as select k1 from test.table1;");
- }
-
- @Test
- public void testNoView() throws Exception {
- String showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv_empty on test.table1;";
- ShowCreateMaterializedViewStmt showStmt =
- (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext);
- ShowExecutor executor = new ShowExecutor(connectContext, showStmt);
- Assert.assertTrue(executor.execute().getResultRows().isEmpty());
- }
-
- @Test
- public void testNoTable() throws Exception {
- String showMvSql = "SHOW CREATE MATERIALIZED VIEW test_mv on test.table1_error;";
- ShowCreateMaterializedViewStmt showStmt =
- (ShowCreateMaterializedViewStmt) UtFrameUtils.parseAndAnalyzeStmt(showMvSql, connectContext);
- ShowExecutor executor = new ShowExecutor(connectContext, showStmt);
- ExceptionChecker.expectThrowsWithMsg(AnalysisException.class,
- "Unknown table 'table1_error' in default_cluster:test", executor::execute);
-
- }
-}
diff --git a/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java b/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java
index c2522aaa5c..be14837e43 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/common/proc/DbsProcDirTest.java
@@ -62,7 +62,7 @@ public class DbsProcDirTest {
public void testRegister() {
DbsProcDir dir;
- dir = new DbsProcDir(catalog);
+ dir = new DbsProcDir(catalog, ds);
Assert.assertFalse(dir.register("db1", new BaseProcDir()));
}
@@ -103,7 +103,7 @@ public class DbsProcDirTest {
DbsProcDir dir;
ProcNodeInterface node;
- dir = new DbsProcDir(catalog);
+ dir = new DbsProcDir(catalog, ds);
try {
node = dir.lookup(String.valueOf(db1.getId()));
Assert.assertNotNull(node);
@@ -112,7 +112,7 @@ public class DbsProcDirTest {
Assert.fail();
}
- dir = new DbsProcDir(catalog);
+ dir = new DbsProcDir(catalog, ds);
try {
node = dir.lookup(String.valueOf(db2.getId()));
Assert.assertNotNull(node);
@@ -121,7 +121,7 @@ public class DbsProcDirTest {
Assert.fail();
}
- dir = new DbsProcDir(catalog);
+ dir = new DbsProcDir(catalog, ds);
node = dir.lookup("10002");
Assert.assertNull(node);
}
@@ -130,7 +130,7 @@ public class DbsProcDirTest {
public void testLookupInvalid() {
DbsProcDir dir;
- dir = new DbsProcDir(catalog);
+ dir = new DbsProcDir(catalog, ds);
try {
dir.lookup(null);
} catch (AnalysisException e) {
@@ -187,7 +187,7 @@ public class DbsProcDirTest {
DbsProcDir dir;
ProcResult result;
- dir = new DbsProcDir(catalog);
+ dir = new DbsProcDir(catalog, ds);
result = dir.fetchResult();
Assert.assertNotNull(result);
Assert.assertTrue(result instanceof BaseProcResult);
@@ -218,14 +218,14 @@ public class DbsProcDirTest {
DbsProcDir dir;
ProcResult result;
- dir = new DbsProcDir(null);
+ dir = new DbsProcDir(null, ds);
try {
result = dir.fetchResult();
} catch (NullPointerException e) {
e.printStackTrace();
}
- dir = new DbsProcDir(catalog);
+ dir = new DbsProcDir(catalog, ds);
result = dir.fetchResult();
Assert.assertEquals(Lists.newArrayList("DbId", "DbName", "TableNum", "Size", "Quota",
"LastConsistencyCheckTime", "ReplicaCount", "ReplicaQuota"),
diff --git a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadManagerTest.java
index c4fddfe0a5..5949b2d2bf 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadManagerTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/load/routineload/RoutineLoadManagerTest.java
@@ -27,6 +27,7 @@ import org.apache.doris.analysis.StopRoutineLoadStmt;
import org.apache.doris.analysis.UserIdentity;
import org.apache.doris.catalog.Catalog;
import org.apache.doris.catalog.Database;
+import org.apache.doris.catalog.Table;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Config;
import org.apache.doris.common.DdlException;
@@ -587,7 +588,7 @@ public class RoutineLoadManagerTest {
@Test
public void testPauseRoutineLoadJob(@Injectable PauseRoutineLoadStmt pauseRoutineLoadStmt, @Mocked Catalog catalog,
- @Mocked InternalDataSource ds, @Mocked Database database, @Mocked PaloAuth paloAuth,
+ @Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl, @Mocked PaloAuth paloAuth,
@Mocked ConnectContext connectContext) throws UserException {
RoutineLoadManager routineLoadManager = new RoutineLoadManager();
Map<Long, Map<String, List<RoutineLoadJob>>> dbToNameToRoutineLoadJob = Maps.newHashMap();
@@ -620,6 +621,12 @@ public class RoutineLoadManagerTest {
database.getId();
minTimes = 0;
result = 1L;
+ database.getTableOrAnalysisException(anyLong);
+ minTimes = 0;
+ result = tbl;
+ tbl.getName();
+ minTimes = 0;
+ result = "tbl";
catalog.getAuth();
minTimes = 0;
result = paloAuth;
@@ -652,7 +659,7 @@ public class RoutineLoadManagerTest {
@Test
public void testResumeRoutineLoadJob(@Injectable ResumeRoutineLoadStmt resumeRoutineLoadStmt,
- @Mocked Catalog catalog, @Mocked InternalDataSource ds, @Mocked Database database,
+ @Mocked Catalog catalog, @Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl,
@Mocked PaloAuth paloAuth, @Mocked ConnectContext connectContext) throws UserException {
RoutineLoadManager routineLoadManager = new RoutineLoadManager();
Map<Long, Map<String, List<RoutineLoadJob>>> dbToNameToRoutineLoadJob = Maps.newHashMap();
@@ -681,6 +688,12 @@ public class RoutineLoadManagerTest {
database.getId();
minTimes = 0;
result = 1L;
+ database.getTableOrAnalysisException(anyLong);
+ minTimes = 0;
+ result = tbl;
+ tbl.getName();
+ minTimes = 0;
+ result = "tbl";
catalog.getAuth();
minTimes = 0;
result = paloAuth;
@@ -697,7 +710,7 @@ public class RoutineLoadManagerTest {
@Test
public void testStopRoutineLoadJob(@Injectable StopRoutineLoadStmt stopRoutineLoadStmt, @Mocked Catalog catalog,
- @Mocked InternalDataSource ds, @Mocked Database database, @Mocked PaloAuth paloAuth,
+ @Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl, @Mocked PaloAuth paloAuth,
@Mocked ConnectContext connectContext) throws UserException {
RoutineLoadManager routineLoadManager = new RoutineLoadManager();
Map<Long, Map<String, List<RoutineLoadJob>>> dbToNameToRoutineLoadJob = Maps.newHashMap();
@@ -726,6 +739,12 @@ public class RoutineLoadManagerTest {
database.getId();
minTimes = 0;
result = 1L;
+ database.getTableOrAnalysisException(anyLong);
+ minTimes = 0;
+ result = tbl;
+ tbl.getName();
+ minTimes = 0;
+ result = "tbl";
catalog.getAuth();
minTimes = 0;
result = paloAuth;
@@ -894,7 +913,7 @@ public class RoutineLoadManagerTest {
@Test
public void testAlterRoutineLoadJob(@Injectable StopRoutineLoadStmt stopRoutineLoadStmt, @Mocked Catalog catalog,
- @Mocked InternalDataSource ds, @Mocked Database database, @Mocked PaloAuth paloAuth,
+ @Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl, @Mocked PaloAuth paloAuth,
@Mocked ConnectContext connectContext) throws UserException {
RoutineLoadManager routineLoadManager = new RoutineLoadManager();
Map<Long, Map<String, List<RoutineLoadJob>>> dbToNameToRoutineLoadJob = Maps.newHashMap();
@@ -923,6 +942,12 @@ public class RoutineLoadManagerTest {
database.getId();
minTimes = 0;
result = 1L;
+ database.getTableOrAnalysisException(anyLong);
+ minTimes = 0;
+ result = tbl;
+ tbl.getName();
+ minTimes = 0;
+ result = "tbl";
catalog.getAuth();
minTimes = 0;
result = paloAuth;
@@ -940,7 +965,7 @@ public class RoutineLoadManagerTest {
@Test
public void testPauseAndResumeAllRoutineLoadJob(@Injectable PauseRoutineLoadStmt pauseRoutineLoadStmt,
@Injectable ResumeRoutineLoadStmt resumeRoutineLoadStmt, @Mocked Catalog catalog,
- @Mocked InternalDataSource ds, @Mocked Database database, @Mocked PaloAuth paloAuth,
+ @Mocked InternalDataSource ds, @Mocked Database database, @Mocked Table tbl, @Mocked PaloAuth paloAuth,
@Mocked ConnectContext connectContext) throws UserException {
RoutineLoadManager routineLoadManager = new RoutineLoadManager();
Map<Long, Map<String, List<RoutineLoadJob>>> dbToNameToRoutineLoadJob = Maps.newHashMap();
@@ -981,6 +1006,12 @@ public class RoutineLoadManagerTest {
database.getId();
minTimes = 0;
result = 1L;
+ database.getTableOrAnalysisException(anyLong);
+ minTimes = 0;
+ result = tbl;
+ tbl.getName();
+ minTimes = 0;
+ result = "tbl";
catalog.getAuth();
minTimes = 0;
result = paloAuth;
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org