You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2022/10/19 08:02:20 UTC
[doris] branch master updated: [Improvement](multi-catalog)Support refresh external catalog. (#13363)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new e65a4a9f9f [Improvement](multi-catalog)Support refresh external catalog. (#13363)
e65a4a9f9f is described below
commit e65a4a9f9f0556637bb32402553e0a8a813f4f4f
Author: Jibing-Li <64...@users.noreply.github.com>
AuthorDate: Wed Oct 19 16:02:14 2022 +0800
[Improvement](multi-catalog)Support refresh external catalog. (#13363)
Support manually refresh external catalog metadata.
1. refresh catalog external_catalog_name
2. refresh database catalog.db OR refresh database db (current catalog)
3. refresh table catalog.db.table OR refresh table db.table (current catalog) OR refresh table table_name (current db)
And the refresh operations above keep the database and table ids unchanged.
---
fe/fe-core/src/main/cup/sql_parser.cup | 4 +
.../org/apache/doris/analysis/RefreshDbStmt.java | 19 +++-
.../apache/doris/analysis/RefreshTableStmt.java | 10 +-
.../org/apache/doris/catalog/RefreshManager.java | 104 +++++++++++++++++----
.../doris/catalog/external/EsExternalDatabase.java | 30 +++++-
.../doris/catalog/external/EsExternalTable.java | 1 -
.../doris/catalog/external/ExternalDatabase.java | 5 +
.../doris/catalog/external/ExternalTable.java | 6 ++
.../catalog/external/HMSExternalDatabase.java | 25 +++--
.../doris/catalog/external/HMSExternalTable.java | 1 -
.../apache/doris/datasource/EsExternalCatalog.java | 14 ++-
.../doris/datasource/HMSExternalCatalog.java | 26 ++++--
12 files changed, 194 insertions(+), 51 deletions(-)
diff --git a/fe/fe-core/src/main/cup/sql_parser.cup b/fe/fe-core/src/main/cup/sql_parser.cup
index e2f33038c0..9e617a454c 100644
--- a/fe/fe-core/src/main/cup/sql_parser.cup
+++ b/fe/fe-core/src/main/cup/sql_parser.cup
@@ -1131,6 +1131,10 @@ refresh_stmt ::=
{:
RESULT = new RefreshDbStmt(db);
:}
+ | KW_REFRESH KW_DATABASE ident:ctl DOT ident:db
+ {:
+ RESULT = new RefreshDbStmt(ctl, db);
+ :}
| KW_REFRESH KW_MATERIALIZED KW_VIEW table_name:mv
{:
RESULT = new RefreshMaterializedViewStmt(mv, MVRefreshInfo.RefreshMethod.COMPLETE);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshDbStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshDbStmt.java
index b3823eb015..5118088916 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshDbStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshDbStmt.java
@@ -34,19 +34,32 @@ import org.apache.logging.log4j.Logger;
public class RefreshDbStmt extends DdlStmt {
private static final Logger LOG = LogManager.getLogger(RefreshDbStmt.class);
+ private String catalogName;
private String dbName;
public RefreshDbStmt(String dbName) {
this.dbName = dbName;
}
+ public RefreshDbStmt(String catalogName, String dbName) {
+ this.catalogName = catalogName;
+ this.dbName = dbName;
+ }
+
public String getDbName() {
return dbName;
}
+ public String getCatalogName() {
+ return catalogName;
+ }
+
@Override
public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
super.analyze(analyzer);
+ if (Strings.isNullOrEmpty(catalogName)) {
+ catalogName = ConnectContext.get().getCurrentCatalog().getName();
+ }
if (Strings.isNullOrEmpty(dbName)) {
ErrorReport.reportAnalysisException(ErrorCode.ERR_WRONG_DB_NAME, dbName);
}
@@ -74,7 +87,11 @@ public class RefreshDbStmt extends DdlStmt {
@Override
public String toSql() {
StringBuilder sb = new StringBuilder();
- sb.append("REFRESH DATABASE ").append("`").append(dbName).append("`");
+ sb.append("REFRESH DATABASE ");
+ if (catalogName != null) {
+ sb.append("`").append(catalogName).append("`.");
+ }
+ sb.append("`").append(dbName).append("`");
return sb.toString();
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshTableStmt.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshTableStmt.java
index a2aacb014c..8a92ff25e7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshTableStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/RefreshTableStmt.java
@@ -18,11 +18,9 @@
package org.apache.doris.analysis;
import org.apache.doris.catalog.Env;
-import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.ErrorReport;
import org.apache.doris.common.UserException;
-import org.apache.doris.common.util.Util;
import org.apache.doris.mysql.privilege.PrivPredicate;
import org.apache.doris.qe.ConnectContext;
@@ -38,6 +36,10 @@ public class RefreshTableStmt extends DdlStmt {
this.tableName = tableName;
}
+ public String getCtl() {
+ return tableName.getCtl();
+ }
+
public String getDbName() {
return tableName.getDb();
}
@@ -51,11 +53,9 @@ public class RefreshTableStmt extends DdlStmt {
}
@Override
- public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
+ public void analyze(Analyzer analyzer) throws UserException {
super.analyze(analyzer);
tableName.analyze(analyzer);
- // disallow external catalog
- Util.prohibitExternalCatalog(tableName.getCtl(), this.getClass().getSimpleName());
// check access
if (!Env.getCurrentEnv().getAuth().checkTblPriv(ConnectContext.get(), tableName.getDb(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
index 99065309b0..31cbc266f5 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
@@ -22,8 +22,13 @@ import org.apache.doris.analysis.DropTableStmt;
import org.apache.doris.analysis.RefreshDbStmt;
import org.apache.doris.analysis.RefreshTableStmt;
import org.apache.doris.analysis.TableName;
+import org.apache.doris.catalog.external.ExternalDatabase;
+import org.apache.doris.catalog.external.ExternalTable;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.CatalogIf;
+import org.apache.doris.datasource.ExternalCatalog;
+import org.apache.doris.datasource.InternalCatalog;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -35,38 +40,48 @@ public class RefreshManager {
private static final Logger LOG = LogManager.getLogger(RefreshManager.class);
public void handleRefreshTable(RefreshTableStmt stmt) throws UserException {
+ String catalogName = stmt.getCtl();
String dbName = stmt.getDbName();
String tableName = stmt.getTblName();
Env env = Env.getCurrentEnv();
- // 0. check table type
- Database db = env.getInternalCatalog().getDbOrDdlException(dbName);
- Table table = db.getTableNullable(tableName);
- if (!(table instanceof IcebergTable)) {
- throw new DdlException("Only support refresh Iceberg table.");
- }
+ CatalogIf catalog = catalogName != null ? env.getCatalogMgr().getCatalog(catalogName) : env.getCurrentCatalog();
- // 1. get iceberg properties
- Map<String, String> icebergProperties = ((IcebergTable) table).getIcebergProperties();
- icebergProperties.put(IcebergProperty.ICEBERG_TABLE, ((IcebergTable) table).getIcebergTbl());
- icebergProperties.put(IcebergProperty.ICEBERG_DATABASE, ((IcebergTable) table).getIcebergDb());
-
- // 2. drop old table
- DropTableStmt dropTableStmt = new DropTableStmt(true, stmt.getTableName(), true);
- env.dropTable(dropTableStmt);
-
- // 3. create new table
- CreateTableStmt createTableStmt = new CreateTableStmt(true, true,
- stmt.getTableName(), "ICEBERG", icebergProperties, "");
- env.createTable(createTableStmt);
+ if (catalog == null) {
+ throw new DdlException("Catalog " + catalogName + " doesn't exist.");
+ }
+ if (catalog.getName().equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ // Process internal catalog iceberg external table refresh.
+ refreshInternalCtlIcebergTable(stmt, env);
+ } else {
+ // Process external catalog table refresh
+ refreshExternalCtlTable(dbName, tableName, catalog);
+ }
LOG.info("Successfully refresh table: {} from db: {}", tableName, dbName);
}
public void handleRefreshDb(RefreshDbStmt stmt) throws DdlException {
+ String catalogName = stmt.getCatalogName();
String dbName = stmt.getDbName();
Env env = Env.getCurrentEnv();
+ CatalogIf catalog = catalogName != null ? env.getCatalogMgr().getCatalog(catalogName) : env.getCurrentCatalog();
+
+ if (catalog == null) {
+ throw new DdlException("Catalog " + catalogName + " doesn't exist.");
+ }
+ if (catalog.getName().equals(InternalCatalog.INTERNAL_CATALOG_NAME)) {
+ // Process internal catalog iceberg external db refresh.
+ refreshInternalCtlIcebergDb(dbName, env);
+ } else {
+ // Process external catalog db refresh
+ refreshExternalCtlDb(dbName, catalog);
+ }
+ LOG.info("Successfully refresh db: {}", dbName);
+ }
+
+ private void refreshInternalCtlIcebergDb(String dbName, Env env) throws DdlException {
Database db = env.getInternalCatalog().getDbOrDdlException(dbName);
// 0. build iceberg property
@@ -90,7 +105,56 @@ public class RefreshManager {
// 3. register iceberg database to recreate iceberg table
env.getIcebergTableCreationRecordMgr().registerDb(db);
+ }
- LOG.info("Successfully refresh db: {}", dbName);
+ private void refreshExternalCtlDb(String dbName, CatalogIf catalog) throws DdlException {
+ if (!(catalog instanceof ExternalCatalog)) {
+ throw new DdlException("Only support refresh ExternalCatalog Database");
+ }
+
+ DatabaseIf db = catalog.getDbNullable(dbName);
+ if (db == null) {
+ throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
+ }
+ ((ExternalDatabase) db).setUnInitialized();
+ }
+
+ private void refreshInternalCtlIcebergTable(RefreshTableStmt stmt, Env env) throws UserException {
+ // 0. check table type
+ Database db = env.getInternalCatalog().getDbOrDdlException(stmt.getDbName());
+ Table table = db.getTableNullable(stmt.getTblName());
+ if (!(table instanceof IcebergTable)) {
+ throw new DdlException("Only support refresh Iceberg table.");
+ }
+
+ // 1. get iceberg properties
+ Map<String, String> icebergProperties = ((IcebergTable) table).getIcebergProperties();
+ icebergProperties.put(IcebergProperty.ICEBERG_TABLE, ((IcebergTable) table).getIcebergTbl());
+ icebergProperties.put(IcebergProperty.ICEBERG_DATABASE, ((IcebergTable) table).getIcebergDb());
+
+ // 2. drop old table
+ DropTableStmt dropTableStmt = new DropTableStmt(true, stmt.getTableName(), true);
+ env.dropTable(dropTableStmt);
+
+ // 3. create new table
+ CreateTableStmt createTableStmt = new CreateTableStmt(true, true,
+ stmt.getTableName(), "ICEBERG", icebergProperties, "");
+ env.createTable(createTableStmt);
+ }
+
+ private void refreshExternalCtlTable(String dbName, String tableName, CatalogIf catalog) throws DdlException {
+ if (!(catalog instanceof ExternalCatalog)) {
+ throw new DdlException("Only support refresh ExternalCatalog Tables");
+ }
+ DatabaseIf db = catalog.getDbNullable(dbName);
+ if (db == null) {
+ throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
+ }
+
+ TableIf table = db.getTableNullable(tableName);
+ if (table == null) {
+ throw new DdlException("Table " + tableName + " does not exist in db " + dbName);
+ }
+ ((ExternalTable) table).setUnInitialized();
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalDatabase.java
index 8ec33b4402..4241d75988 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalDatabase.java
@@ -51,17 +51,36 @@ public class EsExternalDatabase extends ExternalDatabase<EsExternalTable> {
*/
public EsExternalDatabase(ExternalCatalog extCatalog, long id, String name) {
super(extCatalog, id, name);
- init();
+ }
+
+ private synchronized void makeSureInitialized() {
+ if (!initialized) {
+ init();
+ initialized = true;
+ }
}
private void init() {
List<String> tableNames = extCatalog.listTableNames(null, name);
if (tableNames != null) {
+ Map<String, Long> tmpTableNameToId = Maps.newConcurrentMap();
+ Map<Long, EsExternalTable> tmpIdToTbl = Maps.newHashMap();
for (String tableName : tableNames) {
- long tblId = Env.getCurrentEnv().getNextId();
- tableNameToId.put(tableName, tblId);
- idToTbl.put(tblId, new EsExternalTable(tblId, tableName, name, (EsExternalCatalog) extCatalog));
+ long tblId;
+ if (tableNameToId.containsKey(tableName)) {
+ tblId = tableNameToId.get(tableName);
+ tmpTableNameToId.put(tableName, tblId);
+ EsExternalTable table = idToTbl.get(tblId);
+ table.setUnInitialized();
+ tmpIdToTbl.put(tblId, table);
+ } else {
+ tblId = Env.getCurrentEnv().getNextId();
+ tmpTableNameToId.put(tableName, tblId);
+ tmpIdToTbl.put(tblId, new EsExternalTable(tblId, tableName, name, (EsExternalCatalog) extCatalog));
+ }
}
+ tableNameToId = tmpTableNameToId;
+ idToTbl = tmpIdToTbl;
}
}
@@ -73,11 +92,13 @@ public class EsExternalDatabase extends ExternalDatabase<EsExternalTable> {
@Override
public List<EsExternalTable> getTables() {
+ makeSureInitialized();
return new ArrayList<>(idToTbl.values());
}
@Override
public EsExternalTable getTableNullable(String tableName) {
+ makeSureInitialized();
if (!tableNameToId.containsKey(tableName)) {
return null;
}
@@ -86,6 +107,7 @@ public class EsExternalDatabase extends ExternalDatabase<EsExternalTable> {
@Override
public EsExternalTable getTableNullable(long tableId) {
+ makeSureInitialized();
return idToTbl.get(tableId);
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java
index 10eac207f4..ea44552896 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/EsExternalTable.java
@@ -39,7 +39,6 @@ public class EsExternalTable extends ExternalTable {
private final EsExternalCatalog catalog;
private final String dbName;
- private boolean initialized = false;
private EsTable esTable;
/**
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java
index bd100f033a..d76ed380a7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalDatabase.java
@@ -48,6 +48,7 @@ public class ExternalDatabase<T extends ExternalTable> implements DatabaseIf<T>
protected String name;
protected ExternalCatalog extCatalog;
protected DatabaseProperty dbProperties;
+ protected boolean initialized = false;
/**
* Create external database.
@@ -62,6 +63,10 @@ public class ExternalDatabase<T extends ExternalTable> implements DatabaseIf<T>
this.name = name;
}
+ public synchronized void setUnInitialized() {
+ this.initialized = false;
+ }
+
@Override
public void readLock() {
this.rwLock.readLock().lock();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
index 870fc2b3a5..33acbc0ec7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/ExternalTable.java
@@ -45,6 +45,7 @@ public class ExternalTable implements TableIf {
protected ReentrantReadWriteLock rwLock = new ReentrantReadWriteLock(true);
protected TableType type = null;
protected volatile List<Column> fullSchema = null;
+ protected boolean initialized = false;
/**
* Create external table.
@@ -74,6 +75,11 @@ public class ExternalTable implements TableIf {
return false;
}
+ public synchronized void setUnInitialized() {
+ this.initialized = false;
+ this.fullSchema = null;
+ }
+
@Override
public void readLock() {
this.rwLock.readLock().lock();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java
index e3752ff2c8..4e50a4c59a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalDatabase.java
@@ -41,9 +41,8 @@ public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
private static final Logger LOG = LogManager.getLogger(HMSExternalDatabase.class);
// Cache of table name to table id.
- private final Map<String, Long> tableNameToId = Maps.newConcurrentMap();
- private final Map<Long, HMSExternalTable> idToTbl = Maps.newHashMap();
- private boolean initialized = false;
+ private Map<String, Long> tableNameToId = Maps.newConcurrentMap();
+ private Map<Long, HMSExternalTable> idToTbl = Maps.newHashMap();
/**
* Create HMS external database.
@@ -66,11 +65,25 @@ public class HMSExternalDatabase extends ExternalDatabase<HMSExternalTable> {
private void init() {
List<String> tableNames = extCatalog.listTableNames(null, name);
if (tableNames != null) {
+ Map<String, Long> tmpTableNameToId = Maps.newConcurrentMap();
+ Map<Long, HMSExternalTable> tmpIdToTbl = Maps.newHashMap();
for (String tableName : tableNames) {
- long tblId = Env.getCurrentEnv().getNextId();
- tableNameToId.put(tableName, tblId);
- idToTbl.put(tblId, new HMSExternalTable(tblId, tableName, name, (HMSExternalCatalog) extCatalog));
+ long tblId;
+ if (tableNameToId.containsKey(tableName)) {
+ tblId = tableNameToId.get(tableName);
+ tmpTableNameToId.put(tableName, tblId);
+ HMSExternalTable table = idToTbl.get(tblId);
+ table.setUnInitialized();
+ tmpIdToTbl.put(tblId, table);
+ } else {
+ tblId = Env.getCurrentEnv().getNextId();
+ tmpTableNameToId.put(tableName, tblId);
+ tmpIdToTbl.put(tblId,
+ new HMSExternalTable(tblId, tableName, name, (HMSExternalCatalog) extCatalog));
+ }
}
+ tableNameToId = tmpTableNameToId;
+ idToTbl = tmpIdToTbl;
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
index 59b08849e2..f13181987b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/external/HMSExternalTable.java
@@ -51,7 +51,6 @@ public class HMSExternalTable extends ExternalTable {
private volatile org.apache.hadoop.hive.metastore.api.Table remoteTable = null;
private DLAType dlaType = DLAType.UNKNOWN;
- private boolean initialized = false;
public enum DLAType {
UNKNOWN, HIVE, HUDI, ICEBERG
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
index 3b7055d687..baa2c6e169 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/EsExternalCatalog.java
@@ -151,12 +151,16 @@ public class EsExternalCatalog extends ExternalCatalog {
} catch (DdlException e) {
LOG.warn("validate error", e);
}
- dbNameToId = Maps.newConcurrentMap();
- idToDb = Maps.newConcurrentMap();
this.esRestClient = new EsRestClient(this.nodes, this.username, this.password, this.enableSsl);
- long defaultDbId = Env.getCurrentEnv().getNextId();
- dbNameToId.put(DEFAULT_DB, defaultDbId);
- idToDb.put(defaultDbId, new EsExternalDatabase(this, defaultDbId, DEFAULT_DB));
+ if (dbNameToId != null && dbNameToId.containsKey(DEFAULT_DB)) {
+ idToDb.get(dbNameToId.get(DEFAULT_DB)).setUnInitialized();
+ } else {
+ dbNameToId = Maps.newConcurrentMap();
+ idToDb = Maps.newConcurrentMap();
+ long defaultDbId = Env.getCurrentEnv().getNextId();
+ dbNameToId.put(DEFAULT_DB, defaultDbId);
+ idToDb.put(defaultDbId, new EsExternalDatabase(this, defaultDbId, DEFAULT_DB));
+ }
}
@Override
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
index e3a676fe64..e15226cad7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/HMSExternalCatalog.java
@@ -42,8 +42,8 @@ public class HMSExternalCatalog extends ExternalCatalog {
private static final Logger LOG = LogManager.getLogger(HMSExternalCatalog.class);
// Cache of db name to db id.
- private Map<String, Long> dbNameToId;
- private Map<Long, HMSExternalDatabase> idToDb;
+ private Map<String, Long> dbNameToId = Maps.newConcurrentMap();
+ private Map<Long, HMSExternalDatabase> idToDb = Maps.newConcurrentMap();
protected HiveMetaStoreClient client;
/**
@@ -62,9 +62,8 @@ public class HMSExternalCatalog extends ExternalCatalog {
}
private void init() {
- // Must set here. Because after replay from image, these 2 map will become null again.
- dbNameToId = Maps.newConcurrentMap();
- idToDb = Maps.newConcurrentMap();
+ Map<String, Long> tmpDbNameToId = Maps.newConcurrentMap();
+ Map<Long, HMSExternalDatabase> tmpIdToDb = Maps.newConcurrentMap();
HiveConf hiveConf = new HiveConf();
hiveConf.setVar(HiveConf.ConfVars.METASTOREURIS, getHiveMetastoreUris());
try {
@@ -84,10 +83,21 @@ public class HMSExternalCatalog extends ExternalCatalog {
return;
}
for (String dbName : allDatabases) {
- long dbId = Env.getCurrentEnv().getNextId();
- dbNameToId.put(dbName, dbId);
- idToDb.put(dbId, new HMSExternalDatabase(this, dbId, dbName));
+ long dbId;
+ if (dbNameToId.containsKey(dbName)) {
+ dbId = dbNameToId.get(dbName);
+ tmpDbNameToId.put(dbName, dbId);
+ HMSExternalDatabase db = idToDb.get(dbId);
+ db.setUnInitialized();
+ tmpIdToDb.put(dbId, db);
+ } else {
+ dbId = Env.getCurrentEnv().getNextId();
+ tmpDbNameToId.put(dbName, dbId);
+ tmpIdToDb.put(dbId, new HMSExternalDatabase(this, dbId, dbName));
+ }
}
+ dbNameToId = tmpDbNameToId;
+ idToDb = tmpIdToDb;
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org