You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@doris.apache.org by mo...@apache.org on 2023/05/27 14:43:17 UTC
[doris] 05/24: [Fix](multi-catalog) Fix sync hms event failed when start FE soon. (#19344)
This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch branch-1.2-lts
in repository https://gitbox.apache.org/repos/asf/doris.git
commit 47b2097dd025965e8dc6ab016d6b6728e6e8e2bc
Author: Xiangyu Wang <du...@gmail.com>
AuthorDate: Thu May 11 01:00:55 2023 +0800
[Fix](multi-catalog) Fix sync hms event failed when start FE soon. (#19344)
* [Fix](multi-catalog) Fix sync hms event failed when start FE soon after.
* [Fix](multi-catalog) Fix sync hms event failed when start FE soon after.
---------
Co-authored-by: wangxiangyu@360shuke.com <wa...@360shuke.com>
---
.../org/apache/doris/catalog/RefreshManager.java | 2 +-
.../org/apache/doris/datasource/CatalogMgr.java | 79 ++++++++++++++++------
.../datasource/hive/event/AddPartitionEvent.java | 2 +-
.../datasource/hive/event/AlterPartitionEvent.java | 6 +-
.../datasource/hive/event/AlterTableEvent.java | 10 +--
.../datasource/hive/event/CreateDatabaseEvent.java | 2 +-
.../datasource/hive/event/CreateTableEvent.java | 11 +--
.../datasource/hive/event/DropDatabaseEvent.java | 2 +-
.../datasource/hive/event/DropPartitionEvent.java | 2 +-
.../datasource/hive/event/DropTableEvent.java | 2 +-
.../doris/datasource/hive/event/InsertEvent.java | 2 +-
11 files changed, 77 insertions(+), 43 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
index ef95b3e5f8..cf6a490b28 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/RefreshManager.java
@@ -56,7 +56,7 @@ public class RefreshManager {
refreshInternalCtlIcebergTable(stmt, env);
} else {
// Process external catalog table refresh
- env.getCatalogMgr().refreshExternalTable(dbName, tableName, catalogName);
+ env.getCatalogMgr().refreshExternalTable(dbName, tableName, catalogName, false);
}
LOG.info("Successfully refresh table: {} from db: {}", tableName, dbName);
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
index 1222f90432..df74d9a123 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java
@@ -554,7 +554,8 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
}
}
- public void refreshExternalTable(String dbName, String tableName, String catalogName) throws DdlException {
+ public void refreshExternalTable(String dbName, String tableName, String catalogName, boolean ignoreIfNotExists)
+ throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
throw new DdlException("No catalog found with name: " + catalogName);
@@ -564,12 +565,18 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
- throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
+ if (!ignoreIfNotExists) {
+ throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
+ }
+ return;
}
TableIf table = db.getTableNullable(tableName);
if (table == null) {
- throw new DdlException("Table " + tableName + " does not exist in db " + dbName);
+ if (!ignoreIfNotExists) {
+ throw new DdlException("Table " + tableName + " does not exist in db " + dbName);
+ }
+ return;
}
if (table instanceof ExternalTable) {
((ExternalTable) table).unsetObjectCreated();
@@ -603,7 +610,8 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
.invalidateTableCache(catalog.getId(), db.getFullName(), table.getName());
}
- public void dropExternalTable(String dbName, String tableName, String catalogName) throws DdlException {
+ public void dropExternalTable(String dbName, String tableName, String catalogName, boolean ignoreIfExists)
+ throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
throw new DdlException("No catalog found with name: " + catalogName);
@@ -613,12 +621,18 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
- throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
+ if (!ignoreIfExists) {
+ throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
+ }
+ return;
}
TableIf table = db.getTableNullable(tableName);
if (table == null) {
- throw new DdlException("Table " + tableName + " does not exist in db " + dbName);
+ if (!ignoreIfExists) {
+ throw new DdlException("Table " + tableName + " does not exist in db " + dbName);
+ }
+ return;
}
ExternalObjectLog log = new ExternalObjectLog();
log.setCatalogId(catalog.getId());
@@ -668,7 +682,8 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
return ((ExternalCatalog) catalog).tableExistInLocal(dbName, tableName);
}
- public void createExternalTable(String dbName, String tableName, String catalogName) throws DdlException {
+ public void createExternalTable(String dbName, String tableName, String catalogName, boolean ignoreIfExists)
+ throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
throw new DdlException("No catalog found with name: " + catalogName);
@@ -678,12 +693,18 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
- throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
+ if (!ignoreIfExists) {
+ throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
+ }
+ return;
}
TableIf table = db.getTableNullable(tableName);
if (table != null) {
- throw new DdlException("Table " + tableName + " has exist in db " + dbName);
+ if (!ignoreIfExists) {
+ throw new DdlException("Table " + tableName + " has exist in db " + dbName);
+ }
+ return;
}
ExternalObjectLog log = new ExternalObjectLog();
log.setCatalogId(catalog.getId());
@@ -715,7 +736,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
}
}
- public void dropExternalDatabase(String dbName, String catalogName) throws DdlException {
+ public void dropExternalDatabase(String dbName, String catalogName, boolean ignoreIfNotExists) throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
throw new DdlException("No catalog found with name: " + catalogName);
@@ -725,7 +746,10 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
- throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
+ if (!ignoreIfNotExists) {
+ throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
+ }
+ return;
}
ExternalObjectLog log = new ExternalObjectLog();
@@ -758,7 +782,7 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
}
}
- public void createExternalDatabase(String dbName, String catalogName) throws DdlException {
+ public void createExternalDatabase(String dbName, String catalogName, boolean ignoreIfExists) throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
throw new DdlException("No catalog found with name: " + catalogName);
@@ -768,7 +792,10 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db != null) {
- throw new DdlException("Database " + dbName + " has exist in catalog " + catalog.getName());
+ if (!ignoreIfExists) {
+ throw new DdlException("Database " + dbName + " has exist in catalog " + catalog.getName());
+ }
+ return;
}
ExternalObjectLog log = new ExternalObjectLog();
@@ -795,7 +822,8 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
}
}
- public void addExternalPartitions(String catalogName, String dbName, String tableName, List<String> partitionNames)
+ public void addExternalPartitions(String catalogName, String dbName, String tableName, List<String> partitionNames,
+ boolean ignoreIfNotExists)
throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
@@ -806,12 +834,18 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
- throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
+ if (!ignoreIfNotExists) {
+ throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
+ }
+ return;
}
TableIf table = db.getTableNullable(tableName);
if (table == null) {
- throw new DdlException("Table " + tableName + " does not exist in db " + dbName);
+ if (!ignoreIfNotExists) {
+ throw new DdlException("Table " + tableName + " does not exist in db " + dbName);
+ }
+ return;
}
ExternalObjectLog log = new ExternalObjectLog();
@@ -845,7 +879,8 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
.addPartitionsCache(catalog.getId(), table, log.getPartitionNames());
}
- public void dropExternalPartitions(String catalogName, String dbName, String tableName, List<String> partitionNames)
+ public void dropExternalPartitions(String catalogName, String dbName, String tableName, List<String> partitionNames,
+ boolean ignoreIfNotExists)
throws DdlException {
CatalogIf catalog = nameToCatalog.get(catalogName);
if (catalog == null) {
@@ -856,12 +891,18 @@ public class CatalogMgr implements Writable, GsonPostProcessable {
}
DatabaseIf db = catalog.getDbNullable(dbName);
if (db == null) {
- throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
+ if (!ignoreIfNotExists) {
+ throw new DdlException("Database " + dbName + " does not exist in catalog " + catalog.getName());
+ }
+ return;
}
TableIf table = db.getTableNullable(tableName);
if (table == null) {
- throw new DdlException("Table " + tableName + " does not exist in db " + dbName);
+ if (!ignoreIfNotExists) {
+ throw new DdlException("Table " + tableName + " does not exist in db " + dbName);
+ }
+ return;
}
ExternalObjectLog log = new ExternalObjectLog();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
index e5f8d1bb71..9ced60b322 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
@@ -79,7 +79,7 @@ public class AddPartitionEvent extends MetastoreTableEvent {
return;
}
Env.getCurrentEnv().getCatalogMgr()
- .addExternalPartitions(catalogName, dbName, hmsTbl.getTableName(), partitionNames);
+ .addExternalPartitions(catalogName, dbName, hmsTbl.getTableName(), partitionNames, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
index 5209edccab..1e2eb6d06c 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
@@ -79,9 +79,11 @@ public class AlterPartitionEvent extends MetastoreTableEvent {
catalogName, dbName, tblName, partitionNameBefore, partitionNameAfter);
if (isRename) {
Env.getCurrentEnv().getCatalogMgr()
- .dropExternalPartitions(catalogName, dbName, tblName, Lists.newArrayList(partitionNameBefore));
+ .dropExternalPartitions(catalogName, dbName, tblName,
+ Lists.newArrayList(partitionNameBefore), true);
Env.getCurrentEnv().getCatalogMgr()
- .addExternalPartitions(catalogName, dbName, tblName, Lists.newArrayList(partitionNameAfter));
+ .addExternalPartitions(catalogName, dbName, tblName,
+ Lists.newArrayList(partitionNameAfter), true);
} else {
Env.getCurrentEnv().getCatalogMgr()
.refreshExternalPartitions(catalogName, dbName, hmsTbl.getTableName(),
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
index 9b731461cd..902ceacfa7 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
@@ -73,9 +73,9 @@ public class AlterTableEvent extends MetastoreTableEvent {
return;
}
Env.getCurrentEnv().getCatalogMgr()
- .dropExternalTable(tableBefore.getDbName(), tableBefore.getTableName(), catalogName);
+ .dropExternalTable(tableBefore.getDbName(), tableBefore.getTableName(), catalogName, true);
Env.getCurrentEnv().getCatalogMgr()
- .createExternalTable(tableAfter.getDbName(), tableAfter.getTableName(), catalogName);
+ .createExternalTable(tableAfter.getDbName(), tableAfter.getTableName(), catalogName, true);
}
private void processRename() throws DdlException {
@@ -91,9 +91,9 @@ public class AlterTableEvent extends MetastoreTableEvent {
return;
}
Env.getCurrentEnv().getCatalogMgr()
- .dropExternalTable(tableBefore.getDbName(), tableBefore.getTableName(), catalogName);
+ .dropExternalTable(tableBefore.getDbName(), tableBefore.getTableName(), catalogName, true);
Env.getCurrentEnv().getCatalogMgr()
- .createExternalTable(tableAfter.getDbName(), tableAfter.getTableName(), catalogName);
+ .createExternalTable(tableAfter.getDbName(), tableAfter.getTableName(), catalogName, true);
}
@@ -118,7 +118,7 @@ public class AlterTableEvent extends MetastoreTableEvent {
}
//The scope of refresh can be narrowed in the future
Env.getCurrentEnv().getCatalogMgr()
- .refreshExternalTable(tableBefore.getDbName(), tableBefore.getTableName(), catalogName);
+ .refreshExternalTable(tableBefore.getDbName(), tableBefore.getTableName(), catalogName, true);
} catch (Exception e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
index 48476bce57..2205356caf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
@@ -48,7 +48,7 @@ public class CreateDatabaseEvent extends MetastoreEvent {
try {
infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName);
Env.getCurrentEnv().getCatalogMgr()
- .createExternalDatabase(dbName, catalogName);
+ .createExternalDatabase(dbName, catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
index 2d364c4c14..1dbfd08ccf 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
@@ -59,16 +59,7 @@ public class CreateTableEvent extends MetastoreTableEvent {
protected void process() throws MetastoreNotificationException {
try {
infoLog("catalogName:[{}],dbName:[{}],tableName:[{}]", catalogName, dbName, tblName);
- boolean hasExist = Env.getCurrentEnv().getCatalogMgr()
- .externalTableExistInLocal(dbName, hmsTbl.getTableName(), catalogName);
- if (hasExist) {
- infoLog(
- "CreateExternalTable canceled,because table has exist,"
- + "catalogName:[{}],dbName:[{}],tableName:[{}]",
- catalogName, dbName, tblName);
- return;
- }
- Env.getCurrentEnv().getCatalogMgr().createExternalTable(dbName, hmsTbl.getTableName(), catalogName);
+ Env.getCurrentEnv().getCatalogMgr().createExternalTable(dbName, hmsTbl.getTableName(), catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
index b51145a258..a11e893ef2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
@@ -48,7 +48,7 @@ public class DropDatabaseEvent extends MetastoreEvent {
try {
infoLog("catalogName:[{}],dbName:[{}]", catalogName, dbName);
Env.getCurrentEnv().getCatalogMgr()
- .dropExternalDatabase(dbName, catalogName);
+ .dropExternalDatabase(dbName, catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
index 1dce403f55..59254fc4f2 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
@@ -79,7 +79,7 @@ public class DropPartitionEvent extends MetastoreTableEvent {
return;
}
Env.getCurrentEnv().getCatalogMgr()
- .dropExternalPartitions(catalogName, dbName, hmsTbl.getTableName(), partitionNames);
+ .dropExternalPartitions(catalogName, dbName, hmsTbl.getTableName(), partitionNames, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java
index aa74c67512..d42de68cbb 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java
@@ -59,7 +59,7 @@ public class DropTableEvent extends MetastoreTableEvent {
protected void process() throws MetastoreNotificationException {
try {
infoLog("catalogName:[{}],dbName:[{}],tableName:[{}]", catalogName, dbName, tableName);
- Env.getCurrentEnv().getCatalogMgr().dropExternalTable(dbName, tableName, catalogName);
+ Env.getCurrentEnv().getCatalogMgr().dropExternalTable(dbName, tableName, catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
index cf4ba1d5b0..2b53e86639 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
@@ -66,7 +66,7 @@ public class InsertEvent extends MetastoreTableEvent {
* the file cache of this table,
* but <a href="https://github.com/apache/doris/pull/17932">this PR</a> has fixed it.
*/
- Env.getCurrentEnv().getCatalogMgr().refreshExternalTable(dbName, tblName, catalogName);
+ Env.getCurrentEnv().getCatalogMgr().refreshExternalTable(dbName, tblName, catalogName, true);
} catch (DdlException e) {
throw new MetastoreNotificationException(
debugString("Failed to process event"));
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@doris.apache.org
For additional commands, e-mail: commits-help@doris.apache.org