You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by rb...@apache.org on 2021/08/05 22:13:22 UTC
[hive] branch master updated: HIVE-25341. Reduce FileSystem calls
in case drop database cascade. (#2491)
This is an automated email from the ASF dual-hosted git repository.
rbalamohan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2f46592 HIVE-25341. Reduce FileSystem calls in case drop database cascade. (#2491)
2f46592 is described below
commit 2f465921290fc60e80695644b28cc37143fa29f7
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Fri Aug 6 03:43:11 2021 +0530
HIVE-25341. Reduce FileSystem calls in case drop database cascade. (#2491)
* HIVE-25341. Reduce FileSystem calls in case drop database cascade.
* Add check for managed table.
---
.../ql/exec/repl/bootstrap/load/LoadDatabase.java | 4 +-
.../apache/hadoop/hive/metastore/HMSHandler.java | 56 ++++++++++++----------
2 files changed, 31 insertions(+), 29 deletions(-)
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java
index 7f18438..ba7979d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/bootstrap/load/LoadDatabase.java
@@ -146,9 +146,7 @@ public class LoadDatabase {
}
private boolean isDbEmpty(String dbName) throws HiveException {
- List<String> allTables = context.hiveDb.getAllTables(dbName);
- List<String> allFunctions = context.hiveDb.getFunctions(dbName, "*");
- return allTables.isEmpty() && allFunctions.isEmpty();
+ return context.hiveDb.getAllTables(dbName).isEmpty() && context.hiveDb.getFunctions(dbName, "*").isEmpty();
}
private Task<?> createDbTask(Database dbObj) throws MetaException {
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
index 353a193..2e2fc93 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/HMSHandler.java
@@ -1710,14 +1710,14 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
for (Table materializedView : materializedViews) {
if (materializedView.getSd().getLocation() != null) {
Path materializedViewPath = wh.getDnsPath(new Path(materializedView.getSd().getLocation()));
- if (!wh.isWritable(materializedViewPath.getParent())) {
- throw new MetaException("Database metadata not deleted since table: " +
- materializedView.getTableName() + " has a parent location " + materializedViewPath.getParent() +
- " which is not writable by " + SecurityUtils.getUser());
- }
if (!FileUtils.isSubdirectory(databasePath.toString(),
materializedViewPath.toString())) {
+ if (!wh.isWritable(materializedViewPath.getParent())) {
+ throw new MetaException("Database metadata not deleted since table: " +
+ materializedView.getTableName() + " has a parent location " + materializedViewPath.getParent() +
+ " which is not writable by " + SecurityUtils.getUser());
+ }
tablePaths.add(materializedViewPath);
}
}
@@ -1751,15 +1751,15 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
// add it's locations to the list of paths to delete
Path tablePath = null;
boolean tableDataShouldBeDeleted = checkTableDataShouldBeDeleted(table, deleteData);
+ boolean isManagedTable = table.getTableType().equals(TableType.MANAGED_TABLE.toString());
if (table.getSd().getLocation() != null && tableDataShouldBeDeleted) {
tablePath = wh.getDnsPath(new Path(table.getSd().getLocation()));
- if (!wh.isWritable(tablePath.getParent())) {
- throw new MetaException("Database metadata not deleted since table: " +
- table.getTableName() + " has a parent location " + tablePath.getParent() +
- " which is not writable by " + SecurityUtils.getUser());
- }
-
- if (!FileUtils.isSubdirectory(databasePath.toString(), tablePath.toString())) {
+ if (!isManagedTable) {
+ if (!wh.isWritable(tablePath.getParent())) {
+ throw new MetaException(
+ "Database metadata not deleted since table: " + table.getTableName() + " has a parent location "
+ + tablePath.getParent() + " which is not writable by " + SecurityUtils.getUser());
+ }
tablePaths.add(tablePath);
}
}
@@ -1770,8 +1770,9 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
tablePath, tableDataShouldBeDeleted);
// Drop the table but not its data
- drop_table(MetaStoreUtils.prependCatalogToDbName(table.getCatName(), table.getDbName(), conf),
- table.getTableName(), false);
+ drop_table_with_environment_context(
+ MetaStoreUtils.prependCatalogToDbName(table.getCatName(), table.getDbName(), conf),
+ table.getTableName(), false, null, false);
}
}
@@ -2926,7 +2927,7 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
private boolean drop_table_core(final RawStore ms, final String catName, final String dbname,
final String name, final boolean deleteData,
- final EnvironmentContext envContext, final String indexName)
+ final EnvironmentContext envContext, final String indexName, boolean dropPartitions)
throws NoSuchObjectException, MetaException, IOException, InvalidObjectException,
InvalidInputException {
boolean success = false;
@@ -2983,9 +2984,10 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
}
// Drop the partitions and get a list of locations which need to be deleted
- partPaths = dropPartitionsAndGetLocations(ms, catName, dbname, name, tblPath,
- tableDataShouldBeDeleted);
-
+ // In case of drop database cascade we need not to drop the partitions, they are already dropped.
+ if (dropPartitions) {
+ partPaths = dropPartitionsAndGetLocations(ms, catName, dbname, name, tblPath, tableDataShouldBeDeleted);
+ }
// Drop any constraints on the table
ms.dropConstraint(catName, dbname, name, null, true);
@@ -3218,26 +3220,28 @@ public class HMSHandler extends FacebookBase implements IHMSHandler {
}
@Override
- public void drop_table_with_environment_context(final String dbname, final String name,
- final boolean deleteData, final EnvironmentContext envContext)
- throws NoSuchObjectException, MetaException {
+ public void drop_table_with_environment_context(final String dbname, final String name, final boolean deleteData,
+ final EnvironmentContext envContext) throws NoSuchObjectException, MetaException {
+ drop_table_with_environment_context(dbname, name, deleteData, envContext, true);
+ }
+
+ private void drop_table_with_environment_context(final String dbname, final String name, final boolean deleteData,
+ final EnvironmentContext envContext, boolean dropPartitions) throws MetaException {
String[] parsedDbName = parseDbName(dbname, conf);
startTableFunction("drop_table", parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name);
boolean success = false;
Exception ex = null;
try {
- success = drop_table_core(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name,
- deleteData, envContext, null);
+ success =
+ drop_table_core(getMS(), parsedDbName[CAT_NAME], parsedDbName[DB_NAME], name, deleteData, envContext, null, dropPartitions);
} catch (Exception e) {
ex = e;
throw handleException(e).throwIfInstance(MetaException.class, NoSuchObjectException.class)
- .convertIfInstance(IOException.class, MetaException.class)
- .defaultMetaException();
+ .convertIfInstance(IOException.class, MetaException.class).defaultMetaException();
} finally {
endFunction("drop_table", success, ex, name);
}
-
}
private void updateStatsForTruncate(Map<String,String> props, EnvironmentContext environmentContext) {