You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ct...@apache.org on 2016/03/15 16:41:51 UTC
hive git commit: HIVE-13243: Hive drop table on encyption zone fails
for external tables (Chaoyu Tang, reviewed by Sergio Pena)
Repository: hive
Updated Branches:
refs/heads/master f87b2b637 -> d5b1adbb6
HIVE-13243: Hive drop table on encyption zone fails for external tables (Chaoyu Tang, reviewed by Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d5b1adbb
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d5b1adbb
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d5b1adbb
Branch: refs/heads/master
Commit: d5b1adbb6638acd74fd4162a6ed408ebd72abeae
Parents: f87b2b6
Author: ctang <ct...@cloudera.com>
Authored: Tue Mar 15 11:41:41 2016 -0400
Committer: ctang <ct...@cloudera.com>
Committed: Tue Mar 15 11:41:41 2016 -0400
----------------------------------------------------------------------
.../hadoop/hive/metastore/HiveMetaStore.java | 34 ++++++---
.../clientpositive/encryption_drop_partition.q | 10 +++
.../clientpositive/encryption_drop_table.q | 9 ++-
.../encrypted/encryption_drop_partition.q.out | 76 ++++++++++++++++++++
.../encrypted/encryption_drop_table.q.out | 27 +++++++
5 files changed, 144 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index f0bc560..2fa0e9a 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -1508,8 +1508,7 @@ public class HiveMetaStore extends ThriftHiveMetastore {
}
}
- // tblPath will be null when tbl is a view. We skip the following if block in that case.
- checkTrashPurgeCombination(tblPath, dbname + "." + name, ifPurge);
+ checkTrashPurgeCombination(tblPath, dbname + "." + name, ifPurge, deleteData && !isExternal);
// Drop the partitions and get a list of locations which need to be deleted
partPaths = dropPartitionsAndGetLocations(ms, dbname, name, tblPath,
tbl.getPartitionKeys(), deleteData && !isExternal);
@@ -1546,15 +1545,20 @@ public class HiveMetaStore extends ThriftHiveMetastore {
* @param objectName db.table, or db.table.part
* @param ifPurge if PURGE options is specified
*/
- private void checkTrashPurgeCombination(Path pathToData, String objectName, boolean ifPurge)
- throws MetaException {
- if (!(pathToData != null && !ifPurge)) {//pathToData may be NULL for a view
+ private void checkTrashPurgeCombination(Path pathToData, String objectName, boolean ifPurge,
+ boolean deleteData) throws MetaException {
+ // There is no need to check TrashPurgeCombination in following cases since Purge/Trash
+ // is not applicable:
+ // a) deleteData is false -- drop an external table
+ // b) pathToData is null -- a view
+ // c) ifPurge is true -- force delete without Trash
+ if (!deleteData || pathToData == null || ifPurge) {
return;
}
boolean trashEnabled = false;
try {
- trashEnabled = 0 < hiveConf.getFloat("fs.trash.interval", -1);
+ trashEnabled = 0 < hiveConf.getFloat("fs.trash.interval", -1);
} catch(NumberFormatException ex) {
// nothing to do
}
@@ -2644,11 +2648,13 @@ public class HiveMetaStore extends ThriftHiveMetastore {
boolean isArchived = false;
Path archiveParentDir = null;
boolean mustPurge = false;
+ boolean isExternalTbl = false;
try {
ms.openTransaction();
part = ms.getPartition(db_name, tbl_name, part_vals);
tbl = get_table_core(db_name, tbl_name);
+ isExternalTbl = isExternal(tbl);
firePreEvent(new PreDropPartitionEvent(tbl, part, deleteData, this));
mustPurge = isMustPurge(envContext, tbl);
@@ -2661,7 +2667,8 @@ public class HiveMetaStore extends ThriftHiveMetastore {
if (isArchived) {
archiveParentDir = MetaStoreUtils.getOriginalLocation(part);
verifyIsWritablePath(archiveParentDir);
- checkTrashPurgeCombination(archiveParentDir, db_name + "." + tbl_name + "." + part_vals, mustPurge);
+ checkTrashPurgeCombination(archiveParentDir, db_name + "." + tbl_name + "." + part_vals,
+ mustPurge, deleteData && !isExternalTbl);
}
if (!ms.dropPartition(db_name, tbl_name, part_vals)) {
throw new MetaException("Unable to drop partition");
@@ -2670,13 +2677,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
if ((part.getSd() != null) && (part.getSd().getLocation() != null)) {
partPath = new Path(part.getSd().getLocation());
verifyIsWritablePath(partPath);
- checkTrashPurgeCombination(partPath, db_name + "." + tbl_name + "." + part_vals, mustPurge);
+ checkTrashPurgeCombination(partPath, db_name + "." + tbl_name + "." + part_vals,
+ mustPurge, deleteData && !isExternalTbl);
}
} finally {
if (!success) {
ms.rollbackTransaction();
} else if (deleteData && ((partPath != null) || (archiveParentDir != null))) {
- if (tbl != null && !isExternal(tbl)) {
+ if (!isExternalTbl) {
if (mustPurge) {
LOG.info("dropPartition() will purge " + partPath + " directly, skipping trash.");
}
@@ -2761,10 +2769,12 @@ public class HiveMetaStore extends ThriftHiveMetastore {
Table tbl = null;
List<Partition> parts = null;
boolean mustPurge = false;
+ boolean isExternalTbl = false;
try {
// We need Partition-s for firing events and for result; DN needs MPartition-s to drop.
// Great... Maybe we could bypass fetching MPartitions by issuing direct SQL deletes.
tbl = get_table_core(dbName, tblName);
+ isExternalTbl = isExternal(tbl);
mustPurge = isMustPurge(envContext, tbl);
int minCount = 0;
RequestPartsSpec spec = request.getParts();
@@ -2829,13 +2839,15 @@ public class HiveMetaStore extends ThriftHiveMetastore {
if (MetaStoreUtils.isArchived(part)) {
Path archiveParentDir = MetaStoreUtils.getOriginalLocation(part);
verifyIsWritablePath(archiveParentDir);
- checkTrashPurgeCombination(archiveParentDir, dbName + "." + tblName + "." + part.getValues(), mustPurge);
+ checkTrashPurgeCombination(archiveParentDir, dbName + "." + tblName + "." +
+ part.getValues(), mustPurge, deleteData && !isExternalTbl);
archToDelete.add(archiveParentDir);
}
if ((part.getSd() != null) && (part.getSd().getLocation() != null)) {
Path partPath = new Path(part.getSd().getLocation());
verifyIsWritablePath(partPath);
- checkTrashPurgeCombination(partPath, dbName + "." + tblName + "." + part.getValues(), mustPurge);
+ checkTrashPurgeCombination(partPath, dbName + "." + tblName + "." + part.getValues(),
+ mustPurge, deleteData && !isExternalTbl);
dirsToDelete.add(new PathAndPartValSize(partPath, part.getValues().size()));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/ql/src/test/queries/clientpositive/encryption_drop_partition.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_drop_partition.q b/ql/src/test/queries/clientpositive/encryption_drop_partition.q
index a26aa19..57dfabd 100644
--- a/ql/src/test/queries/clientpositive/encryption_drop_partition.q
+++ b/ql/src/test/queries/clientpositive/encryption_drop_partition.q
@@ -12,7 +12,17 @@ CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.
INSERT INTO encrypted_table_dp PARTITION(p)(p,key,value) values('2014-09-23', 1, 'foo'),('2014-09-24', 2, 'bar');
SELECT * FROM encrypted_table_dp;
+
+CREATE EXTERNAL TABLE encrypted_ext_table_dp (key INT, value STRING) partitioned by (p STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table_dp';
+ALTER TABLE encrypted_ext_table_dp ADD PARTITION (p='2014-09-23') LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table_dp/p=2014-09-23';
+SELECT * FROM encrypted_ext_table_dp;
+ALTER TABLE encrypted_ext_table_dp DROP PARTITION (p='2014-09-23');
+SELECT * FROM encrypted_ext_table_dp;
+DROP TABLE encrypted_ext_table_dp;
+
+SELECT * FROM encrypted_table_dp;
ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23');
SELECT * FROM encrypted_table_dp;
ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23') PURGE;
SELECT * FROM encrypted_table_dp;
+DROP TABLE encrypted_table_dp PURGE;
http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/ql/src/test/queries/clientpositive/encryption_drop_table.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_drop_table.q b/ql/src/test/queries/clientpositive/encryption_drop_table.q
index 193beea..2ae3c69 100644
--- a/ql/src/test/queries/clientpositive/encryption_drop_table.q
+++ b/ql/src/test/queries/clientpositive/encryption_drop_table.q
@@ -10,9 +10,16 @@ CRYPTO CREATE_KEY --keyName key_128 --bitLength 128;
CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table;
INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src;
+
+CREATE EXTERNAL TABLE encrypted_ext_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';
+SHOW TABLES;
+
+DROP TABLE default.encrypted_ext_table;
SHOW TABLES;
+
DROP TABLE default.encrypted_table;
SHOW TABLES;
+
DROP TABLE default.encrypted_table PURGE;
SHOW TABLES;
-CRYPTO DELETE_KEY --keyName key_128;
\ No newline at end of file
+CRYPTO DELETE_KEY --keyName key_128;
http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
index 067bf82..2643006 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
@@ -41,6 +41,74 @@ POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24
#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
1 foo 2014-09-23
2 bar 2014-09-24
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_ext_table_dp
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_ext_table_dp
+#### A masked pattern was here ####
+PREHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+PREHOOK: Output: default@encrypted_ext_table_dp
+#### A masked pattern was here ####
+POSTHOOK: type: ALTERTABLE_ADDPARTS
+#### A masked pattern was here ####
+POSTHOOK: Output: default@encrypted_ext_table_dp
+POSTHOOK: Output: default@encrypted_ext_table_dp@p=2014-09-23
+PREHOOK: query: SELECT * FROM encrypted_ext_table_dp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_ext_table_dp
+PREHOOK: Input: default@encrypted_ext_table_dp@p=2014-09-23
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+POSTHOOK: query: SELECT * FROM encrypted_ext_table_dp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_ext_table_dp
+POSTHOOK: Input: default@encrypted_ext_table_dp@p=2014-09-23
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+1 foo 2014-09-23
+PREHOOK: query: ALTER TABLE encrypted_ext_table_dp DROP PARTITION (p='2014-09-23')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@encrypted_ext_table_dp
+PREHOOK: Output: default@encrypted_ext_table_dp@p=2014-09-23
+POSTHOOK: query: ALTER TABLE encrypted_ext_table_dp DROP PARTITION (p='2014-09-23')
+POSTHOOK: type: ALTERTABLE_DROPPARTS
+POSTHOOK: Input: default@encrypted_ext_table_dp
+POSTHOOK: Output: default@encrypted_ext_table_dp@p=2014-09-23
+PREHOOK: query: SELECT * FROM encrypted_ext_table_dp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_ext_table_dp
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+POSTHOOK: query: SELECT * FROM encrypted_ext_table_dp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_ext_table_dp
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+PREHOOK: query: DROP TABLE encrypted_ext_table_dp
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_ext_table_dp
+PREHOOK: Output: default@encrypted_ext_table_dp
+POSTHOOK: query: DROP TABLE encrypted_ext_table_dp
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_ext_table_dp
+POSTHOOK: Output: default@encrypted_ext_table_dp
+PREHOOK: query: SELECT * FROM encrypted_table_dp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_table_dp
+PREHOOK: Input: default@encrypted_table_dp@p=2014-09-23
+PREHOOK: Input: default@encrypted_table_dp@p=2014-09-24
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+POSTHOOK: query: SELECT * FROM encrypted_table_dp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_table_dp
+POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-23
+POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+1 foo 2014-09-23
+2 bar 2014-09-24
PREHOOK: query: ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23')
PREHOOK: type: ALTERTABLE_DROPPARTS
PREHOOK: Input: default@encrypted_table_dp
@@ -79,3 +147,11 @@ POSTHOOK: Input: default@encrypted_table_dp
POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24
#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
2 bar 2014-09-24
+PREHOOK: query: DROP TABLE encrypted_table_dp PURGE
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_table_dp
+PREHOOK: Output: default@encrypted_table_dp
+POSTHOOK: query: DROP TABLE encrypted_table_dp PURGE
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_table_dp
+POSTHOOK: Output: default@encrypted_table_dp
http://git-wip-us.apache.org/repos/asf/hive/blob/d5b1adbb/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
index 55eefa0..c5007ee 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
@@ -24,6 +24,33 @@ POSTHOOK: Input: default@src
POSTHOOK: Output: default@encrypted_table
POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_ext_table
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_ext_table
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:default
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:default
+encrypted_ext_table
+encrypted_table
+src
+PREHOOK: query: DROP TABLE default.encrypted_ext_table
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_ext_table
+PREHOOK: Output: default@encrypted_ext_table
+POSTHOOK: query: DROP TABLE default.encrypted_ext_table
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_ext_table
+POSTHOOK: Output: default@encrypted_ext_table
PREHOOK: query: SHOW TABLES
PREHOOK: type: SHOWTABLES
PREHOOK: Input: database:default