You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/09/12 20:24:43 UTC
[11/31] hive git commit: HIVE-14626: Support Trash in Truncate Table
(Chaoyu Tang, reviewed by Sergio Pena)
HIVE-14626: Support Trash in Truncate Table (Chaoyu Tang, reviewed by Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/943a361d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/943a361d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/943a361d
Branch: refs/heads/hive-14535
Commit: 943a361da621ebe587c613c811231c606c66e515
Parents: 4c543b1
Author: ctang <ct...@cloudera.com>
Authored: Thu Sep 8 14:09:15 2016 -0400
Committer: ctang <ct...@cloudera.com>
Committed: Thu Sep 8 14:09:15 2016 -0400
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 33 ++++--
.../apache/hadoop/hive/ql/metadata/Hive.java | 2 +-
.../clientpositive/encryption_drop_partition.q | 7 ++
.../clientpositive/encryption_drop_table.q | 23 ++++
.../encrypted/encryption_drop_partition.q.out | 33 ++++++
.../encrypted/encryption_drop_table.q.out | 110 +++++++++++++++++++
6 files changed, 198 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/943a361d/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 66cbdd3..569c19e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -225,6 +225,8 @@ import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.hive.shims.HadoopShims;
+import org.apache.hadoop.hive.shims.ShimLoader;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.tools.HadoopArchives;
@@ -4269,15 +4271,28 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
// this is not transactional
for (Path location : getLocations(db, table, partSpec)) {
FileSystem fs = location.getFileSystem(conf);
- HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, location);
- FileStatus targetStatus = fs.getFileStatus(location);
- String targetGroup = targetStatus == null ? null : targetStatus.getGroup();
- fs.delete(location, true);
- fs.mkdirs(location);
- try {
- HdfsUtils.setFullFileStatus(conf, status, targetGroup, fs, location, false);
- } catch (Exception e) {
- LOG.warn("Error setting permissions of " + location, e);
+ HadoopShims.HdfsEncryptionShim shim
+ = ShimLoader.getHadoopShims().createHdfsEncryptionShim(fs, conf);
+ if (!shim.isPathEncrypted(location)) {
+ HdfsUtils.HadoopFileStatus status = new HdfsUtils.HadoopFileStatus(conf, fs, location);
+ FileStatus targetStatus = fs.getFileStatus(location);
+ String targetGroup = targetStatus == null ? null : targetStatus.getGroup();
+ FileUtils.moveToTrash(fs, location, conf);
+ fs.mkdirs(location);
+ try {
+ HdfsUtils.setFullFileStatus(conf, status, targetGroup, fs, location, false);
+ } catch (Exception e) {
+ LOG.warn("Error setting permissions of " + location, e);
+ }
+ } else {
+ FileStatus[] statuses = fs.listStatus(location, FileUtils.HIDDEN_FILES_PATH_FILTER);
+ if (statuses == null || statuses.length == 0) {
+ continue;
+ }
+ boolean success = Hive.trashFiles(fs, statuses, conf);
+ if (!success) {
+ throw new HiveException("Error in deleting the contents of " + location.toString());
+ }
}
}
} catch (Exception e) {
http://git-wip-us.apache.org/repos/asf/hive/blob/943a361d/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 5f53aef..da46854 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -3365,7 +3365,7 @@ private void constructOneLBLocationMap(FileStatus fSta,
* @return true if deletion successful
* @throws IOException
*/
- private boolean trashFiles(final FileSystem fs, final FileStatus[] statuses, final Configuration conf)
+ public static boolean trashFiles(final FileSystem fs, final FileStatus[] statuses, final Configuration conf)
throws IOException {
boolean result = true;
http://git-wip-us.apache.org/repos/asf/hive/blob/943a361d/ql/src/test/queries/clientpositive/encryption_drop_partition.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_drop_partition.q b/ql/src/test/queries/clientpositive/encryption_drop_partition.q
index 57dfabd..d968459 100644
--- a/ql/src/test/queries/clientpositive/encryption_drop_partition.q
+++ b/ql/src/test/queries/clientpositive/encryption_drop_partition.q
@@ -25,4 +25,11 @@ ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23');
SELECT * FROM encrypted_table_dp;
ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-23') PURGE;
SELECT * FROM encrypted_table_dp;
+
+TRUNCATE TABLE encrypted_table_dp PARTITION (p='2014-09-24');
+SHOW PARTITIONS encrypted_table_dp;
+SELECT * FROM encrypted_table_dp;
+
+ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-24');
+DROP TABLE encrypted_table_dp;
DROP TABLE encrypted_table_dp PURGE;
http://git-wip-us.apache.org/repos/asf/hive/blob/943a361d/ql/src/test/queries/clientpositive/encryption_drop_table.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_drop_table.q b/ql/src/test/queries/clientpositive/encryption_drop_table.q
index 2ae3c69..76be118 100644
--- a/ql/src/test/queries/clientpositive/encryption_drop_table.q
+++ b/ql/src/test/queries/clientpositive/encryption_drop_table.q
@@ -5,6 +5,8 @@
set hive.cli.errors.ignore=true;
DROP TABLE IF EXISTS encrypted_table PURGE;
+DROP TABLE IF EXISTS encrypted_ext_table PURGE;
+
CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';
CRYPTO CREATE_KEY --keyName key_128 --bitLength 128;
CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table;
@@ -22,4 +24,25 @@ SHOW TABLES;
DROP TABLE default.encrypted_table PURGE;
SHOW TABLES;
+
+DROP TABLE IF EXISTS encrypted_table1 PURGE;
+CREATE TABLE encrypted_table1 (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table1';
+CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table1;
+INSERT OVERWRITE TABLE encrypted_table1 SELECT * FROM src;
+
+SELECT COUNT(*) FROM encrypted_table1;
+TRUNCATE TABLE encrypted_table1;
+SELECT COUNT(*) FROM encrypted_table1;
+
+INSERT OVERWRITE TABLE encrypted_table1 SELECT * FROM src;
+DROP TABLE default.encrypted_table1;
+SHOW TABLES;
+
+TRUNCATE TABLE encrypted_table1;
+DROP TABLE default.encrypted_table1;
+SHOW TABLES;
+
+DROP TABLE default.encrypted_table1 PURGE;
+SHOW TABLES;
+
CRYPTO DELETE_KEY --keyName key_128;
http://git-wip-us.apache.org/repos/asf/hive/blob/943a361d/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
index 2643006..c2f6b03 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_partition.q.out
@@ -147,6 +147,39 @@ POSTHOOK: Input: default@encrypted_table_dp
POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24
#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
2 bar 2014-09-24
+PREHOOK: query: TRUNCATE TABLE encrypted_table_dp PARTITION (p='2014-09-24')
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@encrypted_table_dp@p=2014-09-24
+POSTHOOK: query: TRUNCATE TABLE encrypted_table_dp PARTITION (p='2014-09-24')
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@encrypted_table_dp@p=2014-09-24
+PREHOOK: query: SHOW PARTITIONS encrypted_table_dp
+PREHOOK: type: SHOWPARTITIONS
+PREHOOK: Input: default@encrypted_table_dp
+POSTHOOK: query: SHOW PARTITIONS encrypted_table_dp
+POSTHOOK: type: SHOWPARTITIONS
+POSTHOOK: Input: default@encrypted_table_dp
+p=2014-09-24
+PREHOOK: query: SELECT * FROM encrypted_table_dp
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_table_dp
+PREHOOK: Input: default@encrypted_table_dp@p=2014-09-24
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+POSTHOOK: query: SELECT * FROM encrypted_table_dp
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_table_dp
+POSTHOOK: Input: default@encrypted_table_dp@p=2014-09-24
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table_dp/.hive-staging
+PREHOOK: query: ALTER TABLE encrypted_table_dp DROP PARTITION (p='2014-09-24')
+PREHOOK: type: ALTERTABLE_DROPPARTS
+PREHOOK: Input: default@encrypted_table_dp
+PREHOOK: Output: default@encrypted_table_dp@p=2014-09-24
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to drop default.encrypted_table_dp.[2014-09-24] because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash.
+PREHOOK: query: DROP TABLE encrypted_table_dp
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_table_dp
+PREHOOK: Output: default@encrypted_table_dp
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Unable to drop default.encrypted_table_dp because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash.)
PREHOOK: query: DROP TABLE encrypted_table_dp PURGE
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@encrypted_table_dp
http://git-wip-us.apache.org/repos/asf/hive/blob/943a361d/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
index c5007ee..a3f41d8 100644
--- a/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_drop_table.q.out
@@ -2,6 +2,10 @@ PREHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
PREHOOK: type: DROPTABLE
POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table PURGE
POSTHOOK: type: DROPTABLE
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_ext_table PURGE
+POSTHOOK: type: DROPTABLE
#### A masked pattern was here ####
PREHOOK: type: CREATETABLE
#### A masked pattern was here ####
@@ -87,3 +91,109 @@ POSTHOOK: query: SHOW TABLES
POSTHOOK: type: SHOWTABLES
POSTHOOK: Input: database:default
src
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_table1 PURGE
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table1 PURGE
+POSTHOOK: type: DROPTABLE
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_table1
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_table1
+Encryption zone created: '/build/ql/test/data/warehouse/default/encrypted_table1' using key: 'key_128'
+PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table1 SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@encrypted_table1
+POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table1 SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@encrypted_table1
+POSTHOOK: Lineage: encrypted_table1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: encrypted_table1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SELECT COUNT(*) FROM encrypted_table1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_table1
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table1/.hive-staging
+POSTHOOK: query: SELECT COUNT(*) FROM encrypted_table1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_table1
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table1/.hive-staging
+500
+PREHOOK: query: TRUNCATE TABLE encrypted_table1
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@encrypted_table1
+POSTHOOK: query: TRUNCATE TABLE encrypted_table1
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@encrypted_table1
+PREHOOK: query: SELECT COUNT(*) FROM encrypted_table1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@encrypted_table1
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table1/.hive-staging
+POSTHOOK: query: SELECT COUNT(*) FROM encrypted_table1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@encrypted_table1
+#### A PARTIAL masked pattern was here #### data/warehouse/default/encrypted_table1/.hive-staging
+0
+PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table1 SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@encrypted_table1
+POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table1 SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@encrypted_table1
+POSTHOOK: Lineage: encrypted_table1.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: encrypted_table1.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: DROP TABLE default.encrypted_table1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_table1
+PREHOOK: Output: default@encrypted_table1
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Unable to drop default.encrypted_table1 because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash.)
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:default
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:default
+encrypted_table1
+src
+PREHOOK: query: TRUNCATE TABLE encrypted_table1
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@encrypted_table1
+POSTHOOK: query: TRUNCATE TABLE encrypted_table1
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@encrypted_table1
+PREHOOK: query: DROP TABLE default.encrypted_table1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_table1
+PREHOOK: Output: default@encrypted_table1
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. MetaException(message:Unable to drop default.encrypted_table1 because it is in an encryption zone and trash is enabled. Use PURGE option to skip trash.)
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:default
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:default
+encrypted_table1
+src
+PREHOOK: query: DROP TABLE default.encrypted_table1 PURGE
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@encrypted_table1
+PREHOOK: Output: default@encrypted_table1
+POSTHOOK: query: DROP TABLE default.encrypted_table1 PURGE
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@encrypted_table1
+POSTHOOK: Output: default@encrypted_table1
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+PREHOOK: Input: database:default
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+POSTHOOK: Input: database:default
+src