You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2016/10/20 22:12:59 UTC
[22/23] hive git commit: HIVE-15020 : handle truncate for MM tables
(not atomic yet) (Sergey Shelukhin)
HIVE-15020 : handle truncate for MM tables (not atomic yet) (Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ff1ea20c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ff1ea20c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ff1ea20c
Branch: refs/heads/hive-14535
Commit: ff1ea20cb72e7e754fca1fc00e712adec257a204
Parents: 8004f71
Author: Sergey Shelukhin <se...@apache.org>
Authored: Thu Oct 20 15:07:39 2016 -0700
Committer: Sergey Shelukhin <se...@apache.org>
Committed: Thu Oct 20 15:07:39 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/ql/exec/TaskFactory.java | 1 +
.../hive/ql/parse/DDLSemanticAnalyzer.java | 9 +-
.../queries/clientnegative/mm_truncate_cols.q | 3 +
ql/src/test/queries/clientpositive/mm_all.q | 20 ++-
.../clientnegative/mm_truncate_cols.q.out | 9 ++
.../results/clientpositive/llap/mm_all.q.out | 152 +++++++++++++------
6 files changed, 136 insertions(+), 58 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ff1ea20c/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
index 822ff41..1cd20e3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskFactory.java
@@ -147,6 +147,7 @@ public final class TaskFactory {
throw new RuntimeException("No task for work class " + workClass.getName());
}
+ @SafeVarargs
public static <T extends Serializable> Task<T> get(T work, HiveConf conf,
Task<? extends Serializable>... tasklist) {
Task<T> ret = get((Class<T>) work.getClass(), conf);
http://git-wip-us.apache.org/repos/asf/hive/blob/ff1ea20c/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index a018b54..038cbbf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -963,6 +963,10 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
if (indexes != null && indexes.size() > 0) {
throw new SemanticException(ErrorMsg.TRUNCATE_COLUMN_INDEXED_TABLE.getMsg());
}
+ // It would be possible to support this, but this is such a pointless command.
+ if (MetaStoreUtils.isMmTable(table.getParameters())) {
+ throw new SemanticException("Truncating MM table columns not presently supported");
+ }
List<String> bucketCols = null;
Class<? extends InputFormat> inputFormatClass = null;
@@ -1060,12 +1064,11 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
// so the operation is atomic.
Path queryTmpdir = ctx.getExternalTmpPath(newTblPartLoc);
truncateTblDesc.setOutputDir(queryTmpdir);
- // TODO# movetask is created here; handle MM tables
LoadTableDesc ltd = new LoadTableDesc(queryTmpdir, tblDesc,
partSpec == null ? new HashMap<String, String>() : partSpec, null);
ltd.setLbCtx(lbCtx);
- Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false),
- conf);
+ @SuppressWarnings("unchecked")
+ Task<MoveWork> moveTsk = TaskFactory.get(new MoveWork(null, null, ltd, null, false), conf);
truncateTask.addDependentTask(moveTsk);
// Recalculate the HDFS stats if auto gather stats is set
http://git-wip-us.apache.org/repos/asf/hive/blob/ff1ea20c/ql/src/test/queries/clientnegative/mm_truncate_cols.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/mm_truncate_cols.q b/ql/src/test/queries/clientnegative/mm_truncate_cols.q
new file mode 100644
index 0000000..1780118
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/mm_truncate_cols.q
@@ -0,0 +1,3 @@
+CREATE TABLE mm_table(key int, value string) tblproperties('hivecommit'='true');
+
+TRUNCATE TABLE mm_table COLUMNS (value);
http://git-wip-us.apache.org/repos/asf/hive/blob/ff1ea20c/ql/src/test/queries/clientpositive/mm_all.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/mm_all.q b/ql/src/test/queries/clientpositive/mm_all.q
index 6639aaa..d440136 100644
--- a/ql/src/test/queries/clientpositive/mm_all.q
+++ b/ql/src/test/queries/clientpositive/mm_all.q
@@ -21,6 +21,10 @@ insert into table part_mm partition(key_mm='455') select key from intermediate;
insert into table part_mm partition(key_mm='456') select key from intermediate;
insert into table part_mm partition(key_mm='455') select key from intermediate;
select * from part_mm order by key, key_mm;
+truncate table part_mm partition(key_mm='455');
+select * from part_mm order by key, key_mm;
+truncate table part_mm;
+select * from part_mm order by key, key_mm;
drop table part_mm;
drop table simple_mm;
@@ -30,6 +34,8 @@ insert overwrite table simple_mm select key from intermediate;
select * from simple_mm order by key;
insert into table simple_mm select key from intermediate;
select * from simple_mm order by key;
+truncate table simple_mm;
+select * from simple_mm;
drop table simple_mm;
@@ -201,9 +207,9 @@ drop table iow0_mm;
create table iow0_mm(key int) tblproperties('hivecommit'='true');
insert overwrite table iow0_mm select key from intermediate;
insert into table iow0_mm select key + 1 from intermediate;
-select * from iow0_mm;
+select * from iow0_mm order by key;
insert overwrite table iow0_mm select key + 2 from intermediate;
-select * from iow0_mm;
+select * from iow0_mm order by key;
drop table iow0_mm;
@@ -213,13 +219,13 @@ insert overwrite table iow1_mm partition (key2)
select key as k1, key from intermediate union all select key as k1, key from intermediate;
insert into table iow1_mm partition (key2)
select key + 1 as k1, key from intermediate union all select key as k1, key from intermediate;
-select * from iow1_mm;
+select * from iow1_mm order by key, key2;
insert overwrite table iow1_mm partition (key2)
select key + 3 as k1, key from intermediate union all select key + 4 as k1, key from intermediate;
-select * from iow1_mm;
+select * from iow1_mm order by key, key2;
insert overwrite table iow1_mm partition (key2)
select key + 3 as k1, key + 3 from intermediate union all select key + 2 as k1, key + 2 from intermediate;
-select * from iow1_mm;
+select * from iow1_mm order by key, key2;
drop table iow1_mm;
@@ -351,12 +357,10 @@ import table import7_mm from 'ql/test/data/exports/intermmediate_part';
select * from import7_mm order by key, p;
drop table import7_mm;
-
-
set hive.exim.test.mode=false;
--- TODO# multi-insert, truncate
+-- TODO# multi-insert
http://git-wip-us.apache.org/repos/asf/hive/blob/ff1ea20c/ql/src/test/results/clientnegative/mm_truncate_cols.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/mm_truncate_cols.q.out b/ql/src/test/results/clientnegative/mm_truncate_cols.q.out
new file mode 100644
index 0000000..015f251
--- /dev/null
+++ b/ql/src/test/results/clientnegative/mm_truncate_cols.q.out
@@ -0,0 +1,9 @@
+PREHOOK: query: CREATE TABLE mm_table(key int, value string) tblproperties('hivecommit'='true')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mm_table
+POSTHOOK: query: CREATE TABLE mm_table(key int, value string) tblproperties('hivecommit'='true')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mm_table
+FAILED: SemanticException org.apache.hadoop.hive.ql.parse.SemanticException: Truncating MM table columns not presently supported
http://git-wip-us.apache.org/repos/asf/hive/blob/ff1ea20c/ql/src/test/results/clientpositive/llap/mm_all.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/mm_all.q.out b/ql/src/test/results/clientpositive/llap/mm_all.q.out
index 7d77047..bdd365d 100644
--- a/ql/src/test/results/clientpositive/llap/mm_all.q.out
+++ b/ql/src/test/results/clientpositive/llap/mm_all.q.out
@@ -180,6 +180,50 @@ POSTHOOK: Input: default@part_mm@key_mm=456
103 455
103 455
103 456
+PREHOOK: query: truncate table part_mm partition(key_mm='455')
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@part_mm@key_mm=455
+POSTHOOK: query: truncate table part_mm partition(key_mm='455')
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@part_mm@key_mm=455
+PREHOOK: query: select * from part_mm order by key, key_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_mm
+PREHOOK: Input: default@part_mm@key_mm=455
+PREHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part_mm order by key, key_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_mm
+POSTHOOK: Input: default@part_mm@key_mm=455
+POSTHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+0 456
+10 456
+97 456
+98 456
+100 456
+103 456
+PREHOOK: query: truncate table part_mm
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@part_mm@key_mm=455
+PREHOOK: Output: default@part_mm@key_mm=456
+POSTHOOK: query: truncate table part_mm
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@part_mm@key_mm=455
+POSTHOOK: Output: default@part_mm@key_mm=456
+PREHOOK: query: select * from part_mm order by key, key_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@part_mm
+PREHOOK: Input: default@part_mm@key_mm=455
+PREHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
+POSTHOOK: query: select * from part_mm order by key, key_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@part_mm
+POSTHOOK: Input: default@part_mm@key_mm=455
+POSTHOOK: Input: default@part_mm@key_mm=456
+#### A masked pattern was here ####
PREHOOK: query: drop table part_mm
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@part_mm
@@ -279,6 +323,20 @@ POSTHOOK: Input: default@simple_mm
100
103
103
+PREHOOK: query: truncate table simple_mm
+PREHOOK: type: TRUNCATETABLE
+PREHOOK: Output: default@simple_mm
+POSTHOOK: query: truncate table simple_mm
+POSTHOOK: type: TRUNCATETABLE
+POSTHOOK: Output: default@simple_mm
+PREHOOK: query: select * from simple_mm
+PREHOOK: type: QUERY
+PREHOOK: Input: default@simple_mm
+#### A masked pattern was here ####
+POSTHOOK: query: select * from simple_mm
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@simple_mm
+#### A masked pattern was here ####
PREHOOK: query: drop table simple_mm
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@simple_mm
@@ -1299,26 +1357,26 @@ POSTHOOK: Input: default@intermediate@p=456
POSTHOOK: Input: default@intermediate@p=457
POSTHOOK: Output: default@iow0_mm
POSTHOOK: Lineage: iow0_mm.key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from iow0_mm
+PREHOOK: query: select * from iow0_mm order by key
PREHOOK: type: QUERY
PREHOOK: Input: default@iow0_mm
#### A masked pattern was here ####
-POSTHOOK: query: select * from iow0_mm
+POSTHOOK: query: select * from iow0_mm order by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@iow0_mm
#### A masked pattern was here ####
-98
-97
-100
-103
0
+1
10
-99
+11
+97
98
+98
+99
+100
101
+103
104
-1
-11
PREHOOK: query: insert overwrite table iow0_mm select key + 2 from intermediate
PREHOOK: type: QUERY
PREHOOK: Input: default@intermediate
@@ -1334,20 +1392,20 @@ POSTHOOK: Input: default@intermediate@p=456
POSTHOOK: Input: default@intermediate@p=457
POSTHOOK: Output: default@iow0_mm
POSTHOOK: Lineage: iow0_mm.key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from iow0_mm
+PREHOOK: query: select * from iow0_mm order by key
PREHOOK: type: QUERY
PREHOOK: Input: default@iow0_mm
#### A masked pattern was here ####
-POSTHOOK: query: select * from iow0_mm
+POSTHOOK: query: select * from iow0_mm order by key
POSTHOOK: type: QUERY
POSTHOOK: Input: default@iow0_mm
#### A masked pattern was here ####
-100
+2
+12
99
+100
102
105
-2
-12
PREHOOK: query: drop table iow0_mm
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@iow0_mm
@@ -1422,7 +1480,7 @@ POSTHOOK: Lineage: iow1_mm PARTITION(key2=103).key EXPRESSION [(intermediate)int
POSTHOOK: Lineage: iow1_mm PARTITION(key2=10).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
POSTHOOK: Lineage: iow1_mm PARTITION(key2=97).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
POSTHOOK: Lineage: iow1_mm PARTITION(key2=98).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from iow1_mm
+PREHOOK: query: select * from iow1_mm order by key, key2
PREHOOK: type: QUERY
PREHOOK: Input: default@iow1_mm
PREHOOK: Input: default@iow1_mm@key2=0
@@ -1432,7 +1490,7 @@ PREHOOK: Input: default@iow1_mm@key2=103
PREHOOK: Input: default@iow1_mm@key2=97
PREHOOK: Input: default@iow1_mm@key2=98
#### A masked pattern was here ####
-POSTHOOK: query: select * from iow1_mm
+POSTHOOK: query: select * from iow1_mm order by key, key2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@iow1_mm
POSTHOOK: Input: default@iow1_mm@key2=0
@@ -1444,28 +1502,28 @@ POSTHOOK: Input: default@iow1_mm@key2=98
#### A masked pattern was here ####
0 0
0 0
-1 0
0 0
+1 0
10 10
10 10
-11 10
10 10
-100 100
-100 100
-101 100
-100 100
-103 103
-103 103
-104 103
-103 103
+11 10
97 97
97 97
-98 97
97 97
+98 97
98 98
98 98
-99 98
98 98
+99 98
+100 100
+100 100
+100 100
+101 100
+103 103
+103 103
+103 103
+104 103
PREHOOK: query: insert overwrite table iow1_mm partition (key2)
select key + 3 as k1, key from intermediate union all select key + 4 as k1, key from intermediate
PREHOOK: type: QUERY
@@ -1493,7 +1551,7 @@ POSTHOOK: Lineage: iow1_mm PARTITION(key2=103).key EXPRESSION [(intermediate)int
POSTHOOK: Lineage: iow1_mm PARTITION(key2=10).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
POSTHOOK: Lineage: iow1_mm PARTITION(key2=97).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
POSTHOOK: Lineage: iow1_mm PARTITION(key2=98).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from iow1_mm
+PREHOOK: query: select * from iow1_mm order by key, key2
PREHOOK: type: QUERY
PREHOOK: Input: default@iow1_mm
PREHOOK: Input: default@iow1_mm@key2=0
@@ -1503,7 +1561,7 @@ PREHOOK: Input: default@iow1_mm@key2=103
PREHOOK: Input: default@iow1_mm@key2=97
PREHOOK: Input: default@iow1_mm@key2=98
#### A masked pattern was here ####
-POSTHOOK: query: select * from iow1_mm
+POSTHOOK: query: select * from iow1_mm order by key, key2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@iow1_mm
POSTHOOK: Input: default@iow1_mm@key2=0
@@ -1517,14 +1575,14 @@ POSTHOOK: Input: default@iow1_mm@key2=98
4 0
13 10
14 10
-103 100
-104 100
-106 103
-107 103
100 97
101 97
101 98
102 98
+103 100
+104 100
+106 103
+107 103
PREHOOK: query: insert overwrite table iow1_mm partition (key2)
select key + 3 as k1, key + 3 from intermediate union all select key + 2 as k1, key + 2 from intermediate
PREHOOK: type: QUERY
@@ -1562,7 +1620,7 @@ POSTHOOK: Lineage: iow1_mm PARTITION(key2=13).key EXPRESSION [(intermediate)inte
POSTHOOK: Lineage: iow1_mm PARTITION(key2=2).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
POSTHOOK: Lineage: iow1_mm PARTITION(key2=3).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
POSTHOOK: Lineage: iow1_mm PARTITION(key2=99).key EXPRESSION [(intermediate)intermediate.FieldSchema(name:key, type:int, comment:null), ]
-PREHOOK: query: select * from iow1_mm
+PREHOOK: query: select * from iow1_mm order by key, key2
PREHOOK: type: QUERY
PREHOOK: Input: default@iow1_mm
PREHOOK: Input: default@iow1_mm@key2=0
@@ -1581,7 +1639,7 @@ PREHOOK: Input: default@iow1_mm@key2=97
PREHOOK: Input: default@iow1_mm@key2=98
PREHOOK: Input: default@iow1_mm@key2=99
#### A masked pattern was here ####
-POSTHOOK: query: select * from iow1_mm
+POSTHOOK: query: select * from iow1_mm order by key, key2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@iow1_mm
POSTHOOK: Input: default@iow1_mm@key2=0
@@ -1600,26 +1658,26 @@ POSTHOOK: Input: default@iow1_mm@key2=97
POSTHOOK: Input: default@iow1_mm@key2=98
POSTHOOK: Input: default@iow1_mm@key2=99
#### A masked pattern was here ####
+2 2
3 0
+3 3
4 0
+12 12
13 10
+13 13
14 10
+99 99
+100 97
100 100
100 100
+101 97
+101 98
101 101
+102 98
102 102
103 103
105 105
106 106
-12 12
-13 13
-2 2
-3 3
-100 97
-101 97
-101 98
-102 98
-99 99
PREHOOK: query: drop table iow1_mm
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@iow1_mm
@@ -2458,7 +2516,7 @@ POSTHOOK: query: drop table import7_mm
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@import7_mm
POSTHOOK: Output: default@import7_mm
-PREHOOK: query: -- TODO# multi-insert, truncate
+PREHOOK: query: -- TODO# multi-insert
@@ -2466,7 +2524,7 @@ drop table intermediate
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@intermediate
PREHOOK: Output: default@intermediate
-POSTHOOK: query: -- TODO# multi-insert, truncate
+POSTHOOK: query: -- TODO# multi-insert