You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by mm...@apache.org on 2016/05/30 23:58:41 UTC
[1/2] hive git commit: HIVE-13713: We miss vectorization in a case of
count(*) when aggregation mode is COMPLETE (Matt McCline,
reviewed by Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/master d74d5637f -> 8136a10c1
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
index 451947b..f5b5d9d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorGroupByOperator.java
@@ -50,6 +50,8 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.VectorGroupByDesc;
+import org.apache.hadoop.hive.ql.plan.VectorGroupByDesc.ProcessingMode;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
import org.apache.hadoop.hive.serde2.io.ByteWritable;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
@@ -84,6 +86,7 @@ public class TestVectorGroupByOperator {
private static AggregationDesc buildAggregationDesc(
VectorizationContext ctx,
String aggregate,
+ GenericUDAFEvaluator.Mode mode,
String column,
TypeInfo typeInfo) {
@@ -94,6 +97,7 @@ public class TestVectorGroupByOperator {
AggregationDesc agg = new AggregationDesc();
agg.setGenericUDAFName(aggregate);
+ agg.setMode(mode);
agg.setParameters(params);
return agg;
@@ -102,6 +106,7 @@ public class TestVectorGroupByOperator {
VectorizationContext ctx) {
AggregationDesc agg = new AggregationDesc();
agg.setGenericUDAFName("COUNT");
+ agg.setMode(GenericUDAFEvaluator.Mode.PARTIAL1);
agg.setParameters(new ArrayList<ExprNodeDesc>());
return agg;
}
@@ -110,10 +115,11 @@ public class TestVectorGroupByOperator {
private static GroupByDesc buildGroupByDescType(
VectorizationContext ctx,
String aggregate,
+ GenericUDAFEvaluator.Mode mode,
String column,
TypeInfo dataType) {
- AggregationDesc agg = buildAggregationDesc(ctx, aggregate,
+ AggregationDesc agg = buildAggregationDesc(ctx, aggregate, mode,
column, dataType);
ArrayList<AggregationDesc> aggs = new ArrayList<AggregationDesc>();
aggs.add(agg);
@@ -124,6 +130,7 @@ public class TestVectorGroupByOperator {
GroupByDesc desc = new GroupByDesc();
desc.setOutputColumnNames(outputColumnNames);
desc.setAggregators(aggs);
+ desc.getVectorDesc().setProcessingMode(ProcessingMode.GLOBAL);
return desc;
}
@@ -154,7 +161,8 @@ public class TestVectorGroupByOperator {
String key,
TypeInfo keyTypeInfo) {
- GroupByDesc desc = buildGroupByDescType(ctx, aggregate, column, dataTypeInfo);
+ GroupByDesc desc = buildGroupByDescType(ctx, aggregate, GenericUDAFEvaluator.Mode.PARTIAL1, column, dataTypeInfo);
+ desc.getVectorDesc().setProcessingMode(ProcessingMode.HASH);
ExprNodeDesc keyExp = buildColumnDesc(ctx, key, keyTypeInfo);
ArrayList<ExprNodeDesc> keys = new ArrayList<ExprNodeDesc>();
@@ -1716,7 +1724,7 @@ public class TestVectorGroupByOperator {
ArrayList<AggregationDesc> aggs = new ArrayList(1);
aggs.add(
- buildAggregationDesc(ctx, aggregateName,
+ buildAggregationDesc(ctx, aggregateName, GenericUDAFEvaluator.Mode.PARTIAL1,
"value", TypeInfoFactory.getPrimitiveTypeInfo(columnTypes[i])));
for(i=0; i<columnTypes.length - 1; ++i) {
@@ -1730,6 +1738,7 @@ public class TestVectorGroupByOperator {
desc.setOutputColumnNames(outputColumnNames);
desc.setAggregators(aggs);
desc.setKeys(keysDesc);
+ desc.getVectorDesc().setProcessingMode(ProcessingMode.HASH);
CompilationOpContext cCtx = new CompilationOpContext();
VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc);
@@ -1827,7 +1836,7 @@ public class TestVectorGroupByOperator {
VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
Set<Object> keys = new HashSet<Object>();
- AggregationDesc agg = buildAggregationDesc(ctx, aggregateName,
+ AggregationDesc agg = buildAggregationDesc(ctx, aggregateName, GenericUDAFEvaluator.Mode.PARTIAL1,
"Value", TypeInfoFactory.getPrimitiveTypeInfo(data.getTypes()[1]));
ArrayList<AggregationDesc> aggs = new ArrayList<AggregationDesc>();
aggs.add(agg);
@@ -1839,6 +1848,7 @@ public class TestVectorGroupByOperator {
GroupByDesc desc = new GroupByDesc();
desc.setOutputColumnNames(outputColumnNames);
desc.setAggregators(aggs);
+ desc.getVectorDesc().setProcessingMode(ProcessingMode.HASH);
ExprNodeDesc keyExp = buildColumnDesc(ctx, "Key",
TypeInfoFactory.getPrimitiveTypeInfo(data.getTypes()[0]));
@@ -2242,6 +2252,7 @@ public class TestVectorGroupByOperator {
VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
GroupByDesc desc = buildGroupByDescCountStar (ctx);
+ desc.getVectorDesc().setProcessingMode(ProcessingMode.HASH);
CompilationOpContext cCtx = new CompilationOpContext();
VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc);
@@ -2271,9 +2282,9 @@ public class TestVectorGroupByOperator {
mapColumnNames.add("A");
VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
- GroupByDesc desc = buildGroupByDescType(ctx, "count", "A", TypeInfoFactory.longTypeInfo);
+ GroupByDesc desc = buildGroupByDescType(ctx, "count", GenericUDAFEvaluator.Mode.FINAL, "A", TypeInfoFactory.longTypeInfo);
VectorGroupByDesc vectorDesc = desc.getVectorDesc();
- vectorDesc.setIsReduceMergePartial(true);
+ vectorDesc.setProcessingMode(ProcessingMode.GLOBAL); // Use GLOBAL when no key for Reduce.
CompilationOpContext cCtx = new CompilationOpContext();
VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc);
@@ -2303,7 +2314,7 @@ public class TestVectorGroupByOperator {
mapColumnNames.add("A");
VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
- GroupByDesc desc = buildGroupByDescType(ctx, aggregateName, "A",
+ GroupByDesc desc = buildGroupByDescType(ctx, aggregateName, GenericUDAFEvaluator.Mode.PARTIAL1, "A",
TypeInfoFactory.stringTypeInfo);
CompilationOpContext cCtx = new CompilationOpContext();
@@ -2336,7 +2347,7 @@ public class TestVectorGroupByOperator {
VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
GroupByDesc desc =
- buildGroupByDescType(ctx, aggregateName, "A", TypeInfoFactory.getDecimalTypeInfo(30, 4));
+ buildGroupByDescType(ctx, aggregateName, GenericUDAFEvaluator.Mode.PARTIAL1, "A", TypeInfoFactory.getDecimalTypeInfo(30, 4));
CompilationOpContext cCtx = new CompilationOpContext();
VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc);
@@ -2368,7 +2379,7 @@ public class TestVectorGroupByOperator {
mapColumnNames.add("A");
VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
- GroupByDesc desc = buildGroupByDescType (ctx, aggregateName, "A",
+ GroupByDesc desc = buildGroupByDescType (ctx, aggregateName, GenericUDAFEvaluator.Mode.PARTIAL1, "A",
TypeInfoFactory.doubleTypeInfo);
CompilationOpContext cCtx = new CompilationOpContext();
@@ -2400,7 +2411,7 @@ public class TestVectorGroupByOperator {
mapColumnNames.add("A");
VectorizationContext ctx = new VectorizationContext("name", mapColumnNames);
- GroupByDesc desc = buildGroupByDescType(ctx, aggregateName, "A", TypeInfoFactory.longTypeInfo);
+ GroupByDesc desc = buildGroupByDescType(ctx, aggregateName, GenericUDAFEvaluator.Mode.PARTIAL1, "A", TypeInfoFactory.longTypeInfo);
CompilationOpContext cCtx = new CompilationOpContext();
VectorGroupByOperator vgo = new VectorGroupByOperator(cCtx, ctx, desc);
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
index 9d4ca76..3295372 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/optimizer/physical/TestVectorizer.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.gen.VectorUD
import org.apache.hadoop.hive.ql.exec.vector.expressions.gen.FuncAbsLongToLong;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.*;
+import org.apache.hadoop.hive.ql.plan.VectorGroupByDesc.ProcessingMode;
import org.apache.hadoop.hive.ql.udf.generic.*;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -107,6 +108,7 @@ public class TestVectorizer {
GroupByOperator gbyOp = new GroupByOperator(new CompilationOpContext());
gbyOp.setConf(desc);
+ desc.setMode(GroupByDesc.Mode.HASH);
Vectorizer v = new Vectorizer();
Assert.assertTrue(v.validateMapWorkOperator(gbyOp, null, false));
@@ -148,9 +150,9 @@ public class TestVectorizer {
Assert.assertFalse(v.validateExprNodeDesc(andExprDesc, VectorExpressionDescriptor.Mode.FILTER));
Assert.assertFalse(v.validateExprNodeDesc(andExprDesc, VectorExpressionDescriptor.Mode.PROJECTION));
}
-
+
/**
- * prepareAbstractMapJoin prepares a join operator descriptor, used as helper by SMB and Map join tests.
+ * prepareAbstractMapJoin prepares a join operator descriptor, used as helper by SMB and Map join tests.
*/
private void prepareAbstractMapJoin(AbstractMapJoinOperator<? extends MapJoinDesc> map, MapJoinDesc mjdesc) {
mjdesc.setPosBigTable(0);
@@ -189,15 +191,15 @@ public class TestVectorizer {
public void testValidateMapJoinOperator() {
MapJoinOperator map = new MapJoinOperator(new CompilationOpContext());
MapJoinDesc mjdesc = new MapJoinDesc();
-
+
prepareAbstractMapJoin(map, mjdesc);
map.setConf(mjdesc);
-
+
Vectorizer vectorizer = new Vectorizer();
Assert.assertTrue(vectorizer.validateMapWorkOperator(map, null, false));
}
-
+
/**
* testValidateSMBJoinOperator validates that the SMB join operator can be vectorized.
*/
@@ -205,11 +207,11 @@ public class TestVectorizer {
public void testValidateSMBJoinOperator() {
SMBMapJoinOperator map = new SMBMapJoinOperator(new CompilationOpContext());
SMBJoinDesc mjdesc = new SMBJoinDesc();
-
+
prepareAbstractMapJoin(map, mjdesc);
map.setConf(mjdesc);
-
+
Vectorizer vectorizer = new Vectorizer();
- Assert.assertTrue(vectorizer.validateMapWorkOperator(map, null, false));
+ Assert.assertTrue(vectorizer.validateMapWorkOperator(map, null, false));
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/queries/clientpositive/vector_count.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_count.q b/ql/src/test/queries/clientpositive/vector_count.q
new file mode 100644
index 0000000..341db74
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_count.q
@@ -0,0 +1,26 @@
+set hive.mapred.mode=nonstrict;
+set hive.explain.user=false;
+SET hive.vectorized.execution.enabled=true;
+set hive.fetch.task.conversion=none;
+
+-- SORT_QUERY_RESULTS
+
+create table abcd_txt (a int, b int, c int, d int);
+LOAD DATA LOCAL INPATH '../../data/files/in4.txt' INTO TABLE abcd_txt;
+
+create table abcd stored as orc as select * from abcd_txt;
+
+select * from abcd;
+set hive.map.aggr=true;
+explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a;
+select a, count(distinct b), count(distinct c), sum(d) from abcd group by a;
+
+explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd;
+select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd;
+
+set hive.map.aggr=false;
+explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a;
+select a, count(distinct b), count(distinct c), sum(d) from abcd group by a;
+
+explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd;
+select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd;
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/queries/clientpositive/vector_groupby4.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_groupby4.q b/ql/src/test/queries/clientpositive/vector_groupby4.q
new file mode 100644
index 0000000..a59d1a8
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_groupby4.q
@@ -0,0 +1,23 @@
+set hive.explain.user=false;
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+set hive.mapred.mode=nonstrict;
+set hive.map.aggr=false;
+set hive.groupby.skewindata=true;
+SET hive.vectorized.execution.enabled=true;
+set hive.fetch.task.conversion=none;
+
+CREATE TABLE srcorc STORED AS ORC AS SELECT * FROM src;
+
+-- SORT_QUERY_RESULTS
+
+CREATE TABLE dest1(c1 STRING) STORED AS ORC;
+
+EXPLAIN
+FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT substr(srcorc.key,1,1) GROUP BY substr(srcorc.key,1,1);
+
+FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT substr(srcorc.key,1,1) GROUP BY substr(srcorc.key,1,1);
+
+SELECT dest1.* FROM dest1;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/queries/clientpositive/vector_groupby6.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_groupby6.q b/ql/src/test/queries/clientpositive/vector_groupby6.q
new file mode 100644
index 0000000..89c7a19
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_groupby6.q
@@ -0,0 +1,24 @@
+set hive.explain.user=false;
+set hive.input.format=org.apache.hadoop.hive.ql.io.HiveInputFormat;
+set hive.mapred.mode=nonstrict;
+set hive.map.aggr=false;
+set hive.groupby.skewindata=true;
+SET hive.vectorized.execution.enabled=true;
+set hive.fetch.task.conversion=none;
+
+CREATE TABLE srcorc STORED AS ORC AS SELECT * FROM src;
+
+-- SORT_QUERY_RESULTS
+
+CREATE TABLE dest1(c1 STRING) STORED AS ORC;
+
+EXPLAIN
+FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(srcorc.value,5,1);
+
+FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(srcorc.value,5,1);
+
+SELECT dest1.* FROM dest1;
+
+
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out b/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out
index ab627b5..3bbbb46 100644
--- a/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_aggregate_without_gby.q.out
@@ -47,11 +47,11 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized
- File Output Operator [FS_7]
- Group By Operator [GBY_12] (rows=1 width=88)
+ File Output Operator [FS_14]
+ Group By Operator [GBY_13] (rows=1 width=88)
Output:["_col0","_col1"],aggregations:["max(VALUE._col0)","max(VALUE._col1)"]
<-Map 1 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_4]
+ SHUFFLE [RS_12]
Group By Operator [GBY_11] (rows=1 width=88)
Output:["_col0","_col1"],aggregations:["max(dt)","max(greg_dt)"]
Select Operator [SEL_10] (rows=3 width=102)
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out b/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out
index 67ddd9e..d37dc51 100644
--- a/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_auto_smb_mapjoin_14.q.out
@@ -64,7 +64,7 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized
- File Output Operator [FS_14]
+ File Output Operator [FS_22]
Group By Operator [GBY_21] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [SIMPLE_EDGE]
@@ -134,11 +134,11 @@ Stage-0
limit:-1
Stage-1
Reducer 3 vectorized
- File Output Operator [FS_19]
- Group By Operator [GBY_29] (rows=1 width=8)
+ File Output Operator [FS_31]
+ Group By Operator [GBY_30] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Reducer 2 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_16]
+ SHUFFLE [RS_29]
Group By Operator [GBY_28] (rows=1 width=8)
Output:["_col0"],aggregations:["count()"]
Select Operator [SEL_27] (rows=5 width=93)
@@ -365,7 +365,7 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized
- File Output Operator [FS_14]
+ File Output Operator [FS_22]
Group By Operator [GBY_21] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [SIMPLE_EDGE]
@@ -444,7 +444,7 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized
- File Output Operator [FS_14]
+ File Output Operator [FS_22]
Group By Operator [GBY_21] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [SIMPLE_EDGE]
@@ -547,7 +547,7 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized
- File Output Operator [FS_14]
+ File Output Operator [FS_22]
Group By Operator [GBY_21] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [SIMPLE_EDGE]
@@ -640,7 +640,7 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized
- File Output Operator [FS_14]
+ File Output Operator [FS_22]
Group By Operator [GBY_21] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [SIMPLE_EDGE]
@@ -710,7 +710,7 @@ Stage-0
limit:-1
Stage-1
Reducer 3 vectorized
- File Output Operator [FS_14]
+ File Output Operator [FS_29]
Group By Operator [GBY_28] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Reducer 2 [SIMPLE_EDGE]
@@ -781,7 +781,7 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized
- File Output Operator [FS_14]
+ File Output Operator [FS_22]
Group By Operator [GBY_21] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [SIMPLE_EDGE]
@@ -852,7 +852,7 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized
- File Output Operator [FS_18]
+ File Output Operator [FS_32]
Group By Operator [GBY_31] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [SIMPLE_EDGE]
@@ -945,7 +945,7 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized
- File Output Operator [FS_14]
+ File Output Operator [FS_22]
Group By Operator [GBY_21] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
<-Map 1 [SIMPLE_EDGE]
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/results/clientpositive/tez/vector_count.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_count.q.out b/ql/src/test/results/clientpositive/tez/vector_count.q.out
new file mode 100644
index 0000000..c854515
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vector_count.q.out
@@ -0,0 +1,314 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+create table abcd_txt (a int, b int, c int, d int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@abcd_txt
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+create table abcd_txt (a int, b int, c int, d int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@abcd_txt
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in4.txt' INTO TABLE abcd_txt
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@abcd_txt
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in4.txt' INTO TABLE abcd_txt
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@abcd_txt
+PREHOOK: query: create table abcd stored as orc as select * from abcd_txt
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@abcd_txt
+PREHOOK: Output: database:default
+PREHOOK: Output: default@abcd
+POSTHOOK: query: create table abcd stored as orc as select * from abcd_txt
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@abcd_txt
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@abcd
+POSTHOOK: Lineage: abcd.a SIMPLE [(abcd_txt)abcd_txt.FieldSchema(name:a, type:int, comment:null), ]
+POSTHOOK: Lineage: abcd.b SIMPLE [(abcd_txt)abcd_txt.FieldSchema(name:b, type:int, comment:null), ]
+POSTHOOK: Lineage: abcd.c SIMPLE [(abcd_txt)abcd_txt.FieldSchema(name:c, type:int, comment:null), ]
+POSTHOOK: Lineage: abcd.d SIMPLE [(abcd_txt)abcd_txt.FieldSchema(name:d, type:int, comment:null), ]
+PREHOOK: query: select * from abcd
+PREHOOK: type: QUERY
+PREHOOK: Input: default@abcd
+#### A masked pattern was here ####
+POSTHOOK: query: select * from abcd
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@abcd
+#### A masked pattern was here ####
+10 100 45 4
+10 100 NULL 5
+10 1000 50 1
+100 100 10 3
+12 100 75 7
+12 NULL 80 2
+NULL 35 23 6
+PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: abcd
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: int), b (type: int), c (type: int), d (type: int)
+ outputColumnNames: a, b, c, d
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(DISTINCT b), count(DISTINCT c), sum(d)
+ keys: a (type: int), b (type: int), c (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col5 (type: bigint)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(DISTINCT KEY._col1:0._col0), count(DISTINCT KEY._col1:1._col0), sum(VALUE._col2)
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 3 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@abcd
+#### A masked pattern was here ####
+POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@abcd
+#### A masked pattern was here ####
+10 2 2 10
+100 1 1 3
+12 1 2 9
+NULL 1 1 6
+PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: abcd
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: int), b (type: int), c (type: int), d (type: int)
+ outputColumnNames: _col1, _col2, _col3, _col4
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(1), count(), count(_col1), count(_col2), count(_col3), count(_col4), count(DISTINCT _col1), count(DISTINCT _col2), count(DISTINCT _col3), count(DISTINCT _col4), count(DISTINCT _col1, _col2), count(DISTINCT _col2, _col3), count(DISTINCT _col3, _col4), count(DISTINCT _col1, _col4), count(DISTINCT _col1, _col3), count(DISTINCT _col2, _col4), count(DISTINCT _col1, _col2, _col3), count(DISTINCT _col2, _col3, _col4), count(DISTINCT _col1, _col3, _col4), count(DISTINCT _col1, _col2, _col4), count(DISTINCT _col1, _col2, _col3, _col4)
+ keys: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int)
+ sort order: ++++
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: bigint)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1), count(VALUE._col2), count(VALUE._col3), count(VALUE._col4), count(VALUE._col5), count(DISTINCT KEY._col0:0._col0), count(DISTINCT KEY._col0:1._col0), count(DISTINCT KEY._col0:2._col0), count(DISTINCT KEY._col0:3._col0), count(DISTINCT KEY._col0:4._col0, KEY._col0:4._col1), count(DISTINCT KEY._col0:5._col0, KEY._col0:5._col1), count(DISTINCT KEY._col0:6._col0, KEY._col0:6._col1), count(DISTINCT KEY._col0:7._col0, KEY._col0:7._col1), count(DISTINCT KEY._col0:8._col0, KEY._col0:8._col1), count(DISTINCT KEY._col0:9._col0, KEY._col0:9._col1), count(DISTINCT KEY._col0:10._col0, KEY._col0:10._col1, KEY._col0:10._col2), count(DISTINCT KEY._col0:11._col0, KEY._col0:11._col1, KEY._col0:11._col2), count(DISTINCT KEY._col0:12._col0, KEY._col0:12._col1, KEY._col0:12._col2), count(DISTINCT KEY._col0:13._col0, KEY._col0:13._col1, KEY._col0:13._col2), count(DISTINCT KEY._col0:14._col0, KEY._col0:14._col1, KEY._col0:14._col2, KEY.
_col0:14._col3)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+PREHOOK: type: QUERY
+PREHOOK: Input: default@abcd
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@abcd
+#### A masked pattern was here ####
+7 7 6 6 6 7 3 3 6 7 4 5 6 6 5 6 4 5 5 5 4
+PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: abcd
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: int), b (type: int), c (type: int), d (type: int)
+ outputColumnNames: a, b, c, d
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: a (type: int), b (type: int), c (type: int)
+ sort order: +++
+ Map-reduce partition columns: a (type: int)
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ value expressions: d (type: int)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(DISTINCT KEY._col1:0._col0), count(DISTINCT KEY._col1:1._col0), sum(VALUE._col0)
+ keys: KEY._col0 (type: int)
+ mode: complete
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 3 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@abcd
+#### A masked pattern was here ####
+POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@abcd
+#### A masked pattern was here ####
+10 2 2 10
+100 1 1 3
+12 1 2 9
+NULL 1 1 6
+PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: abcd
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: int), b (type: int), c (type: int), d (type: int)
+ outputColumnNames: _col1, _col2, _col3, _col4
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int)
+ sort order: ++++
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(1), count(), count(KEY._col0:0._col0), count(KEY._col0:1._col0), count(KEY._col0:2._col0), count(KEY._col0:3._col0), count(DISTINCT KEY._col0:0._col0), count(DISTINCT KEY._col0:1._col0), count(DISTINCT KEY._col0:2._col0), count(DISTINCT KEY._col0:3._col0), count(DISTINCT KEY._col0:4._col0, KEY._col0:4._col1), count(DISTINCT KEY._col0:5._col0, KEY._col0:5._col1), count(DISTINCT KEY._col0:6._col0, KEY._col0:6._col1), count(DISTINCT KEY._col0:7._col0, KEY._col0:7._col1), count(DISTINCT KEY._col0:8._col0, KEY._col0:8._col1), count(DISTINCT KEY._col0:9._col0, KEY._col0:9._col1), count(DISTINCT KEY._col0:10._col0, KEY._col0:10._col1, KEY._col0:10._col2), count(DISTINCT KEY._col0:11._col0, KEY._col0:11._col1, KEY._col0:11._col2), count(DISTINCT KEY._col0:12._col0, KEY._col0:12._col1, KEY._col0:12._col2), count(DISTINCT KEY._col0:13._col0, KEY._col0:13._col1, KEY._col0:13._col2), count(DISTINCT KEY._col0:14._col0, KEY._col0:14._col1, KEY._col0:14._col2, K
EY._col0:14._col3)
+ mode: complete
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+PREHOOK: type: QUERY
+PREHOOK: Input: default@abcd
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@abcd
+#### A masked pattern was here ####
+7 7 6 6 6 7 3 3 6 7 4 5 6 6 5 6 4 5 5 5 4
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/results/clientpositive/tez/vector_groupby4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_groupby4.q.out b/ql/src/test/results/clientpositive/tez/vector_groupby4.q.out
new file mode 100644
index 0000000..401ab09
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vector_groupby4.q.out
@@ -0,0 +1,137 @@
+PREHOOK: query: CREATE TABLE srcorc STORED AS ORC AS SELECT * FROM src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcorc
+POSTHOOK: query: CREATE TABLE srcorc STORED AS ORC AS SELECT * FROM src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcorc
+POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE dest1(c1 STRING) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest1
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE dest1(c1 STRING) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dest1
+PREHOOK: query: EXPLAIN
+FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT substr(srcorc.key,1,1) GROUP BY substr(srcorc.key,1,1)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT substr(srcorc.key,1,1) GROUP BY substr(srcorc.key,1,1)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcorc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: substr(key, 1, 1) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: rand() (type: double)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: partial1
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reducer 3
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: final
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.dest1
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.dest1
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT substr(srcorc.key,1,1) GROUP BY substr(srcorc.key,1,1)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcorc
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT substr(srcorc.key,1,1) GROUP BY substr(srcorc.key,1,1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcorc
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(srcorc)srcorc.FieldSchema(name:key, type:string, comment:null), ]
+PREHOOK: query: SELECT dest1.* FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT dest1.* FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+#### A masked pattern was here ####
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/results/clientpositive/tez/vector_groupby6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_groupby6.q.out b/ql/src/test/results/clientpositive/tez/vector_groupby6.q.out
new file mode 100644
index 0000000..c9174e6
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vector_groupby6.q.out
@@ -0,0 +1,137 @@
+PREHOOK: query: CREATE TABLE srcorc STORED AS ORC AS SELECT * FROM src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcorc
+POSTHOOK: query: CREATE TABLE srcorc STORED AS ORC AS SELECT * FROM src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcorc
+POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE dest1(c1 STRING) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest1
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE dest1(c1 STRING) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dest1
+PREHOOK: query: EXPLAIN
+FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(srcorc.value,5,1)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(srcorc.value,5,1)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: srcorc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: substr(value, 5, 1) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: rand() (type: double)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 2
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: partial1
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reducer 3
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: final
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.dest1
+
+ Stage: Stage-2
+ Dependency Collection
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.dest1
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(srcorc.value,5,1)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcorc
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(srcorc.value,5,1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcorc
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(srcorc)srcorc.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: SELECT dest1.* FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT dest1.* FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+#### A masked pattern was here ####
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/results/clientpositive/tez/vector_groupby_mapjoin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_groupby_mapjoin.q.out b/ql/src/test/results/clientpositive/tez/vector_groupby_mapjoin.q.out
index d69012e..2d5e782 100644
--- a/ql/src/test/results/clientpositive/tez/vector_groupby_mapjoin.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_groupby_mapjoin.q.out
@@ -27,8 +27,8 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized
- File Output Operator [FS_33]
- Select Operator [SEL_32] (rows=302 width=10)
+ File Output Operator [FS_36]
+ Select Operator [SEL_35] (rows=302 width=10)
Output:["_col0","_col1"]
<-Map 1 [SIMPLE_EDGE]
SHUFFLE [RS_21]
@@ -48,9 +48,9 @@ Stage-0
<-Map Join Operator [MAPJOIN_27] (rows=550 width=10)
Conds:(Inner),Output:["_col0","_col1"]
<-Reducer 4 [BROADCAST_EDGE] vectorized
- BROADCAST [RS_14]
- Select Operator [SEL_10] (rows=1 width=8)
- Filter Operator [FIL_9] (rows=1 width=8)
+ BROADCAST [RS_34]
+ Select Operator [SEL_33] (rows=1 width=8)
+ Filter Operator [FIL_32] (rows=1 width=8)
predicate:(_col0 = 0)
Group By Operator [GBY_31] (rows=1 width=8)
Output:["_col0"],aggregations:["count(VALUE._col0)"]
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out b/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out
index 93137f1..9492f8a 100644
--- a/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_join_part_col_char.q.out
@@ -113,7 +113,7 @@ Stage-0
Reducer 2
File Output Operator [FS_10]
Merge Join Operator [MERGEJOIN_21] (rows=2 width=112)
- Conds:RS_23._col2=RS_27._col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
+ Conds:RS_23._col2=RS_28._col2(Inner),Output:["_col0","_col1","_col2","_col3","_col4","_col5"]
<-Map 1 [SIMPLE_EDGE] vectorized
SHUFFLE [RS_23]
PartitionCols:_col2
@@ -121,16 +121,16 @@ Stage-0
Output:["_col0","_col1","_col2"]
TableScan [TS_0] (rows=2 width=102)
default@char_tbl1,c1,Tbl:COMPLETE,Col:NONE,Output:["name","age"]
- Dynamic Partitioning Event Operator [EVENT_20] (rows=2 width=102)
+ Dynamic Partitioning Event Operator [EVENT_26] (rows=2 width=102)
Group By Operator [GBY_25] (rows=2 width=102)
Output:["_col0"],keys:_col0
Select Operator [SEL_24] (rows=2 width=102)
Output:["_col0"]
Please refer to the previous Select Operator [SEL_22]
<-Map 3 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_27]
+ SHUFFLE [RS_28]
PartitionCols:_col2
- Select Operator [SEL_26] (rows=2 width=101)
+ Select Operator [SEL_27] (rows=2 width=101)
Output:["_col0","_col1","_col2"]
TableScan [TS_3] (rows=2 width=101)
default@char_tbl2,c2,Tbl:COMPLETE,Col:NONE,Output:["name","age"]
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/results/clientpositive/tez/vectorization_limit.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorization_limit.q.out b/ql/src/test/results/clientpositive/tez/vectorization_limit.q.out
index d5c01d0..b544784 100644
--- a/ql/src/test/results/clientpositive/tez/vectorization_limit.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorization_limit.q.out
@@ -360,6 +360,7 @@ STAGE PLANS:
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Execution mode: vectorized
Reducer 2
+ Execution mode: vectorized
Reduce Operator Tree:
Group By Operator
keys: KEY._col0 (type: tinyint), KEY._col1 (type: double)
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/results/clientpositive/tez/vectorized_timestamp.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorized_timestamp.q.out b/ql/src/test/results/clientpositive/tez/vectorized_timestamp.q.out
index fb272dd..4a19861 100644
--- a/ql/src/test/results/clientpositive/tez/vectorized_timestamp.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorized_timestamp.q.out
@@ -132,13 +132,13 @@ Stage-0
limit:-1
Stage-1
Reducer 2 vectorized
- File Output Operator [FS_6]
- Select Operator [SEL_5] (rows=1 width=80)
+ File Output Operator [FS_12]
+ Select Operator [SEL_11] (rows=1 width=80)
Output:["_col0","_col1","_col2"]
- Group By Operator [GBY_9] (rows=1 width=80)
+ Group By Operator [GBY_10] (rows=1 width=80)
Output:["_col0","_col1"],aggregations:["min(VALUE._col0)","max(VALUE._col1)"]
<-Map 1 [SIMPLE_EDGE] vectorized
- SHUFFLE [RS_3]
+ SHUFFLE [RS_9]
Group By Operator [GBY_8] (rows=1 width=80)
Output:["_col0","_col1"],aggregations:["min(ts)","max(ts)"]
Select Operator [SEL_7] (rows=2 width=40)
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/results/clientpositive/vector_count.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_count.q.out b/ql/src/test/results/clientpositive/vector_count.q.out
new file mode 100644
index 0000000..734ef39
--- /dev/null
+++ b/ql/src/test/results/clientpositive/vector_count.q.out
@@ -0,0 +1,286 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+create table abcd_txt (a int, b int, c int, d int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@abcd_txt
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+create table abcd_txt (a int, b int, c int, d int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@abcd_txt
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in4.txt' INTO TABLE abcd_txt
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@abcd_txt
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in4.txt' INTO TABLE abcd_txt
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@abcd_txt
+PREHOOK: query: create table abcd stored as orc as select * from abcd_txt
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@abcd_txt
+PREHOOK: Output: database:default
+PREHOOK: Output: default@abcd
+POSTHOOK: query: create table abcd stored as orc as select * from abcd_txt
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@abcd_txt
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@abcd
+POSTHOOK: Lineage: abcd.a SIMPLE [(abcd_txt)abcd_txt.FieldSchema(name:a, type:int, comment:null), ]
+POSTHOOK: Lineage: abcd.b SIMPLE [(abcd_txt)abcd_txt.FieldSchema(name:b, type:int, comment:null), ]
+POSTHOOK: Lineage: abcd.c SIMPLE [(abcd_txt)abcd_txt.FieldSchema(name:c, type:int, comment:null), ]
+POSTHOOK: Lineage: abcd.d SIMPLE [(abcd_txt)abcd_txt.FieldSchema(name:d, type:int, comment:null), ]
+PREHOOK: query: select * from abcd
+PREHOOK: type: QUERY
+PREHOOK: Input: default@abcd
+#### A masked pattern was here ####
+POSTHOOK: query: select * from abcd
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@abcd
+#### A masked pattern was here ####
+10 100 45 4
+10 100 NULL 5
+10 1000 50 1
+100 100 10 3
+12 100 75 7
+12 NULL 80 2
+NULL 35 23 6
+PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: abcd
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: int), b (type: int), c (type: int), d (type: int)
+ outputColumnNames: a, b, c, d
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(DISTINCT b), count(DISTINCT c), sum(d)
+ keys: a (type: int), b (type: int), c (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int)
+ sort order: +++
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col5 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(DISTINCT KEY._col1:0._col0), count(DISTINCT KEY._col1:1._col0), sum(VALUE._col2)
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 3 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@abcd
+#### A masked pattern was here ####
+POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@abcd
+#### A masked pattern was here ####
+10 2 2 10
+100 1 1 3
+12 1 2 9
+NULL 1 1 6
+PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: abcd
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: int), b (type: int), c (type: int), d (type: int)
+ outputColumnNames: _col1, _col2, _col3, _col4
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count(1), count(), count(_col1), count(_col2), count(_col3), count(_col4), count(DISTINCT _col1), count(DISTINCT _col2), count(DISTINCT _col3), count(DISTINCT _col4), count(DISTINCT _col1, _col2), count(DISTINCT _col2, _col3), count(DISTINCT _col3, _col4), count(DISTINCT _col1, _col4), count(DISTINCT _col1, _col3), count(DISTINCT _col2, _col4), count(DISTINCT _col1, _col2, _col3), count(DISTINCT _col2, _col3, _col4), count(DISTINCT _col1, _col3, _col4), count(DISTINCT _col1, _col2, _col4), count(DISTINCT _col1, _col2, _col3, _col4)
+ keys: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20, _col21, _col22, _col23, _col24
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col3 (type: int)
+ sort order: ++++
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col4 (type: bigint), _col5 (type: bigint), _col6 (type: bigint), _col7 (type: bigint), _col8 (type: bigint), _col9 (type: bigint)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0), count(VALUE._col1), count(VALUE._col2), count(VALUE._col3), count(VALUE._col4), count(VALUE._col5), count(DISTINCT KEY._col0:0._col0), count(DISTINCT KEY._col0:1._col0), count(DISTINCT KEY._col0:2._col0), count(DISTINCT KEY._col0:3._col0), count(DISTINCT KEY._col0:4._col0, KEY._col0:4._col1), count(DISTINCT KEY._col0:5._col0, KEY._col0:5._col1), count(DISTINCT KEY._col0:6._col0, KEY._col0:6._col1), count(DISTINCT KEY._col0:7._col0, KEY._col0:7._col1), count(DISTINCT KEY._col0:8._col0, KEY._col0:8._col1), count(DISTINCT KEY._col0:9._col0, KEY._col0:9._col1), count(DISTINCT KEY._col0:10._col0, KEY._col0:10._col1, KEY._col0:10._col2), count(DISTINCT KEY._col0:11._col0, KEY._col0:11._col1, KEY._col0:11._col2), count(DISTINCT KEY._col0:12._col0, KEY._col0:12._col1, KEY._col0:12._col2), count(DISTINCT KEY._col0:13._col0, KEY._col0:13._col1, KEY._col0:13._col2), count(DISTINCT KEY._col0:14._col0, KEY._col0:14._col1, KEY._col0:14._col2, KEY._col0:
14._col3)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 200 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+PREHOOK: type: QUERY
+PREHOOK: Input: default@abcd
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@abcd
+#### A masked pattern was here ####
+7 7 6 6 6 7 3 3 6 7 4 5 6 6 5 6 4 5 5 5 4
+PREHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: abcd
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: int), b (type: int), c (type: int), d (type: int)
+ outputColumnNames: a, b, c, d
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: a (type: int), b (type: int), c (type: int)
+ sort order: +++
+ Map-reduce partition columns: a (type: int)
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ value expressions: d (type: int)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(DISTINCT KEY._col1:0._col0), count(DISTINCT KEY._col1:1._col0), sum(VALUE._col0)
+ keys: KEY._col0 (type: int)
+ mode: complete
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 3 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 3 Data size: 42 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+PREHOOK: type: QUERY
+PREHOOK: Input: default@abcd
+#### A masked pattern was here ####
+POSTHOOK: query: select a, count(distinct b), count(distinct c), sum(d) from abcd group by a
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@abcd
+#### A masked pattern was here ####
+10 2 2 10
+100 1 1 3
+12 1 2 9
+NULL 1 1 6
+PREHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: abcd
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: a (type: int), b (type: int), c (type: int), d (type: int)
+ outputColumnNames: _col1, _col2, _col3, _col4
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: int), _col2 (type: int), _col3 (type: int), _col4 (type: int)
+ sort order: ++++
+ Statistics: Num rows: 7 Data size: 100 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(1), count(), count(KEY._col0:0._col0), count(KEY._col0:1._col0), count(KEY._col0:2._col0), count(KEY._col0:3._col0), count(DISTINCT KEY._col0:0._col0), count(DISTINCT KEY._col0:1._col0), count(DISTINCT KEY._col0:2._col0), count(DISTINCT KEY._col0:3._col0), count(DISTINCT KEY._col0:4._col0, KEY._col0:4._col1), count(DISTINCT KEY._col0:5._col0, KEY._col0:5._col1), count(DISTINCT KEY._col0:6._col0, KEY._col0:6._col1), count(DISTINCT KEY._col0:7._col0, KEY._col0:7._col1), count(DISTINCT KEY._col0:8._col0, KEY._col0:8._col1), count(DISTINCT KEY._col0:9._col0, KEY._col0:9._col1), count(DISTINCT KEY._col0:10._col0, KEY._col0:10._col1, KEY._col0:10._col2), count(DISTINCT KEY._col0:11._col0, KEY._col0:11._col1, KEY._col0:11._col2), count(DISTINCT KEY._col0:12._col0, KEY._col0:12._col1, KEY._col0:12._col2), count(DISTINCT KEY._col0:13._col0, KEY._col0:13._col1, KEY._col0:13._col2), count(DISTINCT KEY._col0:14._col0, KEY._col0:14._col1, KEY._col0:14._col2, KEY._co
l0:14._col3)
+ mode: complete
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10, _col11, _col12, _col13, _col14, _col15, _col16, _col17, _col18, _col19, _col20
+ Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 168 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+PREHOOK: type: QUERY
+PREHOOK: Input: default@abcd
+#### A masked pattern was here ####
+POSTHOOK: query: select count(1), count(*), count(a), count(b), count(c), count(d), count(distinct a), count(distinct b), count(distinct c), count(distinct d), count(distinct a,b), count(distinct b,c), count(distinct c,d), count(distinct a,d), count(distinct a,c), count(distinct b,d), count(distinct a,b,c), count(distinct b,c,d), count(distinct a,c,d), count(distinct a,b,d), count(distinct a,b,c,d) from abcd
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@abcd
+#### A masked pattern was here ####
+7 7 6 6 6 7 3 3 6 7 4 5 6 6 5 6 4 5 5 5 4
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/results/clientpositive/vector_groupby4.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_groupby4.q.out b/ql/src/test/results/clientpositive/vector_groupby4.q.out
new file mode 100644
index 0000000..8041511
--- /dev/null
+++ b/ql/src/test/results/clientpositive/vector_groupby4.q.out
@@ -0,0 +1,134 @@
+PREHOOK: query: CREATE TABLE srcorc STORED AS ORC AS SELECT * FROM src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcorc
+POSTHOOK: query: CREATE TABLE srcorc STORED AS ORC AS SELECT * FROM src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcorc
+POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE dest1(c1 STRING) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest1
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE dest1(c1 STRING) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dest1
+PREHOOK: query: EXPLAIN
+FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT substr(srcorc.key,1,1) GROUP BY substr(srcorc.key,1,1)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT substr(srcorc.key,1,1) GROUP BY substr(srcorc.key,1,1)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: srcorc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: substr(key, 1, 1) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: rand() (type: double)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: partial1
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: final
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.dest1
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.dest1
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT substr(srcorc.key,1,1) GROUP BY substr(srcorc.key,1,1)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcorc
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT substr(srcorc.key,1,1) GROUP BY substr(srcorc.key,1,1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcorc
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(srcorc)srcorc.FieldSchema(name:key, type:string, comment:null), ]
+PREHOOK: query: SELECT dest1.* FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT dest1.* FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+#### A masked pattern was here ####
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/test/results/clientpositive/vector_groupby6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_groupby6.q.out b/ql/src/test/results/clientpositive/vector_groupby6.q.out
new file mode 100644
index 0000000..63fe8f3
--- /dev/null
+++ b/ql/src/test/results/clientpositive/vector_groupby6.q.out
@@ -0,0 +1,134 @@
+PREHOOK: query: CREATE TABLE srcorc STORED AS ORC AS SELECT * FROM src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@srcorc
+POSTHOOK: query: CREATE TABLE srcorc STORED AS ORC AS SELECT * FROM src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@srcorc
+POSTHOOK: Lineage: srcorc.key SIMPLE [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: srcorc.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE dest1(c1 STRING) STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@dest1
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE dest1(c1 STRING) STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@dest1
+PREHOOK: query: EXPLAIN
+FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(srcorc.value,5,1)
+PREHOOK: type: QUERY
+POSTHOOK: query: EXPLAIN
+FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(srcorc.value,5,1)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-2 depends on stages: Stage-1
+ Stage-0 depends on stages: Stage-2
+ Stage-3 depends on stages: Stage-0
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: srcorc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: substr(value, 5, 1) (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: rand() (type: double)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: partial1
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Group By Operator
+ keys: KEY._col0 (type: string)
+ mode: final
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.dest1
+
+ Stage: Stage-0
+ Move Operator
+ tables:
+ replace: true
+ table:
+ input format: org.apache.hadoop.hive.ql.io.orc.OrcInputFormat
+ output format: org.apache.hadoop.hive.ql.io.orc.OrcOutputFormat
+ serde: org.apache.hadoop.hive.ql.io.orc.OrcSerde
+ name: default.dest1
+
+ Stage: Stage-3
+ Stats-Aggr Operator
+
+PREHOOK: query: FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(srcorc.value,5,1)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcorc
+PREHOOK: Output: default@dest1
+POSTHOOK: query: FROM srcorc
+INSERT OVERWRITE TABLE dest1 SELECT DISTINCT substr(srcorc.value,5,1)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcorc
+POSTHOOK: Output: default@dest1
+POSTHOOK: Lineage: dest1.c1 EXPRESSION [(srcorc)srcorc.FieldSchema(name:value, type:string, comment:null), ]
+PREHOOK: query: SELECT dest1.* FROM dest1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@dest1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT dest1.* FROM dest1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@dest1
+#### A masked pattern was here ####
+0
+1
+2
+3
+4
+5
+6
+7
+8
+9
[2/2] hive git commit: HIVE-13713: We miss vectorization in a case of
count(*) when aggregation mode is COMPLETE (Matt McCline,
reviewed by Sergey Shelukhin)
Posted by mm...@apache.org.
HIVE-13713: We miss vectorization in a case of count(*) when aggregation mode is COMPLETE (Matt McCline, reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8136a10c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8136a10c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8136a10c
Branch: refs/heads/master
Commit: 8136a10c1f0c4ebd5545225a9ebddf155ce5d933
Parents: d74d563
Author: Matt McCline <mm...@hortonworks.com>
Authored: Mon May 30 16:58:23 2016 -0700
Committer: Matt McCline <mm...@hortonworks.com>
Committed: Mon May 30 16:58:23 2016 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 3 +
.../ql/exec/vector/AggregateDefinition.java | 13 +-
.../ql/exec/vector/VectorGroupByOperator.java | 56 +--
.../ql/exec/vector/VectorizationContext.java | 354 +++++++++++--------
.../hive/ql/optimizer/physical/Vectorizer.java | 227 +++++++-----
.../hadoop/hive/ql/plan/VectorGroupByDesc.java | 80 ++++-
.../exec/vector/TestVectorGroupByOperator.java | 31 +-
.../ql/optimizer/physical/TestVectorizer.java | 18 +-
.../test/queries/clientpositive/vector_count.q | 26 ++
.../queries/clientpositive/vector_groupby4.q | 23 ++
.../queries/clientpositive/vector_groupby6.q | 24 ++
.../tez/vector_aggregate_without_gby.q.out | 6 +-
.../tez/vector_auto_smb_mapjoin_14.q.out | 24 +-
.../clientpositive/tez/vector_count.q.out | 314 ++++++++++++++++
.../clientpositive/tez/vector_groupby4.q.out | 137 +++++++
.../clientpositive/tez/vector_groupby6.q.out | 137 +++++++
.../tez/vector_groupby_mapjoin.q.out | 10 +-
.../tez/vector_join_part_col_char.q.out | 8 +-
.../tez/vectorization_limit.q.out | 1 +
.../tez/vectorized_timestamp.q.out | 8 +-
.../results/clientpositive/vector_count.q.out | 286 +++++++++++++++
.../clientpositive/vector_groupby4.q.out | 134 +++++++
.../clientpositive/vector_groupby6.q.out | 134 +++++++
23 files changed, 1732 insertions(+), 322 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index fd6901c..c4ba277 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -270,6 +270,7 @@ minitez.query.files.shared=acid_globallimit.q,\
vector_coalesce_2.q,\
vector_complex_all.q,\
vector_complex_join.q,\
+ vector_count.q,\
vector_count_distinct.q,\
vector_data_types.q,\
vector_date_1.q,\
@@ -293,6 +294,8 @@ minitez.query.files.shared=acid_globallimit.q,\
vector_decimal_udf2.q,\
vector_distinct_2.q,\
vector_elt.q,\
+ vector_groupby4.q,\
+ vector_groupby6.q,\
vector_groupby_3.q,\
vector_groupby_mapjoin.q,\
vector_groupby_reduce.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/AggregateDefinition.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/AggregateDefinition.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/AggregateDefinition.java
index 3f15c6f..0334c40 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/AggregateDefinition.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/AggregateDefinition.java
@@ -20,19 +20,20 @@ package org.apache.hadoop.hive.ql.exec.vector;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator;
class AggregateDefinition {
private String name;
private VectorExpressionDescriptor.ArgumentType type;
- private GroupByDesc.Mode mode;
+ private GenericUDAFEvaluator.Mode udafEvaluatorMode;
private Class<? extends VectorAggregateExpression> aggClass;
- AggregateDefinition(String name, VectorExpressionDescriptor.ArgumentType type,
- GroupByDesc.Mode mode, Class<? extends VectorAggregateExpression> aggClass) {
+ AggregateDefinition(String name, VectorExpressionDescriptor.ArgumentType type,
+ GenericUDAFEvaluator.Mode udafEvaluatorMode, Class<? extends VectorAggregateExpression> aggClass) {
this.name = name;
this.type = type;
- this.mode = mode;
+ this.udafEvaluatorMode = udafEvaluatorMode;
this.aggClass = aggClass;
}
@@ -42,8 +43,8 @@ class AggregateDefinition {
VectorExpressionDescriptor.ArgumentType getType() {
return type;
}
- GroupByDesc.Mode getMode() {
- return mode;
+ GenericUDAFEvaluator.Mode getUdafEvaluatorMode() {
+ return udafEvaluatorMode;
}
Class<? extends VectorAggregateExpression> getAggClass() {
return aggClass;
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
index 98a9bf6..6e53526 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorGroupByOperator.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.VectorGroupByDesc.ProcessingMode;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
@@ -51,6 +52,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
/**
* Vectorized GROUP BY operator implementation. Consumes the vectorized input and
@@ -542,18 +544,17 @@ public class VectorGroupByOperator extends Operator<GroupByDesc> implements
if (numEntriesHashTable > sumBatchSize * minReductionHashAggr) {
flush(true);
- changeToUnsortedStreamingMode();
+ changeToStreamingMode();
}
}
}
}
/**
- * Unsorted streaming processing mode. Each input VectorizedRowBatch may have
- * a mix of different keys (hence unsorted). Intermediate values are flushed
- * each time key changes.
+ * Streaming processing mode on ALREADY GROUPED data. Each input VectorizedRowBatch may
+ * have a mix of different keys. Intermediate values are flushed each time key changes.
*/
- private class ProcessingModeUnsortedStreaming extends ProcessingModeBase {
+ private class ProcessingModeStreaming extends ProcessingModeBase {
/**
* The aggregation buffers used in streaming mode
@@ -675,7 +676,7 @@ public class VectorGroupByOperator extends Operator<GroupByDesc> implements
* writeGroupRow does this and finally increments outputBatch.size.
*
*/
- private class ProcessingModeReduceMergePartialKeys extends ProcessingModeBase {
+ private class ProcessingModeReduceMergePartial extends ProcessingModeBase {
private boolean inGroup;
private boolean first;
@@ -761,8 +762,7 @@ public class VectorGroupByOperator extends Operator<GroupByDesc> implements
aggregators = new VectorAggregateExpression[aggrDesc.size()];
for (int i = 0; i < aggrDesc.size(); ++i) {
AggregationDesc aggDesc = aggrDesc.get(i);
- aggregators[i] =
- vContext.getAggregatorExpression(aggDesc, desc.getVectorDesc().isReduceMergePartial());
+ aggregators[i] = vContext.getAggregatorExpression(aggDesc);
}
isVectorOutput = desc.getVectorDesc().isVectorOutput();
@@ -810,12 +810,10 @@ public class VectorGroupByOperator extends Operator<GroupByDesc> implements
objectInspectors.add(aggregators[i].getOutputObjectInspector());
}
- if (outputKeyLength > 0 && !conf.getVectorDesc().isReduceMergePartial()) {
- // These data structures are only used by the non Reduce Merge-Partial Keys processing modes.
- keyWrappersBatch = VectorHashKeyWrapperBatch.compileKeyWrapperBatch(keyExpressions);
- aggregationBatchInfo = new VectorAggregationBufferBatch();
- aggregationBatchInfo.compileAggregationBatchInfo(aggregators);
- }
+ keyWrappersBatch = VectorHashKeyWrapperBatch.compileKeyWrapperBatch(keyExpressions);
+ aggregationBatchInfo = new VectorAggregationBufferBatch();
+ aggregationBatchInfo.compileAggregationBatchInfo(aggregators);
+
LOG.info("VectorGroupByOperator is vector output {}", isVectorOutput);
outputObjInspector = ObjectInspectorFactory.getStandardStructObjectInspector(
outputFieldNames, objectInspectors);
@@ -835,29 +833,35 @@ public class VectorGroupByOperator extends Operator<GroupByDesc> implements
forwardCache = new Object[outputKeyLength + aggregators.length];
- if (outputKeyLength == 0) {
- // Hash and MergePartial global aggregation are both handled here.
+ switch (conf.getVectorDesc().getProcessingMode()) {
+ case GLOBAL:
+ Preconditions.checkState(outputKeyLength == 0);
processingMode = this.new ProcessingModeGlobalAggregate();
- } else if (conf.getVectorDesc().isReduceMergePartial()) {
- // Sorted GroupBy of vector batches where an individual batch has the same group key (e.g. reduce).
- processingMode = this.new ProcessingModeReduceMergePartialKeys();
- } else if (conf.getVectorDesc().isReduceStreaming()) {
- processingMode = this.new ProcessingModeUnsortedStreaming();
- } else {
- // We start in hash mode and may dynamically switch to unsorted stream mode.
+ break;
+ case HASH:
processingMode = this.new ProcessingModeHashAggregate();
+ break;
+ case MERGE_PARTIAL:
+ processingMode = this.new ProcessingModeReduceMergePartial();
+ break;
+ case STREAMING:
+ processingMode = this.new ProcessingModeStreaming();
+ break;
+ default:
+ throw new RuntimeException("Unsupported vector GROUP BY processing mode " +
+ conf.getVectorDesc().getProcessingMode().name());
}
processingMode.initialize(hconf);
}
/**
- * changes the processing mode to unsorted streaming
+ * changes the processing mode to streaming
* This is done at the request of the hash agg mode, if the number of keys
* exceeds the minReductionHashAggr factor
* @throws HiveException
*/
- private void changeToUnsortedStreamingMode() throws HiveException {
- processingMode = this.new ProcessingModeUnsortedStreaming();
+ private void changeToStreamingMode() throws HiveException {
+ processingMode = this.new ProcessingModeStreaming();
processingMode.initialize(null);
LOG.trace("switched to streaming mode");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index 58ce063..1a3299b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -47,8 +47,8 @@ import org.apache.hadoop.hive.ql.exec.FunctionInfo;
import org.apache.hadoop.hive.ql.exec.FunctionRegistry;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.hive.ql.exec.vector.ColumnVector.Type;
+import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.ArgumentType;
import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.InputExpressionType;
-import org.apache.hadoop.hive.ql.exec.vector.VectorExpressionDescriptor.Mode;
import org.apache.hadoop.hive.ql.exec.vector.expressions.*;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorAggregateExpression;
import org.apache.hadoop.hive.ql.exec.vector.expressions.aggregates.VectorUDAFAvgDecimal;
@@ -114,6 +114,7 @@ import org.apache.hadoop.hive.ql.udf.UDFToLong;
import org.apache.hadoop.hive.ql.udf.UDFToShort;
import org.apache.hadoop.hive.ql.udf.UDFToString;
import org.apache.hadoop.hive.ql.udf.generic.*;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDAFEvaluator.Mode;
import org.apache.hadoop.hive.serde2.ByteStream.Output;
import org.apache.hadoop.hive.serde2.binarysortable.fast.BinarySortableSerializeWrite;
import org.apache.hadoop.hive.serde2.io.DateWritable;
@@ -404,7 +405,7 @@ public class VectorizationContext {
}
private VectorExpression getColumnVectorExpression(ExprNodeColumnDesc
- exprDesc, Mode mode) throws HiveException {
+ exprDesc, VectorExpressionDescriptor.Mode mode) throws HiveException {
int columnNum = getInputColumnIndex(exprDesc.getColumn());
VectorExpression expr = null;
switch (mode) {
@@ -425,7 +426,7 @@ public class VectorizationContext {
// Ok, try the UDF.
castToBooleanExpr = getVectorExpressionForUdf(null, UDFToBoolean.class, exprAsList,
- Mode.PROJECTION, null);
+ VectorExpressionDescriptor.Mode.PROJECTION, null);
if (castToBooleanExpr == null) {
throw new HiveException("Cannot vectorize converting expression " +
exprDesc.getExprString() + " to boolean");
@@ -443,10 +444,10 @@ public class VectorizationContext {
}
public VectorExpression[] getVectorExpressions(List<ExprNodeDesc> exprNodes) throws HiveException {
- return getVectorExpressions(exprNodes, Mode.PROJECTION);
+ return getVectorExpressions(exprNodes, VectorExpressionDescriptor.Mode.PROJECTION);
}
- public VectorExpression[] getVectorExpressions(List<ExprNodeDesc> exprNodes, Mode mode)
+ public VectorExpression[] getVectorExpressions(List<ExprNodeDesc> exprNodes, VectorExpressionDescriptor.Mode mode)
throws HiveException {
int i = 0;
@@ -461,7 +462,7 @@ public class VectorizationContext {
}
public VectorExpression getVectorExpression(ExprNodeDesc exprDesc) throws HiveException {
- return getVectorExpression(exprDesc, Mode.PROJECTION);
+ return getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
}
/**
@@ -472,7 +473,7 @@ public class VectorizationContext {
* @return {@link VectorExpression}
* @throws HiveException
*/
- public VectorExpression getVectorExpression(ExprNodeDesc exprDesc, Mode mode) throws HiveException {
+ public VectorExpression getVectorExpression(ExprNodeDesc exprDesc, VectorExpressionDescriptor.Mode mode) throws HiveException {
VectorExpression ve = null;
if (exprDesc instanceof ExprNodeColumnDesc) {
ve = getColumnVectorExpression((ExprNodeColumnDesc) exprDesc, mode);
@@ -873,14 +874,14 @@ public class VectorizationContext {
}
private VectorExpression getConstantVectorExpression(Object constantValue, TypeInfo typeInfo,
- Mode mode) throws HiveException {
+ VectorExpressionDescriptor.Mode mode) throws HiveException {
String typeName = typeInfo.getTypeName();
VectorExpressionDescriptor.ArgumentType vectorArgType = VectorExpressionDescriptor.ArgumentType.fromHiveTypeName(typeName);
if (vectorArgType == VectorExpressionDescriptor.ArgumentType.NONE) {
throw new HiveException("No vector argument type for type name " + typeName);
}
int outCol = -1;
- if (mode == Mode.PROJECTION) {
+ if (mode == VectorExpressionDescriptor.Mode.PROJECTION) {
outCol = ocm.allocateOutputColumn(typeName);
}
if (constantValue == null) {
@@ -889,7 +890,7 @@ public class VectorizationContext {
// Boolean is special case.
if (typeName.equalsIgnoreCase("boolean")) {
- if (mode == Mode.FILTER) {
+ if (mode == VectorExpressionDescriptor.Mode.FILTER) {
if (((Boolean) constantValue).booleanValue()) {
return new FilterConstantBooleanVectorExpression(1);
} else {
@@ -961,7 +962,7 @@ public class VectorizationContext {
}
private VectorExpression getVectorExpressionForUdf(GenericUDF genericeUdf,
- Class<?> udfClass, List<ExprNodeDesc> childExpr, Mode mode,
+ Class<?> udfClass, List<ExprNodeDesc> childExpr, VectorExpressionDescriptor.Mode mode,
TypeInfo returnType) throws HiveException {
int numChildren = (childExpr == null) ? 0 : childExpr.size();
@@ -973,13 +974,13 @@ public class VectorizationContext {
Class<?> vclass;
if (genericeUdf instanceof GenericUDFOPOr) {
- if (mode == Mode.PROJECTION) {
+ if (mode == VectorExpressionDescriptor.Mode.PROJECTION) {
vclass = ColOrCol.class;
} else {
vclass = FilterExprOrExpr.class;
}
} else if (genericeUdf instanceof GenericUDFOPAnd) {
- if (mode == Mode.PROJECTION) {
+ if (mode == VectorExpressionDescriptor.Mode.PROJECTION) {
vclass = ColAndCol.class;
} else {
vclass = FilterExprAndExpr.class;
@@ -987,8 +988,8 @@ public class VectorizationContext {
} else {
throw new RuntimeException("Unexpected multi-child UDF");
}
- Mode childrenMode = getChildrenMode(mode, udfClass);
- if (mode == Mode.PROJECTION) {
+ VectorExpressionDescriptor.Mode childrenMode = getChildrenMode(mode, udfClass);
+ if (mode == VectorExpressionDescriptor.Mode.PROJECTION) {
return createVectorMultiAndOrProjectionExpr(vclass, childExpr, childrenMode, returnType);
} else {
return createVectorExpression(vclass, childExpr, childrenMode, returnType);
@@ -1027,12 +1028,12 @@ public class VectorizationContext {
}
return null;
}
- Mode childrenMode = getChildrenMode(mode, udfClass);
+ VectorExpressionDescriptor.Mode childrenMode = getChildrenMode(mode, udfClass);
return createVectorExpression(vclass, childExpr, childrenMode, returnType);
}
private void determineChildrenVectorExprAndArguments(Class<?> vectorClass,
- List<ExprNodeDesc> childExpr, int numChildren, Mode childrenMode,
+ List<ExprNodeDesc> childExpr, int numChildren, VectorExpressionDescriptor.Mode childrenMode,
VectorExpression.Type [] inputTypes, List<VectorExpression> children, Object[] arguments)
throws HiveException {
for (int i = 0; i < numChildren; i++) {
@@ -1048,7 +1049,7 @@ public class VectorizationContext {
arguments[i] = vChild.getOutputColumn();
} else if (child instanceof ExprNodeColumnDesc) {
int colIndex = getInputColumnIndex((ExprNodeColumnDesc) child);
- if (childrenMode == Mode.FILTER) {
+ if (childrenMode == VectorExpressionDescriptor.Mode.FILTER) {
// In filter mode, the column must be a boolean
children.add(new SelectColumnIsTrue(colIndex));
}
@@ -1063,7 +1064,7 @@ public class VectorizationContext {
}
private VectorExpression createVectorExpression(Class<?> vectorClass,
- List<ExprNodeDesc> childExpr, Mode childrenMode, TypeInfo returnType) throws HiveException {
+ List<ExprNodeDesc> childExpr, VectorExpressionDescriptor.Mode childrenMode, TypeInfo returnType) throws HiveException {
int numChildren = childExpr == null ? 0: childExpr.size();
VectorExpression.Type [] inputTypes = new VectorExpression.Type[numChildren];
List<VectorExpression> children = new ArrayList<VectorExpression>();
@@ -1087,7 +1088,7 @@ public class VectorizationContext {
}
private VectorExpression createVectorMultiAndOrProjectionExpr(Class<?> vectorClass,
- List<ExprNodeDesc> childExpr, Mode childrenMode, TypeInfo returnType) throws HiveException {
+ List<ExprNodeDesc> childExpr, VectorExpressionDescriptor.Mode childrenMode, TypeInfo returnType) throws HiveException {
int numChildren = childExpr == null ? 0: childExpr.size();
VectorExpression.Type [] inputTypes = new VectorExpression.Type[numChildren];
List<VectorExpression> children = new ArrayList<VectorExpression>();
@@ -1119,11 +1120,11 @@ public class VectorizationContext {
}
}
- private Mode getChildrenMode(Mode mode, Class<?> udf) {
- if (mode.equals(Mode.FILTER) && (udf.equals(GenericUDFOPAnd.class) || udf.equals(GenericUDFOPOr.class))) {
- return Mode.FILTER;
+ private VectorExpressionDescriptor.Mode getChildrenMode(VectorExpressionDescriptor.Mode mode, Class<?> udf) {
+ if (mode.equals(VectorExpressionDescriptor.Mode.FILTER) && (udf.equals(GenericUDFOPAnd.class) || udf.equals(GenericUDFOPOr.class))) {
+ return VectorExpressionDescriptor.Mode.FILTER;
}
- return Mode.PROJECTION;
+ return VectorExpressionDescriptor.Mode.PROJECTION;
}
private String getNewInstanceArgumentString(Object [] args) {
@@ -1196,7 +1197,7 @@ public class VectorizationContext {
}
private VectorExpression getGenericUdfVectorExpression(GenericUDF udf,
- List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType) throws HiveException {
+ List<ExprNodeDesc> childExpr, VectorExpressionDescriptor.Mode mode, TypeInfo returnType) throws HiveException {
List<ExprNodeDesc> castedChildren = evaluateCastOnConstants(childExpr);
childExpr = castedChildren;
@@ -1204,7 +1205,7 @@ public class VectorizationContext {
//First handle special cases. If one of the special case methods cannot handle it,
// it returns null.
VectorExpression ve = null;
- if (udf instanceof GenericUDFBetween && mode == Mode.FILTER) {
+ if (udf instanceof GenericUDFBetween && mode == VectorExpressionDescriptor.Mode.FILTER) {
ve = getBetweenFilterExpression(childExpr, mode, returnType);
} else if (udf instanceof GenericUDFIn) {
ve = getInExpression(childExpr, mode, returnType);
@@ -1249,13 +1250,13 @@ public class VectorizationContext {
}
private VectorExpression getCastToTimestamp(GenericUDFTimestamp udf,
- List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType) throws HiveException {
+ List<ExprNodeDesc> childExpr, VectorExpressionDescriptor.Mode mode, TypeInfo returnType) throws HiveException {
VectorExpression ve = getVectorExpressionForUdf(udf, udf.getClass(), childExpr, mode, returnType);
// Replace with the milliseconds conversion
if (!udf.isIntToTimestampInSeconds() && ve instanceof CastLongToTimestamp) {
ve = createVectorExpression(CastMillisecondsLongToTimestamp.class,
- childExpr, Mode.PROJECTION, returnType);
+ childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
}
return ve;
@@ -1266,7 +1267,7 @@ public class VectorizationContext {
int[] inputColumns = new int[childExpr.size()];
VectorExpression[] vectorChildren = null;
try {
- vectorChildren = getVectorExpressions(childExpr, Mode.PROJECTION);
+ vectorChildren = getVectorExpressions(childExpr, VectorExpressionDescriptor.Mode.PROJECTION);
int i = 0;
for (VectorExpression ve : vectorChildren) {
@@ -1293,7 +1294,7 @@ public class VectorizationContext {
int[] inputColumns = new int[childExpr.size()];
VectorExpression[] vectorChildren = null;
try {
- vectorChildren = getVectorExpressions(childExpr, Mode.PROJECTION);
+ vectorChildren = getVectorExpressions(childExpr, VectorExpressionDescriptor.Mode.PROJECTION);
int i = 0;
for (VectorExpression ve : vectorChildren) {
@@ -1363,7 +1364,7 @@ public class VectorizationContext {
}
private VectorExpression getStructInExpression(List<ExprNodeDesc> childExpr, ExprNodeDesc colExpr,
- TypeInfo colTypeInfo, List<ExprNodeDesc> inChildren, Mode mode, TypeInfo returnType)
+ TypeInfo colTypeInfo, List<ExprNodeDesc> inChildren, VectorExpressionDescriptor.Mode mode, TypeInfo returnType)
throws HiveException {
VectorExpression expr = null;
@@ -1484,9 +1485,9 @@ public class VectorizationContext {
// generate the serialized keys of the batch.
int scratchBytesCol = ocm.allocateOutputColumn("string");
- Class<?> cl = (mode == Mode.FILTER ? FilterStructColumnInList.class : StructColumnInList.class);
+ Class<?> cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterStructColumnInList.class : StructColumnInList.class);
- expr = createVectorExpression(cl, null, Mode.PROJECTION, returnType);
+ expr = createVectorExpression(cl, null, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
((IStringInExpr) expr).setInListValues(serializedInChildren);
@@ -1500,7 +1501,7 @@ public class VectorizationContext {
/**
* Create a filter or boolean-valued expression for column IN ( <list-of-constants> )
*/
- private VectorExpression getInExpression(List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType)
+ private VectorExpression getInExpression(List<ExprNodeDesc> childExpr, VectorExpressionDescriptor.Mode mode, TypeInfo returnType)
throws HiveException {
ExprNodeDesc colExpr = childExpr.get(0);
List<ExprNodeDesc> inChildren = childExpr.subList(1, childExpr.size());
@@ -1538,53 +1539,53 @@ public class VectorizationContext {
// determine class
Class<?> cl = null;
if (isIntFamily(colType)) {
- cl = (mode == Mode.FILTER ? FilterLongColumnInList.class : LongColumnInList.class);
+ cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterLongColumnInList.class : LongColumnInList.class);
long[] inVals = new long[childrenForInList.size()];
for (int i = 0; i != inVals.length; i++) {
inVals[i] = getIntFamilyScalarAsLong((ExprNodeConstantDesc) childrenForInList.get(i));
}
- expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType);
((ILongInExpr) expr).setInListValues(inVals);
} else if (isTimestampFamily(colType)) {
- cl = (mode == Mode.FILTER ? FilterTimestampColumnInList.class : TimestampColumnInList.class);
+ cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterTimestampColumnInList.class : TimestampColumnInList.class);
Timestamp[] inVals = new Timestamp[childrenForInList.size()];
for (int i = 0; i != inVals.length; i++) {
inVals[i] = getTimestampScalar(childrenForInList.get(i));
}
- expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType);
((ITimestampInExpr) expr).setInListValues(inVals);
} else if (isStringFamily(colType)) {
- cl = (mode == Mode.FILTER ? FilterStringColumnInList.class : StringColumnInList.class);
+ cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterStringColumnInList.class : StringColumnInList.class);
byte[][] inVals = new byte[childrenForInList.size()][];
for (int i = 0; i != inVals.length; i++) {
inVals[i] = getStringScalarAsByteArray((ExprNodeConstantDesc) childrenForInList.get(i));
}
- expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType);
((IStringInExpr) expr).setInListValues(inVals);
} else if (isFloatFamily(colType)) {
- cl = (mode == Mode.FILTER ? FilterDoubleColumnInList.class : DoubleColumnInList.class);
+ cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterDoubleColumnInList.class : DoubleColumnInList.class);
double[] inValsD = new double[childrenForInList.size()];
for (int i = 0; i != inValsD.length; i++) {
inValsD[i] = getNumericScalarAsDouble(childrenForInList.get(i));
}
- expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType);
((IDoubleInExpr) expr).setInListValues(inValsD);
} else if (isDecimalFamily(colType)) {
- cl = (mode == Mode.FILTER ? FilterDecimalColumnInList.class : DecimalColumnInList.class);
+ cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterDecimalColumnInList.class : DecimalColumnInList.class);
HiveDecimal[] inValsD = new HiveDecimal[childrenForInList.size()];
for (int i = 0; i != inValsD.length; i++) {
inValsD[i] = (HiveDecimal) getVectorTypeScalarValue(
(ExprNodeConstantDesc) childrenForInList.get(i));
}
- expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType);
((IDecimalInExpr) expr).setInListValues(inValsD);
} else if (isDateFamily(colType)) {
- cl = (mode == Mode.FILTER ? FilterLongColumnInList.class : LongColumnInList.class);
+ cl = (mode == VectorExpressionDescriptor.Mode.FILTER ? FilterLongColumnInList.class : LongColumnInList.class);
long[] inVals = new long[childrenForInList.size()];
for (int i = 0; i != inVals.length; i++) {
inVals[i] = (Integer) getVectorTypeScalarValue((ExprNodeConstantDesc) childrenForInList.get(i));
}
- expr = createVectorExpression(cl, childExpr.subList(0, 1), Mode.PROJECTION, returnType);
+ expr = createVectorExpression(cl, childExpr.subList(0, 1), VectorExpressionDescriptor.Mode.PROJECTION, returnType);
((ILongInExpr) expr).setInListValues(inVals);
}
@@ -1607,7 +1608,7 @@ public class VectorizationContext {
* descriptor based lookup.
*/
private VectorExpression getGenericUDFBridgeVectorExpression(GenericUDFBridge udf,
- List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType) throws HiveException {
+ List<ExprNodeDesc> childExpr, VectorExpressionDescriptor.Mode mode, TypeInfo returnType) throws HiveException {
Class<? extends UDF> cl = udf.getUdfClass();
VectorExpression ve = null;
if (isCastToIntFamily(cl)) {
@@ -1741,21 +1742,21 @@ public class VectorizationContext {
// Return a constant vector expression
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
HiveDecimal decimalValue = castConstantToDecimal(constantValue, child.getTypeInfo());
- return getConstantVectorExpression(decimalValue, returnType, Mode.PROJECTION);
+ return getConstantVectorExpression(decimalValue, returnType, VectorExpressionDescriptor.Mode.PROJECTION);
}
if (isIntFamily(inputType)) {
- return createVectorExpression(CastLongToDecimal.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastLongToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (isFloatFamily(inputType)) {
- return createVectorExpression(CastDoubleToDecimal.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastDoubleToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (decimalTypePattern.matcher(inputType).matches()) {
- return createVectorExpression(CastDecimalToDecimal.class, childExpr, Mode.PROJECTION,
+ return createVectorExpression(CastDecimalToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION,
returnType);
} else if (isStringFamily(inputType)) {
- return createVectorExpression(CastStringToDecimal.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastStringToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (inputType.equals("timestamp")) {
- return createVectorExpression(CastTimestampToDecimal.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastTimestampToDecimal.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
}
- throw null;
+ return null;
}
private VectorExpression getCastToString(List<ExprNodeDesc> childExpr, TypeInfo returnType)
@@ -1766,19 +1767,19 @@ public class VectorizationContext {
// Return a constant vector expression
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
String strValue = castConstantToString(constantValue, child.getTypeInfo());
- return getConstantVectorExpression(strValue, returnType, Mode.PROJECTION);
+ return getConstantVectorExpression(strValue, returnType, VectorExpressionDescriptor.Mode.PROJECTION);
}
if (inputType.equals("boolean")) {
// Boolean must come before the integer family. It's a special case.
- return createVectorExpression(CastBooleanToStringViaLongToString.class, childExpr, Mode.PROJECTION, null);
+ return createVectorExpression(CastBooleanToStringViaLongToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, null);
} else if (isIntFamily(inputType)) {
- return createVectorExpression(CastLongToString.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastLongToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (isDecimalFamily(inputType)) {
- return createVectorExpression(CastDecimalToString.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastDecimalToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (isDateFamily(inputType)) {
- return createVectorExpression(CastDateToString.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastDateToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (isStringFamily(inputType)) {
- return createVectorExpression(CastStringGroupToString.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastStringGroupToString.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
}
return null;
}
@@ -1794,15 +1795,15 @@ public class VectorizationContext {
}
if (inputType.equals("boolean")) {
// Boolean must come before the integer family. It's a special case.
- return createVectorExpression(CastBooleanToCharViaLongToChar.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastBooleanToCharViaLongToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (isIntFamily(inputType)) {
- return createVectorExpression(CastLongToChar.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastLongToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (isDecimalFamily(inputType)) {
- return createVectorExpression(CastDecimalToChar.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastDecimalToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (isDateFamily(inputType)) {
- return createVectorExpression(CastDateToChar.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastDateToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (isStringFamily(inputType)) {
- return createVectorExpression(CastStringGroupToChar.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastStringGroupToChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
}
return null;
}
@@ -1818,15 +1819,15 @@ public class VectorizationContext {
}
if (inputType.equals("boolean")) {
// Boolean must come before the integer family. It's a special case.
- return createVectorExpression(CastBooleanToVarCharViaLongToVarChar.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastBooleanToVarCharViaLongToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (isIntFamily(inputType)) {
- return createVectorExpression(CastLongToVarChar.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastLongToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (isDecimalFamily(inputType)) {
- return createVectorExpression(CastDecimalToVarChar.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastDecimalToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (isDateFamily(inputType)) {
- return createVectorExpression(CastDateToVarChar.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastDateToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else if (isStringFamily(inputType)) {
- return createVectorExpression(CastStringGroupToVarChar.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastStringGroupToVarChar.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
}
return null;
}
@@ -1839,17 +1840,17 @@ public class VectorizationContext {
// Return a constant vector expression
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
Double doubleValue = castConstantToDouble(constantValue, child.getTypeInfo());
- return getConstantVectorExpression(doubleValue, returnType, Mode.PROJECTION);
+ return getConstantVectorExpression(doubleValue, returnType, VectorExpressionDescriptor.Mode.PROJECTION);
}
if (isIntFamily(inputType)) {
if (udf.equals(UDFToFloat.class)) {
// In order to convert from integer to float correctly, we need to apply the float cast not the double cast (HIVE-13338).
- return createVectorExpression(CastLongToFloatViaLongToDouble.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastLongToFloatViaLongToDouble.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
} else {
- return createVectorExpression(CastLongToDouble.class, childExpr, Mode.PROJECTION, returnType);
+ return createVectorExpression(CastLongToDouble.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
}
} else if (inputType.equals("timestamp")) {
- return createVectorExpression(CastTimestampToDouble.class, childExpr, Mode.PROJECTION,
+ return createVectorExpression(CastTimestampToDouble.class, childExpr, VectorExpressionDescriptor.Mode.PROJECTION,
returnType);
} else if (isFloatFamily(inputType)) {
@@ -1865,7 +1866,7 @@ public class VectorizationContext {
String inputType = childExpr.get(0).getTypeString();
if (child instanceof ExprNodeConstantDesc) {
if (null == ((ExprNodeConstantDesc)child).getValue()) {
- return getConstantVectorExpression(null, TypeInfoFactory.booleanTypeInfo, Mode.PROJECTION);
+ return getConstantVectorExpression(null, TypeInfoFactory.booleanTypeInfo, VectorExpressionDescriptor.Mode.PROJECTION);
}
// Don't do constant folding here. Wait until the optimizer is changed to do it.
// Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
@@ -1875,7 +1876,7 @@ public class VectorizationContext {
if (isStringFamily(inputType)) {
// string casts to false if it is 0 characters long, otherwise true
VectorExpression lenExpr = createVectorExpression(StringLength.class, childExpr,
- Mode.PROJECTION, null);
+ VectorExpressionDescriptor.Mode.PROJECTION, null);
int outputCol = ocm.allocateOutputColumn("Long");
VectorExpression lenToBoolExpr =
@@ -1895,7 +1896,7 @@ public class VectorizationContext {
// Return a constant vector expression
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
Long longValue = castConstantToLong(constantValue, child.getTypeInfo());
- return getConstantVectorExpression(longValue, TypeInfoFactory.longTypeInfo, Mode.PROJECTION);
+ return getConstantVectorExpression(longValue, TypeInfoFactory.longTypeInfo, VectorExpressionDescriptor.Mode.PROJECTION);
}
// Float family, timestamp are handled via descriptor based lookup, int family needs
// special handling.
@@ -1912,10 +1913,10 @@ public class VectorizationContext {
* needs to be done differently than the standard way where all arguments are
* passed to the VectorExpression constructor.
*/
- private VectorExpression getBetweenFilterExpression(List<ExprNodeDesc> childExpr, Mode mode, TypeInfo returnType)
+ private VectorExpression getBetweenFilterExpression(List<ExprNodeDesc> childExpr, VectorExpressionDescriptor.Mode mode, TypeInfo returnType)
throws HiveException {
- if (mode == Mode.PROJECTION) {
+ if (mode == VectorExpressionDescriptor.Mode.PROJECTION) {
// Projection mode is not yet supported for [NOT] BETWEEN. Return null so Vectorizer
// knows to revert to row-at-a-time execution.
@@ -2000,17 +2001,17 @@ public class VectorizationContext {
} else if (isDateFamily(colType) && notKeywordPresent) {
cl = FilterLongColumnNotBetween.class;
}
- return createVectorExpression(cl, childrenAfterNot, Mode.PROJECTION, returnType);
+ return createVectorExpression(cl, childrenAfterNot, VectorExpressionDescriptor.Mode.PROJECTION, returnType);
}
/*
* Return vector expression for a custom (i.e. not built-in) UDF.
*/
- private VectorExpression getCustomUDFExpression(ExprNodeGenericFuncDesc expr, Mode mode)
+ private VectorExpression getCustomUDFExpression(ExprNodeGenericFuncDesc expr, VectorExpressionDescriptor.Mode mode)
throws HiveException {
boolean isFilter = false; // Assume.
- if (mode == Mode.FILTER) {
+ if (mode == VectorExpressionDescriptor.Mode.FILTER) {
// Is output type a BOOLEAN?
TypeInfo resultTypeInfo = expr.getTypeInfo();
@@ -2043,7 +2044,7 @@ public class VectorizationContext {
for (int i = 0; i < childExprList.size(); i++) {
ExprNodeDesc child = childExprList.get(i);
if (child instanceof ExprNodeGenericFuncDesc) {
- VectorExpression e = getVectorExpression(child, Mode.PROJECTION);
+ VectorExpression e = getVectorExpression(child, VectorExpressionDescriptor.Mode.PROJECTION);
vectorExprs.add(e);
variableArgPositions.add(i);
exprResultColumnNums.add(e.getOutputColumn());
@@ -2384,67 +2385,125 @@ public class VectorizationContext {
}
}
- // TODO: When we support vectorized STRUCTs and can handle more in the reduce-side (MERGEPARTIAL):
- // TODO: Write reduce-side versions of AVG. Currently, only map-side (HASH) versions are in table.
- // TODO: And, investigate if different reduce-side versions are needed for var* and std*, or if map-side aggregate can be used.. Right now they are conservatively
- // marked map-side (HASH).
+
+ /*
+ * In the aggregatesDefinition table, Mode is GenericUDAFEvaluator.Mode.
+ *
+ * It is the different modes for an aggregate UDAF (User Defined Aggregation Function).
+ *
+ * (Notice the these names are a subset of GroupByDesc.Mode...)
+ *
+ * PARTIAL1 Original data --> Partial aggregation data
+ *
+ * PARTIAL2 Partial aggregation data --> Partial aggregation data
+ *
+ * FINAL Partial aggregation data --> Full aggregation data
+ *
+ * COMPLETE Original data --> Full aggregation data
+ *
+ *
+ * SIMPLEST CASE --> The data type/semantics of original data, partial aggregation
+ * data, and full aggregation data ARE THE SAME. E.g. MIN, MAX, SUM. The different
+ * modes can be handled by one aggregation class.
+ *
+ * This case has a null for the Mode.
+ *
+ * FOR OTHERS --> The data type/semantics of partial aggregation data and full aggregation data
+ * ARE THE SAME but different than original data. This results in 2 aggregation classes:
+ *
+ * 1) A class that takes original rows and outputs partial/full aggregation
+ * (PARTIAL1/COMPLETE)
+ *
+ * and
+ *
+ * 2) A class that takes partial aggregation and produces full aggregation
+ * (PARTIAL2/FINAL).
+ *
+ * E.g. COUNT(*) and COUNT(column)
+ *
+ * OTHERWISE FULL --> The data type/semantics of partial aggregation data is different than
+ * original data and full aggregation data.
+ *
+ * E.g. AVG uses a STRUCT with count and sum for partial aggregation data. It divides
+ * sum by count to produce the average for final aggregation.
+ *
+ */
static ArrayList<AggregateDefinition> aggregatesDefinition = new ArrayList<AggregateDefinition>() {{
- add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.INT_DATE_INTERVAL_YEAR_MONTH, null, VectorUDAFMinLong.class));
- add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null, VectorUDAFMinDouble.class));
- add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, null, VectorUDAFMinString.class));
- add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.DECIMAL, null, VectorUDAFMinDecimal.class));
- add(new AggregateDefinition("min", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, null, VectorUDAFMinTimestamp.class));
- add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.INT_DATE_INTERVAL_YEAR_MONTH, null, VectorUDAFMaxLong.class));
- add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null, VectorUDAFMaxDouble.class));
- add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, null, VectorUDAFMaxString.class));
- add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.DECIMAL, null, VectorUDAFMaxDecimal.class));
- add(new AggregateDefinition("max", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, null, VectorUDAFMaxTimestamp.class));
- add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.NONE, GroupByDesc.Mode.HASH, VectorUDAFCountStar.class));
- add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.INT_DATE_INTERVAL_YEAR_MONTH, GroupByDesc.Mode.HASH, VectorUDAFCount.class));
- add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.MERGEPARTIAL, VectorUDAFCountMerge.class));
- add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFCount.class));
- add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.STRING_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFCount.class));
- add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH, VectorUDAFCount.class));
- add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, GroupByDesc.Mode.HASH, VectorUDAFCount.class));
- add(new AggregateDefinition("count", VectorExpressionDescriptor.ArgumentType.INTERVAL_DAY_TIME, GroupByDesc.Mode.HASH, VectorUDAFCount.class));
- add(new AggregateDefinition("sum", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, null, VectorUDAFSumLong.class));
- add(new AggregateDefinition("sum", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, null, VectorUDAFSumDouble.class));
- add(new AggregateDefinition("sum", VectorExpressionDescriptor.ArgumentType.DECIMAL, null, VectorUDAFSumDecimal.class));
- add(new AggregateDefinition("avg", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFAvgLong.class));
- add(new AggregateDefinition("avg", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFAvgDouble.class));
- add(new AggregateDefinition("avg", VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH, VectorUDAFAvgDecimal.class));
- add(new AggregateDefinition("avg", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, GroupByDesc.Mode.HASH, VectorUDAFAvgTimestamp.class));
- add(new AggregateDefinition("variance", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFVarPopLong.class));
- add(new AggregateDefinition("var_pop", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFVarPopLong.class));
- add(new AggregateDefinition("variance", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFVarPopDouble.class));
- add(new AggregateDefinition("var_pop", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFVarPopDouble.class));
- add(new AggregateDefinition("variance", VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH, VectorUDAFVarPopDecimal.class));
- add(new AggregateDefinition("var_pop", VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH, VectorUDAFVarPopDecimal.class));
- add(new AggregateDefinition("variance", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, GroupByDesc.Mode.HASH, VectorUDAFVarPopTimestamp.class));
- add(new AggregateDefinition("var_pop", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, GroupByDesc.Mode.HASH, VectorUDAFVarPopTimestamp.class));
- add(new AggregateDefinition("var_samp", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFVarSampLong.class));
- add(new AggregateDefinition("var_samp" , VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFVarSampDouble.class));
- add(new AggregateDefinition("var_samp" , VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH, VectorUDAFVarSampDecimal.class));
- add(new AggregateDefinition("var_samp" , VectorExpressionDescriptor.ArgumentType.TIMESTAMP, GroupByDesc.Mode.HASH, VectorUDAFVarSampTimestamp.class));
- add(new AggregateDefinition("std", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFStdPopLong.class));
- add(new AggregateDefinition("stddev", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFStdPopLong.class));
- add(new AggregateDefinition("stddev_pop", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFStdPopLong.class));
- add(new AggregateDefinition("std", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFStdPopDouble.class));
- add(new AggregateDefinition("stddev", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFStdPopDouble.class));
- add(new AggregateDefinition("stddev_pop", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFStdPopDouble.class));
- add(new AggregateDefinition("std", VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH, VectorUDAFStdPopDecimal.class));
- add(new AggregateDefinition("stddev", VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH, VectorUDAFStdPopDecimal.class));
- add(new AggregateDefinition("stddev_pop", VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH, VectorUDAFStdPopDecimal.class));
- add(new AggregateDefinition("std", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, GroupByDesc.Mode.HASH, VectorUDAFStdPopTimestamp.class));
- add(new AggregateDefinition("stddev", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, GroupByDesc.Mode.HASH, VectorUDAFStdPopTimestamp.class));
- add(new AggregateDefinition("stddev_pop", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, GroupByDesc.Mode.HASH, VectorUDAFStdPopTimestamp.class));
- add(new AggregateDefinition("stddev_samp", VectorExpressionDescriptor.ArgumentType.INT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFStdSampLong.class));
- add(new AggregateDefinition("stddev_samp", VectorExpressionDescriptor.ArgumentType.FLOAT_FAMILY, GroupByDesc.Mode.HASH, VectorUDAFStdSampDouble.class));
- add(new AggregateDefinition("stddev_samp", VectorExpressionDescriptor.ArgumentType.DECIMAL, GroupByDesc.Mode.HASH, VectorUDAFStdSampDecimal.class));
- add(new AggregateDefinition("stddev_samp", VectorExpressionDescriptor.ArgumentType.TIMESTAMP, GroupByDesc.Mode.HASH, VectorUDAFStdSampTimestamp.class));
+
+ // MIN, MAX, and SUM have the same representation for partial and full aggregation, so the
+ // same class can be used for all modes (PARTIAL1, PARTIAL2, FINAL, and COMPLETE).
+ add(new AggregateDefinition("min", ArgumentType.INT_DATE_INTERVAL_YEAR_MONTH, null, VectorUDAFMinLong.class));
+ add(new AggregateDefinition("min", ArgumentType.FLOAT_FAMILY, null, VectorUDAFMinDouble.class));
+ add(new AggregateDefinition("min", ArgumentType.STRING_FAMILY, null, VectorUDAFMinString.class));
+ add(new AggregateDefinition("min", ArgumentType.DECIMAL, null, VectorUDAFMinDecimal.class));
+ add(new AggregateDefinition("min", ArgumentType.TIMESTAMP, null, VectorUDAFMinTimestamp.class));
+ add(new AggregateDefinition("max", ArgumentType.INT_DATE_INTERVAL_YEAR_MONTH, null, VectorUDAFMaxLong.class));
+ add(new AggregateDefinition("max", ArgumentType.FLOAT_FAMILY, null, VectorUDAFMaxDouble.class));
+ add(new AggregateDefinition("max", ArgumentType.STRING_FAMILY, null, VectorUDAFMaxString.class));
+ add(new AggregateDefinition("max", ArgumentType.DECIMAL, null, VectorUDAFMaxDecimal.class));
+ add(new AggregateDefinition("max", ArgumentType.TIMESTAMP, null, VectorUDAFMaxTimestamp.class));
+ add(new AggregateDefinition("sum", ArgumentType.INT_FAMILY, null, VectorUDAFSumLong.class));
+ add(new AggregateDefinition("sum", ArgumentType.FLOAT_FAMILY, null, VectorUDAFSumDouble.class));
+ add(new AggregateDefinition("sum", ArgumentType.DECIMAL, null, VectorUDAFSumDecimal.class));
+
+ // COUNT(column) doesn't count rows whose column value is NULL.
+ add(new AggregateDefinition("count", ArgumentType.ALL_FAMILY, Mode.PARTIAL1, VectorUDAFCount.class));
+ add(new AggregateDefinition("count", ArgumentType.ALL_FAMILY, Mode.COMPLETE, VectorUDAFCount.class));
+
+ // COUNT(*) counts all rows regardless of whether the column value(s) are NULL.
+ add(new AggregateDefinition("count", ArgumentType.NONE, Mode.PARTIAL1, VectorUDAFCountStar.class));
+ add(new AggregateDefinition("count", ArgumentType.NONE, Mode.COMPLETE, VectorUDAFCountStar.class));
+
+ // Merge the counts produced by either COUNT(column) or COUNT(*) modes PARTIAL1 or PARTIAL2.
+ add(new AggregateDefinition("count", ArgumentType.INT_FAMILY, Mode.PARTIAL2, VectorUDAFCountMerge.class));
+ add(new AggregateDefinition("count", ArgumentType.INT_FAMILY, Mode.FINAL, VectorUDAFCountMerge.class));
+
+ // Since the partial aggregation produced by AVG is a STRUCT with count and sum and the
+ // STRUCT data type isn't vectorized yet, we currently only support PARTIAL1. When we do
+ // support STRUCTs for average partial aggregation, we'll need 4 variations:
+ //
+ // PARTIAL1 Original data --> STRUCT Average Partial Aggregation
+ // PARTIAL2 STRUCT Average Partial Aggregation --> STRUCT Average Partial Aggregation
+ // FINAL STRUCT Average Partial Aggregation --> Full Aggregation
+ // COMPLETE Original data --> Full Aggregation
+ //
+ add(new AggregateDefinition("avg", ArgumentType.INT_FAMILY, Mode.PARTIAL1, VectorUDAFAvgLong.class));
+ add(new AggregateDefinition("avg", ArgumentType.FLOAT_FAMILY, Mode.PARTIAL1, VectorUDAFAvgDouble.class));
+ add(new AggregateDefinition("avg", ArgumentType.DECIMAL, Mode.PARTIAL1, VectorUDAFAvgDecimal.class));
+ add(new AggregateDefinition("avg", ArgumentType.TIMESTAMP, Mode.PARTIAL1, VectorUDAFAvgTimestamp.class));
+
+ // We haven't had a chance to examine the VAR* and STD* area and expand it beyond PARTIAL1.
+ add(new AggregateDefinition("variance", ArgumentType.INT_FAMILY, Mode.PARTIAL1, VectorUDAFVarPopLong.class));
+ add(new AggregateDefinition("var_pop", ArgumentType.INT_FAMILY, Mode.PARTIAL1, VectorUDAFVarPopLong.class));
+ add(new AggregateDefinition("variance", ArgumentType.FLOAT_FAMILY, Mode.PARTIAL1, VectorUDAFVarPopDouble.class));
+ add(new AggregateDefinition("var_pop", ArgumentType.FLOAT_FAMILY, Mode.PARTIAL1, VectorUDAFVarPopDouble.class));
+ add(new AggregateDefinition("variance", ArgumentType.DECIMAL, Mode.PARTIAL1, VectorUDAFVarPopDecimal.class));
+ add(new AggregateDefinition("var_pop", ArgumentType.DECIMAL, Mode.PARTIAL1, VectorUDAFVarPopDecimal.class));
+ add(new AggregateDefinition("variance", ArgumentType.TIMESTAMP, Mode.PARTIAL1, VectorUDAFVarPopTimestamp.class));
+ add(new AggregateDefinition("var_pop", ArgumentType.TIMESTAMP, Mode.PARTIAL1, VectorUDAFVarPopTimestamp.class));
+ add(new AggregateDefinition("var_samp", ArgumentType.INT_FAMILY, Mode.PARTIAL1, VectorUDAFVarSampLong.class));
+ add(new AggregateDefinition("var_samp" , ArgumentType.FLOAT_FAMILY, Mode.PARTIAL1, VectorUDAFVarSampDouble.class));
+ add(new AggregateDefinition("var_samp" , ArgumentType.DECIMAL, Mode.PARTIAL1, VectorUDAFVarSampDecimal.class));
+ add(new AggregateDefinition("var_samp" , ArgumentType.TIMESTAMP, Mode.PARTIAL1, VectorUDAFVarSampTimestamp.class));
+ add(new AggregateDefinition("std", ArgumentType.INT_FAMILY, Mode.PARTIAL1, VectorUDAFStdPopLong.class));
+ add(new AggregateDefinition("stddev", ArgumentType.INT_FAMILY, Mode.PARTIAL1, VectorUDAFStdPopLong.class));
+ add(new AggregateDefinition("stddev_pop", ArgumentType.INT_FAMILY, Mode.PARTIAL1, VectorUDAFStdPopLong.class));
+ add(new AggregateDefinition("std", ArgumentType.FLOAT_FAMILY, Mode.PARTIAL1, VectorUDAFStdPopDouble.class));
+ add(new AggregateDefinition("stddev", ArgumentType.FLOAT_FAMILY, Mode.PARTIAL1, VectorUDAFStdPopDouble.class));
+ add(new AggregateDefinition("stddev_pop", ArgumentType.FLOAT_FAMILY, Mode.PARTIAL1, VectorUDAFStdPopDouble.class));
+ add(new AggregateDefinition("std", ArgumentType.DECIMAL, Mode.PARTIAL1, VectorUDAFStdPopDecimal.class));
+ add(new AggregateDefinition("stddev", ArgumentType.DECIMAL, Mode.PARTIAL1, VectorUDAFStdPopDecimal.class));
+ add(new AggregateDefinition("stddev_pop", ArgumentType.DECIMAL, Mode.PARTIAL1, VectorUDAFStdPopDecimal.class));
+ add(new AggregateDefinition("std", ArgumentType.TIMESTAMP, Mode.PARTIAL1, VectorUDAFStdPopTimestamp.class));
+ add(new AggregateDefinition("stddev", ArgumentType.TIMESTAMP, Mode.PARTIAL1, VectorUDAFStdPopTimestamp.class));
+ add(new AggregateDefinition("stddev_pop", ArgumentType.TIMESTAMP, Mode.PARTIAL1, VectorUDAFStdPopTimestamp.class));
+ add(new AggregateDefinition("stddev_samp", ArgumentType.INT_FAMILY, Mode.PARTIAL1, VectorUDAFStdSampLong.class));
+ add(new AggregateDefinition("stddev_samp", ArgumentType.FLOAT_FAMILY, Mode.PARTIAL1, VectorUDAFStdSampDouble.class));
+ add(new AggregateDefinition("stddev_samp", ArgumentType.DECIMAL, Mode.PARTIAL1, VectorUDAFStdSampDecimal.class));
+ add(new AggregateDefinition("stddev_samp", ArgumentType.TIMESTAMP, Mode.PARTIAL1, VectorUDAFStdSampTimestamp.class));
}};
- public VectorAggregateExpression getAggregatorExpression(AggregationDesc desc, boolean isReduceMergePartial)
+ public VectorAggregateExpression getAggregatorExpression(AggregationDesc desc)
throws HiveException {
ArrayList<ExprNodeDesc> paramDescList = desc.getParameters();
@@ -2452,7 +2511,7 @@ public class VectorizationContext {
for (int i = 0; i< paramDescList.size(); ++i) {
ExprNodeDesc exprDesc = paramDescList.get(i);
- vectorParams[i] = this.getVectorExpression(exprDesc, Mode.PROJECTION);
+ vectorParams[i] = this.getVectorExpression(exprDesc, VectorExpressionDescriptor.Mode.PROJECTION);
}
String aggregateName = desc.getGenericUDAFName();
@@ -2466,15 +2525,16 @@ public class VectorizationContext {
}
}
+ GenericUDAFEvaluator.Mode udafEvaluatorMode = desc.getMode();
for (AggregateDefinition aggDef : aggregatesDefinition) {
if (aggregateName.equalsIgnoreCase(aggDef.getName()) &&
((aggDef.getType() == VectorExpressionDescriptor.ArgumentType.NONE &&
inputType == VectorExpressionDescriptor.ArgumentType.NONE) ||
(aggDef.getType().isSameTypeOrFamily(inputType)))) {
- if (aggDef.getMode() == GroupByDesc.Mode.HASH && isReduceMergePartial) {
- continue;
- } else if (aggDef.getMode() == GroupByDesc.Mode.MERGEPARTIAL && !isReduceMergePartial) {
+ // A null means all modes are ok.
+ GenericUDAFEvaluator.Mode aggDefUdafEvaluatorMode = aggDef.getUdafEvaluatorMode();
+ if (aggDefUdafEvaluatorMode != null && aggDefUdafEvaluatorMode != udafEvaluatorMode) {
continue;
}
@@ -2495,7 +2555,9 @@ public class VectorizationContext {
}
throw new HiveException("Vector aggregate not implemented: \"" + aggregateName +
- "\" for type: \"" + inputType.name() + " (reduce-merge-partial = " + isReduceMergePartial + ")");
+ "\" for type: \"" + inputType.name() +
+ " (UDAF evaluator mode = " +
+ (udafEvaluatorMode == null ? "NULL" : udafEvaluatorMode.name()) + ")");
}
public int firstOutputColumnIndex() {
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index c1d6582..00203ae 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -33,6 +33,7 @@ import java.util.Set;
import java.util.Stack;
import java.util.regex.Pattern;
+import org.apache.calcite.util.Pair;
import org.apache.commons.lang3.tuple.ImmutablePair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -96,6 +97,8 @@ import org.apache.hadoop.hive.ql.plan.JoinDesc;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
import org.apache.hadoop.hive.ql.plan.MapWork;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
+import org.apache.hadoop.hive.ql.plan.VectorGroupByDesc.ProcessingMode;
+import org.apache.hadoop.hive.ql.plan.VectorPartitionConversion;
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.plan.ReduceSinkDesc;
import org.apache.hadoop.hive.ql.plan.ReduceWork;
@@ -1540,76 +1543,131 @@ public class Vectorizer implements PhysicalPlanResolver {
LOG.info("Pruning grouping set id not supported in vector mode");
return false;
}
+ if (desc.getMode() != GroupByDesc.Mode.HASH && desc.isDistinct()) {
+ LOG.info("DISTINCT not supported in vector mode");
+ return false;
+ }
boolean ret = validateExprNodeDesc(desc.getKeys());
if (!ret) {
- LOG.info("Cannot vectorize groupby key expression");
+ LOG.info("Cannot vectorize groupby key expression " + desc.getKeys().toString());
return false;
}
- if (!isReduce) {
-
- // MapWork
-
- ret = validateHashAggregationDesc(desc.getAggregators());
- if (!ret) {
- return false;
- }
- } else {
-
- // ReduceWork
-
- boolean isComplete = desc.getMode() == GroupByDesc.Mode.COMPLETE;
- if (desc.getMode() != GroupByDesc.Mode.HASH) {
-
- // Reduce Merge-Partial GROUP BY.
-
- // A merge-partial GROUP BY is fed by grouping by keys from reduce-shuffle. It is the
- // first (or root) operator for its reduce task.
- // TODO: Technically, we should also handle FINAL, PARTIAL1, PARTIAL2 and PARTIALS
- // that are not hash or complete, but aren't merge-partial, somehow.
-
- if (desc.isDistinct()) {
- LOG.info("Vectorized Reduce MergePartial GROUP BY does not support DISTINCT");
- return false;
- }
-
- boolean hasKeys = (desc.getKeys().size() > 0);
-
- // Do we support merge-partial aggregation AND the output is primitive?
- ret = validateReduceMergePartialAggregationDesc(desc.getAggregators(), hasKeys);
- if (!ret) {
- return false;
- }
+ /**
+ *
+ * GROUP BY DEFINITIONS:
+ *
+ * GroupByDesc.Mode enumeration:
+ *
+ * The different modes of a GROUP BY operator.
+ *
+ * These descriptions are hopefully less cryptic than the comments for GroupByDesc.Mode.
+ *
+ * COMPLETE Aggregates original rows into full aggregation row(s).
+ *
+ * If the key length is 0, this is also called Global aggregation and
+ * 1 output row is produced.
+ *
+ * When the key length is > 0, the original rows come in ALREADY GROUPED.
+ *
+ * An example for key length > 0 is a GROUP BY being applied to the
+ * ALREADY GROUPED rows coming from an upstream JOIN operator. Or,
+ * ALREADY GROUPED rows coming from upstream MERGEPARTIAL GROUP BY
+ * operator.
+ *
+ * PARTIAL1 The first of 2 (or more) phases that aggregates ALREADY GROUPED
+ * original rows into partial aggregations.
+ *
+ * Subsequent phases PARTIAL2 (optional) and MERGEPARTIAL will merge
+ * the partial aggregations and output full aggregations.
+ *
+ * PARTIAL2 Accept ALREADY GROUPED partial aggregations and merge them into another
+ * partial aggregation. Output the merged partial aggregations.
+ *
+ * (Haven't seen this one used)
+ *
+ * PARTIALS (Behaves for non-distinct the same as PARTIAL2; and behaves for
+ * distinct the same as PARTIAL1.)
+ *
+ * FINAL Accept ALREADY GROUPED original rows and aggregate them into
+ * full aggregations.
+ *
+ * Example is a GROUP BY being applied to rows from a sorted table, where
+ * the group key is the table sort key (or a prefix).
+ *
+ * HASH Accept UNORDERED original rows and aggregate them into a memory table.
+ * Output the partial aggregations on closeOp (or low memory).
+ *
+ * Similar to PARTIAL1 except original rows are UNORDERED.
+ *
+ * Commonly used in both Mapper and Reducer nodes. Always followed by
+ * a Reducer with MERGEPARTIAL GROUP BY.
+ *
+ * MERGEPARTIAL Always first operator of a Reducer. Data is grouped by reduce-shuffle.
+ *
+ * (Behaves for non-distinct aggregations the same as FINAL; and behaves
+ * for distinct aggregations the same as COMPLETE.)
+ *
+ * The output is full aggregation(s).
+ *
+ * Used in Reducers after a stage with a HASH GROUP BY operator.
+ *
+ *
+ * VectorGroupByDesc.ProcessingMode for VectorGroupByOperator:
+ *
+ * GLOBAL No key. All rows --> 1 full aggregation on end of input
+ *
+ * HASH Rows aggregated in to hash table on group key -->
+ * 1 partial aggregation per key (normally, unless there is spilling)
+ *
+ * MERGE_PARTIAL As first operator in a REDUCER, partial aggregations come grouped from
+ * reduce-shuffle -->
+ * aggregate the partial aggregations and emit full aggregation on
+ * endGroup / closeOp
+ *
+ * STREAMING Rows come from PARENT operator ALREADY GROUPED -->
+ * aggregate the rows and emit full aggregation on key change / closeOp
+ *
+ * NOTE: Hash can spill partial result rows prematurely if it runs low on memory.
+ * NOTE: Streaming has to compare keys where MergePartial gets an endGroup call.
+ *
+ *
+ * DECIDER: Which VectorGroupByDesc.ProcessingMode for VectorGroupByOperator?
+ *
+ * Decides using GroupByDesc.Mode and whether there are keys with the
+ * VectorGroupByDesc.groupByDescModeToVectorProcessingMode method.
+ *
+ * Mode.COMPLETE --> (numKeys == 0 ? ProcessingMode.GLOBAL : ProcessingMode.STREAMING)
+ *
+ * Mode.HASH --> ProcessingMode.HASH
+ *
+ * Mode.MERGEPARTIAL --> (numKeys == 0 ? ProcessingMode.GLOBAL : ProcessingMode.MERGE_PARTIAL)
+ *
+ * Mode.PARTIAL1,
+ * Mode.PARTIAL2,
+ * Mode.PARTIALS,
+ * Mode.FINAL --> ProcessingMode.STREAMING
+ *
+ */
+ boolean hasKeys = (desc.getKeys().size() > 0);
- if (hasKeys) {
- if (op.getParentOperators().size() > 0 && !isComplete) {
- LOG.info("Vectorized Reduce MergePartial GROUP BY keys can only handle a key group when it is fed by reduce-shuffle");
- return false;
- }
+ ProcessingMode processingMode =
+ VectorGroupByDesc.groupByDescModeToVectorProcessingMode(desc.getMode(), hasKeys);
- LOG.info("Vectorized Reduce MergePartial GROUP BY will process key groups");
+ Pair<Boolean,Boolean> retPair =
+ validateAggregationDescs(desc.getAggregators(), processingMode, hasKeys);
+ if (!retPair.left) {
+ return false;
+ }
- // Primitive output validation above means we can output VectorizedRowBatch to the
- // children operators.
- vectorDesc.setVectorOutput(true);
- } else {
- LOG.info("Vectorized Reduce MergePartial GROUP BY will do global aggregation");
- }
- if (!isComplete) {
- vectorDesc.setIsReduceMergePartial(true);
- } else {
- vectorDesc.setIsReduceStreaming(true);
- }
- } else {
+ // If all the aggregation outputs are primitive, we can output VectorizedRowBatch.
+ // Otherwise, we the rest of the operator tree will be row mode.
+ vectorDesc.setVectorOutput(retPair.right);
- // Reduce Hash GROUP BY or global aggregation.
+ vectorDesc.setProcessingMode(processingMode);
- ret = validateHashAggregationDesc(desc.getAggregators());
- if (!ret) {
- return false;
- }
- }
- }
+ LOG.info("Vector GROUP BY operator will use processing mode " + processingMode.name() +
+ ", isVectorOutput " + vectorDesc.isVectorOutput());
return true;
}
@@ -1633,23 +1691,19 @@ public class Vectorizer implements PhysicalPlanResolver {
return true;
}
-
- private boolean validateHashAggregationDesc(List<AggregationDesc> descs) {
- return validateAggregationDesc(descs, /* isReduceMergePartial */ false, false);
- }
-
- private boolean validateReduceMergePartialAggregationDesc(List<AggregationDesc> descs, boolean hasKeys) {
- return validateAggregationDesc(descs, /* isReduceMergePartial */ true, hasKeys);
- }
-
- private boolean validateAggregationDesc(List<AggregationDesc> descs, boolean isReduceMergePartial, boolean hasKeys) {
+ private Pair<Boolean,Boolean> validateAggregationDescs(List<AggregationDesc> descs,
+ ProcessingMode processingMode, boolean hasKeys) {
+ boolean outputIsPrimitive = true;
for (AggregationDesc d : descs) {
- boolean ret = validateAggregationDesc(d, isReduceMergePartial, hasKeys);
- if (!ret) {
- return false;
+ Pair<Boolean,Boolean> retPair = validateAggregationDesc(d, processingMode, hasKeys);
+ if (!retPair.left) {
+ return retPair;
+ }
+ if (!retPair.right) {
+ outputIsPrimitive = false;
}
}
- return true;
+ return new Pair<Boolean, Boolean>(true, outputIsPrimitive);
}
private boolean validateExprNodeDescRecursive(ExprNodeDesc desc, VectorExpressionDescriptor.Mode mode) {
@@ -1787,38 +1841,45 @@ public class Vectorizer implements PhysicalPlanResolver {
return (outputObjInspector.getCategory() == ObjectInspector.Category.PRIMITIVE);
}
- private boolean validateAggregationDesc(AggregationDesc aggDesc, boolean isReduceMergePartial,
- boolean hasKeys) {
+ private Pair<Boolean,Boolean> validateAggregationDesc(AggregationDesc aggDesc, ProcessingMode processingMode,
+ boolean hasKeys) {
String udfName = aggDesc.getGenericUDAFName().toLowerCase();
if (!supportedAggregationUdfs.contains(udfName)) {
LOG.info("Cannot vectorize groupby aggregate expression: UDF " + udfName + " not supported");
- return false;
+ return new Pair<Boolean,Boolean>(false, false);
}
if (aggDesc.getParameters() != null && !validateExprNodeDesc(aggDesc.getParameters())) {
LOG.info("Cannot vectorize groupby aggregate expression: UDF parameters not supported");
- return false;
+ return new Pair<Boolean,Boolean>(false, false);
}
// See if we can vectorize the aggregation.
VectorizationContext vc = new ValidatorVectorizationContext();
VectorAggregateExpression vectorAggrExpr;
try {
- vectorAggrExpr = vc.getAggregatorExpression(aggDesc, isReduceMergePartial);
+ vectorAggrExpr = vc.getAggregatorExpression(aggDesc);
} catch (Exception e) {
// We should have already attempted to vectorize in validateAggregationDesc.
if (LOG.isDebugEnabled()) {
LOG.debug("Vectorization of aggreation should have succeeded ", e);
}
- return false;
+ return new Pair<Boolean,Boolean>(false, false);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Aggregation " + aggDesc.getExprString() + " --> " +
+ " vector expression " + vectorAggrExpr.toString());
}
- if (isReduceMergePartial && hasKeys && !validateAggregationIsPrimitive(vectorAggrExpr)) {
+ boolean outputIsPrimitive = validateAggregationIsPrimitive(vectorAggrExpr);
+ if (processingMode == ProcessingMode.MERGE_PARTIAL &&
+ hasKeys &&
+ !outputIsPrimitive) {
LOG.info("Vectorized Reduce MergePartial GROUP BY keys can only handle aggregate outputs that are primitive types");
- return false;
+ return new Pair<Boolean,Boolean>(false, false);
}
- return true;
+ return new Pair<Boolean,Boolean>(true, outputIsPrimitive);
}
private boolean validateDataType(String type, VectorExpressionDescriptor.Mode mode) {
http://git-wip-us.apache.org/repos/asf/hive/blob/8136a10c/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorGroupByDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorGroupByDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorGroupByDesc.java
index e613a4e..08f8ebf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorGroupByDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/VectorGroupByDesc.java
@@ -30,23 +30,45 @@ public class VectorGroupByDesc extends AbstractVectorDesc {
private static long serialVersionUID = 1L;
- private boolean isReduceMergePartial;
+ /**
+ * GLOBAL No key. All rows --> 1 full aggregation on end of input
+ *
+ * HASH Rows aggregated in to hash table on group key -->
+ * 1 partial aggregation per key (normally, unless there is spilling)
+ *
+ * MERGE_PARTIAL As first operator in a REDUCER, partial aggregations come grouped from
+ * reduce-shuffle -->
+ * aggregate the partial aggregations and emit full aggregation on
+ * endGroup / closeOp
+ *
+ * STREAMING Rows come from PARENT operator already grouped -->
+ * aggregate the rows and emit full aggregation on key change / closeOp
+ *
+ * NOTE: Hash can spill partial result rows prematurely if it runs low on memory.
+ * NOTE: Streaming has to compare keys where MergePartial gets an endGroup call.
+ */
+ public static enum ProcessingMode {
+ NONE,
+ GLOBAL,
+ HASH,
+ MERGE_PARTIAL,
+ STREAMING
+ };
- private boolean isVectorOutput;
+ private ProcessingMode processingMode;
- private boolean isReduceStreaming;
+ private boolean isVectorOutput;
public VectorGroupByDesc() {
- this.isReduceMergePartial = false;
+ this.processingMode = ProcessingMode.NONE;
this.isVectorOutput = false;
}
- public boolean isReduceMergePartial() {
- return isReduceMergePartial;
+ public void setProcessingMode(ProcessingMode processingMode) {
+ this.processingMode = processingMode;
}
-
- public void setIsReduceMergePartial(boolean isReduceMergePartial) {
- this.isReduceMergePartial = isReduceMergePartial;
+ public ProcessingMode getProcessingMode() {
+ return processingMode;
}
public boolean isVectorOutput() {
@@ -57,11 +79,39 @@ public class VectorGroupByDesc extends AbstractVectorDesc {
this.isVectorOutput = isVectorOutput;
}
- public void setIsReduceStreaming(boolean isReduceStreaming) {
- this.isReduceStreaming = isReduceStreaming;
- }
-
- public boolean isReduceStreaming() {
- return isReduceStreaming;
+ /**
+ * Which ProcessingMode for VectorGroupByOperator?
+ *
+ * Decides using GroupByDesc.Mode and whether there are keys.
+ *
+ * Mode.COMPLETE --> (numKeys == 0 ? ProcessingMode.GLOBAL : ProcessingMode.STREAMING)
+ *
+ * Mode.HASH --> ProcessingMode.HASH
+ *
+ * Mode.MERGEPARTIAL --> (numKeys == 0 ? ProcessingMode.GLOBAL : ProcessingMode.MERGE_PARTIAL)
+ *
+ * Mode.PARTIAL1,
+ * Mode.PARTIAL2,
+ * Mode.PARTIALS,
+ * Mode.FINAL --> ProcessingMode.STREAMING
+ *
+ */
+ public static ProcessingMode groupByDescModeToVectorProcessingMode(GroupByDesc.Mode mode,
+ boolean hasKeys) {
+ switch (mode) {
+ case COMPLETE:
+ return (hasKeys ? ProcessingMode.STREAMING : ProcessingMode.GLOBAL);
+ case HASH:
+ return ProcessingMode.HASH;
+ case MERGEPARTIAL:
+ return (hasKeys ? ProcessingMode.MERGE_PARTIAL : ProcessingMode.GLOBAL);
+ case PARTIAL1:
+ case PARTIAL2:
+ case PARTIALS:
+ case FINAL:
+ return ProcessingMode.STREAMING;
+ default:
+ throw new RuntimeException("Unexpected GROUP BY mode " + mode.name());
+ }
}
}