You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by sp...@apache.org on 2015/05/20 18:01:20 UTC
[01/50] [abbrv] hive git commit: HIVE-10621: serde typeinfo equals
methods are not symmetric (Alex Pivovarov via Jason Dere)
Repository: hive
Updated Branches:
refs/heads/parquet 4580944d5 -> 5a0ae3c2a
HIVE-10621: serde typeinfo equals methods are not symmetric (Alex Pivovarov via Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b4b79558
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b4b79558
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b4b79558
Branch: refs/heads/parquet
Commit: b4b79558a848feb86c90d18f18b32bf0e01d6754
Parents: 1e9e6fe
Author: Jason Dere <jd...@hortonworks.com>
Authored: Tue May 12 14:30:51 2015 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Tue May 12 14:30:51 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java | 5 ++++-
.../org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java | 5 ++++-
.../apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java | 7 +++++--
.../apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java | 5 ++++-
.../apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java | 5 ++++-
5 files changed, 21 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b4b79558/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
index 6ab64e5..a17d2cc 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveDecimalWritable.java
@@ -146,7 +146,10 @@ public class HiveDecimalWritable implements WritableComparable<HiveDecimalWritab
@Override
public boolean equals(Object other) {
- if (other == null || !(other instanceof HiveDecimalWritable)) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || getClass() != other.getClass()) {
return false;
}
HiveDecimalWritable bdw = (HiveDecimalWritable) other;
http://git-wip-us.apache.org/repos/asf/hive/blob/b4b79558/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java
index 610818e..871c0f0 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/CharTypeInfo.java
@@ -40,7 +40,10 @@ public class CharTypeInfo extends BaseCharTypeInfo {
@Override
public boolean equals(Object other) {
- if (other == null || !(other instanceof CharTypeInfo)) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || getClass() != other.getClass()) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b4b79558/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java
index cbe4802..d00f77d 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/DecimalTypeInfo.java
@@ -51,7 +51,10 @@ public class DecimalTypeInfo extends PrimitiveTypeInfo {
@Override
public boolean equals(Object other) {
- if (other == null || !(other instanceof DecimalTypeInfo)) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || getClass() != other.getClass()) {
return false;
}
@@ -99,7 +102,7 @@ public class DecimalTypeInfo extends PrimitiveTypeInfo {
@Override
public boolean accept(TypeInfo other) {
- if (other == null || !(other instanceof DecimalTypeInfo)) {
+ if (other == null || getClass() != other.getClass()) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b4b79558/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java
index a66b50a..ed6ea50 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/PrimitiveTypeInfo.java
@@ -87,7 +87,10 @@ public class PrimitiveTypeInfo extends TypeInfo implements Serializable {
@Override
public boolean equals(Object other) {
- if (other == null || !(other instanceof PrimitiveTypeInfo)) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || getClass() != other.getClass()) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b4b79558/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java
index 5ac2b46..cb53503 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/typeinfo/VarcharTypeInfo.java
@@ -40,7 +40,10 @@ public class VarcharTypeInfo extends BaseCharTypeInfo {
@Override
public boolean equals(Object other) {
- if (other == null || !(other instanceof VarcharTypeInfo)) {
+ if (this == other) {
+ return true;
+ }
+ if (other == null || getClass() != other.getClass()) {
return false;
}
[37/50] [abbrv] hive git commit: HIVE-10686: CBO -
java.lang.IndexOutOfBoundsException for query with rank() over(partition ...)
(Jesus Camacho Rodriguez via Laljo John Pullokkaran)
Posted by sp...@apache.org.
HIVE-10686: CBO - java.lang.IndexOutOfBoundsException for query with rank() over(partition ...) (Jesus Camacho Rodriguez via Laljo John Pullokkaran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/129c496b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/129c496b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/129c496b
Branch: refs/heads/parquet
Commit: 129c496bdf0f16a16e82d6978187319781c16e3d
Parents: 5818a4a
Author: jpullokk <jp...@apache.org>
Authored: Mon May 18 13:17:19 2015 -0700
Committer: jpullokk <jp...@apache.org>
Committed: Mon May 18 13:18:51 2015 -0700
----------------------------------------------------------------------
.../calcite/translator/PlanModifierUtil.java | 47 +++++++++++++-------
1 file changed, 30 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/129c496b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierUtil.java
index 2c820f0..3e2fae9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/PlanModifierUtil.java
@@ -68,23 +68,9 @@ public class PlanModifierUtil {
if (collationInputRefs.contains(i)) {
RexNode obyExpr = obChild.getChildExps().get(i);
if (obyExpr instanceof RexCall) {
- int a = -1;
- List<RexNode> operands = new ArrayList<>();
- for (int k = 0; k< ((RexCall) obyExpr).operands.size(); k++) {
- RexNode rn = ((RexCall) obyExpr).operands.get(k);
- for (int j = 0; j < resultSchema.size(); j++) {
- if( obChild.getChildExps().get(j).toString().equals(rn.toString())) {
- a = j;
- break;
- }
- } if (a != -1) {
- operands.add(new RexInputRef(a, rn.getType()));
- } else {
- operands.add(rn);
- }
- a = -1;
- }
- obyExpr = obChild.getCluster().getRexBuilder().makeCall(((RexCall)obyExpr).getOperator(), operands);
+ LOG.debug("Old RexCall : " + obyExpr);
+ obyExpr = adjustOBSchema((RexCall) obyExpr, obChild, resultSchema);
+ LOG.debug("New RexCall : " + obyExpr);
}
inputRefToCallMapBldr.put(i, obyExpr);
}
@@ -106,6 +92,33 @@ public class PlanModifierUtil {
obRel.setInputRefToCallMap(inputRefToCallMap);
}
+ private static RexCall adjustOBSchema(RexCall obyExpr, Project obChild,
+ List<FieldSchema> resultSchema) {
+ int a = -1;
+ List<RexNode> operands = new ArrayList<>();
+ for (int k = 0; k < obyExpr.operands.size(); k++) {
+ RexNode rn = obyExpr.operands.get(k);
+ for (int j = 0; j < resultSchema.size(); j++) {
+ if( obChild.getChildExps().get(j).toString().equals(rn.toString())) {
+ a = j;
+ break;
+ }
+ }
+ if (a != -1) {
+ operands.add(new RexInputRef(a, rn.getType()));
+ } else {
+ if (rn instanceof RexCall) {
+ operands.add(adjustOBSchema((RexCall)rn, obChild, resultSchema));
+ } else {
+ operands.add(rn);
+ }
+ }
+ a = -1;
+ }
+ return (RexCall) obChild.getCluster().getRexBuilder().makeCall(
+ obyExpr.getType(), obyExpr.getOperator(), operands);
+ }
+
protected static String generateInvalidSchemaMessage(Project topLevelProj,
List<FieldSchema> resultSchema, int fieldsForOB) {
String errorDesc = "Result Schema didn't match Calcite Optimized Op Tree; schema: ";
[42/50] [abbrv] hive git commit: HIVE-10628: Incorrect result when
vectorized native mapjoin is enabled using null safe operators <=> (Matt
McCline reviewed by Jason Dere)
Posted by sp...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/4a2a2bcd/ql/src/test/results/clientpositive/vector_nullsafe_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_nullsafe_join.q.out b/ql/src/test/results/clientpositive/vector_nullsafe_join.q.out
new file mode 100644
index 0000000..afebc52
--- /dev/null
+++ b/ql/src/test/results/clientpositive/vector_nullsafe_join.q.out
@@ -0,0 +1,1208 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+--
+-- We currently do not support null safes (i.e the <=> operator) in native vector map join.
+-- The explain output will show vectorized execution for both. We verify the query
+-- results are the same (HIVE-10628 shows native will produce the wrong results
+-- otherwise).
+--
+-- This query for "HIVE-3315 join predicate transitive" triggers HIVE-10640-
+-- explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL;
+-- select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL;
+--
+CREATE TABLE myinput1_txt(key int, value int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@myinput1_txt
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+--
+-- We currently do not support null safes (i.e the <=> operator) in native vector map join.
+-- The explain output will show vectorized execution for both. We verify the query
+-- results are the same (HIVE-10628 shows native will produce the wrong results
+-- otherwise).
+--
+-- This query for "HIVE-3315 join predicate transitive" triggers HIVE-10640-
+-- explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL;
+-- select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL;
+--
+CREATE TABLE myinput1_txt(key int, value int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@myinput1_txt
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in8.txt' INTO TABLE myinput1_txt
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@myinput1_txt
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in8.txt' INTO TABLE myinput1_txt
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@myinput1_txt
+PREHOOK: query: CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@myinput1_txt
+PREHOOK: Output: database:default
+PREHOOK: Output: default@myinput1
+POSTHOOK: query: CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@myinput1_txt
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@myinput1
+PREHOOK: query: -- merging
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value
+PREHOOK: type: QUERY
+POSTHOOK: query: -- merging
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-4 is a root stage
+ Stage-3 depends on stages: Stage-4
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-4
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ nullSafes: [true]
+ outputColumnNames: _col0, _col1, _col5, _col6
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-4 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ b
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+ b
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10 10 NULL
+100 100 100 100 100 100
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-4 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ b
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+ b
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+ nullSafes: [true]
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10 10 NULL
+100 100 100 100 100 100
+NULL 10 10 NULL NULL 10
+NULL 10 10 NULL NULL 35
+NULL 10 10 NULL NULL NULL
+NULL 10 48 NULL NULL 10
+NULL 10 48 NULL NULL 35
+NULL 10 48 NULL NULL NULL
+NULL 10 NULL NULL NULL 10
+NULL 10 NULL NULL NULL 35
+NULL 10 NULL NULL NULL NULL
+NULL 35 10 NULL NULL 10
+NULL 35 10 NULL NULL 35
+NULL 35 10 NULL NULL NULL
+NULL 35 48 NULL NULL 10
+NULL 35 48 NULL NULL 35
+NULL 35 48 NULL NULL NULL
+NULL 35 NULL NULL NULL 10
+NULL 35 NULL NULL NULL 35
+NULL 35 NULL NULL NULL NULL
+NULL NULL 10 NULL NULL 10
+NULL NULL 10 NULL NULL 35
+NULL NULL 10 NULL NULL NULL
+NULL NULL 48 NULL NULL 10
+NULL NULL 48 NULL NULL 35
+NULL NULL 48 NULL NULL NULL
+NULL NULL NULL NULL NULL 10
+NULL NULL NULL NULL NULL 35
+NULL NULL NULL NULL NULL NULL
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-4 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ b
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+ b
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+ nullSafes: [true, false]
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+100 100 100 100 100 100
+NULL 10 10 NULL NULL 10
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-4 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ b
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+ b
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+ nullSafes: [true, true]
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10 10 NULL
+100 100 100 100 100 100
+NULL 10 10 NULL NULL 10
+NULL NULL NULL NULL NULL NULL
+PREHOOK: query: -- outer joins
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: -- outer joins
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+48 NULL NULL NULL
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
+PREHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL 35
+NULL NULL NULL NULL
+PREHOOK: query: SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+48 NULL NULL NULL
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL 35
+NULL NULL NULL NULL
+PREHOOK: query: -- map joins
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: -- map joins
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
+PREHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
+PREHOOK: query: -- merging
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value
+PREHOOK: type: QUERY
+POSTHOOK: query: -- merging
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-4 is a root stage
+ Stage-3 depends on stages: Stage-4
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-4
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ nullSafes: [true]
+ outputColumnNames: _col0, _col1, _col5, _col6
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-4 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ b
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+ b
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10 10 NULL
+100 100 100 100 100 100
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-4 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ b
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+ b
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+ nullSafes: [true]
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10 10 NULL
+100 100 100 100 100 100
+NULL 10 10 NULL NULL 10
+NULL 10 10 NULL NULL 35
+NULL 10 10 NULL NULL NULL
+NULL 10 48 NULL NULL 10
+NULL 10 48 NULL NULL 35
+NULL 10 48 NULL NULL NULL
+NULL 10 NULL NULL NULL 10
+NULL 10 NULL NULL NULL 35
+NULL 10 NULL NULL NULL NULL
+NULL 35 10 NULL NULL 10
+NULL 35 10 NULL NULL 35
+NULL 35 10 NULL NULL NULL
+NULL 35 48 NULL NULL 10
+NULL 35 48 NULL NULL 35
+NULL 35 48 NULL NULL NULL
+NULL 35 NULL NULL NULL 10
+NULL 35 NULL NULL NULL 35
+NULL 35 NULL NULL NULL NULL
+NULL NULL 10 NULL NULL 10
+NULL NULL 10 NULL NULL 35
+NULL NULL 10 NULL NULL NULL
+NULL NULL 48 NULL NULL 10
+NULL NULL 48 NULL NULL 35
+NULL NULL 48 NULL NULL NULL
+NULL NULL NULL NULL NULL 10
+NULL NULL NULL NULL NULL 35
+NULL NULL NULL NULL NULL NULL
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-4 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ b
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+ b
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+ nullSafes: [true, false]
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+100 100 100 100 100 100
+NULL 10 10 NULL NULL 10
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-4 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-4
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ a
+ Fetch Operator
+ limit: -1
+ b
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ a
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+ b
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+ nullSafes: [true, true]
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10 10 NULL
+100 100 100 100 100 100
+NULL 10 10 NULL NULL 10
+NULL NULL NULL NULL NULL NULL
+PREHOOK: query: -- outer joins
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: -- outer joins
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+48 NULL NULL NULL
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
+PREHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL 35
+NULL NULL NULL NULL
+PREHOOK: query: SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+48 NULL NULL NULL
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL 35
+NULL NULL NULL NULL
+PREHOOK: query: -- map joins
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: -- map joins
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
+PREHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
[10/50] [abbrv] hive git commit: HIVE-10690: ArrayIndexOutOfBounds
exception in MetaStoreDirectSql.aggrColStatsForPartitions() (Vaibhav Gumashta
reviewed by Jason Dere)
Posted by sp...@apache.org.
HIVE-10690: ArrayIndexOutOfBounds exception in MetaStoreDirectSql.aggrColStatsForPartitions() (Vaibhav Gumashta reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/31383348
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/31383348
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/31383348
Branch: refs/heads/parquet
Commit: 31383348c1bd32b7991a20b793a362cca19987e9
Parents: 03eca77
Author: Vaibhav Gumashta <vg...@apache.org>
Authored: Wed May 13 13:44:41 2015 -0700
Committer: Vaibhav Gumashta <vg...@apache.org>
Committed: Wed May 13 13:44:41 2015 -0700
----------------------------------------------------------------------
.../apache/hadoop/hive/metastore/MetaStoreDirectSql.java | 10 ++++++----
1 file changed, 6 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/31383348/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
index 8bee978..0a56bac 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java
@@ -1129,10 +1129,12 @@ class MetaStoreDirectSql {
colStatsAggrFromDB =
columnStatisticsObjForPartitions(dbName, tableName, partNames, colNamesForDB,
partsFound, useDensityFunctionForNDVEstimation);
- ColumnStatisticsObj colStatsAggr = colStatsAggrFromDB.get(0);
- colStatsList.add(colStatsAggr);
- // Update the cache to add this new aggregate node
- aggrStatsCache.add(dbName, tableName, colName, partsFound, colStatsAggr, bloomFilter);
+ if (!colStatsAggrFromDB.isEmpty()) {
+ ColumnStatisticsObj colStatsAggr = colStatsAggrFromDB.get(0);
+ colStatsList.add(colStatsAggr);
+ // Update the cache to add this new aggregate node
+ aggrStatsCache.add(dbName, tableName, colName, partsFound, colStatsAggr, bloomFilter);
+ }
}
}
}
[06/50] [abbrv] hive git commit: HIVE-10548: Remove dependency to s3
repository in root pom (chengxiang reviewed by szehon)
Posted by sp...@apache.org.
HIVE-10548: Remove dependency to s3 repository in root pom (chengxiang reviewed by szehon)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c13ce55b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c13ce55b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c13ce55b
Branch: refs/heads/parquet
Commit: c13ce55b293f6ff0bbc1a1bc7ef3777dc07b557f
Parents: 5af6e92
Author: chengxiang <ch...@apache.com>
Authored: Wed May 13 13:40:07 2015 +0800
Committer: chengxiang <ch...@apache.com>
Committed: Wed May 13 13:40:07 2015 +0800
----------------------------------------------------------------------
itests/hive-unit/pom.xml | 88 +++++++++++++++++++++++--------------------
pom.xml | 12 +-----
2 files changed, 48 insertions(+), 52 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c13ce55b/itests/hive-unit/pom.xml
----------------------------------------------------------------------
diff --git a/itests/hive-unit/pom.xml b/itests/hive-unit/pom.xml
index 58e3e93..dbc9136 100644
--- a/itests/hive-unit/pom.xml
+++ b/itests/hive-unit/pom.xml
@@ -110,7 +110,7 @@
<classifier>tests</classifier>
</dependency>
<!-- test inter-project -->
- <dependency>
+ <dependency>
<groupId>junit</groupId>
<artifactId>junit</artifactId>
<version>${junit.version}</version>
@@ -126,6 +126,52 @@
<profiles>
<profile>
+ <id>spark-test</id>
+ <activation>
+ <property>
+ <name>!skipSparkTests</name>
+ </property>
+ </activation>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <scope>test</scope>
+ <exclusions>
+ <exclusion>
+ <groupId>com.esotericsoftware.kryo</groupId>
+ <artifactId>kryo</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ </dependencies>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-antrun-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>download-spark</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>run</goal>
+ </goals>
+ <configuration>
+ <target>
+ <exec executable="bash" dir="${basedir}" failonerror="true">
+ <arg line="../target/download.sh"/>
+ </exec>
+ </target>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ <profile>
<id>hadoop-1</id>
<dependencies>
<dependency>
@@ -317,46 +363,6 @@
</dependency>
</dependencies>
</profile>
- <profile>
- <id>spark-test</id>
- <activation>
- <property>
- <name>!skipSparkTests</name>
- </property>
- </activation>
- <dependencies>
- <dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-core_${scala.binary.version}</artifactId>
- <version>${spark.version}</version>
- <scope>test</scope>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-antrun-plugin</artifactId>
- <executions>
- <execution>
- <id>download-spark</id>
- <phase>generate-sources</phase>
- <goals>
- <goal>run</goal>
- </goals>
- <configuration>
- <target>
- <exec executable="bash" dir="${basedir}" failonerror="true">
- <arg line="../target/download.sh"/>
- </exec>
- </target>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
</profiles>
<build>
http://git-wip-us.apache.org/repos/asf/hive/blob/c13ce55b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 1921b06..2e4ca36 100644
--- a/pom.xml
+++ b/pom.xml
@@ -156,7 +156,7 @@
<ST4.version>4.0.4</ST4.version>
<tez.version>0.5.2</tez.version>
<super-csv.version>2.2.0</super-csv.version>
- <spark.version>1.3.0</spark.version>
+ <spark.version>1.3.1</spark.version>
<scala.binary.version>2.10</scala.binary.version>
<scala.version>2.10.4</scala.version>
<tempus-fugit.version>1.1</tempus-fugit.version>
@@ -216,16 +216,6 @@
<enabled>false</enabled>
</snapshots>
</repository>
- <repository>
- <id>spark-1.3</id>
- <url>https://s3-us-west-1.amazonaws.com/hive-spark/maven2/spark_2.10-1.3-rc1/</url>
- <releases>
- <enabled>true</enabled>
- </releases>
- <snapshots>
- <enabled>false</enabled>
- </snapshots>
- </repository>
</repositories>
<!-- Hadoop dependency management is done at the bottom under profiles -->
[20/50] [abbrv] hive git commit: HIVE-10565: Native Vector Map Join
doesn't handle filtering and matching on LEFT OUTER JOIN repeated key
correctly (Matt McCline via Gunther Hagleitner)
Posted by sp...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java
index 37ccf22..f971727 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterLongOperator.java
@@ -24,7 +24,9 @@ import java.util.Arrays;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.ql.exec.JoinUtil;
+import org.apache.hadoop.hive.ql.exec.vector.ColumnVector;
import org.apache.hadoop.hive.ql.exec.vector.VectorizationContext;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedBatchUtil;
import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
import org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpression;
import org.apache.hadoop.hive.ql.metadata.HiveException;
@@ -123,13 +125,6 @@ public class VectorMapJoinOuterLongOperator extends VectorMapJoinOuterGenerateRe
batchCounter++;
- // Do the per-batch setup for an outer join.
-
- outerPerBatchSetup(batch);
-
- // For outer join, DO NOT apply filters yet. It is incorrect for outer join to
- // apply the filter before hash table matching.
-
final int inputLogicalSize = batch.size;
if (inputLogicalSize == 0) {
@@ -139,6 +134,44 @@ public class VectorMapJoinOuterLongOperator extends VectorMapJoinOuterGenerateRe
return;
}
+ // Do the per-batch setup for an outer join.
+
+ outerPerBatchSetup(batch);
+
+ // For outer join, remember our input rows before ON expression filtering or before
+ // hash table matching so we can generate results for all rows (matching and non matching)
+ // later.
+ boolean inputSelectedInUse = batch.selectedInUse;
+ if (inputSelectedInUse) {
+ // if (!verifyMonotonicallyIncreasing(batch.selected, batch.size)) {
+ // throw new HiveException("batch.selected is not in sort order and unique");
+ // }
+ System.arraycopy(batch.selected, 0, inputSelected, 0, inputLogicalSize);
+ }
+
+ // Filtering for outer join just removes rows available for hash table matching.
+ boolean someRowsFilteredOut = false;
+ if (bigTableFilterExpressions.length > 0) {
+ // Since the input
+ for (VectorExpression ve : bigTableFilterExpressions) {
+ ve.evaluate(batch);
+ }
+ someRowsFilteredOut = (batch.size != inputLogicalSize);
+ if (LOG.isDebugEnabled()) {
+ if (batch.selectedInUse) {
+ if (inputSelectedInUse) {
+ LOG.debug(CLASS_NAME +
+ " inputSelected " + intArrayToRangesString(inputSelected, inputLogicalSize) +
+ " filtered batch.selected " + intArrayToRangesString(batch.selected, batch.size));
+ } else {
+ LOG.debug(CLASS_NAME +
+ " inputLogicalSize " + inputLogicalSize +
+ " filtered batch.selected " + intArrayToRangesString(batch.selected, batch.size));
+ }
+ }
+ }
+ }
+
// Perform any key expressions. Results will go into scratch columns.
if (bigTableKeyExpressions != null) {
for (VectorExpression ve : bigTableKeyExpressions) {
@@ -146,9 +179,6 @@ public class VectorMapJoinOuterLongOperator extends VectorMapJoinOuterGenerateRe
}
}
- // We rebuild in-place the selected array with rows destine to be forwarded.
- int numSel = 0;
-
/*
* Single-Column Long specific declarations.
*/
@@ -178,12 +208,16 @@ public class VectorMapJoinOuterLongOperator extends VectorMapJoinOuterGenerateRe
*/
JoinUtil.JoinResult joinResult;
- if (!joinColVector.noNulls && joinColVector.isNull[0]) {
- // Null key is no match for whole batch.
+ if (batch.size == 0) {
+ // Whole repeated key batch was filtered out.
+ joinResult = JoinUtil.JoinResult.NOMATCH;
+ } else if (!joinColVector.noNulls && joinColVector.isNull[0]) {
+ // Any (repeated) null key column is no match for whole batch.
joinResult = JoinUtil.JoinResult.NOMATCH;
} else {
// Handle *repeated* join key, if found.
long key = vector[0];
+ // LOG.debug(CLASS_NAME + " repeated key " + key);
if (useMinMax && (key < min || key > max)) {
// Out of range for whole batch.
joinResult = JoinUtil.JoinResult.NOMATCH;
@@ -199,7 +233,8 @@ public class VectorMapJoinOuterLongOperator extends VectorMapJoinOuterGenerateRe
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
}
- numSel = finishOuterRepeated(batch, joinResult, hashMapResults[0], scratch1);
+ finishOuterRepeated(batch, joinResult, hashMapResults[0], someRowsFilteredOut,
+ inputSelectedInUse, inputLogicalSize);
} else {
/*
@@ -213,14 +248,13 @@ public class VectorMapJoinOuterLongOperator extends VectorMapJoinOuterGenerateRe
int selected[] = batch.selected;
boolean selectedInUse = batch.selectedInUse;
- // For outer join we must apply the filter after match and cause some matches to become
- // non-matches, we do not track non-matches here. Instead we remember all non spilled rows
- // and compute non matches later in finishOuter.
int hashMapResultCount = 0;
- int matchCount = 0;
- int nonSpillCount = 0;
+ int allMatchCount = 0;
+ int equalKeySeriesCount = 0;
int spillCount = 0;
+ boolean atLeastOneNonMatch = someRowsFilteredOut;
+
/*
* Single-Column Long specific variables.
*/
@@ -232,9 +266,11 @@ public class VectorMapJoinOuterLongOperator extends VectorMapJoinOuterGenerateRe
JoinUtil.JoinResult saveJoinResult = JoinUtil.JoinResult.NOMATCH;
// Logical loop over the rows in the batch since the batch may have selected in use.
- for (int logical = 0; logical < inputLogicalSize; logical++) {
+ for (int logical = 0; logical < batch.size; logical++) {
int batchIndex = (selectedInUse ? selected[logical] : logical);
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, taskName + ", " + getOperatorId() + " candidate " + CLASS_NAME + " batch");
+
/*
* Single-Column Long outer null detection.
*/
@@ -250,8 +286,8 @@ public class VectorMapJoinOuterLongOperator extends VectorMapJoinOuterGenerateRe
// Let a current SPILL equal key series keep going, or
// Let a current NOMATCH keep not matching.
- // Remember non-matches for Outer Join.
- nonSpills[nonSpillCount++] = batchIndex;
+ atLeastOneNonMatch = true;
+
// LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " NULL");
} else {
@@ -269,9 +305,12 @@ public class VectorMapJoinOuterLongOperator extends VectorMapJoinOuterGenerateRe
// New key.
if (haveSaveKey) {
- // Move on with our count(s).
+ // Move on with our counts.
switch (saveJoinResult) {
case MATCH:
+ hashMapResultCount++;
+ equalKeySeriesCount++;
+ break;
case SPILL:
hashMapResultCount++;
break;
@@ -300,41 +339,70 @@ public class VectorMapJoinOuterLongOperator extends VectorMapJoinOuterGenerateRe
} else {
saveJoinResult = hashMap.lookup(currentKey, hashMapResults[hashMapResultCount]);
}
- // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " New Key " + saveJoinResult.name());
- } else {
- // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " Key Continues " + saveJoinResult.name());
- }
- /*
- * Common outer join result processing.
- */
+ // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " New Key " + currentKey + " " + saveJoinResult.name());
- switch (saveJoinResult) {
- case MATCH:
- matchs[matchCount] = batchIndex;
- matchHashMapResultIndices[matchCount] = hashMapResultCount;
- matchCount++;
- nonSpills[nonSpillCount++] = batchIndex;
- break;
-
- case SPILL:
- spills[spillCount] = batchIndex;
- spillHashMapResultIndices[spillCount] = hashMapResultCount;
- spillCount++;
- break;
-
- case NOMATCH:
- nonSpills[nonSpillCount++] = batchIndex;
- // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH duplicate");
- break;
+ /*
+ * Common outer join result processing.
+ */
+
+ switch (saveJoinResult) {
+ case MATCH:
+ equalKeySeriesHashMapResultIndices[equalKeySeriesCount] = hashMapResultCount;
+ equalKeySeriesAllMatchIndices[equalKeySeriesCount] = allMatchCount;
+ equalKeySeriesIsSingleValue[equalKeySeriesCount] = hashMapResults[hashMapResultCount].isSingleRow();
+ equalKeySeriesDuplicateCounts[equalKeySeriesCount] = 1;
+ allMatchs[allMatchCount++] = batchIndex;
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " MATCH isSingleValue " + equalKeySeriesIsSingleValue[equalKeySeriesCount] + " currentKey " + currentKey);
+ break;
+
+ case SPILL:
+ spills[spillCount] = batchIndex;
+ spillHashMapResultIndices[spillCount] = hashMapResultCount;
+ spillCount++;
+ break;
+
+ case NOMATCH:
+ atLeastOneNonMatch = true;
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH" + " currentKey " + currentKey);
+ break;
+ }
+ } else {
+ // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " Key Continues " + saveKey + " " + saveJoinResult.name());
+
+ // Series of equal keys.
+
+ switch (saveJoinResult) {
+ case MATCH:
+ equalKeySeriesDuplicateCounts[equalKeySeriesCount]++;
+ allMatchs[allMatchCount++] = batchIndex;
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " MATCH duplicate");
+ break;
+
+ case SPILL:
+ spills[spillCount] = batchIndex;
+ spillHashMapResultIndices[spillCount] = hashMapResultCount;
+ spillCount++;
+ break;
+
+ case NOMATCH:
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH duplicate");
+ break;
+ }
}
+ // if (!verifyMonotonicallyIncreasing(allMatchs, allMatchCount)) {
+ // throw new HiveException("allMatchs is not in sort order and unique");
+ // }
}
}
if (haveSaveKey) {
- // Account for last equal key sequence.
+ // Update our counts for the last key.
switch (saveJoinResult) {
case MATCH:
+ hashMapResultCount++;
+ equalKeySeriesCount++;
+ break;
case SPILL:
hashMapResultCount++;
break;
@@ -345,27 +413,26 @@ public class VectorMapJoinOuterLongOperator extends VectorMapJoinOuterGenerateRe
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter +
- " matchs " + intArrayToRangesString(matchs, matchCount) +
- " matchHashMapResultIndices " + intArrayToRangesString(matchHashMapResultIndices, matchCount) +
- " nonSpills " + intArrayToRangesString(nonSpills, nonSpillCount) +
+ " allMatchs " + intArrayToRangesString(allMatchs,allMatchCount) +
+ " equalKeySeriesHashMapResultIndices " + intArrayToRangesString(equalKeySeriesHashMapResultIndices, equalKeySeriesCount) +
+ " equalKeySeriesAllMatchIndices " + intArrayToRangesString(equalKeySeriesAllMatchIndices, equalKeySeriesCount) +
+ " equalKeySeriesIsSingleValue " + Arrays.toString(Arrays.copyOfRange(equalKeySeriesIsSingleValue, 0, equalKeySeriesCount)) +
+ " equalKeySeriesDuplicateCounts " + Arrays.toString(Arrays.copyOfRange(equalKeySeriesDuplicateCounts, 0, equalKeySeriesCount)) +
+ " atLeastOneNonMatch " + atLeastOneNonMatch +
+ " inputSelectedInUse " + inputSelectedInUse +
+ " inputLogicalSize " + inputLogicalSize +
" spills " + intArrayToRangesString(spills, spillCount) +
" spillHashMapResultIndices " + intArrayToRangesString(spillHashMapResultIndices, spillCount) +
" hashMapResults " + Arrays.toString(Arrays.copyOfRange(hashMapResults, 0, hashMapResultCount)));
}
// We will generate results for all matching and non-matching rows.
- // Note that scratch1 is undefined at this point -- it's preallocated storage.
- numSel = finishOuter(batch,
- matchs, matchHashMapResultIndices, matchCount,
- nonSpills, nonSpillCount,
- spills, spillHashMapResultIndices, spillCount,
- hashMapResults, hashMapResultCount,
- scratch1);
+ finishOuter(batch,
+ allMatchCount, equalKeySeriesCount, atLeastOneNonMatch,
+ inputSelectedInUse, inputLogicalSize,
+ spillCount, hashMapResultCount);
}
- batch.selectedInUse = true;
- batch.size = numSel;
-
if (batch.size > 0) {
// Forward any remaining selected rows.
forwardBigTableBatch(batch);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java
index 23a29f7..bea032a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterMultiKeyOperator.java
@@ -128,13 +128,6 @@ public class VectorMapJoinOuterMultiKeyOperator extends VectorMapJoinOuterGenera
batchCounter++;
- // Do the per-batch setup for an outer join.
-
- outerPerBatchSetup(batch);
-
- // For outer join, DO NOT apply filters yet. It is incorrect for outer join to
- // apply the filter before hash table matching.
-
final int inputLogicalSize = batch.size;
if (inputLogicalSize == 0) {
@@ -144,6 +137,44 @@ public class VectorMapJoinOuterMultiKeyOperator extends VectorMapJoinOuterGenera
return;
}
+ // Do the per-batch setup for an outer join.
+
+ outerPerBatchSetup(batch);
+
+ // For outer join, remember our input rows before ON expression filtering or before
+ // hash table matching so we can generate results for all rows (matching and non matching)
+ // later.
+ boolean inputSelectedInUse = batch.selectedInUse;
+ if (inputSelectedInUse) {
+ // if (!verifyMonotonicallyIncreasing(batch.selected, batch.size)) {
+ // throw new HiveException("batch.selected is not in sort order and unique");
+ // }
+ System.arraycopy(batch.selected, 0, inputSelected, 0, inputLogicalSize);
+ }
+
+ // Filtering for outer join just removes rows available for hash table matching.
+ boolean someRowsFilteredOut = false;
+ if (bigTableFilterExpressions.length > 0) {
+ // Since the input
+ for (VectorExpression ve : bigTableFilterExpressions) {
+ ve.evaluate(batch);
+ }
+ someRowsFilteredOut = (batch.size != inputLogicalSize);
+ if (LOG.isDebugEnabled()) {
+ if (batch.selectedInUse) {
+ if (inputSelectedInUse) {
+ LOG.debug(CLASS_NAME +
+ " inputSelected " + intArrayToRangesString(inputSelected, inputLogicalSize) +
+ " filtered batch.selected " + intArrayToRangesString(batch.selected, batch.size));
+ } else {
+ LOG.debug(CLASS_NAME +
+ " inputLogicalSize " + inputLogicalSize +
+ " filtered batch.selected " + intArrayToRangesString(batch.selected, batch.size));
+ }
+ }
+ }
+ }
+
// Perform any key expressions. Results will go into scratch columns.
if (bigTableKeyExpressions != null) {
for (VectorExpression ve : bigTableKeyExpressions) {
@@ -151,9 +182,6 @@ public class VectorMapJoinOuterMultiKeyOperator extends VectorMapJoinOuterGenera
}
}
- // We rebuild in-place the selected array with rows destine to be forwarded.
- int numSel = 0;
-
/*
* Multi-Key specific declarations.
*/
@@ -199,8 +227,11 @@ public class VectorMapJoinOuterMultiKeyOperator extends VectorMapJoinOuterGenera
*/
JoinUtil.JoinResult joinResult;
- if (someKeyInputColumnIsNull) {
- // Any null key column is no match for whole batch.
+ if (batch.size == 0) {
+ // Whole repeated key batch was filtered out.
+ joinResult = JoinUtil.JoinResult.NOMATCH;
+ } else if (someKeyInputColumnIsNull) {
+ // Any (repeated) null key column is no match for whole batch.
joinResult = JoinUtil.JoinResult.NOMATCH;
} else {
@@ -219,7 +250,8 @@ public class VectorMapJoinOuterMultiKeyOperator extends VectorMapJoinOuterGenera
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
}
- numSel = finishOuterRepeated(batch, joinResult, hashMapResults[0], scratch1);
+ finishOuterRepeated(batch, joinResult, hashMapResults[0], someRowsFilteredOut,
+ inputSelectedInUse, inputLogicalSize);
} else {
/*
@@ -233,14 +265,13 @@ public class VectorMapJoinOuterMultiKeyOperator extends VectorMapJoinOuterGenera
int selected[] = batch.selected;
boolean selectedInUse = batch.selectedInUse;
- // For outer join we must apply the filter after match and cause some matches to become
- // non-matches, we do not track non-matches here. Instead we remember all non spilled rows
- // and compute non matches later in finishOuter.
int hashMapResultCount = 0;
- int matchCount = 0;
- int nonSpillCount = 0;
+ int allMatchCount = 0;
+ int equalKeySeriesCount = 0;
int spillCount = 0;
+ boolean atLeastOneNonMatch = someRowsFilteredOut;
+
/*
* Multi-Key specific variables.
*/
@@ -252,9 +283,11 @@ public class VectorMapJoinOuterMultiKeyOperator extends VectorMapJoinOuterGenera
JoinUtil.JoinResult saveJoinResult = JoinUtil.JoinResult.NOMATCH;
// Logical loop over the rows in the batch since the batch may have selected in use.
- for (int logical = 0; logical < inputLogicalSize; logical++) {
+ for (int logical = 0; logical < batch.size; logical++) {
int batchIndex = (selectedInUse ? selected[logical] : logical);
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, taskName + ", " + getOperatorId() + " candidate " + CLASS_NAME + " batch");
+
/*
* Multi-Key outer null detection.
*/
@@ -272,8 +305,8 @@ public class VectorMapJoinOuterMultiKeyOperator extends VectorMapJoinOuterGenera
// Let a current SPILL equal key series keep going, or
// Let a current NOMATCH keep not matching.
- // Remember non-matches for Outer Join.
- nonSpills[nonSpillCount++] = batchIndex;
+ atLeastOneNonMatch = true;
+
// LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " NULL");
} else {
@@ -292,9 +325,12 @@ public class VectorMapJoinOuterMultiKeyOperator extends VectorMapJoinOuterGenera
// New key.
if (haveSaveKey) {
- // Move on with our count(s).
+ // Move on with our counts.
switch (saveJoinResult) {
case MATCH:
+ hashMapResultCount++;
+ equalKeySeriesCount++;
+ break;
case SPILL:
hashMapResultCount++;
break;
@@ -322,41 +358,68 @@ public class VectorMapJoinOuterMultiKeyOperator extends VectorMapJoinOuterGenera
byte[] keyBytes = saveKeyOutput.getData();
int keyLength = saveKeyOutput.getLength();
saveJoinResult = hashMap.lookup(keyBytes, 0, keyLength, hashMapResults[hashMapResultCount]);
- // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " New Key " + saveJoinResult.name());
- } else {
- // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " Key Continues " + saveJoinResult.name());
- }
- /*
- * Common outer join result processing.
- */
+ /*
+ * Common outer join result processing.
+ */
- switch (saveJoinResult) {
- case MATCH:
- matchs[matchCount] = batchIndex;
- matchHashMapResultIndices[matchCount] = hashMapResultCount;
- matchCount++;
- nonSpills[nonSpillCount++] = batchIndex;
- break;
-
- case SPILL:
- spills[spillCount] = batchIndex;
- spillHashMapResultIndices[spillCount] = hashMapResultCount;
- spillCount++;
- break;
-
- case NOMATCH:
- nonSpills[nonSpillCount++] = batchIndex;
- // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH duplicate");
- break;
+ switch (saveJoinResult) {
+ case MATCH:
+ equalKeySeriesHashMapResultIndices[equalKeySeriesCount] = hashMapResultCount;
+ equalKeySeriesAllMatchIndices[equalKeySeriesCount] = allMatchCount;
+ equalKeySeriesIsSingleValue[equalKeySeriesCount] = hashMapResults[hashMapResultCount].isSingleRow();
+ equalKeySeriesDuplicateCounts[equalKeySeriesCount] = 1;
+ allMatchs[allMatchCount++] = batchIndex;
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " MATCH isSingleValue " + equalKeySeriesIsSingleValue[equalKeySeriesCount] + " currentKey " + currentKey);
+ break;
+
+ case SPILL:
+ spills[spillCount] = batchIndex;
+ spillHashMapResultIndices[spillCount] = hashMapResultCount;
+ spillCount++;
+ break;
+
+ case NOMATCH:
+ atLeastOneNonMatch = true;
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH" + " currentKey " + currentKey);
+ break;
+ }
+ } else {
+ // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " Key Continues " + saveKey + " " + saveJoinResult.name());
+
+ // Series of equal keys.
+
+ switch (saveJoinResult) {
+ case MATCH:
+ equalKeySeriesDuplicateCounts[equalKeySeriesCount]++;
+ allMatchs[allMatchCount++] = batchIndex;
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " MATCH duplicate");
+ break;
+
+ case SPILL:
+ spills[spillCount] = batchIndex;
+ spillHashMapResultIndices[spillCount] = hashMapResultCount;
+ spillCount++;
+ break;
+
+ case NOMATCH:
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH duplicate");
+ break;
+ }
}
+ // if (!verifyMonotonicallyIncreasing(allMatchs, allMatchCount)) {
+ // throw new HiveException("allMatchs is not in sort order and unique");
+ // }
}
}
if (haveSaveKey) {
- // Account for last equal key sequence.
+ // Update our counts for the last key.
switch (saveJoinResult) {
case MATCH:
+ hashMapResultCount++;
+ equalKeySeriesCount++;
+ break;
case SPILL:
hashMapResultCount++;
break;
@@ -367,27 +430,26 @@ public class VectorMapJoinOuterMultiKeyOperator extends VectorMapJoinOuterGenera
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter +
- " matchs " + intArrayToRangesString(matchs, matchCount) +
- " matchHashMapResultIndices " + intArrayToRangesString(matchHashMapResultIndices, matchCount) +
- " nonSpills " + intArrayToRangesString(nonSpills, nonSpillCount) +
+ " allMatchs " + intArrayToRangesString(allMatchs,allMatchCount) +
+ " equalKeySeriesHashMapResultIndices " + intArrayToRangesString(equalKeySeriesHashMapResultIndices, equalKeySeriesCount) +
+ " equalKeySeriesAllMatchIndices " + intArrayToRangesString(equalKeySeriesAllMatchIndices, equalKeySeriesCount) +
+ " equalKeySeriesIsSingleValue " + Arrays.toString(Arrays.copyOfRange(equalKeySeriesIsSingleValue, 0, equalKeySeriesCount)) +
+ " equalKeySeriesDuplicateCounts " + Arrays.toString(Arrays.copyOfRange(equalKeySeriesDuplicateCounts, 0, equalKeySeriesCount)) +
+ " atLeastOneNonMatch " + atLeastOneNonMatch +
+ " inputSelectedInUse " + inputSelectedInUse +
+ " inputLogicalSize " + inputLogicalSize +
" spills " + intArrayToRangesString(spills, spillCount) +
" spillHashMapResultIndices " + intArrayToRangesString(spillHashMapResultIndices, spillCount) +
" hashMapResults " + Arrays.toString(Arrays.copyOfRange(hashMapResults, 0, hashMapResultCount)));
}
// We will generate results for all matching and non-matching rows.
- // Note that scratch1 is undefined at this point -- it's preallocated storage.
- numSel = finishOuter(batch,
- matchs, matchHashMapResultIndices, matchCount,
- nonSpills, nonSpillCount,
- spills, spillHashMapResultIndices, spillCount,
- hashMapResults, hashMapResultCount,
- scratch1);
+ finishOuter(batch,
+ allMatchCount, equalKeySeriesCount, atLeastOneNonMatch,
+ inputSelectedInUse, inputLogicalSize,
+ spillCount, hashMapResultCount);
}
- batch.selectedInUse = true;
- batch.size = numSel;
-
if (batch.size > 0) {
// Forward any remaining selected rows.
forwardBigTableBatch(batch);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java
index f0af3f6..49efe1a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterStringOperator.java
@@ -115,13 +115,6 @@ public class VectorMapJoinOuterStringOperator extends VectorMapJoinOuterGenerate
batchCounter++;
- // Do the per-batch setup for an outer join.
-
- outerPerBatchSetup(batch);
-
- // For outer join, DO NOT apply filters yet. It is incorrect for outer join to
- // apply the filter before hash table matching.
-
final int inputLogicalSize = batch.size;
if (inputLogicalSize == 0) {
@@ -131,6 +124,44 @@ public class VectorMapJoinOuterStringOperator extends VectorMapJoinOuterGenerate
return;
}
+ // Do the per-batch setup for an outer join.
+
+ outerPerBatchSetup(batch);
+
+ // For outer join, remember our input rows before ON expression filtering or before
+ // hash table matching so we can generate results for all rows (matching and non matching)
+ // later.
+ boolean inputSelectedInUse = batch.selectedInUse;
+ if (inputSelectedInUse) {
+ // if (!verifyMonotonicallyIncreasing(batch.selected, batch.size)) {
+ // throw new HiveException("batch.selected is not in sort order and unique");
+ // }
+ System.arraycopy(batch.selected, 0, inputSelected, 0, inputLogicalSize);
+ }
+
+ // Filtering for outer join just removes rows available for hash table matching.
+ boolean someRowsFilteredOut = false;
+ if (bigTableFilterExpressions.length > 0) {
+ // Since the input
+ for (VectorExpression ve : bigTableFilterExpressions) {
+ ve.evaluate(batch);
+ }
+ someRowsFilteredOut = (batch.size != inputLogicalSize);
+ if (LOG.isDebugEnabled()) {
+ if (batch.selectedInUse) {
+ if (inputSelectedInUse) {
+ LOG.debug(CLASS_NAME +
+ " inputSelected " + intArrayToRangesString(inputSelected, inputLogicalSize) +
+ " filtered batch.selected " + intArrayToRangesString(batch.selected, batch.size));
+ } else {
+ LOG.debug(CLASS_NAME +
+ " inputLogicalSize " + inputLogicalSize +
+ " filtered batch.selected " + intArrayToRangesString(batch.selected, batch.size));
+ }
+ }
+ }
+ }
+
// Perform any key expressions. Results will go into scratch columns.
if (bigTableKeyExpressions != null) {
for (VectorExpression ve : bigTableKeyExpressions) {
@@ -138,9 +169,6 @@ public class VectorMapJoinOuterStringOperator extends VectorMapJoinOuterGenerate
}
}
- // We rebuild in-place the selected array with rows destine to be forwarded.
- int numSel = 0;
-
/*
* Single-Column String specific declarations.
*/
@@ -172,8 +200,11 @@ public class VectorMapJoinOuterStringOperator extends VectorMapJoinOuterGenerate
*/
JoinUtil.JoinResult joinResult;
- if (!joinColVector.noNulls && joinColVector.isNull[0]) {
- // Null key is no match for whole batch.
+ if (batch.size == 0) {
+ // Whole repeated key batch was filtered out.
+ joinResult = JoinUtil.JoinResult.NOMATCH;
+ } else if (!joinColVector.noNulls && joinColVector.isNull[0]) {
+ // Any (repeated) null key column is no match for whole batch.
joinResult = JoinUtil.JoinResult.NOMATCH;
} else {
// Handle *repeated* join key, if found.
@@ -190,7 +221,8 @@ public class VectorMapJoinOuterStringOperator extends VectorMapJoinOuterGenerate
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
}
- numSel = finishOuterRepeated(batch, joinResult, hashMapResults[0], scratch1);
+ finishOuterRepeated(batch, joinResult, hashMapResults[0], someRowsFilteredOut,
+ inputSelectedInUse, inputLogicalSize);
} else {
/*
@@ -204,14 +236,13 @@ public class VectorMapJoinOuterStringOperator extends VectorMapJoinOuterGenerate
int selected[] = batch.selected;
boolean selectedInUse = batch.selectedInUse;
- // For outer join we must apply the filter after match and cause some matches to become
- // non-matches, we do not track non-matches here. Instead we remember all non spilled rows
- // and compute non matches later in finishOuter.
int hashMapResultCount = 0;
- int matchCount = 0;
- int nonSpillCount = 0;
+ int allMatchCount = 0;
+ int equalKeySeriesCount = 0;
int spillCount = 0;
+ boolean atLeastOneNonMatch = someRowsFilteredOut;
+
/*
* Single-Column String specific variables.
*/
@@ -223,9 +254,11 @@ public class VectorMapJoinOuterStringOperator extends VectorMapJoinOuterGenerate
JoinUtil.JoinResult saveJoinResult = JoinUtil.JoinResult.NOMATCH;
// Logical loop over the rows in the batch since the batch may have selected in use.
- for (int logical = 0; logical < inputLogicalSize; logical++) {
+ for (int logical = 0; logical < batch.size; logical++) {
int batchIndex = (selectedInUse ? selected[logical] : logical);
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, taskName + ", " + getOperatorId() + " candidate " + CLASS_NAME + " batch");
+
/*
* Single-Column String outer null detection.
*/
@@ -241,8 +274,8 @@ public class VectorMapJoinOuterStringOperator extends VectorMapJoinOuterGenerate
// Let a current SPILL equal key series keep going, or
// Let a current NOMATCH keep not matching.
- // Remember non-matches for Outer Join.
- nonSpills[nonSpillCount++] = batchIndex;
+ atLeastOneNonMatch = true;
+
// LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " NULL");
} else {
@@ -262,9 +295,12 @@ public class VectorMapJoinOuterStringOperator extends VectorMapJoinOuterGenerate
// New key.
if (haveSaveKey) {
- // Move on with our count(s).
+ // Move on with our counts.
switch (saveJoinResult) {
case MATCH:
+ hashMapResultCount++;
+ equalKeySeriesCount++;
+ break;
case SPILL:
hashMapResultCount++;
break;
@@ -290,43 +326,69 @@ public class VectorMapJoinOuterStringOperator extends VectorMapJoinOuterGenerate
byte[] keyBytes = vector[batchIndex];
int keyStart = start[batchIndex];
int keyLength = length[batchIndex];
-
saveJoinResult = hashMap.lookup(keyBytes, keyStart, keyLength, hashMapResults[hashMapResultCount]);
- // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " New Key " + saveJoinResult.name());
- } else {
- // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " Key Continues " + saveJoinResult.name());
- }
- /*
- * Common outer join result processing.
- */
+ /*
+ * Common outer join result processing.
+ */
- switch (saveJoinResult) {
- case MATCH:
- matchs[matchCount] = batchIndex;
- matchHashMapResultIndices[matchCount] = hashMapResultCount;
- matchCount++;
- nonSpills[nonSpillCount++] = batchIndex;
- break;
-
- case SPILL:
- spills[spillCount] = batchIndex;
- spillHashMapResultIndices[spillCount] = hashMapResultCount;
- spillCount++;
- break;
-
- case NOMATCH:
- nonSpills[nonSpillCount++] = batchIndex;
- // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH duplicate");
- break;
+ switch (saveJoinResult) {
+ case MATCH:
+ equalKeySeriesHashMapResultIndices[equalKeySeriesCount] = hashMapResultCount;
+ equalKeySeriesAllMatchIndices[equalKeySeriesCount] = allMatchCount;
+ equalKeySeriesIsSingleValue[equalKeySeriesCount] = hashMapResults[hashMapResultCount].isSingleRow();
+ equalKeySeriesDuplicateCounts[equalKeySeriesCount] = 1;
+ allMatchs[allMatchCount++] = batchIndex;
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " MATCH isSingleValue " + equalKeySeriesIsSingleValue[equalKeySeriesCount] + " currentKey " + currentKey);
+ break;
+
+ case SPILL:
+ spills[spillCount] = batchIndex;
+ spillHashMapResultIndices[spillCount] = hashMapResultCount;
+ spillCount++;
+ break;
+
+ case NOMATCH:
+ atLeastOneNonMatch = true;
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH" + " currentKey " + currentKey);
+ break;
+ }
+ } else {
+ // LOG.debug(CLASS_NAME + " logical " + logical + " batchIndex " + batchIndex + " Key Continues " + saveKey + " " + saveJoinResult.name());
+
+ // Series of equal keys.
+
+ switch (saveJoinResult) {
+ case MATCH:
+ equalKeySeriesDuplicateCounts[equalKeySeriesCount]++;
+ allMatchs[allMatchCount++] = batchIndex;
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " MATCH duplicate");
+ break;
+
+ case SPILL:
+ spills[spillCount] = batchIndex;
+ spillHashMapResultIndices[spillCount] = hashMapResultCount;
+ spillCount++;
+ break;
+
+ case NOMATCH:
+ // VectorizedBatchUtil.debugDisplayOneRow(batch, batchIndex, CLASS_NAME + " NOMATCH duplicate");
+ break;
+ }
}
+ // if (!verifyMonotonicallyIncreasing(allMatchs, allMatchCount)) {
+ // throw new HiveException("allMatchs is not in sort order and unique");
+ // }
}
}
if (haveSaveKey) {
- // Account for last equal key sequence.
+ // Update our counts for the last key.
switch (saveJoinResult) {
case MATCH:
+ hashMapResultCount++;
+ equalKeySeriesCount++;
+ break;
case SPILL:
hashMapResultCount++;
break;
@@ -337,27 +399,26 @@ public class VectorMapJoinOuterStringOperator extends VectorMapJoinOuterGenerate
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter +
- " matchs " + intArrayToRangesString(matchs, matchCount) +
- " matchHashMapResultIndices " + intArrayToRangesString(matchHashMapResultIndices, matchCount) +
- " nonSpills " + intArrayToRangesString(nonSpills, nonSpillCount) +
+ " allMatchs " + intArrayToRangesString(allMatchs,allMatchCount) +
+ " equalKeySeriesHashMapResultIndices " + intArrayToRangesString(equalKeySeriesHashMapResultIndices, equalKeySeriesCount) +
+ " equalKeySeriesAllMatchIndices " + intArrayToRangesString(equalKeySeriesAllMatchIndices, equalKeySeriesCount) +
+ " equalKeySeriesIsSingleValue " + Arrays.toString(Arrays.copyOfRange(equalKeySeriesIsSingleValue, 0, equalKeySeriesCount)) +
+ " equalKeySeriesDuplicateCounts " + Arrays.toString(Arrays.copyOfRange(equalKeySeriesDuplicateCounts, 0, equalKeySeriesCount)) +
+ " atLeastOneNonMatch " + atLeastOneNonMatch +
+ " inputSelectedInUse " + inputSelectedInUse +
+ " inputLogicalSize " + inputLogicalSize +
" spills " + intArrayToRangesString(spills, spillCount) +
" spillHashMapResultIndices " + intArrayToRangesString(spillHashMapResultIndices, spillCount) +
" hashMapResults " + Arrays.toString(Arrays.copyOfRange(hashMapResults, 0, hashMapResultCount)));
}
// We will generate results for all matching and non-matching rows.
- // Note that scratch1 is undefined at this point -- it's preallocated storage.
- numSel = finishOuter(batch,
- matchs, matchHashMapResultIndices, matchCount,
- nonSpills, nonSpillCount,
- spills, spillHashMapResultIndices, spillCount,
- hashMapResults, hashMapResultCount,
- scratch1);
+ finishOuter(batch,
+ allMatchCount, equalKeySeriesCount, atLeastOneNonMatch,
+ inputSelectedInUse, inputLogicalSize,
+ spillCount, hashMapResultCount);
}
- batch.selectedInUse = true;
- batch.size = numSel;
-
if (batch.size > 0) {
// Forward any remaining selected rows.
forwardBigTableBatch(batch);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinRowBytesContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinRowBytesContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinRowBytesContainer.java
index 1c91be6..32b60d0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinRowBytesContainer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinRowBytesContainer.java
@@ -91,7 +91,7 @@ public class VectorMapJoinRowBytesContainer {
}
tmpFile = File.createTempFile("BytesContainer", ".tmp", parentFile);
- LOG.info("BytesContainer created temp file " + tmpFile.getAbsolutePath());
+ LOG.debug("BytesContainer created temp file " + tmpFile.getAbsolutePath());
tmpFile.deleteOnExit();
fileOutputStream = new FileOutputStream(tmpFile);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java
index f9550c9..6afaec3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMap.java
@@ -54,13 +54,13 @@ public abstract class VectorMapJoinFastBytesHashMap
slotTriples[tripleIndex] = keyStore.add(keyBytes, keyStart, keyLength);
slotTriples[tripleIndex + 1] = hashCode;
slotTriples[tripleIndex + 2] = valueStore.addFirst(valueBytes, 0, valueLength);
- // LOG.info("VectorMapJoinFastBytesHashMap add first keyRefWord " + Long.toHexString(slotTriples[tripleIndex]) + " hashCode " + Long.toHexString(slotTriples[tripleIndex + 1]) + " valueRefWord " + Long.toHexString(slotTriples[tripleIndex + 2]));
+ // LOG.debug("VectorMapJoinFastBytesHashMap add first keyRefWord " + Long.toHexString(slotTriples[tripleIndex]) + " hashCode " + Long.toHexString(slotTriples[tripleIndex + 1]) + " valueRefWord " + Long.toHexString(slotTriples[tripleIndex + 2]));
keysAssigned++;
} else {
// Add another value.
- // LOG.info("VectorMapJoinFastBytesHashMap add more keyRefWord " + Long.toHexString(slotTriples[tripleIndex]) + " hashCode " + Long.toHexString(slotTriples[tripleIndex + 1]) + " valueRefWord " + Long.toHexString(slotTriples[tripleIndex + 2]));
+ // LOG.debug("VectorMapJoinFastBytesHashMap add more keyRefWord " + Long.toHexString(slotTriples[tripleIndex]) + " hashCode " + Long.toHexString(slotTriples[tripleIndex + 1]) + " valueRefWord " + Long.toHexString(slotTriples[tripleIndex + 2]));
slotTriples[tripleIndex + 2] = valueStore.addMore(slotTriples[tripleIndex + 2], valueBytes, 0, valueLength);
- // LOG.info("VectorMapJoinFastBytesHashMap add more new valueRefWord " + Long.toHexString(slotTriples[tripleIndex + 2]));
+ // LOG.debug("VectorMapJoinFastBytesHashMap add more new valueRefWord " + Long.toHexString(slotTriples[tripleIndex + 2]));
}
}
@@ -77,7 +77,7 @@ public abstract class VectorMapJoinFastBytesHashMap
if (valueRefWord == -1) {
joinResult = JoinUtil.JoinResult.NOMATCH;
} else {
- // LOG.info("VectorMapJoinFastBytesHashMap lookup hashCode " + Long.toHexString(hashCode) + " valueRefWord " + Long.toHexString(valueRefWord) + " (valueStore != null) " + (valueStore != null));
+ // LOG.debug("VectorMapJoinFastBytesHashMap lookup hashCode " + Long.toHexString(hashCode) + " valueRefWord " + Long.toHexString(valueRefWord) + " (valueStore != null) " + (valueStore != null));
optimizedHashMapResult.set(valueStore, valueRefWord);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java
index 9dcaf8f..dceb99c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashMultiSet.java
@@ -49,11 +49,11 @@ public abstract class VectorMapJoinFastBytesHashMultiSet
slotTriples[tripleIndex] = keyStore.add(keyBytes, keyStart, keyLength);
slotTriples[tripleIndex + 1] = hashCode;
slotTriples[tripleIndex + 2] = 1; // Count.
- // LOG.info("VectorMapJoinFastBytesHashMap add first keyRefWord " + Long.toHexString(slotTriples[tripleIndex]) + " hashCode " + Long.toHexString(slotTriples[tripleIndex + 1]) + " valueRefWord " + Long.toHexString(slotTriples[tripleIndex + 2]));
+ // LOG.debug("VectorMapJoinFastBytesHashMap add first keyRefWord " + Long.toHexString(slotTriples[tripleIndex]) + " hashCode " + Long.toHexString(slotTriples[tripleIndex + 1]) + " valueRefWord " + Long.toHexString(slotTriples[tripleIndex + 2]));
keysAssigned++;
} else {
// Add another value.
- // LOG.info("VectorMapJoinFastBytesHashMap add more keyRefWord " + Long.toHexString(slotTriples[tripleIndex]) + " hashCode " + Long.toHexString(slotTriples[tripleIndex + 1]) + " valueRefWord " + Long.toHexString(slotTriples[tripleIndex + 2]));
+ // LOG.debug("VectorMapJoinFastBytesHashMap add more keyRefWord " + Long.toHexString(slotTriples[tripleIndex]) + " hashCode " + Long.toHexString(slotTriples[tripleIndex + 1]) + " valueRefWord " + Long.toHexString(slotTriples[tripleIndex + 2]));
slotTriples[tripleIndex + 2]++;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java
index b6e6321..91d7fd6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastBytesHashTable.java
@@ -79,13 +79,13 @@ public abstract class VectorMapJoinFastBytesHashTable
while (true) {
int tripleIndex = 3 * slot;
if (slotTriples[tripleIndex] == 0) {
- // LOG.info("VectorMapJoinFastBytesHashMap findWriteSlot slot " + slot + " tripleIndex " + tripleIndex + " empty");
+ // LOG.debug("VectorMapJoinFastBytesHashMap findWriteSlot slot " + slot + " tripleIndex " + tripleIndex + " empty");
isNewKey = true;;
break;
}
if (hashCode == slotTriples[tripleIndex + 1] &&
keyStore.equalKey(slotTriples[tripleIndex], keyBytes, keyStart, keyLength)) {
- // LOG.info("VectorMapJoinFastBytesHashMap findWriteSlot slot " + slot + " tripleIndex " + tripleIndex + " existing");
+ // LOG.debug("VectorMapJoinFastBytesHashMap findWriteSlot slot " + slot + " tripleIndex " + tripleIndex + " existing");
isNewKey = false;
break;
}
@@ -155,7 +155,7 @@ public abstract class VectorMapJoinFastBytesHashTable
}
// Use old value reference word.
- // LOG.info("VectorMapJoinFastLongHashTable expandAndRehash key " + tableKey + " slot " + newSlot + " newPairIndex " + newPairIndex + " empty slot (i = " + i + ")");
+ // LOG.debug("VectorMapJoinFastLongHashTable expandAndRehash key " + tableKey + " slot " + newSlot + " newPairIndex " + newPairIndex + " empty slot (i = " + i + ")");
newSlotTriples[newTripleIndex] = keyRef;
newSlotTriples[newTripleIndex + 1] = hashCode;
@@ -170,7 +170,7 @@ public abstract class VectorMapJoinFastBytesHashTable
largestNumberOfSteps = newLargestNumberOfSteps;
resizeThreshold = (int)(logicalHashBucketCount * loadFactor);
metricExpands++;
- // LOG.info("VectorMapJoinFastLongHashTable expandAndRehash new logicalHashBucketCount " + logicalHashBucketCount + " resizeThreshold " + resizeThreshold + " metricExpands " + metricExpands);
+ // LOG.debug("VectorMapJoinFastLongHashTable expandAndRehash new logicalHashBucketCount " + logicalHashBucketCount + " resizeThreshold " + resizeThreshold + " metricExpands " + metricExpands);
}
protected long findReadSlot(byte[] keyBytes, int keyStart, int keyLength, long hashCode) {
@@ -181,7 +181,7 @@ public abstract class VectorMapJoinFastBytesHashTable
int i = 0;
while (true) {
int tripleIndex = slot * 3;
- // LOG.info("VectorMapJoinFastBytesHashMap findReadSlot slot keyRefWord " + Long.toHexString(slotTriples[tripleIndex]) + " hashCode " + Long.toHexString(hashCode) + " entry hashCode " + Long.toHexString(slotTriples[tripleIndex + 1]) + " valueRefWord " + Long.toHexString(slotTriples[tripleIndex + 2]));
+ // LOG.debug("VectorMapJoinFastBytesHashMap findReadSlot slot keyRefWord " + Long.toHexString(slotTriples[tripleIndex]) + " hashCode " + Long.toHexString(hashCode) + " entry hashCode " + Long.toHexString(slotTriples[tripleIndex + 1]) + " valueRefWord " + Long.toHexString(slotTriples[tripleIndex + 2]));
if (slotTriples[tripleIndex] != 0 && hashCode == slotTriples[tripleIndex + 1]) {
// Finally, verify the key bytes match.
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java
index f2f42ee..9d95d05 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastKeyStore.java
@@ -112,7 +112,7 @@ public class VectorMapJoinFastKeyStore {
}
keyRefWord |= absoluteKeyOffset;
- // LOG.info("VectorMapJoinFastKeyStore add keyLength " + keyLength + " absoluteKeyOffset " + absoluteKeyOffset + " keyRefWord " + Long.toHexString(keyRefWord));
+ // LOG.debug("VectorMapJoinFastKeyStore add keyLength " + keyLength + " absoluteKeyOffset " + absoluteKeyOffset + " keyRefWord " + Long.toHexString(keyRefWord));
return keyRefWord;
}
@@ -122,7 +122,7 @@ public class VectorMapJoinFastKeyStore {
(int) ((keyRefWord & SmallKeyLength.bitMask) >> SmallKeyLength.bitShift);
boolean isKeyLengthSmall = (storedKeyLengthLength != SmallKeyLength.allBitsOn);
- // LOG.info("VectorMapJoinFastKeyStore equalKey keyLength " + keyLength + " isKeyLengthSmall " + isKeyLengthSmall + " storedKeyLengthLength " + storedKeyLengthLength + " keyRefWord " + Long.toHexString(keyRefWord));
+ // LOG.debug("VectorMapJoinFastKeyStore equalKey keyLength " + keyLength + " isKeyLengthSmall " + isKeyLengthSmall + " storedKeyLengthLength " + storedKeyLengthLength + " keyRefWord " + Long.toHexString(keyRefWord));
if (isKeyLengthSmall && storedKeyLengthLength != keyLength) {
return false;
@@ -135,7 +135,7 @@ public class VectorMapJoinFastKeyStore {
// Read big value length we wrote with the value.
storedKeyLengthLength = writeBuffers.readVInt(readPos);
if (storedKeyLengthLength != keyLength) {
- // LOG.info("VectorMapJoinFastKeyStore equalKey no match big length");
+ // LOG.debug("VectorMapJoinFastKeyStore equalKey no match big length");
return false;
}
}
@@ -148,11 +148,11 @@ public class VectorMapJoinFastKeyStore {
for (int i = 0; i < keyLength; i++) {
if (currentBytes[currentStart + i] != keyBytes[keyStart + i]) {
- // LOG.info("VectorMapJoinFastKeyStore equalKey no match on bytes");
+ // LOG.debug("VectorMapJoinFastKeyStore equalKey no match on bytes");
return false;
}
}
- // LOG.info("VectorMapJoinFastKeyStore equalKey match on bytes");
+ // LOG.debug("VectorMapJoinFastKeyStore equalKey match on bytes");
return true;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java
index d6ad028..4725f55 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashMap.java
@@ -68,7 +68,7 @@ public class VectorMapJoinFastLongHashMap
optimizedHashMapResult.forget();
long hashCode = VectorMapJoinFastLongHashUtil.hashKey(key);
- // LOG.info("VectorMapJoinFastLongHashMap lookup " + key + " hashCode " + hashCode);
+ // LOG.debug("VectorMapJoinFastLongHashMap lookup " + key + " hashCode " + hashCode);
long valueRef = findReadSlot(key, hashCode);
JoinUtil.JoinResult joinResult;
if (valueRef == -1) {
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java
index 2137fb7..17855eb 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastLongHashTable.java
@@ -121,13 +121,13 @@ public abstract class VectorMapJoinFastLongHashTable
int pairIndex = 2 * slot;
long valueRef = slotPairs[pairIndex];
if (valueRef == 0) {
- // LOG.info("VectorMapJoinFastLongHashTable add key " + key + " slot " + slot + " pairIndex " + pairIndex + " empty slot (i = " + i + ")");
+ // LOG.debug("VectorMapJoinFastLongHashTable add key " + key + " slot " + slot + " pairIndex " + pairIndex + " empty slot (i = " + i + ")");
isNewKey = true;
break;
}
long tableKey = slotPairs[pairIndex + 1];
if (key == tableKey) {
- // LOG.info("VectorMapJoinFastLongHashTable add key " + key + " slot " + slot + " pairIndex " + pairIndex + " found key (i = " + i + ")");
+ // LOG.debug("VectorMapJoinFastLongHashTable add key " + key + " slot " + slot + " pairIndex " + pairIndex + " found key (i = " + i + ")");
isNewKey = false;
break;
}
@@ -145,7 +145,7 @@ public abstract class VectorMapJoinFastLongHashTable
// debugDumpKeyProbe(keyOffset, keyLength, hashCode, slot);
}
- // LOG.info("VectorMapJoinFastLongHashTable add slot " + slot + " hashCode " + Long.toHexString(hashCode));
+ // LOG.debug("VectorMapJoinFastLongHashTable add slot " + slot + " hashCode " + Long.toHexString(hashCode));
assignSlot(slot, key, isNewKey, currentValue);
@@ -206,7 +206,7 @@ public abstract class VectorMapJoinFastLongHashTable
}
// Use old value reference word.
- // LOG.info("VectorMapJoinFastLongHashTable expandAndRehash key " + tableKey + " slot " + newSlot + " newPairIndex " + newPairIndex + " empty slot (i = " + i + ")");
+ // LOG.debug("VectorMapJoinFastLongHashTable expandAndRehash key " + tableKey + " slot " + newSlot + " newPairIndex " + newPairIndex + " empty slot (i = " + i + ")");
newSlotPairs[newPairIndex] = valueRef;
newSlotPairs[newPairIndex + 1] = tableKey;
@@ -220,7 +220,7 @@ public abstract class VectorMapJoinFastLongHashTable
largestNumberOfSteps = newLargestNumberOfSteps;
resizeThreshold = (int)(logicalHashBucketCount * loadFactor);
metricExpands++;
- // LOG.info("VectorMapJoinFastLongHashTable expandAndRehash new logicalHashBucketCount " + logicalHashBucketCount + " resizeThreshold " + resizeThreshold + " metricExpands " + metricExpands);
+ // LOG.debug("VectorMapJoinFastLongHashTable expandAndRehash new logicalHashBucketCount " + logicalHashBucketCount + " resizeThreshold " + resizeThreshold + " metricExpands " + metricExpands);
}
protected long findReadSlot(long key, long hashCode) {
@@ -235,20 +235,20 @@ public abstract class VectorMapJoinFastLongHashTable
long valueRef = slotPairs[pairIndex];
if (valueRef == 0) {
// Given that we do not delete, an empty slot means no match.
- // LOG.info("VectorMapJoinFastLongHashTable findReadSlot key " + key + " slot " + slot + " pairIndex " + pairIndex + " empty slot (i = " + i + ")");
+ // LOG.debug("VectorMapJoinFastLongHashTable findReadSlot key " + key + " slot " + slot + " pairIndex " + pairIndex + " empty slot (i = " + i + ")");
return -1;
}
long tableKey = slotPairs[pairIndex + 1];
if (key == tableKey) {
- // LOG.info("VectorMapJoinFastLongHashTable findReadSlot key " + key + " slot " + slot + " pairIndex " + pairIndex + " found key (i = " + i + ")");
+ // LOG.debug("VectorMapJoinFastLongHashTable findReadSlot key " + key + " slot " + slot + " pairIndex " + pairIndex + " found key (i = " + i + ")");
return slotPairs[pairIndex];
}
// Some other key (collision) - keep probing.
probeSlot += (++i);
if (i > largestNumberOfSteps) {
- // LOG.info("VectorMapJoinFastLongHashTable findReadSlot returning not found");
+ // LOG.debug("VectorMapJoinFastLongHashTable findReadSlot returning not found");
// We know we never went that far when we were inserting.
- // LOG.info("VectorMapJoinFastLongHashTable findReadSlot key " + key + " slot " + slot + " pairIndex " + pairIndex + " largestNumberOfSteps " + largestNumberOfSteps + " (i = " + i + ")");
+ // LOG.debug("VectorMapJoinFastLongHashTable findReadSlot key " + key + " slot " + slot + " pairIndex " + pairIndex + " largestNumberOfSteps " + largestNumberOfSteps + " (i = " + i + ")");
return -1;
}
slot = (int)(probeSlot & logicalHashBucketMask);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java
index 373b5f4..4b1d6f6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastTableContainer.java
@@ -81,7 +81,7 @@ public class VectorMapJoinFastTableContainer implements VectorMapJoinTableContai
int newThreshold = HashMapWrapper.calculateTableSize(
keyCountAdj, threshold, loadFactor, keyCount);
- // LOG.info("VectorMapJoinFastTableContainer load newThreshold " + newThreshold);
+ // LOG.debug("VectorMapJoinFastTableContainer load newThreshold " + newThreshold);
VectorMapJoinFastHashTable = createHashTable(newThreshold);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java
index caa705c..6491dc6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/fast/VectorMapJoinFastValueStore.java
@@ -142,7 +142,7 @@ public class VectorMapJoinFastValueStore {
}
public void set(VectorMapJoinFastValueStore valueStore, long valueRefWord) {
- // LOG.info("VectorMapJoinFastValueStore set valueRefWord " + Long.toHexString(valueRefWord));
+ // LOG.debug("VectorMapJoinFastValueStore set valueRefWord " + Long.toHexString(valueRefWord));
this.valueStore = valueStore;
this.valueRefWord = valueRefWord;
@@ -473,7 +473,7 @@ public class VectorMapJoinFastValueStore {
valueRefWord |= SmallValueLength.allBitsOnBitShifted;
}
- // LOG.info("VectorMapJoinFastValueStore addFirst valueLength " + valueLength + " newAbsoluteOffset " + newAbsoluteOffset + " valueRefWord " + Long.toHexString(valueRefWord));
+ // LOG.debug("VectorMapJoinFastValueStore addFirst valueLength " + valueLength + " newAbsoluteOffset " + newAbsoluteOffset + " valueRefWord " + Long.toHexString(valueRefWord));
// The lower bits are the absolute value offset.
valueRefWord |= newAbsoluteOffset;
@@ -499,7 +499,7 @@ public class VectorMapJoinFastValueStore {
boolean isOldValueLast =
((oldValueRef & IsLastFlag.flagOnMask) != 0);
- // LOG.info("VectorMapJoinFastValueStore addMore isOldValueLast " + isOldValueLast + " oldSmallValueLength " + oldSmallValueLength + " oldAbsoluteValueOffset " + oldAbsoluteValueOffset + " oldValueRef " + Long.toHexString(oldValueRef));
+ // LOG.debug("VectorMapJoinFastValueStore addMore isOldValueLast " + isOldValueLast + " oldSmallValueLength " + oldSmallValueLength + " oldAbsoluteValueOffset " + oldAbsoluteValueOffset + " oldValueRef " + Long.toHexString(oldValueRef));
/*
* Write information about the old value (which becomes our next) at the beginning
@@ -546,7 +546,7 @@ public class VectorMapJoinFastValueStore {
// The lower bits are the absolute value offset.
newValueRef |= newAbsoluteOffset;
- // LOG.info("VectorMapJoinFastValueStore addMore valueLength " + valueLength + " newAbsoluteOffset " + newAbsoluteOffset + " newValueRef " + Long.toHexString(newValueRef));
+ // LOG.debug("VectorMapJoinFastValueStore addMore valueLength " + valueLength + " newAbsoluteOffset " + newAbsoluteOffset + " newValueRef " + Long.toHexString(newValueRef));
return newValueRef;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedLongCommon.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedLongCommon.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedLongCommon.java
index 60825ce..dc65eaa 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedLongCommon.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/optimized/VectorMapJoinOptimizedLongCommon.java
@@ -113,7 +113,7 @@ public class VectorMapJoinOptimizedLongCommon {
}
// byte[] bytes = Arrays.copyOf(currentKey.get(), currentKey.getLength());
- // LOG.info("VectorMapJoinOptimizedLongCommon adaptPutRow key " + key + " min " + min + " max " + max + " hashTableKeyType " + hashTableKeyType.name() + " hex " + Hex.encodeHexString(bytes));
+ // LOG.debug("VectorMapJoinOptimizedLongCommon adaptPutRow key " + key + " min " + min + " max " + max + " hashTableKeyType " + hashTableKeyType.name() + " hex " + Hex.encodeHexString(bytes));
}
@@ -145,7 +145,7 @@ public class VectorMapJoinOptimizedLongCommon {
}
// byte[] bytes = Arrays.copyOf(output.getData(), output.getLength());
- // LOG.info("VectorMapJoinOptimizedLongCommon serialize key " + key + " hashTableKeyType " + hashTableKeyType.name() + " hex " + Hex.encodeHexString(bytes));
+ // LOG.debug("VectorMapJoinOptimizedLongCommon serialize key " + key + " hashTableKeyType " + hashTableKeyType.name() + " hex " + Hex.encodeHexString(bytes));
serializedBytes.bytes = output.getData();
serializedBytes.offset = 0;
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 096239e..656a5e3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -1069,11 +1069,21 @@ public class Vectorizer implements PhysicalPlanResolver {
private boolean validateMapJoinDesc(MapJoinDesc desc) {
byte posBigTable = (byte) desc.getPosBigTable();
List<ExprNodeDesc> filterExprs = desc.getFilters().get(posBigTable);
+ if (!validateExprNodeDesc(filterExprs, VectorExpressionDescriptor.Mode.FILTER)) {
+ LOG.info("Cannot vectorize map work filter expression");
+ return false;
+ }
List<ExprNodeDesc> keyExprs = desc.getKeys().get(posBigTable);
+ if (!validateExprNodeDesc(keyExprs)) {
+ LOG.info("Cannot vectorize map work key expression");
+ return false;
+ }
List<ExprNodeDesc> valueExprs = desc.getExprs().get(posBigTable);
- return validateExprNodeDesc(filterExprs, VectorExpressionDescriptor.Mode.FILTER) &&
- validateExprNodeDesc(keyExprs) &&
- validateExprNodeDesc(valueExprs);
+ if (!validateExprNodeDesc(valueExprs)) {
+ LOG.info("Cannot vectorize map work value expression");
+ return false;
+ }
+ return true;
}
private boolean validateReduceSinkOperator(ReduceSinkOperator op) {
@@ -1089,6 +1099,7 @@ public class Vectorizer implements PhysicalPlanResolver {
for (ExprNodeDesc desc : descList) {
boolean ret = validateExprNodeDesc(desc);
if (!ret) {
+ LOG.info("Cannot vectorize select expression: " + desc.toString());
return false;
}
}
@@ -1110,10 +1121,12 @@ public class Vectorizer implements PhysicalPlanResolver {
}
boolean ret = validateExprNodeDesc(desc.getKeys());
if (!ret) {
+ LOG.info("Cannot vectorize groupby key expression");
return false;
}
ret = validateAggregationDesc(desc.getAggregators(), isReduce);
if (!ret) {
+ LOG.info("Cannot vectorize groupby aggregate expression");
return false;
}
if (isReduce) {
@@ -1248,10 +1261,13 @@ public class Vectorizer implements PhysicalPlanResolver {
}
private boolean validateAggregationDesc(AggregationDesc aggDesc, boolean isReduce) {
- if (!supportedAggregationUdfs.contains(aggDesc.getGenericUDAFName().toLowerCase())) {
+ String udfName = aggDesc.getGenericUDAFName().toLowerCase();
+ if (!supportedAggregationUdfs.contains(udfName)) {
+ LOG.info("Cannot vectorize groupby aggregate expression: UDF " + udfName + " not supported");
return false;
}
if (aggDesc.getParameters() != null && !validateExprNodeDesc(aggDesc.getParameters())) {
+ LOG.info("Cannot vectorize groupby aggregate expression: UDF parameters not supported");
return false;
}
// See if we can vectorize the aggregation.
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/queries/clientpositive/vector_join30.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_join30.q b/ql/src/test/queries/clientpositive/vector_join30.q
new file mode 100644
index 0000000..2275804
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_join30.q
@@ -0,0 +1,160 @@
+SET hive.vectorized.execution.enabled=true;
+SET hive.vectorized.execution.mapjoin.native.enabled=true;
+set hive.fetch.task.conversion=none;
+SET hive.auto.convert.join=true;
+SET hive.auto.convert.join.noconditionaltask=true;
+SET hive.auto.convert.join.noconditionaltask.size=1000000000;
+
+-- SORT_QUERY_RESULTS
+
+CREATE TABLE orcsrc STORED AS ORC AS SELECT * FROM src;
+
+explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value));
+
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value));
+
+explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value));
+
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value));
+
+explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value));
+
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value));
+
+explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value));
+
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value));
+
+explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value));
+
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value));
+
+explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value));
+
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value));
+
+explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value));
+
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value));
+
+explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value));
+
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value));
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/queries/clientpositive/vector_join_filters.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_join_filters.q b/ql/src/test/queries/clientpositive/vector_join_filters.q
new file mode 100644
index 0000000..adf525c
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_join_filters.q
@@ -0,0 +1,38 @@
+SET hive.vectorized.execution.enabled=true;
+SET hive.vectorized.execution.mapjoin.native.enabled=true;
+set hive.fetch.task.conversion=none;
+SET hive.auto.convert.join=true;
+SET hive.auto.convert.join.noconditionaltask=true;
+SET hive.auto.convert.join.noconditionaltask.size=1000000000;
+
+-- SORT_QUERY_RESULTS
+
+CREATE TABLE myinput1_txt(key int, value int);
+LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE myinput1_txt;
+CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/queries/clientpositive/vector_join_nulls.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_join_nulls.q b/ql/src/test/queries/clientpositive/vector_join_nulls.q
new file mode 100644
index 0000000..6cfb7a8
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_join_nulls.q
@@ -0,0 +1,33 @@
+SET hive.vectorized.execution.enabled=true;
+SET hive.vectorized.execution.mapjoin.native.enabled=true;
+set hive.fetch.task.conversion=none;
+SET hive.auto.convert.join=true;
+SET hive.auto.convert.join.noconditionaltask=true;
+SET hive.auto.convert.join.noconditionaltask.size=1000000000;
+
+-- SORT_QUERY_RESULTS
+
+CREATE TABLE myinput1_txt(key int, value int);
+LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE myinput1_txt;
+CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value;
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value;
+
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value);
+SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/queries/clientpositive/vector_left_outer_join2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_left_outer_join2.q b/ql/src/test/queries/clientpositive/vector_left_outer_join2.q
index 098d002..62ad9ee 100644
--- a/ql/src/test/queries/clientpositive/vector_left_outer_join2.q
+++ b/ql/src/test/queries/clientpositive/vector_left_outer_join2.q
@@ -3,6 +3,8 @@ set hive.auto.convert.join=true;
set hive.auto.convert.join.noconditionaltask=true;
set hive.auto.convert.join.noconditionaltask.size=10000;
+-- SORT_QUERY_RESULTS
+
drop table if exists TJOIN1;
drop table if exists TJOIN2;
create table if not exists TJOIN1 (RNUM int , C1 int, C2 int) STORED AS orc;
[02/50] [abbrv] hive git commit: HIVE-10518 Replace US_ASCII charset
with US-ASCII in encode and decode description
Posted by sp...@apache.org.
HIVE-10518 Replace US_ASCII charset with US-ASCII in encode and decode description
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5333bd39
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5333bd39
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5333bd39
Branch: refs/heads/parquet
Commit: 5333bd39811dc43eae74c7ae4097b54c2d677fc2
Parents: b4b7955
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Mon Apr 27 23:23:59 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Tue May 12 14:51:25 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java | 2 +-
.../org/apache/hadoop/hive/ql/udf/generic/GenericUDFEncode.java | 2 +-
ql/src/test/results/clientpositive/udf_decode.q.out | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5333bd39/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java
index f4366c7..1236085 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFDecode.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn
@Description(name = "decode",
value = "_FUNC_(bin, str) - Decode the first argument using the second argument character set",
- extended = "Possible options for the character set are 'US_ASCII', 'ISO-8859-1',\n" +
+ extended = "Possible options for the character set are 'US-ASCII', 'ISO-8859-1',\n" +
"'UTF-8', 'UTF-16BE', 'UTF-16LE', and 'UTF-16'. If either argument\n" +
"is null, the result will also be null")
public class GenericUDFDecode extends GenericUDF {
http://git-wip-us.apache.org/repos/asf/hive/blob/5333bd39/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEncode.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEncode.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEncode.java
index 64f8a14..56b7685 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEncode.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFEncode.java
@@ -42,7 +42,7 @@ import org.apache.hadoop.io.BytesWritable;
@Description(name = "encode",
value = "_FUNC_(str, str) - Encode the first argument using the second argument character set",
-extended = "Possible options for the character set are 'US_ASCII', 'ISO-8859-1',\n" +
+extended = "Possible options for the character set are 'US-ASCII', 'ISO-8859-1',\n" +
"'UTF-8', 'UTF-16BE', 'UTF-16LE', and 'UTF-16'. If either argument\n" +
"is null, the result will also be null")
public class GenericUDFEncode extends GenericUDF {
http://git-wip-us.apache.org/repos/asf/hive/blob/5333bd39/ql/src/test/results/clientpositive/udf_decode.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_decode.q.out b/ql/src/test/results/clientpositive/udf_decode.q.out
index 97f0acf..107712a 100644
--- a/ql/src/test/results/clientpositive/udf_decode.q.out
+++ b/ql/src/test/results/clientpositive/udf_decode.q.out
@@ -8,7 +8,7 @@ PREHOOK: type: DESCFUNCTION
POSTHOOK: query: DESC FUNCTION EXTENDED decode
POSTHOOK: type: DESCFUNCTION
decode(bin, str) - Decode the first argument using the second argument character set
-Possible options for the character set are 'US_ASCII', 'ISO-8859-1',
+Possible options for the character set are 'US-ASCII', 'ISO-8859-1',
'UTF-8', 'UTF-16BE', 'UTF-16LE', and 'UTF-16'. If either argument
is null, the result will also be null
PREHOOK: query: explain select decode(binary('TestDecode1'), 'UTF-8')
[25/50] [abbrv] hive git commit: HIVE-10639 create SHA1 UDF
(Alexander Pivovarov, reviewed by Jason Dere)
Posted by sp...@apache.org.
HIVE-10639 create SHA1 UDF (Alexander Pivovarov, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/84a2235c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/84a2235c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/84a2235c
Branch: refs/heads/parquet
Commit: 84a2235c5590944120bb9319565f5e113b6275a1
Parents: bf7810a
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Wed May 6 20:13:22 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Fri May 15 10:23:27 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/FunctionRegistry.java | 3 +
.../org/apache/hadoop/hive/ql/udf/UDFSha1.java | 88 ++++++++++++++++++++
.../apache/hadoop/hive/ql/udf/TestUDFSha1.java | 57 +++++++++++++
ql/src/test/queries/clientpositive/udf_sha1.q | 13 +++
.../results/clientpositive/show_functions.q.out | 2 +
.../test/results/clientpositive/udf_sha1.q.out | 61 ++++++++++++++
6 files changed, 224 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/84a2235c/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index 02a604f..606185c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -86,6 +86,7 @@ import org.apache.hadoop.hive.ql.udf.UDFRegExpReplace;
import org.apache.hadoop.hive.ql.udf.UDFRepeat;
import org.apache.hadoop.hive.ql.udf.UDFReverse;
import org.apache.hadoop.hive.ql.udf.UDFSecond;
+import org.apache.hadoop.hive.ql.udf.UDFSha1;
import org.apache.hadoop.hive.ql.udf.UDFSign;
import org.apache.hadoop.hive.ql.udf.UDFSin;
import org.apache.hadoop.hive.ql.udf.UDFSpace;
@@ -226,6 +227,8 @@ public final class FunctionRegistry {
system.registerUDF("base64", UDFBase64.class, false);
system.registerUDF("unbase64", UDFUnbase64.class, false);
system.registerUDF("md5", UDFMd5.class, false);
+ system.registerUDF("sha1", UDFSha1.class, false);
+ system.registerUDF("sha", UDFSha1.class, false);
system.registerGenericUDF("encode", GenericUDFEncode.class);
system.registerGenericUDF("decode", GenericUDFDecode.class);
http://git-wip-us.apache.org/repos/asf/hive/blob/84a2235c/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSha1.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSha1.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSha1.java
new file mode 100644
index 0000000..04e6f81
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSha1.java
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+/**
+ * UDFSha.
+ *
+ */
+@Description(name = "sha1,sha",
+ value = "_FUNC_(str or bin) - Calculates the SHA-1 digest for string or binary "
+ + "and returns the value as a hex string.",
+ extended = "Example:\n"
+ + " > SELECT _FUNC_('ABC');\n"
+ + " '3c01bdbb26f358bab27f267924aa2c9a03fcfdb8'\n"
+ + " > SELECT _FUNC_(binary('ABC'));\n"
+ + " '3c01bdbb26f358bab27f267924aa2c9a03fcfdb8'")
+public class UDFSha1 extends UDF {
+
+ private final Text result = new Text();
+ private final MessageDigest digest;
+
+ public UDFSha1() {
+ try {
+ digest = MessageDigest.getInstance("SHA");
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ /**
+ * Convert String to SHA-1
+ */
+ public Text evaluate(Text n) {
+ if (n == null) {
+ return null;
+ }
+
+ digest.reset();
+ digest.update(n.getBytes(), 0, n.getLength());
+ byte[] shaBytes = digest.digest();
+ String shaHex = Hex.encodeHexString(shaBytes);
+
+ result.set(shaHex);
+ return result;
+ }
+
+ /**
+ * Convert bytes to SHA-1
+ */
+ public Text evaluate(BytesWritable b) {
+ if (b == null) {
+ return null;
+ }
+
+ digest.reset();
+ digest.update(b.getBytes(), 0, b.getLength());
+ byte[] shaBytes = digest.digest();
+ String shaHex = Hex.encodeHexString(shaBytes);
+
+ result.set(shaHex);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/84a2235c/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFSha1.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFSha1.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFSha1.java
new file mode 100644
index 0000000..fefec92
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFSha1.java
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+public class TestUDFSha1 extends TestCase {
+
+ public void testSha1Str() throws HiveException {
+ UDFSha1 udf = new UDFSha1();
+
+ runAndVerifyStr("ABC", "3c01bdbb26f358bab27f267924aa2c9a03fcfdb8", udf);
+ runAndVerifyStr("", "da39a3ee5e6b4b0d3255bfef95601890afd80709", udf);
+ // null
+ runAndVerifyStr(null, null, udf);
+ }
+
+ public void testSha1Bin() throws HiveException {
+ UDFSha1 udf = new UDFSha1();
+
+ runAndVerifyBin(new byte[] { 65, 66, 67 }, "3c01bdbb26f358bab27f267924aa2c9a03fcfdb8", udf);
+ runAndVerifyBin(new byte[0], "da39a3ee5e6b4b0d3255bfef95601890afd80709", udf);
+ // null
+ runAndVerifyBin(null, null, udf);
+ }
+
+ private void runAndVerifyStr(String str, String expResult, UDFSha1 udf) throws HiveException {
+ Text t = str != null ? new Text(str) : null;
+ Text output = (Text) udf.evaluate(t);
+ assertEquals("sha1() test ", expResult, output != null ? output.toString() : null);
+ }
+
+ private void runAndVerifyBin(byte[] binV, String expResult, UDFSha1 udf) throws HiveException {
+ BytesWritable binWr = binV != null ? new BytesWritable(binV) : null;
+ Text output = (Text) udf.evaluate(binWr);
+ assertEquals("sha1() test ", expResult, output != null ? output.toString() : null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/84a2235c/ql/src/test/queries/clientpositive/udf_sha1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udf_sha1.q b/ql/src/test/queries/clientpositive/udf_sha1.q
new file mode 100644
index 0000000..0bfa7ec
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/udf_sha1.q
@@ -0,0 +1,13 @@
+DESCRIBE FUNCTION sha1;
+DESC FUNCTION EXTENDED sha;
+
+explain select sha1('ABC');
+
+select
+sha1('ABC'),
+sha(''),
+sha(binary('ABC')),
+sha1(binary('')),
+sha1(cast(null as string)),
+sha(cast(null as binary)),
+sha1(null);
http://git-wip-us.apache.org/repos/asf/hive/blob/84a2235c/ql/src/test/results/clientpositive/show_functions.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/show_functions.q.out b/ql/src/test/results/clientpositive/show_functions.q.out
index a422760..0948154 100644
--- a/ql/src/test/results/clientpositive/show_functions.q.out
+++ b/ql/src/test/results/clientpositive/show_functions.q.out
@@ -170,6 +170,8 @@ rpad
rtrim
second
sentences
+sha
+sha1
shiftleft
shiftright
shiftrightunsigned
http://git-wip-us.apache.org/repos/asf/hive/blob/84a2235c/ql/src/test/results/clientpositive/udf_sha1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_sha1.q.out b/ql/src/test/results/clientpositive/udf_sha1.q.out
new file mode 100644
index 0000000..ea7c805
--- /dev/null
+++ b/ql/src/test/results/clientpositive/udf_sha1.q.out
@@ -0,0 +1,61 @@
+PREHOOK: query: DESCRIBE FUNCTION sha1
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION sha1
+POSTHOOK: type: DESCFUNCTION
+sha1(str or bin) - Calculates the SHA-1 digest for string or binary and returns the value as a hex string.
+PREHOOK: query: DESC FUNCTION EXTENDED sha
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESC FUNCTION EXTENDED sha
+POSTHOOK: type: DESCFUNCTION
+sha(str or bin) - Calculates the SHA-1 digest for string or binary and returns the value as a hex string.
+Synonyms: sha1
+Example:
+ > SELECT sha('ABC');
+ '3c01bdbb26f358bab27f267924aa2c9a03fcfdb8'
+ > SELECT sha(binary('ABC'));
+ '3c01bdbb26f358bab27f267924aa2c9a03fcfdb8'
+PREHOOK: query: explain select sha1('ABC')
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select sha1('ABC')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: _dummy_table
+ Row Limit Per Split: 1
+ Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+ Select Operator
+ expressions: '3c01bdbb26f358bab27f267924aa2c9a03fcfdb8' (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+ ListSink
+
+PREHOOK: query: select
+sha1('ABC'),
+sha(''),
+sha(binary('ABC')),
+sha1(binary('')),
+sha1(cast(null as string)),
+sha(cast(null as binary)),
+sha1(null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select
+sha1('ABC'),
+sha(''),
+sha(binary('ABC')),
+sha1(binary('')),
+sha1(cast(null as string)),
+sha(cast(null as binary)),
+sha1(null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+3c01bdbb26f358bab27f267924aa2c9a03fcfdb8 da39a3ee5e6b4b0d3255bfef95601890afd80709 3c01bdbb26f358bab27f267924aa2c9a03fcfdb8 da39a3ee5e6b4b0d3255bfef95601890afd80709 NULL NULL NULL
[30/50] [abbrv] hive git commit: HIVE-10726: Hive JDBC
setQueryTimeout should not throw exception to make it work with JMeter
(Prasanth Jayachandran reviewed by Thejas M Nair)
Posted by sp...@apache.org.
HIVE-10726: Hive JDBC setQueryTimeout should not throw exception to make it work with JMeter (Prasanth Jayachandran reviewed by Thejas M Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b28d394b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b28d394b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b28d394b
Branch: refs/heads/parquet
Commit: b28d394b571659e84856e73de789a884c1fdb9b4
Parents: 52308ab
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Sun May 17 21:30:25 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Sun May 17 21:30:25 2015 -0700
----------------------------------------------------------------------
jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b28d394b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
index c2d0c9d..6b3d05c 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveStatement.java
@@ -29,6 +29,8 @@ import java.util.List;
import java.util.Map;
import java.util.concurrent.locks.ReentrantLock;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hive.service.cli.RowSet;
import org.apache.hive.service.cli.RowSetFactory;
import org.apache.hive.service.cli.thrift.TCLIService;
@@ -51,6 +53,7 @@ import org.apache.hive.service.cli.thrift.TFetchOrientation;
*
*/
public class HiveStatement implements java.sql.Statement {
+ public static final Log LOG = LogFactory.getLog(HiveStatement.class.getName());
private final HiveConnection connection;
private TCLIService.Iface client;
private TOperationHandle stmtHandle = null;
@@ -736,7 +739,10 @@ public class HiveStatement implements java.sql.Statement {
@Override
public void setQueryTimeout(int seconds) throws SQLException {
- throw new SQLException("Method not supported");
+ // 0 is supported which means "no limit"
+ if (seconds != 0) {
+ throw new SQLException("Query timeout seconds must be 0");
+ }
}
/*
[18/50] [abbrv] hive git commit: HIVE-10565: Native Vector Map Join
doesn't handle filtering and matching on LEFT OUTER JOIN repeated key
correctly (Matt McCline via Gunther Hagleitner)
Posted by sp...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/results/clientpositive/tez/vector_join_nulls.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_join_nulls.q.out b/ql/src/test/results/clientpositive/tez/vector_join_nulls.q.out
new file mode 100644
index 0000000..2243072
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vector_join_nulls.q.out
@@ -0,0 +1,195 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE myinput1_txt(key int, value int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@myinput1_txt
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE myinput1_txt(key int, value int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@myinput1_txt
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE myinput1_txt
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@myinput1_txt
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE myinput1_txt
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@myinput1_txt
+PREHOOK: query: CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@myinput1_txt
+PREHOOK: Output: database:default
+PREHOOK: Output: default@myinput1
+POSTHOOK: query: CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@myinput1_txt
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@myinput1
+Warning: Map Join MAPJOIN[16][bigTable=?] in task 'Map 1' is a cross product
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+13630578
+Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Map 1' is a cross product
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+13630578
+Warning: Map Join MAPJOIN[15][bigTable=?] in task 'Map 1' is a cross product
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+13630578
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4509856
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3112070
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4542003
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4542038
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4543491
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4542003
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3079923
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4509891
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3113558
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3079923
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3112070
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3113558
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3112070
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/results/clientpositive/tez/vector_left_outer_join2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_left_outer_join2.q.out b/ql/src/test/results/clientpositive/tez/vector_left_outer_join2.q.out
index 929194e..6c781e3 100644
--- a/ql/src/test/results/clientpositive/tez/vector_left_outer_join2.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_left_outer_join2.q.out
@@ -1,6 +1,10 @@
-PREHOOK: query: drop table if exists TJOIN1
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+drop table if exists TJOIN1
PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table if exists TJOIN1
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+drop table if exists TJOIN1
POSTHOOK: type: DROPTABLE
PREHOOK: query: drop table if exists TJOIN2
PREHOOK: type: DROPTABLE
@@ -228,8 +232,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@tjoin1
POSTHOOK: Input: default@tjoin2
#### A masked pattern was here ####
-1 20 25 NULL
0 10 15 NULL
+1 20 25 NULL
2 NULL 50 NULL
PREHOOK: query: explain
select tjoin1.rnum, tjoin1.c1, tjoin1.c2, tjoin2.c2 as c2j2 from tjoin1 left outer join tjoin2 on ( tjoin1.c1 = tjoin2.c1 and tjoin1.c2 > 15 )
@@ -387,8 +391,8 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@tjoin1
POSTHOOK: Input: default@tjoin2
#### A masked pattern was here ####
-1 20 25 NULL
0 10 15 NULL
+1 20 25 NULL
2 NULL 50 NULL
PREHOOK: query: explain
select tjoin1.rnum, tjoin1.c1, tjoin1.c2, tjoin2.c2 as c2j2 from tjoin1 left outer join tjoin2 on ( tjoin1.c1 = tjoin2.c1 and tjoin1.c2 > 15 )
@@ -466,10 +470,9 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@tjoin1
POSTHOOK: Input: default@tjoin2
#### A masked pattern was here ####
+0 10 15 NULL
1 20 25 NULL
2 NULL 50 NULL
-0 10 15 BB
-0 10 15 FF
PREHOOK: query: explain
select tjoin1.rnum, tjoin1.c1, tjoin1.c2, tjoin2.c2 as c2j2 from tjoin1 left outer join tjoin2 on ( tjoin1.c1 = tjoin2.c1 and tjoin1.c2 > 15 )
PREHOOK: type: QUERY
@@ -547,7 +550,6 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: default@tjoin1
POSTHOOK: Input: default@tjoin2
#### A masked pattern was here ####
-2 NULL 50 NULL
+0 10 15 NULL
1 20 25 NULL
-0 10 15 BB
-0 10 15 FF
+2 NULL 50 NULL
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/results/clientpositive/tez/vector_left_outer_join3.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_left_outer_join3.q.out b/ql/src/test/results/clientpositive/tez/vector_left_outer_join3.q.out
new file mode 100644
index 0000000..591e165
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vector_left_outer_join3.q.out
@@ -0,0 +1,222 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+create table vsmb_bucket_1(RNUM int, C1 int, C2 int)
+ CLUSTERED BY (C1)
+ SORTED BY (C1) INTO 1 BUCKETS
+ STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vsmb_bucket_1
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+create table vsmb_bucket_1(RNUM int, C1 int, C2 int)
+ CLUSTERED BY (C1)
+ SORTED BY (C1) INTO 1 BUCKETS
+ STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vsmb_bucket_1
+PREHOOK: query: create table vsmb_bucket_2(RNUM int, C1 int, C2 int)
+ CLUSTERED BY (C1)
+ SORTED BY (C1) INTO 1 BUCKETS
+ STORED AS ORC
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vsmb_bucket_2
+POSTHOOK: query: create table vsmb_bucket_2(RNUM int, C1 int, C2 int)
+ CLUSTERED BY (C1)
+ SORTED BY (C1) INTO 1 BUCKETS
+ STORED AS ORC
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vsmb_bucket_2
+PREHOOK: query: create table vsmb_bucket_TXT_1(RNUM int, C1 int, C2 int)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' LINES TERMINATED BY '\n' STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vsmb_bucket_TXT_1
+POSTHOOK: query: create table vsmb_bucket_TXT_1(RNUM int, C1 int, C2 int)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' LINES TERMINATED BY '\n' STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vsmb_bucket_TXT_1
+PREHOOK: query: create table vsmb_bucket_TXT_2(RNUM int, C1 int, C2 int)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' LINES TERMINATED BY '\n' STORED AS TEXTFILE
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@vsmb_bucket_TXT_2
+POSTHOOK: query: create table vsmb_bucket_TXT_2(RNUM int, C1 int, C2 int)
+ROW FORMAT DELIMITED FIELDS TERMINATED BY '|' LINES TERMINATED BY '\n' STORED AS TEXTFILE
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@vsmb_bucket_TXT_2
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/tjoin1.txt' OVERWRITE INTO TABLE vsmb_bucket_TXT_1
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@vsmb_bucket_txt_1
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/tjoin1.txt' OVERWRITE INTO TABLE vsmb_bucket_TXT_1
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@vsmb_bucket_txt_1
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/tjoin2.txt' OVERWRITE INTO TABLE vsmb_bucket_TXT_2
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@vsmb_bucket_txt_2
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/tjoin2.txt' OVERWRITE INTO TABLE vsmb_bucket_TXT_2
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@vsmb_bucket_txt_2
+PREHOOK: query: insert into table vsmb_bucket_1 select * from vsmb_bucket_TXT_1 order by c1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vsmb_bucket_txt_1
+PREHOOK: Output: default@vsmb_bucket_1
+POSTHOOK: query: insert into table vsmb_bucket_1 select * from vsmb_bucket_TXT_1 order by c1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vsmb_bucket_txt_1
+POSTHOOK: Output: default@vsmb_bucket_1
+POSTHOOK: Lineage: vsmb_bucket_1.c1 SIMPLE [(vsmb_bucket_txt_1)vsmb_bucket_txt_1.FieldSchema(name:c1, type:int, comment:null), ]
+POSTHOOK: Lineage: vsmb_bucket_1.c2 SIMPLE [(vsmb_bucket_txt_1)vsmb_bucket_txt_1.FieldSchema(name:c2, type:int, comment:null), ]
+POSTHOOK: Lineage: vsmb_bucket_1.rnum SIMPLE [(vsmb_bucket_txt_1)vsmb_bucket_txt_1.FieldSchema(name:rnum, type:int, comment:null), ]
+PREHOOK: query: insert into table vsmb_bucket_2 select * from vsmb_bucket_TXT_2 order by c1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vsmb_bucket_txt_2
+PREHOOK: Output: default@vsmb_bucket_2
+POSTHOOK: query: insert into table vsmb_bucket_2 select * from vsmb_bucket_TXT_2 order by c1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vsmb_bucket_txt_2
+POSTHOOK: Output: default@vsmb_bucket_2
+POSTHOOK: Lineage: vsmb_bucket_2.c1 SIMPLE [(vsmb_bucket_txt_2)vsmb_bucket_txt_2.FieldSchema(name:c1, type:int, comment:null), ]
+POSTHOOK: Lineage: vsmb_bucket_2.c2 SIMPLE [(vsmb_bucket_txt_2)vsmb_bucket_txt_2.FieldSchema(name:c2, type:int, comment:null), ]
+POSTHOOK: Lineage: vsmb_bucket_2.rnum SIMPLE [(vsmb_bucket_txt_2)vsmb_bucket_txt_2.FieldSchema(name:rnum, type:int, comment:null), ]
+PREHOOK: query: explain
+select tjoin1.rnum, tjoin1.c1, tjoin1.c2, tjoin2.c2 from vsmb_bucket_1 tjoin1 left outer join vsmb_bucket_2 tjoin2 on tjoin1.c1 = tjoin2.c1 and tjoin1.c2 > 15
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select tjoin1.rnum, tjoin1.c1, tjoin1.c2, tjoin2.c2 from vsmb_bucket_1 tjoin1 left outer join vsmb_bucket_2 tjoin2 on tjoin1.c1 = tjoin2.c1 and tjoin1.c2 > 15
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: tjoin2
+ Statistics: Num rows: 4 Data size: 28 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: tjoin1
+ Statistics: Num rows: 3 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {(c2 > 15)}
+ 1
+ keys:
+ 0 c1 (type: int)
+ 1 c1 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col8
+ Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col8 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select tjoin1.rnum, tjoin1.c1, tjoin1.c2, tjoin2.c2 from vsmb_bucket_1 tjoin1 left outer join vsmb_bucket_2 tjoin2 on tjoin1.c1 = tjoin2.c1 and tjoin1.c2 > 15
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vsmb_bucket_1
+PREHOOK: Input: default@vsmb_bucket_2
+#### A masked pattern was here ####
+POSTHOOK: query: select tjoin1.rnum, tjoin1.c1, tjoin1.c2, tjoin2.c2 from vsmb_bucket_1 tjoin1 left outer join vsmb_bucket_2 tjoin2 on tjoin1.c1 = tjoin2.c1 and tjoin1.c2 > 15
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vsmb_bucket_1
+POSTHOOK: Input: default@vsmb_bucket_2
+#### A masked pattern was here ####
+0 10 15 NULL
+1 20 25 NULL
+2 NULL 50 NULL
+PREHOOK: query: explain
+select tjoin1.rnum, tjoin1.c1, tjoin1.c2, tjoin2.c2 from vsmb_bucket_1 tjoin1 left outer join vsmb_bucket_2 tjoin2 on tjoin1.c1 = tjoin2.c1 and tjoin1.c2 > 15
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select tjoin1.rnum, tjoin1.c1, tjoin1.c2, tjoin2.c2 from vsmb_bucket_1 tjoin1 left outer join vsmb_bucket_2 tjoin2 on tjoin1.c1 = tjoin2.c1 and tjoin1.c2 > 15
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: tjoin2
+ Statistics: Num rows: 4 Data size: 28 Basic stats: COMPLETE Column stats: NONE
+ Map Operator Tree:
+ TableScan
+ alias: tjoin1
+ Statistics: Num rows: 3 Data size: 32 Basic stats: COMPLETE Column stats: NONE
+ Merge Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {(c2 > 15)}
+ 1
+ keys:
+ 0 c1 (type: int)
+ 1 c1 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col8
+ Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col2 (type: int), _col8 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 4 Data size: 30 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select tjoin1.rnum, tjoin1.c1, tjoin1.c2, tjoin2.c2 from vsmb_bucket_1 tjoin1 left outer join vsmb_bucket_2 tjoin2 on tjoin1.c1 = tjoin2.c1 and tjoin1.c2 > 15
+PREHOOK: type: QUERY
+PREHOOK: Input: default@vsmb_bucket_1
+PREHOOK: Input: default@vsmb_bucket_2
+#### A masked pattern was here ####
+POSTHOOK: query: select tjoin1.rnum, tjoin1.c1, tjoin1.c2, tjoin2.c2 from vsmb_bucket_1 tjoin1 left outer join vsmb_bucket_2 tjoin2 on tjoin1.c1 = tjoin2.c1 and tjoin1.c2 > 15
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@vsmb_bucket_1
+POSTHOOK: Input: default@vsmb_bucket_2
+#### A masked pattern was here ####
+0 10 15 NULL
+1 20 25 NULL
+2 NULL 50 NULL
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/results/clientpositive/tez/vector_outer_join5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_outer_join5.q.out b/ql/src/test/results/clientpositive/tez/vector_outer_join5.q.out
new file mode 100644
index 0000000..e77903a
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vector_outer_join5.q.out
@@ -0,0 +1,1328 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+create table sorted_mod_4 stored as orc
+as select ctinyint, pmod(cint, 4) as cmodint from alltypesorc
+where cint is not null and ctinyint is not null
+order by ctinyint
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: database:default
+PREHOOK: Output: default@sorted_mod_4
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+create table sorted_mod_4 stored as orc
+as select ctinyint, pmod(cint, 4) as cmodint from alltypesorc
+where cint is not null and ctinyint is not null
+order by ctinyint
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@sorted_mod_4
+PREHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sorted_mod_4
+PREHOOK: Output: default@sorted_mod_4
+POSTHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@sorted_mod_4
+POSTHOOK: Output: default@sorted_mod_4
+PREHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS FOR COLUMNS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+POSTHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS FOR COLUMNS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+PREHOOK: query: create table small_table stored
+as orc as select ctinyint, cbigint from alltypesorc limit 100
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: database:default
+PREHOOK: Output: default@small_table
+POSTHOOK: query: create table small_table stored
+as orc as select ctinyint, cbigint from alltypesorc limit 100
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@small_table
+PREHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table
+PREHOOK: Output: default@small_table
+POSTHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table
+POSTHOOK: Output: default@small_table
+PREHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS FOR COLUMNS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table
+#### A masked pattern was here ####
+POSTHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS FOR COLUMNS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table
+#### A masked pattern was here ####
+PREHOOK: query: explain
+select count(*) from (select s.*, st.*
+from sorted_mod_4 s
+left outer join small_table st
+on s.ctinyint = st.ctinyint
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.*, st.*
+from sorted_mod_4 s
+left outer join small_table st
+on s.ctinyint = st.ctinyint
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 3 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: st
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: tinyint)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: tinyint)
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.*, st.*
+from sorted_mod_4 s
+left outer join small_table st
+on s.ctinyint = st.ctinyint
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table
+PREHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.*, st.*
+from sorted_mod_4 s
+left outer join small_table st
+on s.ctinyint = st.ctinyint
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table
+POSTHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+6876
+PREHOOK: query: explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.cmodint = 2
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.cmodint = 2
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 3 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint), cmodint (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {(_col1 = 2)}
+ 1
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: tinyint)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: tinyint)
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.cmodint = 2
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table
+PREHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.cmodint = 2
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table
+POSTHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+6058
+PREHOOK: query: explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and pmod(s.ctinyint, 4) = s.cmodint
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and pmod(s.ctinyint, 4) = s.cmodint
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 3 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint), cmodint (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {((UDFToInteger(_col0) pmod 4) = _col1)}
+ 1
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: tinyint)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: tinyint)
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and pmod(s.ctinyint, 4) = s.cmodint
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table
+PREHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and pmod(s.ctinyint, 4) = s.cmodint
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table
+POSTHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+6248
+PREHOOK: query: explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.ctinyint < 100
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.ctinyint < 100
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 3 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {(_col0 < 100)}
+ 1
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: tinyint)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: tinyint)
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.ctinyint < 100
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table
+PREHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.ctinyint < 100
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table
+POSTHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+6876
+PREHOOK: query: explain
+select count(*) from (select s.*, sm.*, s2.*
+from sorted_mod_4 s
+left outer join small_table sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join sorted_mod_4 s2
+ on s2.ctinyint = s.ctinyint
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.*, sm.*, s2.*
+from sorted_mod_4 s
+left outer join small_table sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join sorted_mod_4 s2
+ on s2.ctinyint = s.ctinyint
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 4 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint), cmodint (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 UDFToLong(_col1) (type: bigint)
+ 1 (_col0 pmod UDFToLong(8)) (type: bigint)
+ outputColumnNames: _col0
+ input vertices:
+ 1 Map 4
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Reduce Output Operator
+ key expressions: _col0 (type: tinyint)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: tinyint)
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cbigint (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: (_col0 pmod UDFToLong(8)) (type: bigint)
+ sort order: +
+ Map-reduce partition columns: (_col0 pmod UDFToLong(8)) (type: bigint)
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: tinyint)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: tinyint)
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+ Statistics: Num rows: 7329 Data size: 58641 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.*, sm.*, s2.*
+from sorted_mod_4 s
+left outer join small_table sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join sorted_mod_4 s2
+ on s2.ctinyint = s.ctinyint
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table
+PREHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.*, sm.*, s2.*
+from sorted_mod_4 s
+left outer join small_table sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join sorted_mod_4 s2
+ on s2.ctinyint = s.ctinyint
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table
+POSTHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+3268334
+PREHOOK: query: create table mod_8_mod_4 stored as orc
+as select pmod(ctinyint, 8) as cmodtinyint, pmod(cint, 4) as cmodint from alltypesorc
+where cint is not null and ctinyint is not null
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mod_8_mod_4
+POSTHOOK: query: create table mod_8_mod_4 stored as orc
+as select pmod(ctinyint, 8) as cmodtinyint, pmod(cint, 4) as cmodint from alltypesorc
+where cint is not null and ctinyint is not null
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mod_8_mod_4
+PREHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mod_8_mod_4
+PREHOOK: Output: default@mod_8_mod_4
+POSTHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mod_8_mod_4
+POSTHOOK: Output: default@mod_8_mod_4
+PREHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS FOR COLUMNS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mod_8_mod_4
+#### A masked pattern was here ####
+POSTHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS FOR COLUMNS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mod_8_mod_4
+#### A masked pattern was here ####
+PREHOOK: query: create table small_table2 stored
+as orc as select pmod(ctinyint, 16) as cmodtinyint, cbigint from alltypesorc limit 100
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: database:default
+PREHOOK: Output: default@small_table2
+POSTHOOK: query: create table small_table2 stored
+as orc as select pmod(ctinyint, 16) as cmodtinyint, cbigint from alltypesorc limit 100
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@small_table2
+PREHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table2
+PREHOOK: Output: default@small_table2
+POSTHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table2
+POSTHOOK: Output: default@small_table2
+PREHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS FOR COLUMNS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+POSTHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS FOR COLUMNS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+PREHOOK: query: explain
+select count(*) from (select s.*, st.*
+from mod_8_mod_4 s
+left outer join small_table2 st
+on s.cmodtinyint = st.cmodtinyint
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.*, st.*
+from mod_8_mod_4 s
+left outer join small_table2 st
+on s.cmodtinyint = st.cmodtinyint
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 3 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: st
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.*, st.*
+from mod_8_mod_4 s
+left outer join small_table2 st
+on s.cmodtinyint = st.cmodtinyint
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mod_8_mod_4
+PREHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.*, st.*
+from mod_8_mod_4 s
+left outer join small_table2 st
+on s.cmodtinyint = st.cmodtinyint
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mod_8_mod_4
+POSTHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+39112
+PREHOOK: query: explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodint = 2
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodint = 2
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 3 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int), cmodint (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {(_col1 = 2)}
+ 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodint = 2
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mod_8_mod_4
+PREHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodint = 2
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mod_8_mod_4
+POSTHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+11171
+PREHOOK: query: explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and pmod(s.cmodtinyint, 4) = s.cmodint
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and pmod(s.cmodtinyint, 4) = s.cmodint
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 3 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int), cmodint (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {((_col0 pmod 4) = _col1)}
+ 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and pmod(s.cmodtinyint, 4) = s.cmodint
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mod_8_mod_4
+PREHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and pmod(s.cmodtinyint, 4) = s.cmodint
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mod_8_mod_4
+POSTHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+14371
+PREHOOK: query: explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodtinyint < 3
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodtinyint < 3
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 3 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {(_col0 < 3)}
+ 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodtinyint < 3
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mod_8_mod_4
+PREHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodtinyint < 3
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mod_8_mod_4
+POSTHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+17792
+PREHOOK: query: explain
+select count(*) from (select s.*, sm.*, s2.*
+from mod_8_mod_4 s
+left outer join small_table2 sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join mod_8_mod_4 s2
+ on s2.cmodtinyint = s.cmodtinyint
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.*, sm.*, s2.*
+from mod_8_mod_4 s
+left outer join small_table2 sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join mod_8_mod_4 s2
+ on s2.cmodtinyint = s.cmodtinyint
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 4 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Map 5 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int), cmodint (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 UDFToLong(_col1) (type: bigint)
+ 1 (_col0 pmod UDFToLong(8)) (type: bigint)
+ outputColumnNames: _col0
+ input vertices:
+ 1 Map 4
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cbigint (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: (_col0 pmod UDFToLong(8)) (type: bigint)
+ sort order: +
+ Map-reduce partition columns: (_col0 pmod UDFToLong(8)) (type: bigint)
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 7329 Data size: 58641 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.*, sm.*, s2.*
+from mod_8_mod_4 s
+left outer join small_table2 sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join mod_8_mod_4 s2
+ on s2.cmodtinyint = s.cmodtinyint
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mod_8_mod_4
+PREHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.*, sm.*, s2.*
+from mod_8_mod_4 s
+left outer join small_table2 sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join mod_8_mod_4 s2
+ on s2.cmodtinyint = s.cmodtinyint
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mod_8_mod_4
+POSTHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+6524438
[07/50] [abbrv] hive git commit: HIVE-10669: The HMS upgrade test is
not testing postgres nor derby upgrades (Sergio Pena, reviewed by Szehon Ho)
Posted by sp...@apache.org.
HIVE-10669: The HMS upgrade test is not testing postgres nor derby upgrades (Sergio Pena, reviewed by Szehon Ho)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fbdd7dad
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fbdd7dad
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fbdd7dad
Branch: refs/heads/parquet
Commit: fbdd7dad17d7a46d8c77b9fb24bbca0cc709767d
Parents: c13ce55
Author: Sergio Pena <se...@cloudera.com>
Authored: Wed May 13 10:37:50 2015 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed May 13 10:37:50 2015 -0500
----------------------------------------------------------------------
metastore/dbs/derby/execute.sh | 37 ------------
metastore/dbs/derby/prepare.sh | 63 ---------------------
metastore/dbs/postgres/execute.sh | 29 ----------
metastore/dbs/postgres/prepare.sh | 72 ------------------------
testutils/metastore/dbs/derby/execute.sh | 37 ++++++++++++
testutils/metastore/dbs/derby/prepare.sh | 63 +++++++++++++++++++++
testutils/metastore/dbs/postgres/execute.sh | 29 ++++++++++
testutils/metastore/dbs/postgres/prepare.sh | 72 ++++++++++++++++++++++++
8 files changed, 201 insertions(+), 201 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/fbdd7dad/metastore/dbs/derby/execute.sh
----------------------------------------------------------------------
diff --git a/metastore/dbs/derby/execute.sh b/metastore/dbs/derby/execute.sh
deleted file mode 100644
index d60f05b..0000000
--- a/metastore/dbs/derby/execute.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script executes all hive metastore upgrade scripts on an specific
-# database server in order to verify that upgrade scripts are working
-# properly.
-
-cd $(dirname $1)
-
-echo "####################################################"
-echo "Executing script for Derby SQL: $1"
-echo "####################################################"
-
-export DERBY_HOME=/usr/share/javadb
-export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
-export PATH=$PATH:$DERBY_HOME/bin:$JAVA_HOME/bin
-export CLASSPATH=$CLASSPATH:$DERBY_HOME/lib/derby.jar:$DERBY_HOME/lib/derbytools.jar:$DERBY_HOME/lib/derbyclient.jar
-
-echo "connect 'jdbc:derby:/tmp/hive_hms_testing;create=true';" > /tmp/derbyRun.sql
-echo "run '$1';" >> /tmp/derbyRun.sql
-echo "quit;" >> /tmp/derbyRun.sql
-
-ij /tmp/derbyRun.sql
http://git-wip-us.apache.org/repos/asf/hive/blob/fbdd7dad/metastore/dbs/derby/prepare.sh
----------------------------------------------------------------------
diff --git a/metastore/dbs/derby/prepare.sh b/metastore/dbs/derby/prepare.sh
deleted file mode 100644
index fe4b2c3..0000000
--- a/metastore/dbs/derby/prepare.sh
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script executes all hive metastore upgrade scripts on an specific
-# database server in order to verify that upgrade scripts are working
-# properly.
-
-export DEBIAN_FRONTEND=noninteractive
-OS_VERSION=`lsb_release -c | cut -d":" -f2`
-
-echo "####################################################"
-echo "Begin for OS version $OS_VERSION"
-echo "####################################################"
-
-HTTPS_INFO=($(dpkg -l apt-transport-https | grep ^i | tr -s ' '))
-if [[ ${HTTPS_INFO[1]} == "apt-transport-https" ]]
-then
- echo "apt-transport-https package installed"
-else
- echo "apt-transport-https package not installed"
- apt-get install -y --force-yes apt-transport-https
-fi
-
-INSTALL_INFO=($(dpkg -l \*javadb-core\* | grep ^ii | tr -s ' '))
-
-if [[ ${INSTALL_INFO[1]} == "sun-javadb-core" ]]
-then
- echo "Derby already installed...Skipping"
-else
- echo "Derby not installed"
- # Cleanup existing installation + configuration.
- apt-get purge -y --force-yes derby-tools sun-javadb-client sun-javadb-core sun-javadb-common libderby-java openjdk-7-jre openjdk-7-jre openjdk-7-jre-headless || /bin/true
- echo "####################################################"
- echo "Installing Derby dependencies:"
- echo "####################################################"
- apt-get update || /bin/true
- apt-get install -y --force-yes -o Dpkg::Options::="--force-overwrite" sun-javadb-core sun-javadb-client derby-tools
-fi
-
-export DERBY_HOME=/usr/share/javadb
-export JAVA_HOME=//usr/lib/jvm/java-7-openjdk-amd64
-export PATH=$PATH:/usr/share/javadb/bin:$JAVA_HOME/bin
-export CLASSPATH=$CLASSPATH:$DERBY_HOME/lib/derby.jar:$DERBY_HOME/lib/derbytools.jar:$DERBY_HOME/lib/derbyclient.jar
-rm -rf /tmp/hive_hms_testing;
-
-echo "connect 'jdbc:derby:/tmp/hive_hms_testing;create=true';" > /tmp/derbyInit.sql
-ij /tmp/derbyInit.sql
-
-echo "DONE!!!"
-
http://git-wip-us.apache.org/repos/asf/hive/blob/fbdd7dad/metastore/dbs/postgres/execute.sh
----------------------------------------------------------------------
diff --git a/metastore/dbs/postgres/execute.sh b/metastore/dbs/postgres/execute.sh
deleted file mode 100644
index cabcae4..0000000
--- a/metastore/dbs/postgres/execute.sh
+++ /dev/null
@@ -1,29 +0,0 @@
-#!/bin/bash
-
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script executes all hive metastore upgrade scripts on an specific
-# database server in order to verify that upgrade scripts are working
-# properly.
-
-cd $(dirname $1)
-
-echo "####################################################"
-echo "Executing script for PostgreSQL: $1"
-echo "####################################################"
-
-export PGPASSWORD=hivepw
-psql -h localhost -U hiveuser -d hive_hms_testing -f $1
http://git-wip-us.apache.org/repos/asf/hive/blob/fbdd7dad/metastore/dbs/postgres/prepare.sh
----------------------------------------------------------------------
diff --git a/metastore/dbs/postgres/prepare.sh b/metastore/dbs/postgres/prepare.sh
deleted file mode 100644
index 2036354..0000000
--- a/metastore/dbs/postgres/prepare.sh
+++ /dev/null
@@ -1,72 +0,0 @@
-#!/bin/bash
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This script executes all hive metastore upgrade scripts on an specific
-# database server in order to verify that upgrade scripts are working
-# properly.
-
-export DEBIAN_FRONTEND=noninteractive
-OS_VERSION=`lsb_release -c | cut -d":" -f2`
-echo "$OS_VERSION"
-
-echo "####################################################"
-echo "Begin for OS version $OS_VERSION"
-echo "####################################################"
-
-HTTPS_INFO=($(dpkg -l apt-transport-https | grep ^i | tr -s ' '))
-if [[ ${HTTPS_INFO[1]} == "apt-transport-https" ]]
-then
- echo "apt-transport-https package installed"
-else
- echo "apt-transport-https package not installed"
- apt-get install -y --force-yes apt-transport-https
-fi
-
-INSTALL_INFO=($(dpkg -l postgresql-9.4\* | grep ^i | tr -s ' '))
-
-if [[ ${INSTALL_INFO[1]} == "postgresql-9.4" ]]
-then
- echo "PostgreSQL already installed...Skipping"
-else
- echo "PostgreSQL not installed"
- # Cleanup existing installation + configuration.
- apt-get purge -y --force-yes postgressql-9.4 || /bin/true
- echo "####################################################"
- echo "Installing PostgreSQL dependencies:"
- echo "####################################################"
- if grep -q "deb http://apt.postgresql.org/pub/repos/apt/ $OS_VERSION-pgdg main" /etc/apt/sources.list.d/postgreSQL.list
- then
- echo "Sources already listed"
- else
- echo "deb http://apt.postgresql.org/pub/repos/apt/ $OS_VERSION-pgdg main" >> /etc/apt/sources.list.d/postgreSQL.list
- fi
-
- wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
- apt-get update || /bin/true
- apt-get install -y --force-yes postgresql-9.4
-fi
-
-echo "####################################################"
-echo "Configuring PostgreSQL Environment:"
-echo "####################################################"
-echo "drop database if exists hive_hms_testing;" > /tmp/postgresInit.sql
-echo "drop user if exists hiveuser;" >> /tmp/postgresInit.sql
-echo "create user hiveuser createdb createuser password 'hivepw';" >> /tmp/postgresInit.sql
-echo "create database hive_hms_testing owner hiveuser;" >> /tmp/postgresInit.sql
-sudo -u postgres psql -f /tmp/postgresInit.sql
-
-echo "DONE!!!"
-
http://git-wip-us.apache.org/repos/asf/hive/blob/fbdd7dad/testutils/metastore/dbs/derby/execute.sh
----------------------------------------------------------------------
diff --git a/testutils/metastore/dbs/derby/execute.sh b/testutils/metastore/dbs/derby/execute.sh
new file mode 100644
index 0000000..d60f05b
--- /dev/null
+++ b/testutils/metastore/dbs/derby/execute.sh
@@ -0,0 +1,37 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script executes all hive metastore upgrade scripts on an specific
+# database server in order to verify that upgrade scripts are working
+# properly.
+
+cd $(dirname $1)
+
+echo "####################################################"
+echo "Executing script for Derby SQL: $1"
+echo "####################################################"
+
+export DERBY_HOME=/usr/share/javadb
+export JAVA_HOME=/usr/lib/jvm/java-7-openjdk-amd64
+export PATH=$PATH:$DERBY_HOME/bin:$JAVA_HOME/bin
+export CLASSPATH=$CLASSPATH:$DERBY_HOME/lib/derby.jar:$DERBY_HOME/lib/derbytools.jar:$DERBY_HOME/lib/derbyclient.jar
+
+echo "connect 'jdbc:derby:/tmp/hive_hms_testing;create=true';" > /tmp/derbyRun.sql
+echo "run '$1';" >> /tmp/derbyRun.sql
+echo "quit;" >> /tmp/derbyRun.sql
+
+ij /tmp/derbyRun.sql
http://git-wip-us.apache.org/repos/asf/hive/blob/fbdd7dad/testutils/metastore/dbs/derby/prepare.sh
----------------------------------------------------------------------
diff --git a/testutils/metastore/dbs/derby/prepare.sh b/testutils/metastore/dbs/derby/prepare.sh
new file mode 100644
index 0000000..fe4b2c3
--- /dev/null
+++ b/testutils/metastore/dbs/derby/prepare.sh
@@ -0,0 +1,63 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script executes all hive metastore upgrade scripts on an specific
+# database server in order to verify that upgrade scripts are working
+# properly.
+
+export DEBIAN_FRONTEND=noninteractive
+OS_VERSION=`lsb_release -c | cut -d":" -f2`
+
+echo "####################################################"
+echo "Begin for OS version $OS_VERSION"
+echo "####################################################"
+
+HTTPS_INFO=($(dpkg -l apt-transport-https | grep ^i | tr -s ' '))
+if [[ ${HTTPS_INFO[1]} == "apt-transport-https" ]]
+then
+ echo "apt-transport-https package installed"
+else
+ echo "apt-transport-https package not installed"
+ apt-get install -y --force-yes apt-transport-https
+fi
+
+INSTALL_INFO=($(dpkg -l \*javadb-core\* | grep ^ii | tr -s ' '))
+
+if [[ ${INSTALL_INFO[1]} == "sun-javadb-core" ]]
+then
+ echo "Derby already installed...Skipping"
+else
+ echo "Derby not installed"
+ # Cleanup existing installation + configuration.
+ apt-get purge -y --force-yes derby-tools sun-javadb-client sun-javadb-core sun-javadb-common libderby-java openjdk-7-jre openjdk-7-jre openjdk-7-jre-headless || /bin/true
+ echo "####################################################"
+ echo "Installing Derby dependencies:"
+ echo "####################################################"
+ apt-get update || /bin/true
+ apt-get install -y --force-yes -o Dpkg::Options::="--force-overwrite" sun-javadb-core sun-javadb-client derby-tools
+fi
+
+export DERBY_HOME=/usr/share/javadb
+export JAVA_HOME=//usr/lib/jvm/java-7-openjdk-amd64
+export PATH=$PATH:/usr/share/javadb/bin:$JAVA_HOME/bin
+export CLASSPATH=$CLASSPATH:$DERBY_HOME/lib/derby.jar:$DERBY_HOME/lib/derbytools.jar:$DERBY_HOME/lib/derbyclient.jar
+rm -rf /tmp/hive_hms_testing;
+
+echo "connect 'jdbc:derby:/tmp/hive_hms_testing;create=true';" > /tmp/derbyInit.sql
+ij /tmp/derbyInit.sql
+
+echo "DONE!!!"
+
http://git-wip-us.apache.org/repos/asf/hive/blob/fbdd7dad/testutils/metastore/dbs/postgres/execute.sh
----------------------------------------------------------------------
diff --git a/testutils/metastore/dbs/postgres/execute.sh b/testutils/metastore/dbs/postgres/execute.sh
new file mode 100644
index 0000000..cabcae4
--- /dev/null
+++ b/testutils/metastore/dbs/postgres/execute.sh
@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script executes all hive metastore upgrade scripts on an specific
+# database server in order to verify that upgrade scripts are working
+# properly.
+
+cd $(dirname $1)
+
+echo "####################################################"
+echo "Executing script for PostgreSQL: $1"
+echo "####################################################"
+
+export PGPASSWORD=hivepw
+psql -h localhost -U hiveuser -d hive_hms_testing -f $1
http://git-wip-us.apache.org/repos/asf/hive/blob/fbdd7dad/testutils/metastore/dbs/postgres/prepare.sh
----------------------------------------------------------------------
diff --git a/testutils/metastore/dbs/postgres/prepare.sh b/testutils/metastore/dbs/postgres/prepare.sh
new file mode 100644
index 0000000..2036354
--- /dev/null
+++ b/testutils/metastore/dbs/postgres/prepare.sh
@@ -0,0 +1,72 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script executes all hive metastore upgrade scripts on an specific
+# database server in order to verify that upgrade scripts are working
+# properly.
+
+export DEBIAN_FRONTEND=noninteractive
+OS_VERSION=`lsb_release -c | cut -d":" -f2`
+echo "$OS_VERSION"
+
+echo "####################################################"
+echo "Begin for OS version $OS_VERSION"
+echo "####################################################"
+
+HTTPS_INFO=($(dpkg -l apt-transport-https | grep ^i | tr -s ' '))
+if [[ ${HTTPS_INFO[1]} == "apt-transport-https" ]]
+then
+ echo "apt-transport-https package installed"
+else
+ echo "apt-transport-https package not installed"
+ apt-get install -y --force-yes apt-transport-https
+fi
+
+INSTALL_INFO=($(dpkg -l postgresql-9.4\* | grep ^i | tr -s ' '))
+
+if [[ ${INSTALL_INFO[1]} == "postgresql-9.4" ]]
+then
+ echo "PostgreSQL already installed...Skipping"
+else
+ echo "PostgreSQL not installed"
+ # Cleanup existing installation + configuration.
+ apt-get purge -y --force-yes postgressql-9.4 || /bin/true
+ echo "####################################################"
+ echo "Installing PostgreSQL dependencies:"
+ echo "####################################################"
+ if grep -q "deb http://apt.postgresql.org/pub/repos/apt/ $OS_VERSION-pgdg main" /etc/apt/sources.list.d/postgreSQL.list
+ then
+ echo "Sources already listed"
+ else
+ echo "deb http://apt.postgresql.org/pub/repos/apt/ $OS_VERSION-pgdg main" >> /etc/apt/sources.list.d/postgreSQL.list
+ fi
+
+ wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add -
+ apt-get update || /bin/true
+ apt-get install -y --force-yes postgresql-9.4
+fi
+
+echo "####################################################"
+echo "Configuring PostgreSQL Environment:"
+echo "####################################################"
+echo "drop database if exists hive_hms_testing;" > /tmp/postgresInit.sql
+echo "drop user if exists hiveuser;" >> /tmp/postgresInit.sql
+echo "create user hiveuser createdb createuser password 'hivepw';" >> /tmp/postgresInit.sql
+echo "create database hive_hms_testing owner hiveuser;" >> /tmp/postgresInit.sql
+sudo -u postgres psql -f /tmp/postgresInit.sql
+
+echo "DONE!!!"
+
[21/50] [abbrv] hive git commit: HIVE-10565: Native Vector Map Join
doesn't handle filtering and matching on LEFT OUTER JOIN repeated key
correctly (Matt McCline via Gunther Hagleitner)
Posted by sp...@apache.org.
HIVE-10565: Native Vector Map Join doesn't handle filtering and matching on LEFT OUTER JOIN repeated key correctly (Matt McCline via Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2b9f2f5e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2b9f2f5e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2b9f2f5e
Branch: refs/heads/parquet
Commit: 2b9f2f5e2574e6e64ce9496dfe9ff6e085036fb1
Parents: 3fa7489
Author: Gunther Hagleitner <gu...@apache.org>
Authored: Thu May 14 15:42:04 2015 -0700
Committer: Gunther Hagleitner <gu...@apache.org>
Committed: Thu May 14 15:42:04 2015 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 10 +
.../ql/exec/vector/VectorizedBatchUtil.java | 5 +-
.../mapjoin/VectorMapJoinCommonOperator.java | 8 +-
.../VectorMapJoinGenerateResultOperator.java | 47 +-
...pJoinInnerBigOnlyGenerateResultOperator.java | 53 +-
.../VectorMapJoinInnerBigOnlyLongOperator.java | 15 +-
...ctorMapJoinInnerBigOnlyMultiKeyOperator.java | 15 +-
...VectorMapJoinInnerBigOnlyStringOperator.java | 12 +-
...ectorMapJoinInnerGenerateResultOperator.java | 39 +-
.../mapjoin/VectorMapJoinInnerLongOperator.java | 17 +-
.../VectorMapJoinInnerMultiKeyOperator.java | 19 +-
.../VectorMapJoinInnerStringOperator.java | 17 +-
...orMapJoinLeftSemiGenerateResultOperator.java | 40 +-
.../VectorMapJoinLeftSemiLongOperator.java | 13 +-
.../VectorMapJoinLeftSemiMultiKeyOperator.java | 17 +-
.../VectorMapJoinLeftSemiStringOperator.java | 17 +-
...ectorMapJoinOuterGenerateResultOperator.java | 805 ++++---
.../mapjoin/VectorMapJoinOuterLongOperator.java | 189 +-
.../VectorMapJoinOuterMultiKeyOperator.java | 184 +-
.../VectorMapJoinOuterStringOperator.java | 185 +-
.../mapjoin/VectorMapJoinRowBytesContainer.java | 2 +-
.../fast/VectorMapJoinFastBytesHashMap.java | 8 +-
.../VectorMapJoinFastBytesHashMultiSet.java | 4 +-
.../fast/VectorMapJoinFastBytesHashTable.java | 10 +-
.../mapjoin/fast/VectorMapJoinFastKeyStore.java | 10 +-
.../fast/VectorMapJoinFastLongHashMap.java | 2 +-
.../fast/VectorMapJoinFastLongHashTable.java | 18 +-
.../fast/VectorMapJoinFastTableContainer.java | 2 +-
.../fast/VectorMapJoinFastValueStore.java | 8 +-
.../VectorMapJoinOptimizedLongCommon.java | 4 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 24 +-
.../test/queries/clientpositive/vector_join30.q | 160 ++
.../clientpositive/vector_join_filters.q | 38 +
.../queries/clientpositive/vector_join_nulls.q | 33 +
.../clientpositive/vector_left_outer_join2.q | 2 +
.../queries/clientpositive/vector_outer_join5.q | 173 ++
.../tez/acid_vectorization_partition.q.out | 20 +-
.../clientpositive/tez/vector_join30.q.out | 1367 +++++++++++
.../tez/vector_join_filters.q.out | 222 ++
.../clientpositive/tez/vector_join_nulls.q.out | 195 ++
.../tez/vector_left_outer_join2.q.out | 20 +-
.../tez/vector_left_outer_join3.q.out | 222 ++
.../clientpositive/tez/vector_outer_join5.q.out | 1328 +++++++++++
.../tez/vectorized_timestamp_ints_casts.q.out | 234 ++
.../results/clientpositive/vector_join30.q.out | 2194 ++++++++++++++++++
.../clientpositive/vector_join_filters.q.out | 222 ++
.../clientpositive/vector_join_nulls.q.out | 195 ++
.../vector_left_outer_join2.q.out | 8 +-
.../clientpositive/vector_outer_join5.q.out | 1300 +++++++++++
49 files changed, 8936 insertions(+), 796 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index f9c9351..c79c36c 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -220,8 +220,12 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
vector_groupby_3.q,\
vector_groupby_reduce.q,\
vector_if_expr.q,\
+ vector_inner_join.q,\
vector_interval_1.q,\
vector_interval_2.q,\
+ vector_join30.q,\
+ vector_join_filters.q,\
+ vector_join_nulls.q,\
vector_left_outer_join.q,\
vector_left_outer_join2.q,\
vector_leftsemi_mapjoin.q,\
@@ -230,6 +234,12 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
vector_multi_insert.q,\
vector_non_string_partition.q,\
vector_orderby_5.q,\
+ vector_outer_join0.q,\
+ vector_outer_join1.q,\
+ vector_outer_join2.q,\
+ vector_outer_join3.q,\
+ vector_outer_join4.q,\
+ vector_outer_join5.q,\
vector_partition_diff_num_cols.q,\
vector_partitioned_date_time.q,\
vector_reduce_groupby_decimal.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
index dcea8ae..4a16b4c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
@@ -645,8 +645,7 @@ public class VectorizedBatchUtil {
public static void debugDisplayOneRow(VectorizedRowBatch batch, int index, String prefix) {
StringBuffer sb = new StringBuffer();
sb.append(prefix + " row " + index + " ");
- for (int i = 0; i < batch.projectionSize; i++) {
- int column = batch.projectedColumns[i];
+ for (int column = 0; column < batch.cols.length; column++) {
ColumnVector colVector = batch.cols[column];
if (colVector == null) {
sb.append("(null colVector " + column + ")");
@@ -666,7 +665,7 @@ public class VectorizedBatchUtil {
if (bytes == null) {
sb.append("(Unexpected null bytes with start " + start + " length " + length + ")");
} else {
- sb.append(displayBytes(bytes, start, length));
+ sb.append("bytes: '" + displayBytes(bytes, start, length) + "'");
}
} else if (colVector instanceof DecimalColumnVector) {
sb.append(((DecimalColumnVector) colVector).vector[index].toString());
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
index a9082eb..af78776 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinCommonOperator.java
@@ -470,8 +470,8 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator implem
LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor bigTableByteColumnVectorColumns " + Arrays.toString(bigTableByteColumnVectorColumns));
LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor smallTableByteColumnVectorColumns " + Arrays.toString(smallTableByteColumnVectorColumns));
- LOG.info(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor outputProjection " + Arrays.toString(outputProjection));
- LOG.info(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor outputTypeNames " + Arrays.toString(outputTypeNames));
+ LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor outputProjection " + Arrays.toString(outputProjection));
+ LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor outputTypeNames " + Arrays.toString(outputTypeNames));
}
setupVOutContext(conf.getOutputColumnNames());
@@ -503,7 +503,7 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator implem
*/
protected void setupVOutContext(List<String> outputColumnNames) {
if (LOG.isDebugEnabled()) {
- LOG.info(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor outputColumnNames " + outputColumnNames);
+ LOG.debug(taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator constructor outputColumnNames " + outputColumnNames);
}
if (outputColumnNames.size() != outputProjection.length) {
throw new RuntimeException("Output column names " + outputColumnNames + " length and output projection " + Arrays.toString(outputProjection) + " / " + Arrays.toString(outputTypeNames) + " length mismatch");
@@ -729,9 +729,9 @@ public abstract class VectorMapJoinCommonOperator extends MapJoinOperator implem
* Common one time setup by native vectorized map join operator's processOp.
*/
protected void commonSetup(VectorizedRowBatch batch) throws HiveException {
- LOG.info("VectorMapJoinInnerCommonOperator commonSetup begin...");
if (LOG.isDebugEnabled()) {
+ LOG.debug("VectorMapJoinInnerCommonOperator commonSetup begin...");
displayBatchColumns(batch, "batch");
displayBatchColumns(overflowBatch, "overflowBatch");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java
index 860ebb5..32c126c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinGenerateResultOperator.java
@@ -373,10 +373,8 @@ public abstract class VectorMapJoinGenerateResultOperator extends VectorMapJoinC
* The big table batch.
* @param hashMapResult
* The hash map results for the repeated key.
- * @return
- * The new count of selected rows.
*/
- protected int generateHashMapResultRepeatedAll(VectorizedRowBatch batch,
+ protected void generateHashMapResultRepeatedAll(VectorizedRowBatch batch,
VectorMapJoinHashMapResult hashMapResult) throws IOException, HiveException {
int[] selected = batch.selected;
@@ -400,7 +398,7 @@ public abstract class VectorMapJoinGenerateResultOperator extends VectorMapJoinC
batch.selected, 0, batch.size);
}
- return numSel;
+ batch.size = numSel;
}
//-----------------------------------------------------------------------------------------------
@@ -462,7 +460,7 @@ public abstract class VectorMapJoinGenerateResultOperator extends VectorMapJoinC
// int length = output.getLength() - offset;
rowBytesContainer.finishRow();
-// LOG.info("spillSerializeRow spilled batchIndex " + batchIndex + ", length " + length);
+// LOG.debug("spillSerializeRow spilled batchIndex " + batchIndex + ", length " + length);
}
protected void spillHashMapBatch(VectorizedRowBatch batch,
@@ -514,14 +512,18 @@ public abstract class VectorMapJoinGenerateResultOperator extends VectorMapJoinC
smallTable);
needHashTableSetup = true;
- LOG.info(CLASS_NAME + " reloadHashTable!");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(CLASS_NAME + " reloadHashTable!");
+ }
}
@Override
protected void reProcessBigTable(int partitionId)
throws HiveException {
- LOG.info(CLASS_NAME + " reProcessBigTable enter...");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(CLASS_NAME + " reProcessBigTable enter...");
+ }
if (spillReplayBatch == null) {
// The process method was not called -- no big table rows.
@@ -544,14 +546,14 @@ public abstract class VectorMapJoinGenerateResultOperator extends VectorMapJoinC
int offset = bigTable.currentOffset();
int length = bigTable.currentLength();
-// LOG.info(CLASS_NAME + " reProcessBigTable serialized row #" + rowCount + ", offset " + offset + ", length " + length);
+// LOG.debug(CLASS_NAME + " reProcessBigTable serialized row #" + rowCount + ", offset " + offset + ", length " + length);
bigTableVectorDeserializeRow.setBytes(bytes, offset, length);
bigTableVectorDeserializeRow.deserializeByValue(spillReplayBatch, spillReplayBatch.size);
spillReplayBatch.size++;
if (spillReplayBatch.size == VectorizedRowBatch.DEFAULT_SIZE) {
- LOG.info("reProcessBigTable going to call process with spillReplayBatch.size " + spillReplayBatch.size + " rows");
+ // LOG.debug("reProcessBigTable going to call process with spillReplayBatch.size " + spillReplayBatch.size + " rows");
process(spillReplayBatch, posBigTable); // call process once we have a full batch
spillReplayBatch.reset();
batchCount++;
@@ -559,7 +561,7 @@ public abstract class VectorMapJoinGenerateResultOperator extends VectorMapJoinC
}
// Process the row batch that has less than DEFAULT_SIZE rows
if (spillReplayBatch.size > 0) {
- LOG.info("reProcessBigTable going to call process with spillReplayBatch.size " + spillReplayBatch.size + " rows");
+ // LOG.debug("reProcessBigTable going to call process with spillReplayBatch.size " + spillReplayBatch.size + " rows");
process(spillReplayBatch, posBigTable);
spillReplayBatch.reset();
batchCount++;
@@ -570,7 +572,9 @@ public abstract class VectorMapJoinGenerateResultOperator extends VectorMapJoinC
throw new HiveException(e);
}
- LOG.info(CLASS_NAME + " reProcessBigTable exit! " + rowCount + " row processed and " + batchCount + " batches processed");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug(CLASS_NAME + " reProcessBigTable exit! " + rowCount + " row processed and " + batchCount + " batches processed");
+ }
}
@@ -632,7 +636,9 @@ public abstract class VectorMapJoinGenerateResultOperator extends VectorMapJoinC
if (!aborted && overflowBatch.size > 0) {
forwardOverflow();
}
- LOG.info("VectorMapJoinInnerLongOperator closeOp " + batchCounter + " batches processed");
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("VectorMapJoinInnerLongOperator closeOp " + batchCounter + " batches processed");
+ }
}
//-----------------------------------------------------------------------------------------------
@@ -641,6 +647,23 @@ public abstract class VectorMapJoinGenerateResultOperator extends VectorMapJoinC
* Debug.
*/
+ public boolean verifyMonotonicallyIncreasing(int[] selected, int size) {
+
+ if (size == 0) {
+ return true;
+ }
+ int prevBatchIndex = selected[0];
+
+ for (int i = 1; i < size; i++) {
+ int batchIndex = selected[i];
+ if (batchIndex <= prevBatchIndex) {
+ return false;
+ }
+ prevBatchIndex = batchIndex;
+ }
+ return true;
+ }
+
public static String intArrayToRangesString(int selection[], int size) {
if (size == 0) {
return "[]";
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyGenerateResultOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyGenerateResultOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyGenerateResultOperator.java
index 3132531..f18b982 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyGenerateResultOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyGenerateResultOperator.java
@@ -129,22 +129,10 @@ public abstract class VectorMapJoinInnerBigOnlyGenerateResultOperator
* @param batch
* The big table batch with any matching and any non matching rows both as
* selected in use.
- * @param allMatchs
- * A subset of the rows of the batch that are matches.
* @param allMatchCount
* Number of matches in allMatchs.
- * @param equalKeySeriesValueCounts
- * For each equal key series, whether the number of (empty) small table values.
- * @param equalKeySeriesAllMatchIndices
- * For each equal key series, the logical index into allMatchs.
- * @param equalKeySeriesDuplicateCounts
- * For each equal key series, the number of duplicates or equal keys.
* @param equalKeySeriesCount
* Number of single value matches.
- * @param spills
- * A subset of the rows of the batch that are spills.
- * @param spillHashMapResultIndices
- * For each entry in spills, the index into the hashMapResult.
* @param spillCount
* Number of spills in spills.
* @param hashTableResults
@@ -154,15 +142,16 @@ public abstract class VectorMapJoinInnerBigOnlyGenerateResultOperator
* Number of entries in hashMapResults.
*
**/
- protected int finishInnerBigOnly(VectorizedRowBatch batch,
- int[] allMatchs, int allMatchCount,
- long[] equalKeySeriesValueCounts, int[] equalKeySeriesAllMatchIndices,
- int[] equalKeySeriesDuplicateCounts, int equalKeySeriesCount,
- int[] spills, int[] spillHashMapResultIndices, int spillCount,
+ protected void finishInnerBigOnly(VectorizedRowBatch batch,
+ int allMatchCount, int equalKeySeriesCount, int spillCount,
VectorMapJoinHashTableResult[] hashTableResults, int hashMapResultCount)
throws HiveException, IOException {
- int numSel = 0;
+ // Get rid of spills before we start modifying the batch.
+ if (spillCount > 0) {
+ spillHashMapBatch(batch, hashTableResults,
+ spills, spillHashMapResultIndices, spillCount);
+ }
/*
* Optimize by running value expressions only over the matched rows.
@@ -171,6 +160,7 @@ public abstract class VectorMapJoinInnerBigOnlyGenerateResultOperator
performValueExpressions(batch, allMatchs, allMatchCount);
}
+ int numSel = 0;
for (int i = 0; i < equalKeySeriesCount; i++) {
long count = equalKeySeriesValueCounts[i];
int allMatchesIndex = equalKeySeriesAllMatchIndices[i];
@@ -185,13 +175,8 @@ public abstract class VectorMapJoinInnerBigOnlyGenerateResultOperator
duplicateCount, count);
}
}
-
- if (spillCount > 0) {
- spillHashMapBatch(batch, hashTableResults,
- spills, spillHashMapResultIndices, spillCount);
- }
-
- return numSel;
+ batch.size = numSel;
+ batch.selectedInUse = true;
}
/**
@@ -215,11 +200,11 @@ public abstract class VectorMapJoinInnerBigOnlyGenerateResultOperator
int[] allMatchs, int allMatchesIndex, int duplicateCount, int numSel)
throws HiveException, IOException {
- // LOG.info("generateHashMultiSetResultSingleValue enter...");
+ // LOG.debug("generateHashMultiSetResultSingleValue enter...");
// Generate result within big table batch itself.
- // LOG.info("generateHashMultiSetResultSingleValue with big table...");
+ // LOG.debug("generateHashMultiSetResultSingleValue with big table...");
for (int i = 0; i < duplicateCount; i++) {
@@ -250,7 +235,7 @@ public abstract class VectorMapJoinInnerBigOnlyGenerateResultOperator
int[] allMatchs, int allMatchesIndex,
int duplicateCount, long count) throws HiveException, IOException {
- // LOG.info("generateHashMultiSetResultMultiValue allMatchesIndex " + allMatchesIndex + " duplicateCount " + duplicateCount + " count " + count);
+ // LOG.debug("generateHashMultiSetResultMultiValue allMatchesIndex " + allMatchesIndex + " duplicateCount " + duplicateCount + " count " + count);
// TODO: Look at repeating optimizations...
@@ -309,11 +294,9 @@ public abstract class VectorMapJoinInnerBigOnlyGenerateResultOperator
return 0;
}
- protected int finishInnerBigOnlyRepeated(VectorizedRowBatch batch, JoinUtil.JoinResult joinResult,
+ protected void finishInnerBigOnlyRepeated(VectorizedRowBatch batch, JoinUtil.JoinResult joinResult,
VectorMapJoinHashMultiSetResult hashMultiSetResult) throws HiveException, IOException {
- int numSel = 0;
-
switch (joinResult) {
case MATCH:
@@ -325,19 +308,21 @@ public abstract class VectorMapJoinInnerBigOnlyGenerateResultOperator
}
// Generate special repeated case.
- numSel = generateHashMultiSetResultRepeatedAll(batch, hashMultiSetResult);
+ int numSel = generateHashMultiSetResultRepeatedAll(batch, hashMultiSetResult);
+ batch.size = numSel;
+ batch.selectedInUse = true;
break;
case SPILL:
// Whole batch is spilled.
spillBatchRepeated(batch, (VectorMapJoinHashTableResult) hashMultiSetResult);
+ batch.size = 0;
break;
case NOMATCH:
// No match for entire batch.
+ batch.size = 0;
break;
}
-
- return numSel;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java
index 53a91d8..bb7efda 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyLongOperator.java
@@ -151,9 +151,6 @@ public class VectorMapJoinInnerBigOnlyLongOperator extends VectorMapJoinInnerBig
}
}
- // We rebuild in-place the selected array with rows destine to be forwarded.
- int numSel = 0;
-
/*
* Single-Column Long specific declarations.
*/
@@ -198,7 +195,7 @@ public class VectorMapJoinInnerBigOnlyLongOperator extends VectorMapJoinInnerBig
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
}
- numSel = finishInnerBigOnlyRepeated(batch, joinResult, hashMultiSetResults[0]);
+ finishInnerBigOnlyRepeated(batch, joinResult, hashMultiSetResults[0]);
} else {
/*
@@ -358,17 +355,11 @@ public class VectorMapJoinInnerBigOnlyLongOperator extends VectorMapJoinInnerBig
" hashMapResults " + Arrays.toString(Arrays.copyOfRange(hashMultiSetResults, 0, hashMultiSetResultCount)));
}
- numSel = finishInnerBigOnly(batch,
- allMatchs, allMatchCount,
- equalKeySeriesValueCounts, equalKeySeriesAllMatchIndices,
- equalKeySeriesDuplicateCounts, equalKeySeriesCount,
- spills, spillHashMapResultIndices, spillCount,
+ finishInnerBigOnly(batch,
+ allMatchCount, equalKeySeriesCount, spillCount,
(VectorMapJoinHashTableResult[]) hashMultiSetResults, hashMultiSetResultCount);
}
- batch.selectedInUse = true;
- batch.size = numSel;
-
if (batch.size > 0) {
// Forward any remaining selected rows.
forwardBigTableBatch(batch);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java
index 9553fa0..c36f668 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyMultiKeyOperator.java
@@ -156,9 +156,6 @@ public class VectorMapJoinInnerBigOnlyMultiKeyOperator extends VectorMapJoinInne
}
}
- // We rebuild in-place the selected array with rows destine to be forwarded.
- int numSel = 0;
-
/*
* Multi-Key specific declarations.
*/
@@ -210,7 +207,7 @@ public class VectorMapJoinInnerBigOnlyMultiKeyOperator extends VectorMapJoinInne
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
}
- numSel = finishInnerBigOnlyRepeated(batch, joinResult, hashMultiSetResults[0]);
+ finishInnerBigOnlyRepeated(batch, joinResult, hashMultiSetResults[0]);
} else {
/*
@@ -371,17 +368,11 @@ public class VectorMapJoinInnerBigOnlyMultiKeyOperator extends VectorMapJoinInne
" hashMapResults " + Arrays.toString(Arrays.copyOfRange(hashMultiSetResults, 0, hashMultiSetResultCount)));
}
- numSel = finishInnerBigOnly(batch,
- allMatchs, allMatchCount,
- equalKeySeriesValueCounts, equalKeySeriesAllMatchIndices,
- equalKeySeriesDuplicateCounts, equalKeySeriesCount,
- spills, spillHashMapResultIndices, spillCount,
+ finishInnerBigOnly(batch,
+ allMatchCount, equalKeySeriesCount, spillCount,
(VectorMapJoinHashTableResult[]) hashMultiSetResults, hashMultiSetResultCount);
}
- batch.selectedInUse = true;
- batch.size = numSel;
-
if (batch.size > 0) {
// Forward any remaining selected rows.
forwardBigTableBatch(batch);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java
index 17d0b63..87a11c0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerBigOnlyStringOperator.java
@@ -187,7 +187,7 @@ public class VectorMapJoinInnerBigOnlyStringOperator extends VectorMapJoinInnerB
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
}
- numSel = finishInnerBigOnlyRepeated(batch, joinResult, hashMultiSetResults[0]);
+ finishInnerBigOnlyRepeated(batch, joinResult, hashMultiSetResults[0]);
} else {
/*
@@ -347,17 +347,11 @@ public class VectorMapJoinInnerBigOnlyStringOperator extends VectorMapJoinInnerB
" hashMapResults " + Arrays.toString(Arrays.copyOfRange(hashMultiSetResults, 0, hashMultiSetResultCount)));
}
- numSel = finishInnerBigOnly(batch,
- allMatchs, allMatchCount,
- equalKeySeriesValueCounts, equalKeySeriesAllMatchIndices,
- equalKeySeriesDuplicateCounts, equalKeySeriesCount,
- spills, spillHashMapResultIndices, spillCount,
+ finishInnerBigOnly(batch,
+ allMatchCount, equalKeySeriesCount, spillCount,
(VectorMapJoinHashTableResult[]) hashMultiSetResults, hashMultiSetResultCount);
}
- batch.selectedInUse = true;
- batch.size = numSel;
-
if (batch.size > 0) {
// Forward any remaining selected rows.
forwardBigTableBatch(batch);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerGenerateResultOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerGenerateResultOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerGenerateResultOperator.java
index 3a5e4b2..ee1abd3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerGenerateResultOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerGenerateResultOperator.java
@@ -147,38 +147,17 @@ public abstract class VectorMapJoinInnerGenerateResultOperator
* @param batch
* The big table batch with any matching and any non matching rows both as
* selected in use.
- * @param allMatchs
- * A subset of the rows of the batch that are matches.
* @param allMatchCount
* Number of matches in allMatchs.
- * @param equalKeySeriesHashMapResultIndices
- * For each equal key series, the index into the hashMapResult.
- * @param equalKeySeriesAllMatchIndices
- * For each equal key series, the logical index into allMatchs.
- * @param equalKeySeriesIsSingleValue
- * For each equal key series, whether there is 1 or multiple small table values.
- * @param equalKeySeriesDuplicateCounts
- * For each equal key series, the number of duplicates or equal keys.
* @param equalKeySeriesCount
* Number of single value matches.
- * @param spills
- * A subset of the rows of the batch that are spills.
- * @param spillHashMapResultIndices
- * For each entry in spills, the index into the hashMapResult.
* @param spillCount
* Number of spills in spills.
- * @param hashMapResults
- * The array of all hash map results for the batch.
* @param hashMapResultCount
* Number of entries in hashMapResults.
*/
- protected int finishInner(VectorizedRowBatch batch,
- int[] allMatchs, int allMatchCount,
- int[] equalKeySeriesHashMapResultIndices, int[] equalKeySeriesAllMatchIndices,
- boolean[] equalKeySeriesIsSingleValue, int[] equalKeySeriesDuplicateCounts,
- int equalKeySeriesCount,
- int[] spills, int[] spillHashMapResultIndices, int spillCount,
- VectorMapJoinHashMapResult[] hashMapResults, int hashMapResultCount)
+ protected void finishInner(VectorizedRowBatch batch,
+ int allMatchCount, int equalKeySeriesCount, int spillCount, int hashMapResultCount)
throws HiveException, IOException {
int numSel = 0;
@@ -211,10 +190,11 @@ public abstract class VectorMapJoinInnerGenerateResultOperator
spills, spillHashMapResultIndices, spillCount);
}
- return numSel;
+ batch.size = numSel;
+ batch.selectedInUse = true;
}
- protected int finishInnerRepeated(VectorizedRowBatch batch, JoinUtil.JoinResult joinResult,
+ protected void finishInnerRepeated(VectorizedRowBatch batch, JoinUtil.JoinResult joinResult,
VectorMapJoinHashTableResult hashMapResult) throws HiveException, IOException {
int numSel = 0;
@@ -230,22 +210,19 @@ public abstract class VectorMapJoinInnerGenerateResultOperator
}
// Generate special repeated case.
- numSel = generateHashMapResultRepeatedAll(batch, hashMapResults[0]);
+ generateHashMapResultRepeatedAll(batch, hashMapResults[0]);
break;
case SPILL:
// Whole batch is spilled.
spillBatchRepeated(batch, (VectorMapJoinHashTableResult) hashMapResults[0]);
+ batch.size = 0;
break;
case NOMATCH:
// No match for entire batch.
+ batch.size = 0;
break;
}
- /*
- * Common repeated join result processing.
- */
-
- return numSel;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java
index b77a93c..9005d00 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerLongOperator.java
@@ -149,9 +149,6 @@ public class VectorMapJoinInnerLongOperator extends VectorMapJoinInnerGenerateRe
}
}
- // We rebuild in-place the selected array with rows destine to be forwarded.
- int numSel = 0;
-
/*
* Single-Column Long specific declarations.
*/
@@ -196,7 +193,7 @@ public class VectorMapJoinInnerLongOperator extends VectorMapJoinInnerGenerateRe
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
}
- numSel = finishInnerRepeated(batch, joinResult, hashMapResults[0]);
+ finishInnerRepeated(batch, joinResult, hashMapResults[0]);
} else {
/*
@@ -356,18 +353,10 @@ public class VectorMapJoinInnerLongOperator extends VectorMapJoinInnerGenerateRe
" hashMapResults " + Arrays.toString(Arrays.copyOfRange(hashMapResults, 0, hashMapResultCount)));
}
- numSel = finishInner(batch,
- allMatchs, allMatchCount,
- equalKeySeriesHashMapResultIndices, equalKeySeriesAllMatchIndices,
- equalKeySeriesIsSingleValue, equalKeySeriesDuplicateCounts,
- equalKeySeriesCount,
- spills, spillHashMapResultIndices, spillCount,
- hashMapResults, hashMapResultCount);
+ finishInner(batch,
+ allMatchCount, equalKeySeriesCount, spillCount, hashMapResultCount);
}
- batch.selectedInUse = true;
- batch.size = numSel;
-
if (batch.size > 0) {
// Forward any remaining selected rows.
forwardBigTableBatch(batch);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java
index 938506b..b13ded6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerMultiKeyOperator.java
@@ -153,9 +153,6 @@ public class VectorMapJoinInnerMultiKeyOperator extends VectorMapJoinInnerGenera
}
}
- // We rebuild in-place the selected array with rows destine to be forwarded.
- int numSel = 0;
-
/*
* Multi-Key specific declarations.
*/
@@ -207,7 +204,7 @@ public class VectorMapJoinInnerMultiKeyOperator extends VectorMapJoinInnerGenera
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
}
- numSel = finishInnerRepeated(batch, joinResult, hashMapResults[0]);
+ finishInnerRepeated(batch, joinResult, hashMapResults[0]);
} else {
/*
@@ -279,7 +276,7 @@ public class VectorMapJoinInnerMultiKeyOperator extends VectorMapJoinInnerGenera
haveSaveKey = true;
/*
- * Multi-Key specific save key and lookup.
+ * Multi-Key specific save key.
*/
temp = saveKeyOutput;
@@ -368,18 +365,10 @@ public class VectorMapJoinInnerMultiKeyOperator extends VectorMapJoinInnerGenera
" hashMapResults " + Arrays.toString(Arrays.copyOfRange(hashMapResults, 0, hashMapResultCount)));
}
- numSel = finishInner(batch,
- allMatchs, allMatchCount,
- equalKeySeriesHashMapResultIndices, equalKeySeriesAllMatchIndices,
- equalKeySeriesIsSingleValue, equalKeySeriesDuplicateCounts,
- equalKeySeriesCount,
- spills, spillHashMapResultIndices, spillCount,
- hashMapResults, hashMapResultCount);
+ finishInner(batch,
+ allMatchCount, equalKeySeriesCount, spillCount, hashMapResultCount);
}
- batch.selectedInUse = true;
- batch.size = numSel;
-
if (batch.size > 0) {
// Forward any remaining selected rows.
forwardBigTableBatch(batch);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java
index f7dd8e2..9f10ff1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinInnerStringOperator.java
@@ -140,9 +140,6 @@ public class VectorMapJoinInnerStringOperator extends VectorMapJoinInnerGenerate
}
}
- // We rebuild in-place the selected array with rows destine to be forwarded.
- int numSel = 0;
-
/*
* Single-Column String specific declarations.
*/
@@ -185,7 +182,7 @@ public class VectorMapJoinInnerStringOperator extends VectorMapJoinInnerGenerate
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
}
- numSel = finishInnerRepeated(batch, joinResult, hashMapResults[0]);
+ finishInnerRepeated(batch, joinResult, hashMapResults[0]);
} else {
/*
@@ -345,18 +342,10 @@ public class VectorMapJoinInnerStringOperator extends VectorMapJoinInnerGenerate
" hashMapResults " + Arrays.toString(Arrays.copyOfRange(hashMapResults, 0, hashMapResultCount)));
}
- numSel = finishInner(batch,
- allMatchs, allMatchCount,
- equalKeySeriesHashMapResultIndices, equalKeySeriesAllMatchIndices,
- equalKeySeriesIsSingleValue, equalKeySeriesDuplicateCounts,
- equalKeySeriesCount,
- spills, spillHashMapResultIndices, spillCount,
- hashMapResults, hashMapResultCount);
+ finishInner(batch,
+ allMatchCount, equalKeySeriesCount, spillCount, hashMapResultCount);
}
- batch.selectedInUse = true;
- batch.size = numSel;
-
if (batch.size > 0) {
// Forward any remaining selected rows.
forwardBigTableBatch(batch);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiGenerateResultOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiGenerateResultOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiGenerateResultOperator.java
index 230f9fe..07393b2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiGenerateResultOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiGenerateResultOperator.java
@@ -111,26 +111,23 @@ public abstract class VectorMapJoinLeftSemiGenerateResultOperator
* @param batch
* The big table batch with any matching and any non matching rows both as
* selected in use.
- * @param allMatchs
- * A subset of the rows of the batch that are matches.
* @param allMatchCount
* Number of matches in allMatchs.
- * @param spills
- * A subset of the rows of the batch that are spills.
- * @param spillHashMapResultIndices
- * For each entry in spills, the index into the hashTableResults.
* @param spillCount
* Number of spills in spills.
* @param hashTableResults
* The array of all hash table results for the batch. We need the
* VectorMapJoinHashTableResult for the spill information.
*/
- protected int finishLeftSemi(VectorizedRowBatch batch,
- int[] allMatchs, int allMatchCount,
- int[] spills, int[] spillHashMapResultIndices, int spillCount,
+ protected void finishLeftSemi(VectorizedRowBatch batch,
+ int allMatchCount, int spillCount,
VectorMapJoinHashTableResult[] hashTableResults) throws HiveException, IOException {
- int numSel;
+ // Get rid of spills before we start modifying the batch.
+ if (spillCount > 0) {
+ spillHashMapBatch(batch, hashTableResults,
+ spills, spillHashMapResultIndices, spillCount);
+ }
/*
* Optimize by running value expressions only over the matched rows.
@@ -139,14 +136,9 @@ public abstract class VectorMapJoinLeftSemiGenerateResultOperator
performValueExpressions(batch, allMatchs, allMatchCount);
}
- numSel = generateHashSetResults(batch, allMatchs, allMatchCount);
-
- if (spillCount > 0) {
- spillHashMapBatch(batch, hashTableResults,
- spills, spillHashMapResultIndices, spillCount);
- }
-
- return numSel;
+ int numSel = generateHashSetResults(batch, allMatchs, allMatchCount);
+ batch.size = numSel;
+ batch.selectedInUse = true;
}
/**
@@ -199,11 +191,9 @@ public abstract class VectorMapJoinLeftSemiGenerateResultOperator
return batch.size;
}
- protected int finishLeftSemiRepeated(VectorizedRowBatch batch, JoinUtil.JoinResult joinResult,
+ protected void finishLeftSemiRepeated(VectorizedRowBatch batch, JoinUtil.JoinResult joinResult,
VectorMapJoinHashTableResult hashSetResult) throws HiveException, IOException {
- int numSel = 0;
-
switch (joinResult) {
case MATCH:
@@ -215,19 +205,21 @@ public abstract class VectorMapJoinLeftSemiGenerateResultOperator
}
// Generate special repeated case.
- numSel = generateHashSetResultRepeatedAll(batch);
+ int numSel = generateHashSetResultRepeatedAll(batch);
+ batch.size = numSel;
+ batch.selectedInUse = true;
break;
case SPILL:
// Whole batch is spilled.
spillBatchRepeated(batch, (VectorMapJoinHashTableResult) hashSetResult);
+ batch.size = 0;
break;
case NOMATCH:
// No match for entire batch.
+ batch.size = 0;
break;
}
-
- return numSel;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java
index 75aeefb..712978a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiLongOperator.java
@@ -151,9 +151,6 @@ public class VectorMapJoinLeftSemiLongOperator extends VectorMapJoinLeftSemiGene
}
}
- // We rebuild in-place the selected array with rows destine to be forwarded.
- int numSel = 0;
-
/*
* Single-Column Long specific declarations.
*/
@@ -198,7 +195,7 @@ public class VectorMapJoinLeftSemiLongOperator extends VectorMapJoinLeftSemiGene
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
}
- numSel = finishLeftSemiRepeated(batch, joinResult, hashSetResults[0]);
+ finishLeftSemiRepeated(batch, joinResult, hashSetResults[0]);
} else {
/*
@@ -348,15 +345,11 @@ public class VectorMapJoinLeftSemiLongOperator extends VectorMapJoinLeftSemiGene
" hashMapResults " + Arrays.toString(Arrays.copyOfRange(hashSetResults, 0, hashSetResultCount)));
}
- numSel = finishLeftSemi(batch,
- allMatchs, allMatchCount,
- spills, spillHashMapResultIndices, spillCount,
+ finishLeftSemi(batch,
+ allMatchCount, spillCount,
(VectorMapJoinHashTableResult[]) hashSetResults);
}
- batch.selectedInUse = true;
- batch.size = numSel;
-
if (batch.size > 0) {
// Forward any remaining selected rows.
forwardBigTableBatch(batch);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java
index ea287f4..b941431 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiMultiKeyOperator.java
@@ -155,9 +155,6 @@ public class VectorMapJoinLeftSemiMultiKeyOperator extends VectorMapJoinLeftSemi
}
}
- // We rebuild in-place the selected array with rows destine to be forwarded.
- int numSel = 0;
-
/*
* Multi-Key specific declarations.
*/
@@ -210,7 +207,7 @@ public class VectorMapJoinLeftSemiMultiKeyOperator extends VectorMapJoinLeftSemi
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
}
- numSel = finishLeftSemiRepeated(batch, joinResult, hashSetResults[0]);
+ finishLeftSemiRepeated(batch, joinResult, hashSetResults[0]);
} else {
/*
@@ -291,6 +288,10 @@ public class VectorMapJoinLeftSemiMultiKeyOperator extends VectorMapJoinLeftSemi
saveKeyOutput = currentKeyOutput;
currentKeyOutput = temp;
+ /*
+ * Multi-key specific lookup key.
+ */
+
byte[] keyBytes = saveKeyOutput.getData();
int keyLength = saveKeyOutput.getLength();
saveJoinResult = hashSet.contains(keyBytes, 0, keyLength, hashSetResults[hashSetResultCount]);
@@ -360,15 +361,11 @@ public class VectorMapJoinLeftSemiMultiKeyOperator extends VectorMapJoinLeftSemi
" hashMapResults " + Arrays.toString(Arrays.copyOfRange(hashSetResults, 0, hashSetResultCount)));
}
- numSel = finishLeftSemi(batch,
- allMatchs, allMatchCount,
- spills, spillHashMapResultIndices, spillCount,
+ finishLeftSemi(batch,
+ allMatchCount, spillCount,
(VectorMapJoinHashTableResult[]) hashSetResults);
}
- batch.selectedInUse = true;
- batch.size = numSel;
-
if (batch.size > 0) {
// Forward any remaining selected rows.
forwardBigTableBatch(batch);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java
index 116cb81..9ff1141 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinLeftSemiStringOperator.java
@@ -142,9 +142,6 @@ public class VectorMapJoinLeftSemiStringOperator extends VectorMapJoinLeftSemiGe
}
}
- // We rebuild in-place the selected array with rows destine to be forwarded.
- int numSel = 0;
-
/*
* Single-Column String specific declarations.
*/
@@ -187,7 +184,7 @@ public class VectorMapJoinLeftSemiStringOperator extends VectorMapJoinLeftSemiGe
if (LOG.isDebugEnabled()) {
LOG.debug(CLASS_NAME + " batch #" + batchCounter + " repeated joinResult " + joinResult.name());
}
- numSel = finishLeftSemiRepeated(batch, joinResult, hashSetResults[0]);
+ finishLeftSemiRepeated(batch, joinResult, hashSetResults[0]);
} else {
/*
@@ -263,6 +260,10 @@ public class VectorMapJoinLeftSemiStringOperator extends VectorMapJoinLeftSemiGe
saveKeyBatchIndex = batchIndex;
+ /*
+ * Single-Column String specific lookup key.
+ */
+
byte[] keyBytes = vector[batchIndex];
int keyStart = start[batchIndex];
int keyLength = length[batchIndex];
@@ -333,15 +334,11 @@ public class VectorMapJoinLeftSemiStringOperator extends VectorMapJoinLeftSemiGe
" hashMapResults " + Arrays.toString(Arrays.copyOfRange(hashSetResults, 0, hashSetResultCount)));
}
- numSel = finishLeftSemi(batch,
- allMatchs, allMatchCount,
- spills, spillHashMapResultIndices, spillCount,
+ finishLeftSemi(batch,
+ allMatchCount, spillCount,
(VectorMapJoinHashTableResult[]) hashSetResults);
}
- batch.selectedInUse = true;
- batch.size = numSel;
-
if (batch.size > 0) {
// Forward any remaining selected rows.
forwardBigTableBatch(batch);
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterGenerateResultOperator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterGenerateResultOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterGenerateResultOperator.java
index 7ef5574..57814fd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterGenerateResultOperator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/mapjoin/VectorMapJoinOuterGenerateResultOperator.java
@@ -70,15 +70,34 @@ public abstract class VectorMapJoinOuterGenerateResultOperator
// generation.
protected transient VectorMapJoinHashMapResult hashMapResults[];
- // Pre-allocated member for storing any matching row indexes during a processOp call.
- protected transient int[] matchs;
+ // Pre-allocated member for remembering the big table's selected array at the beginning of
+ // the process method before applying any filter. For outer join we need to remember which
+ // rows did not match since they will appear the in outer join result with NULLs for the
+ // small table.
+ protected transient int[] inputSelected;
- // Pre-allocated member for storing the mapping to the row batchIndex of the first of a series of
- // equal keys that was looked up during a processOp call.
- protected transient int[] matchHashMapResultIndices;
+ // Pre-allocated member for storing the (physical) batch index of matching row (single- or
+ // multi-small-table-valued) indexes during a process call.
+ protected transient int[] allMatchs;
- // All matching and non-matching big table rows.
- protected transient int[] nonSpills;
+ /*
+ * Pre-allocated members for storing information equal key series for small-table matches.
+ *
+ * ~HashMapResultIndices
+ * Index into the hashMapResults array for the match.
+ * ~AllMatchIndices
+ * (Logical) indices into allMatchs to the first row of a match of a
+ * possible series of duplicate keys.
+ * ~IsSingleValue
+ * Whether there is 1 or multiple small table values.
+ * ~DuplicateCounts
+ * The duplicate count for each matched key.
+ *
+ */
+ protected transient int[] equalKeySeriesHashMapResultIndices;
+ protected transient int[] equalKeySeriesAllMatchIndices;
+ protected transient boolean[] equalKeySeriesIsSingleValue;
+ protected transient int[] equalKeySeriesDuplicateCounts;
// Pre-allocated member for storing the (physical) batch index of rows that need to be spilled.
protected transient int[] spills;
@@ -86,8 +105,11 @@ public abstract class VectorMapJoinOuterGenerateResultOperator
// Pre-allocated member for storing index into the hashSetResults for each spilled row.
protected transient int[] spillHashMapResultIndices;
- // Pre-allocated member for storing any non-matching row indexes during a processOp call.
- protected transient int[] scratch1;
+ // Pre-allocated member for storing any non-spills, non-matches, or merged row indexes during a
+ // process method call.
+ protected transient int[] nonSpills;
+ protected transient int[] noMatchs;
+ protected transient int[] merged;
public VectorMapJoinOuterGenerateResultOperator() {
super();
@@ -111,12 +133,23 @@ public abstract class VectorMapJoinOuterGenerateResultOperator
for (int i = 0; i < hashMapResults.length; i++) {
hashMapResults[i] = baseHashMap.createHashMapResult();
}
- matchs = new int[batch.DEFAULT_SIZE];
- matchHashMapResultIndices = new int[batch.DEFAULT_SIZE];
- nonSpills = new int[batch.DEFAULT_SIZE];
+
+ inputSelected = new int[batch.DEFAULT_SIZE];
+
+ allMatchs = new int[batch.DEFAULT_SIZE];
+
+ equalKeySeriesHashMapResultIndices = new int[batch.DEFAULT_SIZE];
+ equalKeySeriesAllMatchIndices = new int[batch.DEFAULT_SIZE];
+ equalKeySeriesIsSingleValue = new boolean[batch.DEFAULT_SIZE];
+ equalKeySeriesDuplicateCounts = new int[batch.DEFAULT_SIZE];
+
spills = new int[batch.DEFAULT_SIZE];
spillHashMapResultIndices = new int[batch.DEFAULT_SIZE];
- scratch1 = new int[batch.DEFAULT_SIZE];
+
+ nonSpills = new int[batch.DEFAULT_SIZE];
+ noMatchs = new int[batch.DEFAULT_SIZE];
+ merged = new int[batch.DEFAULT_SIZE];
+
}
//-----------------------------------------------------------------------------------------------
@@ -145,260 +178,372 @@ public abstract class VectorMapJoinOuterGenerateResultOperator
}
/**
- * Generate the outer join output results for one vectorized row batch.
- *
- * Any filter expressions will apply now since hash map lookup for outer join is complete.
+ * Apply the value expression to rows in the (original) input selected array.
*
* @param batch
- * The big table batch with any matching and any non matching rows both as
- * selected in use.
- * @param matchs
- * A subset of the rows of the batch that are matches.
- * @param matchHashMapResultIndices
- * For each entry in matches, the index into the hashMapResult.
- * @param matchSize
- * Number of matches in matchs.
- * @param nonSpills
- * The rows of the batch that are both matches and non-matches.
- * @param nonspillCount
- * Number of rows in nonSpills.
- * @param spills
- * A subset of the rows of the batch that are spills.
- * @param spillHashMapResultIndices
- * For each entry in spills, the index into the hashMapResult.
- * @param spillCount
- * Number of spills in spills.
- * @param hashMapResults
- * The array of all hash map results for the batch.
- * @param hashMapResultCount
- * Number of entries in hashMapResults.
- * @param scratch1
- * Pre-allocated storage to internal use.
+ * The vectorized row batch.
+ * @param inputSelectedInUse
+ * Whether the (original) input batch is selectedInUse.
+ * @param inputLogicalSize
+ * The (original) input batch size.
*/
- public int finishOuter(VectorizedRowBatch batch,
- int[] matchs, int[] matchHashMapResultIndices, int matchCount,
- int[] nonSpills, int nonSpillCount,
- int[] spills, int[] spillHashMapResultIndices, int spillCount,
- VectorMapJoinHashMapResult[] hashMapResults, int hashMapResultCount,
- int[] scratch1) throws IOException, HiveException {
-
- int numSel = 0;
-
- // At this point we have determined the matching rows only for the ON equality condition(s).
- // Implicitly, non-matching rows are those in the selected array minus matchs.
+ private void doValueExprOnInputSelected(VectorizedRowBatch batch,
+ boolean inputSelectedInUse, int inputLogicalSize) {
- // Next, for outer join, apply any ON predicates to filter down the matches.
- if (matchCount > 0 && bigTableFilterExpressions.length > 0) {
+ int saveBatchSize = batch.size;
+ int[] saveSelected = batch.selected;
+ boolean saveSelectedInUse = batch.selectedInUse;
- System.arraycopy(matchs, 0, batch.selected, 0, matchCount);
- batch.size = matchCount;
+ batch.size = inputLogicalSize;
+ batch.selected = inputSelected;
+ batch.selectedInUse = inputSelectedInUse;
- // Non matches will be removed from the selected array.
- for (VectorExpression ve : bigTableFilterExpressions) {
+ if (bigTableValueExpressions != null) {
+ for(VectorExpression ve: bigTableValueExpressions) {
ve.evaluate(batch);
}
+ }
- // LOG.info("finishOuter" +
- // " filtered batch.selected " + Arrays.toString(Arrays.copyOfRange(batch.selected, 0, batch.size)));
-
- // Fixup the matchHashMapResultIndices array.
- if (batch.size < matchCount) {
- int numMatch = 0;
- int[] selected = batch.selected;
- for (int i = 0; i < batch.size; i++) {
- if (selected[i] == matchs[numMatch]) {
- matchHashMapResultIndices[numMatch] = matchHashMapResultIndices[i];
- numMatch++;
- if (numMatch == matchCount) {
- break;
- }
- }
- }
- System.arraycopy(batch.selected, 0, matchs, 0, matchCount);
+ batch.size = saveBatchSize;
+ batch.selected = saveSelected;
+ batch.selectedInUse = saveSelectedInUse;
+ }
+
+ /**
+ * Apply the value expression to rows specified by a selected array.
+ *
+ * @param batch
+ * The vectorized row batch.
+ * @param selected
+ * The (physical) batch indices to apply the expression to.
+ * @param size
+ * The size of selected.
+ */
+ private void doValueExpr(VectorizedRowBatch batch,
+ int[] selected, int size) {
+
+ int saveBatchSize = batch.size;
+ int[] saveSelected = batch.selected;
+ boolean saveSelectedInUse = batch.selectedInUse;
+
+ batch.size = size;
+ batch.selected = selected;
+ batch.selectedInUse = true;
+
+ if (bigTableValueExpressions != null) {
+ for(VectorExpression ve: bigTableValueExpressions) {
+ ve.evaluate(batch);
}
}
- // LOG.info("finishOuter" +
- // " matchs[" + matchCount + "] " + intArrayToRangesString(matchs, matchCount) +
- // " matchHashMapResultIndices " + Arrays.toString(Arrays.copyOfRange(matchHashMapResultIndices, 0, matchCount)));
- // Big table value expressions apply to ALL matching and non-matching rows.
- if (bigTableValueExpressions != null) {
+ batch.size = saveBatchSize;
+ batch.selected = saveSelected;
+ batch.selectedInUse = saveSelectedInUse;
+ }
- System.arraycopy(nonSpills, 0, batch.selected, 0, nonSpillCount);
- batch.size = nonSpillCount;
+ /**
+ * Remove (subtract) members from the input selected array and produce the results into
+ * a difference array.
+ *
+ * @param inputSelectedInUse
+ * Whether the (original) input batch is selectedInUse.
+ * @param inputLogicalSize
+ * The (original) input batch size.
+ * @param remove
+ * The indices to remove. They must all be present in input selected array.
+ * @param removeSize
+ * The size of remove.
+ * @param difference
+ * The resulting difference -- the input selected array indices not in the
+ * remove array.
+ * @return
+ * The resulting size of the difference array.
+ * @throws HiveException
+ */
+ private int subtractFromInputSelected(boolean inputSelectedInUse, int inputLogicalSize,
+ int[] remove, int removeSize, int[] difference) throws HiveException {
- for (VectorExpression ve: bigTableValueExpressions) {
- ve.evaluate(batch);
+ // if (!verifyMonotonicallyIncreasing(remove, removeSize)) {
+ // throw new HiveException("remove is not in sort order and unique");
+ // }
+
+ int differenceCount = 0;
+
+ // Determine which rows are left.
+ int removeIndex = 0;
+ if (inputSelectedInUse) {
+ for (int i = 0; i < inputLogicalSize; i++) {
+ int candidateIndex = inputSelected[i];
+ if (removeIndex < removeSize && candidateIndex == remove[removeIndex]) {
+ removeIndex++;
+ } else {
+ difference[differenceCount++] = candidateIndex;
+ }
+ }
+ } else {
+ for (int candidateIndex = 0; candidateIndex < inputLogicalSize; candidateIndex++) {
+ if (removeIndex < removeSize && candidateIndex == remove[removeIndex]) {
+ removeIndex++;
+ } else {
+ difference[differenceCount++] = candidateIndex;
+ }
+ }
+ }
+
+ if (removeIndex != removeSize) {
+ throw new HiveException("Not all batch indices removed");
+ }
+
+ // if (!verifyMonotonicallyIncreasing(difference, differenceCount)) {
+ // throw new HiveException("difference is not in sort order and unique");
+ // }
+
+ return differenceCount;
+ }
+
+ /**
+ * Remove (subtract) members from an array and produce the results into
+ * a difference array.
+
+ * @param all
+ * The selected array containing all members.
+ * @param allSize
+ * The size of all.
+ * @param remove
+ * The indices to remove. They must all be present in input selected array.
+ * @param removeSize
+ * The size of remove.
+ * @param difference
+ * The resulting difference -- the all array indices not in the
+ * remove array.
+ * @return
+ * The resulting size of the difference array.
+ * @throws HiveException
+ */
+ private int subtract(int[] all, int allSize,
+ int[] remove, int removeSize, int[] difference) throws HiveException {
+
+ // if (!verifyMonotonicallyIncreasing(remove, removeSize)) {
+ // throw new HiveException("remove is not in sort order and unique");
+ // }
+
+ int differenceCount = 0;
+
+ // Determine which rows are left.
+ int removeIndex = 0;
+ for (int i = 0; i < allSize; i++) {
+ int candidateIndex = all[i];
+ if (removeIndex < removeSize && candidateIndex == remove[removeIndex]) {
+ removeIndex++;
+ } else {
+ difference[differenceCount++] = candidateIndex;
}
}
- // Determine which rows are non matches by determining the delta between selected and
- // matchs.
- int[] noMatchs = scratch1;
- int noMatchCount = 0;
- if (matchCount < nonSpillCount) {
- // Determine which rows are non matches.
- int matchIndex = 0;
- for (int i = 0; i < nonSpillCount; i++) {
- int candidateIndex = nonSpills[i];
- if (matchIndex < matchCount && candidateIndex == matchs[matchIndex]) {
- matchIndex++;
+ if (removeIndex != removeSize) {
+ throw new HiveException("Not all batch indices removed");
+ }
+
+ return differenceCount;
+ }
+
+ /**
+ * Sort merge two select arrays so the resulting array is ordered by (batch) index.
+ *
+ * @param selected1
+ * @param selected1Count
+ * @param selected2
+ * @param selected2Count
+ * @param sortMerged
+ * The resulting sort merge of selected1 and selected2.
+ * @return
+ * The resulting size of the sortMerged array.
+ * @throws HiveException
+ */
+ private int sortMerge(int[] selected1, int selected1Count,
+ int[] selected2, int selected2Count, int[] sortMerged) throws HiveException {
+
+ // if (!verifyMonotonicallyIncreasing(selected1, selected1Count)) {
+ // throw new HiveException("selected1 is not in sort order and unique");
+ // }
+
+ // if (!verifyMonotonicallyIncreasing(selected2, selected2Count)) {
+ // throw new HiveException("selected1 is not in sort order and unique");
+ // }
+
+
+ int sortMergeCount = 0;
+
+ int selected1Index = 0;
+ int selected2Index = 0;
+ for (int i = 0; i < selected1Count + selected2Count; i++) {
+ if (selected1Index < selected1Count && selected2Index < selected2Count) {
+ if (selected1[selected1Index] < selected2[selected2Index]) {
+ sortMerged[sortMergeCount++] = selected1[selected1Index++];
} else {
- noMatchs[noMatchCount++] = candidateIndex;
+ sortMerged[sortMergeCount++] = selected2[selected2Index++];
}
+ } else if (selected1Index < selected1Count) {
+ sortMerged[sortMergeCount++] = selected1[selected1Index++];
+ } else {
+ sortMerged[sortMergeCount++] = selected2[selected2Index++];
}
}
- // LOG.info("finishOuter" +
- // " noMatchs[" + noMatchCount + "] " + intArrayToRangesString(noMatchs, noMatchCount));
+ // if (!verifyMonotonicallyIncreasing(sortMerged, sortMergeCount)) {
+ // throw new HiveException("sortMerged is not in sort order and unique");
+ // }
- // When we generate results into the overflow batch, we may still end up with fewer rows
- // in the big table batch. So, nulSel and the batch's selected array will be rebuilt with
- // just the big table rows that need to be forwarded, minus any rows processed with the
- // overflow batch.
- if (matchCount > 0) {
- numSel = generateOuterHashMapMatchResults(batch,
- matchs, matchHashMapResultIndices, matchCount,
- hashMapResults, numSel);
- }
+ return sortMergeCount;
+ }
- if (noMatchCount > 0) {
- numSel = generateOuterHashMapNoMatchResults(batch, noMatchs, noMatchCount, numSel);
- }
+ /**
+ * Generate the outer join output results for one vectorized row batch.
+ *
+ * @param batch
+ * The big table batch with any matching and any non matching rows both as
+ * selected in use.
+ * @param allMatchCount
+ * Number of matches in allMatchs.
+ * @param equalKeySeriesCount
+ * Number of single value matches.
+ * @param atLeastOneNonMatch
+ * Whether at least one row was a non-match.
+ * @param inputSelectedInUse
+ * A copy of the batch's selectedInUse flag on input to the process method.
+ * @param inputLogicalSize
+ * The batch's size on input to the process method.
+ * @param spillCount
+ * Number of spills in spills.
+ * @param hashMapResultCount
+ * Number of entries in hashMapResults.
+ */
+ public void finishOuter(VectorizedRowBatch batch,
+ int allMatchCount, int equalKeySeriesCount, boolean atLeastOneNonMatch,
+ boolean inputSelectedInUse, int inputLogicalSize,
+ int spillCount, int hashMapResultCount) throws IOException, HiveException {
+ // Get rid of spills before we start modifying the batch.
if (spillCount > 0) {
spillHashMapBatch(batch, (VectorMapJoinHashTableResult[]) hashMapResults,
spills, spillHashMapResultIndices, spillCount);
}
- return numSel;
- }
-
- /**
- * Generate the matching outer join output results for one row of a vectorized row batch into
- * the overflow batch.
- *
- * @param batch
- * The big table batch.
- * @param batchIndex
- * Index of the big table row.
- * @param hashMapResult
- * The hash map result with the small table values.
- */
- private void copyOuterHashMapResultToOverflow(VectorizedRowBatch batch, int batchIndex,
- VectorMapJoinHashMapResult hashMapResult) throws HiveException, IOException {
-
- // if (hashMapResult.isCappedCountAvailable()) {
- // LOG.info("copyOuterHashMapResultToOverflow cappedCount " + hashMapResult.cappedCount());
- // }
- ByteSegmentRef byteSegmentRef = hashMapResult.first();
- while (byteSegmentRef != null) {
-
- // Copy the BigTable values into the overflow batch. Since the overflow batch may
- // not get flushed here, we must copy by value.
- if (bigTableRetainedVectorCopy != null) {
- bigTableRetainedVectorCopy.copyByValue(batch, batchIndex,
- overflowBatch, overflowBatch.size);
- }
-
- // Reference the keys we just copied above.
- if (bigTableVectorCopyOuterKeys != null) {
- bigTableVectorCopyOuterKeys.copyByReference(overflowBatch, overflowBatch.size,
- overflowBatch, overflowBatch.size);
- }
-
- if (smallTableVectorDeserializeRow != null) {
-
- byte[] bytes = byteSegmentRef.getBytes();
- int offset = (int) byteSegmentRef.getOffset();
- int length = byteSegmentRef.getLength();
- smallTableVectorDeserializeRow.setBytes(bytes, offset, length);
-
- smallTableVectorDeserializeRow.deserializeByValue(overflowBatch, overflowBatch.size);
- }
+ int noMatchCount = 0;
+ if (spillCount > 0) {
- ++overflowBatch.size;
- if (overflowBatch.size == VectorizedRowBatch.DEFAULT_SIZE) {
- forwardOverflow();
- }
+ // Subtract the spills to get all match and non-match rows.
+ int nonSpillCount = subtractFromInputSelected(
+ inputSelectedInUse, inputLogicalSize, spills, spillCount, nonSpills);
- byteSegmentRef = hashMapResult.next();
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("finishOuter spillCount > 0" +
+ " nonSpills " + intArrayToRangesString(nonSpills, nonSpillCount));
+ }
+
+ // Big table value expressions apply to ALL matching and non-matching rows.
+ if (bigTableValueExpressions != null) {
+
+ doValueExpr(batch, nonSpills, nonSpillCount);
+
}
- // LOG.info("copyOuterHashMapResultToOverflow overflowBatch.size " + overflowBatch.size);
+
+ if (atLeastOneNonMatch) {
+ noMatchCount = subtract(nonSpills, nonSpillCount, allMatchs, allMatchCount,
+ noMatchs);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("finishOuter spillCount > 0" +
+ " noMatchs " + intArrayToRangesString(noMatchs, noMatchCount));
+ }
- }
+ }
+ } else {
- /**
- * Generate the matching outer join output results for one vectorized row batch.
- *
- * For each matching row specified by parameter, get the one or more small table values and
- * form join results.
- *
- * (Note: Since all matching and non-matching rows are selected and output for outer joins,
- * we cannot use selected as the matching rows).
- *
- * @param batch
- * The big table batch with any matching and any non matching rows both as
- * selected in use.
- * @param matchs
- * A subset of the rows of the batch that are matches.
- * @param matchHashMapResultIndices
- * For each entry in matches, the index into the hashMapResult.
- * @param matchSize
- * Number of matches in matchs.
- * @param hashMapResults
- * The array of all hash map results for the batch.
- * @param numSel
- * The current count of rows in the rebuilding of the selected array.
- *
- * @return
- * The new count of selected rows.
- */
- protected int generateOuterHashMapMatchResults(VectorizedRowBatch batch,
- int[] matchs, int[] matchHashMapResultIndices, int matchSize,
- VectorMapJoinHashMapResult[] hashMapResults, int numSel)
- throws IOException, HiveException {
+ // Run value expressions over original (whole) input batch.
+ doValueExprOnInputSelected(batch, inputSelectedInUse, inputLogicalSize);
- int[] selected = batch.selected;
+ if (atLeastOneNonMatch) {
+ noMatchCount = subtractFromInputSelected(
+ inputSelectedInUse, inputLogicalSize, allMatchs, allMatchCount, noMatchs);
- // Generate result within big table batch when single small table value. Otherwise, copy
- // to overflow batch.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("finishOuter spillCount == 0" +
+ " noMatchs " + intArrayToRangesString(noMatchs, noMatchCount));
+ }
+ }
+ }
- for (int i = 0; i < matchSize; i++) {
- int batchIndex = matchs[i];
+ // When we generate results into the overflow batch, we may still end up with fewer rows
+ // in the big table batch. So, nulSel and the batch's selected array will be rebuilt with
+ // just the big table rows that need to be forwarded, minus any rows processed with the
+ // overflow batch.
+ if (allMatchCount > 0) {
+
+ int numSel = 0;
+ for (int i = 0; i < equalKeySeriesCount; i++) {
+ int hashMapResultIndex = equalKeySeriesHashMapResultIndices[i];
+ VectorMapJoinHashMapResult hashMapResult = hashMapResults[hashMapResultIndex];
+ int allMatchesIndex = equalKeySeriesAllMatchIndices[i];
+ boolean isSingleValue = equalKeySeriesIsSingleValue[i];
+ int duplicateCount = equalKeySeriesDuplicateCounts[i];
+
+ if (isSingleValue) {
+ numSel = generateHashMapResultSingleValue(
+ batch, hashMapResult, allMatchs, allMatchesIndex, duplicateCount, numSel);
+ } else {
+ generateHashMapResultMultiValue(
+ batch, hashMapResult, allMatchs, allMatchesIndex, duplicateCount);
+ }
+ }
- int hashMapResultIndex = matchHashMapResultIndices[i];
- VectorMapJoinHashMapResult hashMapResult = hashMapResults[hashMapResultIndex];
+ // The number of single value rows that were generated in the big table batch.
+ batch.size = numSel;
+ batch.selectedInUse = true;
- if (!hashMapResult.isSingleRow()) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("finishOuter allMatchCount > 0" +
+ " batch.selected " + intArrayToRangesString(batch.selected, batch.size));
+ }
- // Multiple small table rows require use of the overflow batch.
- copyOuterHashMapResultToOverflow(batch, batchIndex, hashMapResult);
- } else {
+ } else {
+ batch.size = 0;
+ }
- // Generate join result in big table batch.
- ByteSegmentRef byteSegmentRef = hashMapResult.first();
+ if (noMatchCount > 0) {
+ if (batch.size > 0) {
+
+ generateOuterNulls(batch, noMatchs, noMatchCount);
+
+ // Merge noMatchs and (match) selected.
+ int mergeCount = sortMerge(
+ noMatchs, noMatchCount, batch.selected, batch.size, merged);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("finishOuter noMatchCount > 0 && batch.size > 0" +
+ " merged " + intArrayToRangesString(merged, mergeCount));
+ }
- if (bigTableVectorCopyOuterKeys != null) {
- bigTableVectorCopyOuterKeys.copyByReference(batch, batchIndex, batch, batchIndex);
- }
+ System.arraycopy(merged, 0, batch.selected, 0, mergeCount);
+ batch.size = mergeCount;
+ batch.selectedInUse = true;
+ } else {
- if (smallTableVectorDeserializeRow != null) {
+ // We can use the whole batch for output of no matches.
- byte[] bytes = byteSegmentRef.getBytes();
- int offset = (int) byteSegmentRef.getOffset();
- int length = byteSegmentRef.getLength();
- smallTableVectorDeserializeRow.setBytes(bytes, offset, length);
+ generateOuterNullsRepeatedAll(batch);
- smallTableVectorDeserializeRow.deserializeByValue(batch, batchIndex);
- }
+ System.arraycopy(noMatchs, 0, batch.selected, 0, noMatchCount);
+ batch.size = noMatchCount;
+ batch.selectedInUse = true;
- // Remember this big table row was used for an output result.
- selected[numSel++] = batchIndex;
- }
- }
- return numSel;
- }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("finishOuter noMatchCount > 0 && batch.size == 0" +
+ " batch.selected " + intArrayToRangesString(batch.selected, batch.size));
+ }
+ }
+ }
+ }
/**
* Generate the non matching outer join output results for one vectorized row batch.
@@ -412,72 +557,30 @@ public abstract class VectorMapJoinOuterGenerateResultOperator
* A subset of the rows of the batch that are non matches.
* @param noMatchSize
* Number of non matches in noMatchs.
- * @param numSel
- * The current count of rows in the rebuilding of the selected array.
- *
- * @return
- * The new count of selected rows.
*/
- protected int generateOuterHashMapNoMatchResults(VectorizedRowBatch batch, int[] noMatchs,
- int noMatchSize, int numSel) throws IOException, HiveException {
- int[] selected = batch.selected;
-
- // Generate result within big table batch with null small table results, using isRepeated
- // if possible.
+ protected void generateOuterNulls(VectorizedRowBatch batch, int[] noMatchs,
+ int noMatchSize) throws IOException, HiveException {
- if (numSel == 0) {
+ // Set null information in the small table results area.
- // There were 0 matching rows -- so we can use the isRepeated optimization for the non
- // matching rows.
+ for (int i = 0; i < noMatchSize; i++) {
+ int batchIndex = noMatchs[i];
// Mark any scratch small table scratch columns that would normally receive a copy of the
- // key as null and repeating.
+ // key as null, too.
for (int column : bigTableOuterKeyOutputVectorColumns) {
ColumnVector colVector = batch.cols[column];
- colVector.isRepeating = true;
colVector.noNulls = false;
- colVector.isNull[0] = true;
+ colVector.isNull[batchIndex] = true;
}
- // Small table values are set to null and repeating.
+ // Small table values are set to null.
for (int column : smallTableOutputVectorColumns) {
ColumnVector colVector = batch.cols[column];
- colVector.isRepeating = true;
colVector.noNulls = false;
- colVector.isNull[0] = true;
- }
-
- // Rebuild the selected array.
- for (int i = 0; i < noMatchSize; i++) {
- int batchIndex = noMatchs[i];
- selected[numSel++] = batchIndex;
- }
- } else {
-
- // Set null information in the small table results area.
-
- for (int i = 0; i < noMatchSize; i++) {
- int batchIndex = noMatchs[i];
-
- // Mark any scratch small table scratch columns that would normally receive a copy of the
- // key as null, too.
- for (int column : bigTableOuterKeyOutputVectorColumns) {
- ColumnVector colVector = batch.cols[column];
- colVector.noNulls = false;
- colVector.isNull[batchIndex] = true;
- }
-
- // Small table values are set to null.
- for (int column : smallTableOutputVectorColumns) {
- ColumnVector colVector = batch.cols[column];
- colVector.noNulls = false;
- colVector.isNull[batchIndex] = true;
- }
-
- selected[numSel++] = batchIndex;
+ colVector.isNull[batchIndex] = true;
}
}
- return numSel;
}
/**
@@ -492,65 +595,114 @@ public abstract class VectorMapJoinOuterGenerateResultOperator
* The hash map lookup result for the repeated key.
* @param hashMapResults
* The array of all hash map results for the batch.
+ * @param someRowsFilteredOut
+ * Whether some rows of the repeated key batch were knocked out by the filter.
+ * @param inputSelectedInUse
+ * A copy of the batch's selectedInUse flag on input to the process method.
+ * @param inputLogicalSize
+ * The batch's size on input to the process method.
* @param scratch1
* Pre-allocated storage to internal use.
+ * @param scratch2
+ * Pre-allocated storage to internal use.
*/
- public int finishOuterRepeated(VectorizedRowBatch batch, JoinUtil.JoinResult joinResult,
- VectorMapJoinHashMapResult hashMapResult, int[] scratch1)
+ public void finishOuterRepeated(VectorizedRowBatch batch, JoinUtil.JoinResult joinResult,
+ VectorMapJoinHashMapResult hashMapResult, boolean someRowsFilteredOut,
+ boolean inputSelectedInUse, int inputLogicalSize)
throws IOException, HiveException {
- int numSel = 0;
+ // LOG.debug("finishOuterRepeated batch #" + batchCounter + " " + joinResult.name() + " batch.size " + batch.size + " someRowsFilteredOut " + someRowsFilteredOut);
- if (joinResult == JoinUtil.JoinResult.MATCH && bigTableFilterExpressions.length > 0) {
+ switch (joinResult) {
+ case MATCH:
- // Since it is repeated, the evaluation of the filter will knock the whole batch out.
- // But since we are doing outer join, we want to keep non-matches.
+ // Rows we looked up as one repeated key are a match. But filtered out rows
+ // need to be generated as non-matches, too.
- // First, remember selected;
- int[] rememberSelected = scratch1;
- int rememberBatchSize = batch.size;
- if (batch.selectedInUse) {
- System.arraycopy(batch.selected, 0, rememberSelected, 0, batch.size);
- }
+ if (someRowsFilteredOut) {
- // Filter.
- for (VectorExpression ve : bigTableFilterExpressions) {
- ve.evaluate(batch);
- }
+ // For the filtered out rows that didn't (logically) get looked up in the hash table,
+ // we need to generate no match results for those too...
- // Convert a filter out to a non match.
- if (batch.size == 0) {
- joinResult = JoinUtil.JoinResult.NOMATCH;
- if (batch.selectedInUse) {
- System.arraycopy(rememberSelected, 0, batch.selected, 0, rememberBatchSize);
- // LOG.info("finishOuterRepeated batch #" + batchCounter + " filter out converted to no matchs " +
- // Arrays.toString(Arrays.copyOfRange(batch.selected, 0, rememberBatchSize)));
- } else {
- // LOG.info("finishOuterRepeated batch #" + batchCounter + " filter out converted to no matchs batch size " +
- // rememberBatchSize);
- }
- batch.size = rememberBatchSize;
- }
- }
+ // Run value expressions over original (whole) input batch.
+ doValueExprOnInputSelected(batch, inputSelectedInUse, inputLogicalSize);
- // LOG.info("finishOuterRepeated batch #" + batchCounter + " " + joinResult.name() + " batch.size " + batch.size);
- switch (joinResult) {
- case MATCH:
- // Run our value expressions over whole batch.
- if (bigTableValueExpressions != null) {
- for(VectorExpression ve: bigTableValueExpressions) {
- ve.evaluate(batch);
+ // Now calculate which rows were filtered out (they are logically no matches).
+
+ // Determine which rows are non matches by determining the delta between inputSelected and
+ // (current) batch selected.
+
+ int noMatchCount = subtractFromInputSelected(
+ inputSelectedInUse, inputLogicalSize, batch.selected, batch.size, noMatchs);
+
+ generateOuterNulls(batch, noMatchs, noMatchCount);
+
+ // Now generate the matchs. Single small table values will be put into the big table
+ // batch and come back in matchs. Any multiple small table value results will go into
+ // the overflow batch.
+ generateHashMapResultRepeatedAll(batch, hashMapResult);
+
+ // Merge noMatchs and (match) selected.
+ int mergeCount = sortMerge(
+ noMatchs, noMatchCount, batch.selected, batch.size, merged);
+
+ System.arraycopy(merged, 0, batch.selected, 0, mergeCount);
+ batch.size = mergeCount;
+ batch.selectedInUse = true;
+ } else {
+
+ // Just run our value expressions over input batch.
+
+ if (bigTableValueExpressions != null) {
+ for(VectorExpression ve: bigTableValueExpressions) {
+ ve.evaluate(batch);
+ }
}
- }
- // Use a common method applicable for inner and outer.
- numSel = generateHashMapResultRepeatedAll(batch, hashMapResult);
+ generateHashMapResultRepeatedAll(batch, hashMapResult);
+ }
break;
+
case SPILL:
- // Whole batch is spilled.
+
+ // Rows we looked up as one repeated key need to spill. But filtered out rows
+ // need to be generated as non-matches, too.
+
spillBatchRepeated(batch, (VectorMapJoinHashTableResult) hashMapResult);
+
+ // After using selected to generate spills, generate non-matches, if any.
+ if (someRowsFilteredOut) {
+
+ // Determine which rows are non matches by determining the delta between inputSelected and
+ // (current) batch selected.
+
+ int noMatchCount = subtractFromInputSelected(
+ inputSelectedInUse, inputLogicalSize, batch.selected, batch.size, noMatchs);
+
+ System.arraycopy(noMatchs, 0, batch.selected, 0, noMatchCount);
+ batch.size = noMatchCount;
+ batch.selectedInUse = true;
+
+ generateOuterNullsRepeatedAll(batch);
+ } else {
+ batch.size = 0;
+ }
+
break;
+
case NOMATCH:
+
+ if (someRowsFilteredOut) {
+
+ // When the repeated no match is due to filtering, we need to restore the
+ // selected information.
+
+ if (inputSelectedInUse) {
+ System.arraycopy(inputSelected, 0, batch.selected, 0, inputLogicalSize);
+ }
+ batch.size = inputLogicalSize;
+ }
+
// Run our value expressions over whole batch.
if (bigTableValueExpressions != null) {
for(VectorExpression ve: bigTableValueExpressions) {
@@ -558,11 +710,9 @@ public abstract class VectorMapJoinOuterGenerateResultOperator
}
}
- numSel = generateOuterNullsRepeatedAll(batch);
+ generateOuterNullsRepeatedAll(batch);
break;
}
-
- return numSel;
}
/**
@@ -573,24 +723,8 @@ public abstract class VectorMapJoinOuterGenerateResultOperator
*
* @param batch
* The big table batch.
- * @return
- * The new count of selected rows.
*/
- protected int generateOuterNullsRepeatedAll(VectorizedRowBatch batch) throws HiveException {
-
- int[] selected = batch.selected;
- boolean selectedInUse = batch.selectedInUse;
-
- // Generate result within big table batch using is repeated for null small table results.
-
- if (batch.selectedInUse) {
- // The selected array is already filled in as we want it.
- } else {
- for (int i = 0; i < batch.size; i++) {
- selected[i] = i;
- }
- batch.selectedInUse = true;
- }
+ protected void generateOuterNullsRepeatedAll(VectorizedRowBatch batch) throws HiveException {
for (int column : smallTableOutputVectorColumns) {
ColumnVector colVector = batch.cols[column];
@@ -607,12 +741,5 @@ public abstract class VectorMapJoinOuterGenerateResultOperator
colVector.isNull[0] = true;
colVector.isRepeating = true;
}
-
- // for (int i = 0; i < batch.size; i++) {
- // int bigTableIndex = selected[i];
- // VectorizedBatchUtil.debugDisplayOneRow(batch, bigTableIndex, taskName + ", " + getOperatorId() + " VectorMapJoinCommonOperator generate generateOuterNullsRepeatedAll batch");
- // }
-
- return batch.size;
}
}
\ No newline at end of file
[32/50] [abbrv] hive git commit: HIVE-9974: Sensitive data redaction:
data appears in name of mapreduce job (Sergio Pena, reviewed by Xuefu Zhang)
Posted by sp...@apache.org.
HIVE-9974: Sensitive data redaction: data appears in name of mapreduce job (Sergio Pena, reviewed by Xuefu Zhang)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/65d04bc2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/65d04bc2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/65d04bc2
Branch: refs/heads/parquet
Commit: 65d04bc22838c4bd7a1a95d21d7d1a1a1eb85ebf
Parents: 037559f
Author: Sergio Pena <se...@cloudera.com>
Authored: Mon May 18 13:06:03 2015 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Mon May 18 13:06:03 2015 -0500
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/Driver.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/65d04bc2/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
index 338e755..f046c65 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Driver.java
@@ -1317,7 +1317,9 @@ public class Driver implements CommandProcessor {
int maxlen = conf.getIntVar(HiveConf.ConfVars.HIVEJOBNAMELENGTH);
String queryId = plan.getQueryId();
- String queryStr = plan.getQueryStr();
+ // Get the query string from the conf file as the compileInternal() method might
+ // hide sensitive information during query redaction.
+ String queryStr = HiveConf.getVar(conf, HiveConf.ConfVars.HIVEQUERYSTRING);
maxthreads = HiveConf.getIntVar(conf, HiveConf.ConfVars.EXECPARALLETHREADNUMBER);
[04/50] [abbrv] hive git commit: HIV-10584 fix Integer references
comparison in hcat OutputJobInfo (122)
Posted by sp...@apache.org.
HIV-10584 fix Integer references comparison in hcat OutputJobInfo (122)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/317df03b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/317df03b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/317df03b
Branch: refs/heads/parquet
Commit: 317df03beda8c986b3b9a90a16c327567189ef67
Parents: 500f298
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Sun May 3 01:31:18 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Tue May 12 16:24:21 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hive/hcatalog/mapreduce/OutputJobInfo.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/317df03b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/OutputJobInfo.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/OutputJobInfo.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/OutputJobInfo.java
index d50f43b..ce0d6d1 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/OutputJobInfo.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/mapreduce/OutputJobInfo.java
@@ -119,7 +119,7 @@ public class OutputJobInfo implements Serializable {
Collections.sort(posOfPartCols, new Comparator<Integer>() {
@Override
public int compare(Integer earlier, Integer later) {
- return (earlier > later) ? -1 : ((earlier == later) ? 0 : 1);
+ return later.compareTo(earlier);
}
});
this.posOfPartCols = posOfPartCols;
[48/50] [abbrv] hive git commit: HIVE-10644 create SHA2 UDF
(Alexander Pivovarov, reviewed by Jason Dere)
Posted by sp...@apache.org.
HIVE-10644 create SHA2 UDF (Alexander Pivovarov, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d703c222
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d703c222
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d703c222
Branch: refs/heads/parquet
Commit: d703c22212845946cd252f5ac6a6bd28484371c1
Parents: 0d0757b
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Thu May 7 17:10:12 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Tue May 19 23:24:03 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/FunctionRegistry.java | 1 +
.../ql/udf/generic/GenericUDFParamUtils.java | 71 +++++
.../hive/ql/udf/generic/GenericUDFSha2.java | 137 ++++++++++
.../hive/ql/udf/generic/TestGenericUDFSha2.java | 271 +++++++++++++++++++
ql/src/test/queries/clientpositive/udf_sha2.q | 41 +++
.../results/clientpositive/show_functions.q.out | 1 +
.../test/results/clientpositive/udf_sha2.q.out | 134 +++++++++
7 files changed, 656 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d703c222/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index 9abe15e..94a3b17 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -227,6 +227,7 @@ public final class FunctionRegistry {
system.registerUDF("unhex", UDFUnhex.class, false);
system.registerUDF("base64", UDFBase64.class, false);
system.registerUDF("unbase64", UDFUnbase64.class, false);
+ system.registerGenericUDF("sha2", GenericUDFSha2.class);
system.registerUDF("md5", UDFMd5.class, false);
system.registerUDF("sha1", UDFSha1.class, false);
system.registerUDF("sha", UDFSha1.class, false);
http://git-wip-us.apache.org/repos/asf/hive/blob/d703c222/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFParamUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFParamUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFParamUtils.java
new file mode 100644
index 0000000..cdbc6ea
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFParamUtils.java
@@ -0,0 +1,71 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+/**
+ * Generic UDF params utility class
+ */
+public class GenericUDFParamUtils {
+
+ private GenericUDFParamUtils() {
+ }
+
+ public static BytesWritable getBinaryValue(DeferredObject[] arguments, int i,
+ Converter[] converters) throws HiveException {
+ Object obj;
+ if ((obj = arguments[i].get()) == null) {
+ return null;
+ }
+ Object writableValue = converters[i].convert(obj);
+ return (BytesWritable) writableValue;
+ }
+
+ public static Text getTextValue(DeferredObject[] arguments, int i, Converter[] converters)
+ throws HiveException {
+ Object obj;
+ if ((obj = arguments[i].get()) == null) {
+ return null;
+ }
+ Object writableValue = converters[i].convert(obj);
+ return (Text) writableValue;
+ }
+
+ public static void obtainBinaryConverter(ObjectInspector[] arguments, int i,
+ PrimitiveCategory[] inputTypes, Converter[] converters) throws UDFArgumentTypeException {
+ PrimitiveObjectInspector inOi = (PrimitiveObjectInspector) arguments[i];
+ PrimitiveCategory inputType = inOi.getPrimitiveCategory();
+
+ Converter converter = ObjectInspectorConverters.getConverter(arguments[i],
+ PrimitiveObjectInspectorFactory.writableBinaryObjectInspector);
+ converters[i] = converter;
+ inputTypes[i] = inputType;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d703c222/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSha2.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSha2.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSha2.java
new file mode 100644
index 0000000..296a666
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFSha2.java
@@ -0,0 +1,137 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.BINARY_GROUP;
+import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.NUMERIC_GROUP;
+import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.STRING_GROUP;
+
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import org.apache.commons.codec.binary.Hex;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
+
+/**
+ * GenericUDFSha2.
+ *
+ */
+@Description(name = "sha2", value = "_FUNC_(string/binary, len) - Calculates the SHA-2 family of hash functions "
+ + "(SHA-224, SHA-256, SHA-384, and SHA-512).",
+ extended = "The first argument is the string or binary to be hashed. "
+ + "The second argument indicates the desired bit length of the result, "
+ + "which must have a value of 224, 256, 384, 512, or 0 (which is equivalent to 256). "
+ + "SHA-224 is supported starting from Java 8. "
+ + "If either argument is NULL or the hash length is not one of the permitted values, the return value is NULL.\n"
+ + "Example: > SELECT _FUNC_('ABC', 256);\n 'b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78'")
+public class GenericUDFSha2 extends GenericUDF {
+ private transient Converter[] converters = new Converter[2];
+ private transient PrimitiveCategory[] inputTypes = new PrimitiveCategory[2];
+ private final Text output = new Text();
+ private transient boolean isStr;
+ private transient MessageDigest digest;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+ checkArgsSize(arguments, 2, 2);
+
+ checkArgPrimitive(arguments, 0);
+ checkArgPrimitive(arguments, 1);
+
+ // the function should support both string and binary input types
+ checkArgGroups(arguments, 0, inputTypes, STRING_GROUP, BINARY_GROUP);
+ checkArgGroups(arguments, 1, inputTypes, NUMERIC_GROUP);
+
+ if (PrimitiveObjectInspectorUtils.getPrimitiveGrouping(inputTypes[0]) == STRING_GROUP) {
+ obtainStringConverter(arguments, 0, inputTypes, converters);
+ isStr = true;
+ } else {
+ GenericUDFParamUtils.obtainBinaryConverter(arguments, 0, inputTypes, converters);
+ isStr = false;
+ }
+
+ if (arguments[1] instanceof ConstantObjectInspector) {
+ Integer lenObj = getConstantIntValue(arguments, 1);
+ if (lenObj != null) {
+ int len = lenObj.intValue();
+ if (len == 0) {
+ len = 256;
+ }
+ try {
+ digest = MessageDigest.getInstance("SHA-" + len);
+ } catch (NoSuchAlgorithmException e) {
+ // ignore
+ }
+ }
+ } else {
+ throw new UDFArgumentTypeException(1, getFuncName() + " only takes constant as "
+ + getArgOrder(1) + " argument");
+ }
+
+ ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ return outputOI;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ if (digest == null) {
+ return null;
+ }
+
+ digest.reset();
+ if (isStr) {
+ Text n = GenericUDFParamUtils.getTextValue(arguments, 0, converters);
+ if (n == null) {
+ return null;
+ }
+ digest.update(n.getBytes(), 0, n.getLength());
+ } else {
+ BytesWritable bWr = GenericUDFParamUtils.getBinaryValue(arguments, 0, converters);
+ if (bWr == null) {
+ return null;
+ }
+ digest.update(bWr.getBytes(), 0, bWr.getLength());
+ }
+ byte[] resBin = digest.digest();
+ String resStr = Hex.encodeHexString(resBin);
+
+ output.set(resStr);
+ return output;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return getStandardDisplayString(getFuncName(), children);
+ }
+
+ @Override
+ protected String getFuncName() {
+ return "sha2";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d703c222/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFSha2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFSha2.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFSha2.java
new file mode 100644
index 0000000..c9e97eb
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFSha2.java
@@ -0,0 +1,271 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+
+public class TestGenericUDFSha2 extends TestCase {
+
+ public void testSha0Str() throws HiveException {
+ GenericUDFSha2 udf = new GenericUDFSha2();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ IntWritable lenWr = new IntWritable(0);
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.intTypeInfo, lenWr);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ runAndVerifyStr("ABC", lenWr,
+ "b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78", udf);
+ runAndVerifyStr("", lenWr, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ udf);
+ // null
+ runAndVerifyStr(null, lenWr, null, udf);
+ }
+
+ public void testSha0Bin() throws HiveException {
+ GenericUDFSha2 udf = new GenericUDFSha2();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
+ IntWritable lenWr = new IntWritable(0);
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.intTypeInfo, lenWr);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ runAndVerifyBin(new byte[] { 65, 66, 67 }, lenWr,
+ "b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78", udf);
+ runAndVerifyBin(new byte[0], lenWr,
+ "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", udf);
+ // null
+ runAndVerifyBin(null, lenWr, null, udf);
+ }
+
+ public void testSha200Str() throws HiveException {
+ GenericUDFSha2 udf = new GenericUDFSha2();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ IntWritable lenWr = new IntWritable(200);
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.intTypeInfo, lenWr);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ runAndVerifyStr("ABC", lenWr, null, udf);
+ }
+
+ public void testSha200Bin() throws HiveException {
+ GenericUDFSha2 udf = new GenericUDFSha2();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
+ IntWritable lenWr = new IntWritable(200);
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.intTypeInfo, lenWr);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ runAndVerifyBin(new byte[] { 65, 66, 67 }, lenWr, null, udf);
+ }
+
+ public void testSha256Str() throws HiveException {
+ GenericUDFSha2 udf = new GenericUDFSha2();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ IntWritable lenWr = new IntWritable(256);
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.intTypeInfo, lenWr);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ runAndVerifyStr("ABC", lenWr,
+ "b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78", udf);
+ runAndVerifyStr("", lenWr, "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855",
+ udf);
+ // null
+ runAndVerifyStr(null, lenWr, null, udf);
+ }
+
+ public void testSha256Bin() throws HiveException {
+ GenericUDFSha2 udf = new GenericUDFSha2();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
+ IntWritable lenWr = new IntWritable(256);
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.intTypeInfo, lenWr);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ runAndVerifyBin(new byte[] { 65, 66, 67 }, lenWr,
+ "b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78", udf);
+ runAndVerifyBin(new byte[0], lenWr,
+ "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", udf);
+ // null
+ runAndVerifyBin(null, lenWr, null, udf);
+ }
+
+ public void testSha384Str() throws HiveException {
+ GenericUDFSha2 udf = new GenericUDFSha2();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ IntWritable lenWr = new IntWritable(384);
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.intTypeInfo, lenWr);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ runAndVerifyStr(
+ "ABC",
+ lenWr,
+ "1e02dc92a41db610c9bcdc9b5935d1fb9be5639116f6c67e97bc1a3ac649753baba7ba021c813e1fe20c0480213ad371",
+ udf);
+ runAndVerifyStr(
+ "",
+ lenWr,
+ "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b",
+ udf);
+ // null
+ runAndVerifyStr(null, lenWr, null, udf);
+ }
+
+ public void testSha384Bin() throws HiveException {
+ GenericUDFSha2 udf = new GenericUDFSha2();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
+ IntWritable lenWr = new IntWritable(384);
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.intTypeInfo, lenWr);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ runAndVerifyBin(
+ new byte[] { 65, 66, 67 },
+ lenWr,
+ "1e02dc92a41db610c9bcdc9b5935d1fb9be5639116f6c67e97bc1a3ac649753baba7ba021c813e1fe20c0480213ad371",
+ udf);
+ runAndVerifyBin(
+ new byte[0],
+ lenWr,
+ "38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b",
+ udf);
+ // null
+ runAndVerifyBin(null, lenWr, null, udf);
+ }
+
+ public void testSha512Str() throws HiveException {
+ GenericUDFSha2 udf = new GenericUDFSha2();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ IntWritable lenWr = new IntWritable(512);
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.intTypeInfo, lenWr);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ runAndVerifyStr(
+ "ABC",
+ lenWr,
+ "397118fdac8d83ad98813c50759c85b8c47565d8268bf10da483153b747a74743a58a90e85aa9f705ce6984ffc128db567489817e4092d050d8a1cc596ddc119",
+ udf);
+ runAndVerifyStr(
+ "",
+ lenWr,
+ "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
+ udf);
+ // null
+ runAndVerifyStr(null, lenWr, null, udf);
+ }
+
+ public void testSha512Bin() throws HiveException {
+ GenericUDFSha2 udf = new GenericUDFSha2();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
+ IntWritable lenWr = new IntWritable(512);
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.intTypeInfo, lenWr);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ runAndVerifyBin(
+ new byte[] { 65, 66, 67 },
+ lenWr,
+ "397118fdac8d83ad98813c50759c85b8c47565d8268bf10da483153b747a74743a58a90e85aa9f705ce6984ffc128db567489817e4092d050d8a1cc596ddc119",
+ udf);
+ runAndVerifyBin(
+ new byte[0],
+ lenWr,
+ "cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e",
+ udf);
+ // null
+ runAndVerifyBin(null, lenWr, null, udf);
+ }
+
+ public void testShaNullStr() throws HiveException {
+ GenericUDFSha2 udf = new GenericUDFSha2();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ IntWritable lenWr = null;
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.intTypeInfo, lenWr);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ runAndVerifyStr("ABC", lenWr, null, udf);
+ }
+
+ public void testShaNullBin() throws HiveException {
+ GenericUDFSha2 udf = new GenericUDFSha2();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableBinaryObjectInspector;
+ IntWritable lenWr = null;
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.intTypeInfo, lenWr);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ runAndVerifyBin(new byte[] { 65, 66, 67 }, lenWr, null, udf);
+ }
+
+ private void runAndVerifyStr(String str, IntWritable lenWr, String expResult, GenericUDFSha2 udf)
+ throws HiveException {
+ DeferredObject valueObj0 = new DeferredJavaObject(str != null ? new Text(str) : null);
+ DeferredObject valueObj1 = new DeferredJavaObject(lenWr);
+ DeferredObject[] args = { valueObj0, valueObj1 };
+ Text output = (Text) udf.evaluate(args);
+ assertEquals("sha2() test ", expResult, output != null ? output.toString() : null);
+ }
+
+ private void runAndVerifyBin(byte[] b, IntWritable lenWr, String expResult, GenericUDFSha2 udf)
+ throws HiveException {
+ DeferredObject valueObj0 = new DeferredJavaObject(b != null ? new BytesWritable(b) : null);
+ DeferredObject valueObj1 = new DeferredJavaObject(lenWr);
+ DeferredObject[] args = { valueObj0, valueObj1 };
+ Text output = (Text) udf.evaluate(args);
+ assertEquals("sha2() test ", expResult, output != null ? output.toString() : null);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/d703c222/ql/src/test/queries/clientpositive/udf_sha2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udf_sha2.q b/ql/src/test/queries/clientpositive/udf_sha2.q
new file mode 100644
index 0000000..0a3443b
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/udf_sha2.q
@@ -0,0 +1,41 @@
+DESCRIBE FUNCTION sha2;
+DESC FUNCTION EXTENDED sha2;
+
+explain select sha2('ABC', 256);
+
+select
+sha2('ABC', 0),
+sha2('', 0),
+sha2(binary('ABC'), 0),
+sha2(binary(''), 0),
+sha2(cast(null as string), 0),
+sha2(cast(null as binary), 0);
+
+select
+sha2('ABC', 256),
+sha2('', 256),
+sha2(binary('ABC'), 256),
+sha2(binary(''), 256),
+sha2(cast(null as string), 256),
+sha2(cast(null as binary), 256);
+
+select
+sha2('ABC', 384),
+sha2('', 384),
+sha2(binary('ABC'), 384),
+sha2(binary(''), 384),
+sha2(cast(null as string), 384),
+sha2(cast(null as binary), 384);
+
+select
+sha2('ABC', 512),
+sha2('', 512),
+sha2(binary('ABC'), 512),
+sha2(binary(''), 512),
+sha2(cast(null as string), 512),
+sha2(cast(null as binary), 512);
+
+--null
+select
+sha2('ABC', 200),
+sha2('ABC', cast(null as int));
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/d703c222/ql/src/test/results/clientpositive/show_functions.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/show_functions.q.out b/ql/src/test/results/clientpositive/show_functions.q.out
index 2c138a3..16820ca 100644
--- a/ql/src/test/results/clientpositive/show_functions.q.out
+++ b/ql/src/test/results/clientpositive/show_functions.q.out
@@ -173,6 +173,7 @@ second
sentences
sha
sha1
+sha2
shiftleft
shiftright
shiftrightunsigned
http://git-wip-us.apache.org/repos/asf/hive/blob/d703c222/ql/src/test/results/clientpositive/udf_sha2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_sha2.q.out b/ql/src/test/results/clientpositive/udf_sha2.q.out
new file mode 100644
index 0000000..e39da08
--- /dev/null
+++ b/ql/src/test/results/clientpositive/udf_sha2.q.out
@@ -0,0 +1,134 @@
+PREHOOK: query: DESCRIBE FUNCTION sha2
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION sha2
+POSTHOOK: type: DESCFUNCTION
+sha2(string/binary, len) - Calculates the SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384, and SHA-512).
+PREHOOK: query: DESC FUNCTION EXTENDED sha2
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESC FUNCTION EXTENDED sha2
+POSTHOOK: type: DESCFUNCTION
+sha2(string/binary, len) - Calculates the SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384, and SHA-512).
+The first argument is the string or binary to be hashed. The second argument indicates the desired bit length of the result, which must have a value of 224, 256, 384, 512, or 0 (which is equivalent to 256). SHA-224 is supported starting from Java 8. If either argument is NULL or the hash length is not one of the permitted values, the return value is NULL.
+Example: > SELECT sha2('ABC', 256);
+ 'b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78'
+PREHOOK: query: explain select sha2('ABC', 256)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select sha2('ABC', 256)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: _dummy_table
+ Row Limit Per Split: 1
+ Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+ Select Operator
+ expressions: 'b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78' (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+ ListSink
+
+PREHOOK: query: select
+sha2('ABC', 0),
+sha2('', 0),
+sha2(binary('ABC'), 0),
+sha2(binary(''), 0),
+sha2(cast(null as string), 0),
+sha2(cast(null as binary), 0)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select
+sha2('ABC', 0),
+sha2('', 0),
+sha2(binary('ABC'), 0),
+sha2(binary(''), 0),
+sha2(cast(null as string), 0),
+sha2(cast(null as binary), 0)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 NULL NULL
+PREHOOK: query: select
+sha2('ABC', 256),
+sha2('', 256),
+sha2(binary('ABC'), 256),
+sha2(binary(''), 256),
+sha2(cast(null as string), 256),
+sha2(cast(null as binary), 256)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select
+sha2('ABC', 256),
+sha2('', 256),
+sha2(binary('ABC'), 256),
+sha2(binary(''), 256),
+sha2(cast(null as string), 256),
+sha2(cast(null as binary), 256)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 b5d4045c3f466fa91fe2cc6abe79232a1a57cdf104f7a26e716e0a1e2789df78 e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855 NULL NULL
+PREHOOK: query: select
+sha2('ABC', 384),
+sha2('', 384),
+sha2(binary('ABC'), 384),
+sha2(binary(''), 384),
+sha2(cast(null as string), 384),
+sha2(cast(null as binary), 384)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select
+sha2('ABC', 384),
+sha2('', 384),
+sha2(binary('ABC'), 384),
+sha2(binary(''), 384),
+sha2(cast(null as string), 384),
+sha2(cast(null as binary), 384)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+1e02dc92a41db610c9bcdc9b5935d1fb9be5639116f6c67e97bc1a3ac649753baba7ba021c813e1fe20c0480213ad371 38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b 1e02dc92a41db610c9bcdc9b5935d1fb9be5639116f6c67e97bc1a3ac649753baba7ba021c813e1fe20c0480213ad371 38b060a751ac96384cd9327eb1b1e36a21fdb71114be07434c0cc7bf63f6e1da274edebfe76f65fbd51ad2f14898b95b NULL NULL
+PREHOOK: query: select
+sha2('ABC', 512),
+sha2('', 512),
+sha2(binary('ABC'), 512),
+sha2(binary(''), 512),
+sha2(cast(null as string), 512),
+sha2(cast(null as binary), 512)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select
+sha2('ABC', 512),
+sha2('', 512),
+sha2(binary('ABC'), 512),
+sha2(binary(''), 512),
+sha2(cast(null as string), 512),
+sha2(cast(null as binary), 512)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+397118fdac8d83ad98813c50759c85b8c47565d8268bf10da483153b747a74743a58a90e85aa9f705ce6984ffc128db567489817e4092d050d8a1cc596ddc119 cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e 397118fdac8d83ad98813c50759c85b8c47565d8268bf10da483153b747a74743a58a90e85aa9f705ce6984ffc128db567489817e4092d050d8a1cc596ddc119 cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e NULL NULL
+PREHOOK: query: --null
+select
+sha2('ABC', 200),
+sha2('ABC', cast(null as int))
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: --null
+select
+sha2('ABC', 200),
+sha2('ABC', cast(null as int))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+NULL NULL
[05/50] [abbrv] hive git commit: HIVE-10618 Fix invocation of
toString on byteArray in VerifyFast (250, 254)
Posted by sp...@apache.org.
HIVE-10618 Fix invocation of toString on byteArray in VerifyFast (250, 254)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5af6e92b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5af6e92b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5af6e92b
Branch: refs/heads/parquet
Commit: 5af6e92bc1a38d05ec42907123a15776f0c78588
Parents: 317df03
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Tue May 5 20:52:34 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Tue May 12 16:26:54 2015 -0700
----------------------------------------------------------------------
.../test/org/apache/hadoop/hive/serde2/VerifyFast.java | 10 ++++------
1 file changed, 4 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5af6e92b/serde/src/test/org/apache/hadoop/hive/serde2/VerifyFast.java
----------------------------------------------------------------------
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/VerifyFast.java b/serde/src/test/org/apache/hadoop/hive/serde2/VerifyFast.java
index a3472ad..fa46c9e 100644
--- a/serde/src/test/org/apache/hadoop/hive/serde2/VerifyFast.java
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/VerifyFast.java
@@ -29,15 +29,11 @@ import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
import org.apache.hadoop.hive.common.type.HiveIntervalYearMonth;
import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.hive.serde2.binarysortable.MyTestPrimitiveClass.ExtraTypeInfo;
import org.apache.hadoop.hive.serde2.fast.DeserializeRead;
-import org.apache.hadoop.hive.serde2.io.HiveCharWritable;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.fast.SerializeWrite;
-import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
import org.apache.hadoop.io.Text;
/**
@@ -247,11 +243,13 @@ public class VerifyFast {
byte[] byteArray = Arrays.copyOfRange(readBinaryResults.bytes, readBinaryResults.start, readBinaryResults.start + readBinaryResults.length);
byte[] expected = (byte[]) object;
if (byteArray.length != expected.length){
- TestCase.fail("Byte Array field mismatch (expected " + expected + " found " + byteArray + ")");
+ TestCase.fail("Byte Array field mismatch (expected " + Arrays.toString(expected)
+ + " found " + Arrays.toString(byteArray) + ")");
}
for (int b = 0; b < byteArray.length; b++) {
if (byteArray[b] != expected[b]) {
- TestCase.fail("Byte Array field mismatch (expected " + expected + " found " + byteArray + ")");
+ TestCase.fail("Byte Array field mismatch (expected " + Arrays.toString(expected)
+ + " found " + Arrays.toString(byteArray) + ")");
}
}
}
[50/50] [abbrv] hive git commit: HIVE-10769: Merge master to parquet
05/20/2015 [Parquet branch]
Posted by sp...@apache.org.
HIVE-10769: Merge master to parquet 05/20/2015 [Parquet branch]
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5a0ae3c2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5a0ae3c2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5a0ae3c2
Branch: refs/heads/parquet
Commit: 5a0ae3c2a3a8ce0d332eae9b9d3f5a0ecca3e013
Parents: 4580944 1253a8d
Author: Sergio Pena <se...@cloudera.com>
Authored: Wed May 20 10:57:42 2015 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed May 20 10:57:42 2015 -0500
----------------------------------------------------------------------
.reviewboardrc | 2 +-
README.txt | 36 +-
RELEASE_NOTES.txt | 463 +
accumulo-handler/pom.xml | 2 +-
.../src/test/templates/TestAccumuloCliDriver.vm | 19 +-
ant/pom.xml | 2 +-
.../apache/hadoop/hive/ant/GenVectorCode.java | 2 +
.../apache/hadoop/hive/ant/QTestGenTask.java | 25 +
beeline/pom.xml | 2 +-
.../java/org/apache/hive/beeline/BeeLine.java | 10 +-
bin/beeline.cmd | 11 +-
bin/ext/hiveserver2.cmd | 2 +-
bin/ext/orcfiledump.sh | 9 +-
bin/hive | 9 +-
cli/pom.xml | 2 +-
.../org/apache/hadoop/hive/cli/CliDriver.java | 2 -
common/pom.xml | 2 +-
.../apache/hadoop/hive/common/JavaUtils.java | 8 +
.../hadoop/hive/common/StatsSetupConst.java | 5 +
.../hive/common/jsonexplain/tez/Vertex.java | 2 +-
.../hadoop/hive/common/type/HiveDecimal.java | 19 +-
.../org/apache/hadoop/hive/conf/HiveConf.java | 146 +-
.../apache/hive/common/util/BloomFilter.java | 291 +
.../org/apache/hive/common/util/Murmur3.java | 334 +
.../apache/hive/common/util/ReflectionUtil.java | 118 +
.../hive/common/util/TestBloomFilter.java | 458 +
.../apache/hive/common/util/TestMurmur3.java | 189 +
contrib/pom.xml | 2 +-
data/files/tjoin1.txt | 3 +
data/files/tjoin2.txt | 4 +
dev-support/jenkins-execute-build.sh | 6 +-
dev-support/jenkins-execute-hms-test.sh | 20 +-
hbase-handler/pom.xml | 2 +-
.../hadoop/hive/hbase/HBaseStorageHandler.java | 8 +
.../hive/hbase/HBaseTestStructSerializer.java | 20 +-
.../src/test/queries/positive/hbase_timestamp.q | 22 +-
.../test/results/positive/hbase_timestamp.q.out | 60 +-
.../src/test/templates/TestHBaseCliDriver.vm | 18 +-
.../templates/TestHBaseNegativeCliDriver.vm | 19 +-
hcatalog/core/pom.xml | 2 +-
.../apache/hive/hcatalog/common/HCatUtil.java | 37 +-
.../hive/hcatalog/common/HiveClientCache.java | 94 +-
.../data/HCatRecordObjectInspectorFactory.java | 2 +-
.../hive/hcatalog/data/HCatRecordSerDe.java | 9 +-
.../apache/hive/hcatalog/data/JsonSerDe.java | 7 +-
.../hcatalog/data/schema/HCatFieldSchema.java | 6 +-
.../DefaultOutputCommitterContainer.java | 6 +-
.../mapreduce/FileOutputCommitterContainer.java | 22 +-
.../mapreduce/FileOutputFormatContainer.java | 8 +-
.../hcatalog/mapreduce/HCatBaseInputFormat.java | 50 +-
.../hcatalog/mapreduce/HCatOutputFormat.java | 6 +-
.../hcatalog/mapreduce/HCatRecordReader.java | 4 +-
.../hive/hcatalog/mapreduce/HCatSplit.java | 21 +-
.../hive/hcatalog/mapreduce/HCatTableInfo.java | 12 +
.../hcatalog/mapreduce/InitializeInput.java | 6 +-
.../hive/hcatalog/mapreduce/InputJobInfo.java | 5 +
.../hive/hcatalog/mapreduce/InternalUtil.java | 2 +
.../hive/hcatalog/mapreduce/OutputJobInfo.java | 2 +-
.../hive/hcatalog/mapreduce/PartInfo.java | 117 +-
.../hive/hcatalog/mapreduce/Security.java | 10 +-
.../mapreduce/TaskCommitContextRegistry.java | 6 +-
.../hcatalog/common/TestHiveClientCache.java | 37 +-
.../hcatalog/mapreduce/HCatMapReduceTest.java | 3 +-
.../mapreduce/TestHCatOutputFormat.java | 5 +-
.../hcatalog/mapreduce/TestHCatPartitioned.java | 32 +-
.../hcatalog/mapreduce/TestPassProperties.java | 5 +-
hcatalog/hcatalog-pig-adapter/pom.xml | 2 +-
.../apache/hive/hcatalog/pig/PigHCatUtil.java | 10 +-
.../hcatalog/pig/TestHCatLoaderEncryption.java | 3 +-
hcatalog/pom.xml | 2 +-
hcatalog/server-extensions/pom.xml | 2 +-
.../listener/DbNotificationListener.java | 6 +-
.../hcatalog/listener/NotificationListener.java | 23 +-
.../messaging/AlterPartitionMessage.java | 6 +-
.../hive/hcatalog/messaging/InsertMessage.java | 7 +-
.../hive/hcatalog/messaging/MessageFactory.java | 10 +-
.../json/JSONAlterPartitionMessage.java | 14 +-
.../messaging/json/JSONAlterTableMessage.java | 5 +
.../messaging/json/JSONInsertMessage.java | 15 +-
.../messaging/json/JSONMessageFactory.java | 14 +-
.../listener/TestNotificationListener.java | 8 +-
hcatalog/src/test/e2e/templeton/README.txt | 13 +
hcatalog/src/test/e2e/templeton/build.xml | 29 +
.../deployers/config/hive/hive-log4j.properties | 88 +
.../deployers/config/hive/hive-site.mysql.xml | 77 +
.../webhcat/webhcat-site.updateConfig.xml | 118 +
.../deployers/config/webhcat/webhcat-site.xml | 9 +-
.../templeton/deployers/deploy_e2e_artifacts.sh | 6 +
.../src/test/e2e/templeton/deployers/env.sh | 7 +
.../deployers/modify_webhcat_config.sh | 40 +
.../deployers/restore_webhcat_config.sh | 36 +
.../templeton/deployers/start_hive_services.sh | 7 +
.../src/test/e2e/templeton/inpdir/xmlmapper.py | 15 +
.../src/test/e2e/templeton/inpdir/xmlreducer.py | 16 +
.../templeton/tests/modifyConfiguration.conf | 67 +
hcatalog/streaming/pom.xml | 2 +-
.../streaming/AbstractRecordWriter.java | 11 +-
.../hive/hcatalog/streaming/HiveEndPoint.java | 9 +-
hcatalog/webhcat/java-client/pom.xml | 2 +-
.../apache/hive/hcatalog/api/HCatClient.java | 18 +
.../hive/hcatalog/api/HCatClientHMSImpl.java | 22 +-
.../hcatalog/api/HCatNotificationEvent.java | 2 +-
.../apache/hive/hcatalog/api/HCatPartition.java | 12 +
.../api/repl/HCatReplicationTaskIterator.java | 81 +-
.../hcatalog/api/repl/NoopReplicationTask.java | 9 +-
.../hive/hcatalog/api/repl/ReplicationTask.java | 103 +-
.../hcatalog/api/repl/ReplicationUtils.java | 31 +-
.../api/repl/commands/DropDatabaseCommand.java | 99 +
.../api/repl/commands/DropPartitionCommand.java | 122 +
.../api/repl/commands/DropTableCommand.java | 113 +
.../api/repl/commands/ExportCommand.java | 131 +
.../api/repl/commands/ImportCommand.java | 139 +
.../hcatalog/api/repl/commands/NoopCommand.java | 22 +-
.../repl/exim/AddPartitionReplicationTask.java | 111 +
.../exim/AlterPartitionReplicationTask.java | 88 +
.../repl/exim/AlterTableReplicationTask.java | 85 +
.../exim/CreateDatabaseReplicationTask.java | 38 +
.../repl/exim/CreateTableReplicationTask.java | 85 +
.../repl/exim/DropDatabaseReplicationTask.java | 56 +
.../repl/exim/DropPartitionReplicationTask.java | 77 +
.../api/repl/exim/DropTableReplicationTask.java | 56 +
.../repl/exim/EximReplicationTaskFactory.java | 63 +
.../api/repl/exim/InsertReplicationTask.java | 99 +
.../hive/hcatalog/api/TestHCatClient.java | 124 +-
.../hcatalog/api/repl/CommandTestUtils.java | 107 +
.../hcatalog/api/repl/TestReplicationTask.java | 92 +
.../api/repl/commands/TestCommands.java | 590 +
.../api/repl/commands/TestNoopCommand.java | 42 +
.../api/repl/exim/TestEximReplicationTasks.java | 600 +
hcatalog/webhcat/svr/pom.xml | 15 +-
.../svr/src/main/config/webhcat-default.xml | 10 +-
.../hcatalog/templeton/CompleteDelegator.java | 6 +-
.../hcatalog/templeton/SecureProxySupport.java | 9 +-
.../hcatalog/templeton/tool/LaunchMapper.java | 2 +-
.../templeton/tool/TempletonControllerJob.java | 15 +-
hwi/pom.xml | 2 +-
itests/custom-serde/pom.xml | 2 +-
itests/hcatalog-unit/pom.xml | 2 +-
.../listener/TestDbNotificationListener.java | 75 +-
itests/hive-jmh/pom.xml | 2 +-
.../vectorization/VectorizationBench.java | 146 +-
itests/hive-minikdc/pom.xml | 2 +-
.../hive/minikdc/TestJdbcWithMiniKdcCookie.java | 102 +
itests/hive-unit-hadoop2/pom.xml | 2 +-
.../apache/hive/jdbc/TestSchedulerQueue.java | 3 +-
itests/hive-unit/pom.xml | 135 +-
.../org/apache/hive/jdbc/miniHS2/MiniHS2.java | 57 +-
.../hive/metastore/TestHiveMetaStore.java | 30 +
.../hadoop/hive/metastore/TestHiveMetaTool.java | 17 +-
.../metastore/TestMetaStoreEventListener.java | 14 +-
.../hive/beeline/TestBeeLineWithArgs.java | 12 +
.../jdbc/TestJdbcWithLocalClusterSpark.java | 3 +-
.../apache/hive/jdbc/TestJdbcWithMiniMr.java | 3 +-
...stMultiSessionsHS2WithLocalClusterSpark.java | 3 +-
.../TestJdbcWithSQLAuthorization.java | 25 +
.../operation/OperationLoggingAPITestBase.java | 235 +
.../cli/operation/TestOperationLoggingAPI.java | 379 -
.../TestOperationLoggingAPIWithMr.java | 168 +
.../TestOperationLoggingAPIWithTez.java | 54 +
.../cli/thrift/TestThriftHttpCLIService.java | 74 +-
itests/pom.xml | 102 +-
itests/qtest-spark/pom.xml | 2 +-
itests/qtest/pom.xml | 60 +-
.../test/resources/testconfiguration.properties | 59 +-
itests/test-serde/pom.xml | 2 +-
itests/util/pom.xml | 2 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 78 +-
.../generic/GenericUDFTestGetJavaBoolean.java | 8 +-
jdbc/pom.xml | 2 +-
.../org/apache/hive/jdbc/HiveConnection.java | 101 +-
.../apache/hive/jdbc/HiveQueryResultSet.java | 5 +
.../org/apache/hive/jdbc/HiveStatement.java | 8 +-
.../hive/jdbc/HttpBasicAuthInterceptor.java | 49 +-
.../jdbc/HttpKerberosRequestInterceptor.java | 62 +-
.../hive/jdbc/HttpRequestInterceptorBase.java | 89 +
jdbc/src/java/org/apache/hive/jdbc/Utils.java | 22 +-
.../hive/jdbc/ZooKeeperHiveClientHelper.java | 2 +-
metastore/bin/.gitignore | 3 +-
metastore/pom.xml | 2 +-
.../upgrade/derby/hive-schema-1.3.0.derby.sql | 336 +
.../derby/upgrade-1.2.0-to-1.3.0.derby.sql | 3 +
.../scripts/upgrade/derby/upgrade.order.derby | 1 +
.../upgrade/mssql/006-HIVE-9456.mssql.sql | 323 +
.../upgrade/mssql/hive-schema-1.2.0.mssql.sql | 256 +-
.../upgrade/mssql/hive-schema-1.3.0.mssql.sql | 947 ++
.../mssql/upgrade-1.1.0-to-1.2.0.mssql.sql | 1 +
.../mssql/upgrade-1.2.0-to-1.3.0.mssql.sql | 5 +
.../scripts/upgrade/mssql/upgrade.order.mssql | 1 +
.../upgrade/mysql/021-HIVE-7018.mysql.sql | 53 -
.../upgrade/mysql/hive-schema-1.2.0.mysql.sql | 10 +-
.../upgrade/mysql/hive-schema-1.3.0.mysql.sql | 840 +
.../mysql/upgrade-1.1.0-to-1.2.0.mysql.sql | 2 +-
.../mysql/upgrade-1.2.0-to-1.3.0.mysql.sql | 4 +
.../scripts/upgrade/mysql/upgrade.order.mysql | 1 +
.../upgrade/oracle/hive-schema-1.3.0.oracle.sql | 788 +
.../oracle/upgrade-1.2.0-to-1.3.0.oracle.sql | 4 +
.../scripts/upgrade/oracle/upgrade.order.oracle | 1 +
.../postgres/hive-schema-1.3.0.postgres.sql | 1493 ++
.../upgrade-1.2.0-to-1.3.0.postgres.sql | 12 +
.../upgrade/postgres/upgrade.order.postgres | 1 +
.../hive/metastore/AggregateStatsCache.java | 572 +
.../hadoop/hive/metastore/HiveAlterHandler.java | 32 +-
.../hadoop/hive/metastore/HiveMetaStore.java | 56 +-
.../hive/metastore/HiveMetaStoreClient.java | 26 +-
.../hadoop/hive/metastore/IMetaStoreClient.java | 11 +
.../hive/metastore/MetaStoreDirectSql.java | 81 +-
.../hadoop/hive/metastore/MetaStoreUtils.java | 27 +-
.../hadoop/hive/metastore/ObjectStore.java | 107 +-
.../hive/metastore/RetryingMetaStoreClient.java | 89 +-
.../metastore/events/DropPartitionEvent.java | 14 +-
.../hive/metastore/events/InsertEvent.java | 31 +-
.../metastore/events/PreDropPartitionEvent.java | 23 +-
.../hive/metastore/tools/HiveMetaTool.java | 35 +-
.../hadoop/hive/metastore/txn/TxnHandler.java | 221 +-
.../hive/metastore/TestAggregateStatsCache.java | 266 +
.../hive/metastore/TestHiveMetastoreCli.java | 63 +
.../hive/metastore/txn/TestTxnHandler.java | 40 +-
odbc/pom.xml | 2 +-
packaging/pom.xml | 2 +-
packaging/src/main/assembly/bin.xml | 1 +
pom.xml | 12 +-
ql/.gitignore | 2 +
ql/pom.xml | 18 +-
.../ExpressionTemplates/ColumnDivideColumn.txt | 26 +-
.../java/org/apache/hadoop/hive/ql/Context.java | 10 +-
.../java/org/apache/hadoop/hive/ql/Driver.java | 6 +-
.../org/apache/hadoop/hive/ql/ErrorMsg.java | 1 +
.../hive/ql/exec/CommonMergeJoinOperator.java | 104 +-
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 100 +-
.../hive/ql/exec/DefaultFetchFormatter.java | 5 +-
.../hadoop/hive/ql/exec/DemuxOperator.java | 6 +-
.../hive/ql/exec/ExprNodeEvaluatorFactory.java | 12 +-
.../ql/exec/ExprNodeGenericFuncEvaluator.java | 3 +-
.../hive/ql/exec/ExprNodeNullEvaluator.java | 47 -
.../hadoop/hive/ql/exec/FetchOperator.java | 15 +-
.../apache/hadoop/hive/ql/exec/FetchTask.java | 2 +
.../hadoop/hive/ql/exec/FileSinkOperator.java | 47 +-
.../hadoop/hive/ql/exec/FunctionRegistry.java | 27 +-
.../hadoop/hive/ql/exec/HashTableLoader.java | 4 +-
.../hadoop/hive/ql/exec/JoinOperator.java | 3 +-
.../apache/hadoop/hive/ql/exec/JoinUtil.java | 4 +-
.../hadoop/hive/ql/exec/MapJoinOperator.java | 293 +-
.../apache/hadoop/hive/ql/exec/ObjectCache.java | 7 +
.../apache/hadoop/hive/ql/exec/Operator.java | 4 +
.../hadoop/hive/ql/exec/OperatorFactory.java | 22 +-
.../hadoop/hive/ql/exec/ReduceSinkOperator.java | 3 +-
.../apache/hadoop/hive/ql/exec/Registry.java | 29 +-
.../apache/hadoop/hive/ql/exec/RowSchema.java | 9 +
.../hadoop/hive/ql/exec/SMBMapJoinOperator.java | 4 +-
.../hadoop/hive/ql/exec/SecureCmdDoAs.java | 5 +-
.../ql/exec/SparkHashTableSinkOperator.java | 35 +-
.../hadoop/hive/ql/exec/StatsNoJobTask.java | 6 +-
.../apache/hadoop/hive/ql/exec/TaskRunner.java | 1 +
.../apache/hadoop/hive/ql/exec/Utilities.java | 71 +-
.../hadoop/hive/ql/exec/mr/ExecDriver.java | 1 +
.../hadoop/hive/ql/exec/mr/HashTableLoader.java | 2 +-
.../hadoop/hive/ql/exec/mr/ObjectCache.java | 5 +
.../persistence/BytesBytesMultiHashMap.java | 1 +
.../exec/persistence/HybridHashTableConf.java | 86 +
.../persistence/HybridHashTableContainer.java | 348 +-
.../ql/exec/persistence/KeyValueContainer.java | 31 +-
.../persistence/MapJoinBytesTableContainer.java | 88 +-
.../hive/ql/exec/persistence/MapJoinKey.java | 14 +
.../MapJoinTableContainerDirectAccess.java | 31 +
.../ql/exec/persistence/ObjectContainer.java | 31 +-
.../ReusableGetAdaptorDirectAccess.java | 30 +
.../hive/ql/exec/persistence/RowContainer.java | 4 +-
.../hive/ql/exec/spark/HashTableLoader.java | 2 +-
.../ql/exec/spark/HiveSparkClientFactory.java | 28 +-
.../hive/ql/exec/spark/KryoSerializer.java | 4 +-
.../ql/exec/spark/RemoteHiveSparkClient.java | 28 +-
.../ql/exec/spark/SparkMapRecordHandler.java | 1 +
.../hive/ql/exec/spark/SparkPlanGenerator.java | 35 +-
.../ql/exec/spark/SparkReduceRecordHandler.java | 1 +
.../hadoop/hive/ql/exec/spark/SparkTask.java | 16 +
.../hive/ql/exec/spark/SparkUtilities.java | 27 +-
.../spark/status/impl/JobMetricsListener.java | 12 +
.../hadoop/hive/ql/exec/tez/DagUtils.java | 1 +
.../hive/ql/exec/tez/HashTableLoader.java | 73 +-
.../hadoop/hive/ql/exec/tez/ObjectCache.java | 6 +
.../hive/ql/exec/tez/ReduceRecordSource.java | 148 +-
.../hadoop/hive/ql/exec/tez/TezJobMonitor.java | 4 +-
.../hive/ql/exec/tez/TezSessionState.java | 13 +-
.../apache/hadoop/hive/ql/exec/tez/TezTask.java | 3 +-
.../vector/VectorAppMasterEventOperator.java | 123 +-
.../ql/exec/vector/VectorColumnMapping.java | 6 +-
.../ql/exec/vector/VectorColumnOrderedMap.java | 17 +-
.../exec/vector/VectorColumnOutputMapping.java | 4 +
.../ql/exec/vector/VectorColumnSetInfo.java | 3 +-
.../exec/vector/VectorColumnSourceMapping.java | 4 +
.../hive/ql/exec/vector/VectorCopyRow.java | 2 +-
.../ql/exec/vector/VectorDeserializeRow.java | 13 +-
.../ql/exec/vector/VectorFileSinkOperator.java | 78 +-
.../ql/exec/vector/VectorGroupByOperator.java | 26 +-
.../exec/vector/VectorMapJoinBaseOperator.java | 185 +
.../ql/exec/vector/VectorMapJoinOperator.java | 171 +-
.../VectorMapJoinOuterFilteredOperator.java | 122 +
.../exec/vector/VectorReduceSinkOperator.java | 83 +-
.../exec/vector/VectorSMBMapJoinOperator.java | 21 +-
.../hive/ql/exec/vector/VectorSerializeRow.java | 17 +
.../exec/vector/VectorSerializeRowNoNulls.java | 17 +
.../ql/exec/vector/VectorizationContext.java | 89 +-
.../ql/exec/vector/VectorizedBatchUtil.java | 54 +-
.../ql/exec/vector/VectorizedRowBatchCtx.java | 4 +-
.../expressions/ConstantVectorExpression.java | 3 -
.../vector/expressions/IdentityExpression.java | 9 +
.../ql/exec/vector/expressions/MathExpr.java | 9 +-
.../VectorExpressionWriterFactory.java | 29 +-
.../mapjoin/VectorMapJoinCommonOperator.java | 775 +
.../VectorMapJoinGenerateResultOperator.java | 852 +
...pJoinInnerBigOnlyGenerateResultOperator.java | 328 +
.../VectorMapJoinInnerBigOnlyLongOperator.java | 374 +
...ctorMapJoinInnerBigOnlyMultiKeyOperator.java | 387 +
...VectorMapJoinInnerBigOnlyStringOperator.java | 366 +
...ectorMapJoinInnerGenerateResultOperator.java | 228 +
.../mapjoin/VectorMapJoinInnerLongOperator.java | 371 +
.../VectorMapJoinInnerMultiKeyOperator.java | 383 +
.../VectorMapJoinInnerStringOperator.java | 360 +
...orMapJoinLeftSemiGenerateResultOperator.java | 225 +
.../VectorMapJoinLeftSemiLongOperator.java | 364 +
.../VectorMapJoinLeftSemiMultiKeyOperator.java | 380 +
.../VectorMapJoinLeftSemiStringOperator.java | 353 +
...ectorMapJoinOuterGenerateResultOperator.java | 745 +
.../mapjoin/VectorMapJoinOuterLongOperator.java | 447 +
.../VectorMapJoinOuterMultiKeyOperator.java | 464 +
.../VectorMapJoinOuterStringOperator.java | 433 +
.../mapjoin/VectorMapJoinRowBytesContainer.java | 318 +
.../fast/VectorMapJoinFastBytesHashMap.java | 101 +
.../VectorMapJoinFastBytesHashMultiSet.java | 93 +
.../fast/VectorMapJoinFastBytesHashSet.java | 85 +
.../fast/VectorMapJoinFastBytesHashTable.java | 221 +
.../fast/VectorMapJoinFastBytesHashUtil.java | 41 +
.../mapjoin/fast/VectorMapJoinFastHashMap.java | 38 +
.../fast/VectorMapJoinFastHashMultiSet.java | 48 +
.../mapjoin/fast/VectorMapJoinFastHashSet.java | 44 +
.../fast/VectorMapJoinFastHashTable.java | 68 +
.../fast/VectorMapJoinFastHashTableLoader.java | 114 +
.../fast/VectorMapJoinFastIntHashUtil.java | 32 +
.../mapjoin/fast/VectorMapJoinFastKeyStore.java | 173 +
.../fast/VectorMapJoinFastLongHashMap.java | 94 +
.../fast/VectorMapJoinFastLongHashMultiSet.java | 91 +
.../fast/VectorMapJoinFastLongHashSet.java | 84 +
.../fast/VectorMapJoinFastLongHashTable.java | 284 +
.../fast/VectorMapJoinFastLongHashUtil.java | 63 +
.../fast/VectorMapJoinFastMultiKeyHashMap.java | 39 +
.../VectorMapJoinFastMultiKeyHashMultiSet.java | 32 +
.../fast/VectorMapJoinFastMultiKeyHashSet.java | 32 +
.../fast/VectorMapJoinFastStringCommon.java | 67 +
.../fast/VectorMapJoinFastStringHashMap.java | 44 +
.../VectorMapJoinFastStringHashMultiSet.java | 44 +
.../fast/VectorMapJoinFastStringHashSet.java | 44 +
.../fast/VectorMapJoinFastTableContainer.java | 222 +
.../fast/VectorMapJoinFastValueStore.java | 557 +
.../hashtable/VectorMapJoinBytesHashMap.java | 51 +
.../VectorMapJoinBytesHashMultiSet.java | 51 +
.../hashtable/VectorMapJoinBytesHashSet.java | 51 +
.../hashtable/VectorMapJoinBytesHashTable.java | 26 +
.../mapjoin/hashtable/VectorMapJoinHashMap.java | 34 +
.../hashtable/VectorMapJoinHashMapResult.java | 63 +
.../hashtable/VectorMapJoinHashMultiSet.java | 31 +
.../VectorMapJoinHashMultiSetResult.java | 34 +
.../mapjoin/hashtable/VectorMapJoinHashSet.java | 34 +
.../hashtable/VectorMapJoinHashSetResult.java | 28 +
.../hashtable/VectorMapJoinHashTable.java | 43 +
.../hashtable/VectorMapJoinHashTableResult.java | 81 +
.../hashtable/VectorMapJoinLongHashMap.java | 46 +
.../VectorMapJoinLongHashMultiSet.java | 46 +
.../hashtable/VectorMapJoinLongHashSet.java | 46 +
.../hashtable/VectorMapJoinLongHashTable.java | 31 +
.../hashtable/VectorMapJoinTableContainer.java | 28 +
.../VectorMapJoinOptimizedCreateHashTable.java | 129 +
.../VectorMapJoinOptimizedHashMap.java | 128 +
.../VectorMapJoinOptimizedHashMultiSet.java | 103 +
.../VectorMapJoinOptimizedHashSet.java | 78 +
.../VectorMapJoinOptimizedHashTable.java | 95 +
.../VectorMapJoinOptimizedLongCommon.java | 171 +
.../VectorMapJoinOptimizedLongHashMap.java | 82 +
.../VectorMapJoinOptimizedLongHashMultiSet.java | 83 +
.../VectorMapJoinOptimizedLongHashSet.java | 83 +
.../VectorMapJoinOptimizedMultiKeyHashMap.java | 36 +
...torMapJoinOptimizedMultiKeyHashMultiSet.java | 36 +
.../VectorMapJoinOptimizedMultiKeyHashSet.java | 36 +
.../VectorMapJoinOptimizedStringCommon.java | 98 +
.../VectorMapJoinOptimizedStringHashMap.java | 63 +
...ectorMapJoinOptimizedStringHashMultiSet.java | 64 +
.../VectorMapJoinOptimizedStringHashSet.java | 63 +
.../hadoop/hive/ql/history/HiveHistory.java | 3 +-
.../hive/ql/io/BucketizedHiveInputSplit.java | 5 +-
.../hive/ql/io/CombineHiveInputFormat.java | 4 +
.../hadoop/hive/ql/io/HiveFileFormatUtils.java | 4 +-
.../hadoop/hive/ql/io/HiveInputFormat.java | 8 +-
.../hadoop/hive/ql/io/filters/BloomFilter.java | 298 -
.../hive/ql/io/filters/BloomFilterIO.java | 44 +
.../hadoop/hive/ql/io/filters/Murmur3.java | 334 -
.../hive/ql/io/orc/ColumnStatisticsImpl.java | 35 +-
.../ql/io/orc/ConversionTreeReaderFactory.java | 38 +
.../hive/ql/io/orc/DateColumnStatistics.java | 6 +-
.../apache/hadoop/hive/ql/io/orc/FileDump.java | 99 +-
.../hadoop/hive/ql/io/orc/IntegerWriter.java | 5 -
.../hadoop/hive/ql/io/orc/JsonFileDump.java | 365 +
.../hadoop/hive/ql/io/orc/MetadataReader.java | 1 -
.../apache/hadoop/hive/ql/io/orc/OrcFile.java | 4 +-
.../hadoop/hive/ql/io/orc/OrcInputFormat.java | 8 +-
.../hadoop/hive/ql/io/orc/OrcRecordUpdater.java | 24 +-
.../apache/hadoop/hive/ql/io/orc/OrcUtils.java | 2 +-
.../hive/ql/io/orc/RecordReaderFactory.java | 269 +
.../hadoop/hive/ql/io/orc/RecordReaderImpl.java | 236 +-
.../hive/ql/io/orc/RecordReaderUtils.java | 12 +-
.../hive/ql/io/orc/RunLengthIntegerWriter.java | 9 +-
.../ql/io/orc/RunLengthIntegerWriterV2.java | 11 +-
.../hive/ql/io/orc/TreeReaderFactory.java | 98 +-
.../hadoop/hive/ql/io/orc/WriterImpl.java | 15 +-
.../ql/io/parquet/MapredParquetInputFormat.java | 7 +-
.../io/parquet/MapredParquetOutputFormat.java | 7 +-
.../parquet/VectorizedParquetInputFormat.java | 2 +-
.../convert/HiveCollectionConverter.java | 18 +
.../hive/ql/io/parquet/convert/Repeated.java | 18 +
.../read/ParquetRecordReaderWrapper.java | 7 +-
.../write/ParquetRecordWriterWrapper.java | 7 +-
.../hive/ql/io/sarg/SearchArgumentImpl.java | 85 +-
.../hadoop/hive/ql/lib/PreOrderWalker.java | 8 +
.../hadoop/hive/ql/lockmgr/DbLockManager.java | 41 +-
.../hadoop/hive/ql/lockmgr/DbTxnManager.java | 22 +-
.../ql/lockmgr/zookeeper/ZooKeeperHiveLock.java | 22 +
.../apache/hadoop/hive/ql/metadata/Hive.java | 30 +-
.../apache/hadoop/hive/ql/metadata/Table.java | 18 +-
.../BucketingSortingReduceSinkOptimizer.java | 13 +
.../ql/optimizer/ColumnPrunerProcFactory.java | 25 +-
.../optimizer/ConstantPropagateProcFactory.java | 144 +-
.../hive/ql/optimizer/ConvertJoinMapJoin.java | 27 +-
.../hive/ql/optimizer/GroupByOptimizer.java | 7 +-
.../ql/optimizer/IdentityProjectRemover.java | 12 +
.../ql/optimizer/LimitPushdownOptimizer.java | 9 +-
.../ql/optimizer/NonBlockingOpDeDupProc.java | 13 +-
.../hadoop/hive/ql/optimizer/Optimizer.java | 16 +-
.../PrunerExpressionOperatorFactory.java | 3 -
.../optimizer/RemoveDynamicPruningBySize.java | 19 +-
.../hive/ql/optimizer/SimpleFetchOptimizer.java | 2 -
.../ql/optimizer/calcite/HiveCalciteUtil.java | 412 +-
.../ql/optimizer/calcite/HiveConfigContext.java | 37 +
.../calcite/HiveDefaultRelMetadataProvider.java | 55 +-
.../ql/optimizer/calcite/HiveRelCollation.java | 34 +
.../optimizer/calcite/HiveRelDistribution.java | 80 +
.../ql/optimizer/calcite/HiveRelOptUtil.java | 414 +
.../ql/optimizer/calcite/RelOptHiveTable.java | 217 +-
.../calcite/cost/HiveAlgorithmsConf.java | 39 +
.../calcite/cost/HiveAlgorithmsUtil.java | 363 +
.../ql/optimizer/calcite/cost/HiveCost.java | 52 +-
.../optimizer/calcite/cost/HiveCostModel.java | 102 +
.../ql/optimizer/calcite/cost/HiveCostUtil.java | 43 -
.../calcite/cost/HiveDefaultCostModel.java | 123 +
.../calcite/cost/HiveOnTezCostModel.java | 635 +
.../optimizer/calcite/cost/HiveRelMdCost.java | 71 +
.../calcite/cost/HiveVolcanoPlanner.java | 9 +-
.../calcite/reloperators/HiveAggregate.java | 10 +-
.../calcite/reloperators/HiveFilter.java | 4 +-
.../calcite/reloperators/HiveGroupingID.java | 17 +
.../calcite/reloperators/HiveJoin.java | 151 +-
.../calcite/reloperators/HiveLimit.java | 4 +-
.../calcite/reloperators/HiveProject.java | 4 +-
.../calcite/reloperators/HiveSortExchange.java | 84 +
.../calcite/reloperators/HiveTableScan.java | 126 +-
.../rules/HiveExpandDistinctAggregatesRule.java | 278 +
.../rules/HiveInsertExchange4JoinRule.java | 134 +
.../calcite/rules/HiveJoinAddNotNullRule.java | 197 +
.../calcite/rules/HiveJoinToMultiJoinRule.java | 333 +
.../calcite/rules/HiveWindowingFixRule.java | 163 +
.../calcite/stats/HiveRelMdCollation.java | 67 +
.../calcite/stats/HiveRelMdDistribution.java | 56 +
.../calcite/stats/HiveRelMdMemory.java | 102 +
.../calcite/stats/HiveRelMdParallelism.java | 117 +
.../calcite/stats/HiveRelMdRowCount.java | 1 -
.../optimizer/calcite/stats/HiveRelMdSize.java | 148 +
.../calcite/stats/HiveRelMdUniqueKeys.java | 1 -
.../calcite/translator/ASTBuilder.java | 3 +-
.../calcite/translator/ASTConverter.java | 24 +-
.../calcite/translator/ExprNodeConverter.java | 245 +-
.../calcite/translator/HiveGBOpConvUtil.java | 1226 ++
.../calcite/translator/HiveOpConverter.java | 986 ++
.../translator/HiveOpConverterPostProc.java | 175 +
.../translator/PlanModifierForASTConv.java | 111 +-
.../translator/PlanModifierForReturnPath.java | 41 +
.../calcite/translator/PlanModifierUtil.java | 138 +
.../calcite/translator/RexNodeConverter.java | 5 +-
.../calcite/translator/TypeConverter.java | 4 +-
.../ql/optimizer/lineage/ExprProcFactory.java | 3 +-
.../ql/optimizer/pcr/PcrExprProcFactory.java | 3 +-
.../physical/GenSparkSkewJoinProcessor.java | 9 +-
.../physical/SparkMapJoinResolver.java | 185 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 405 +-
.../spark/SparkSkewJoinProcFactory.java | 57 +-
.../optimizer/spark/SparkSkewJoinResolver.java | 5 +-
.../stats/annotation/StatsRulesProcFactory.java | 4 +-
.../hive/ql/parse/BaseSemanticAnalyzer.java | 96 +-
.../hadoop/hive/ql/parse/CalcitePlanner.java | 306 +-
.../ql/parse/ColumnStatsSemanticAnalyzer.java | 24 +-
.../hive/ql/parse/DDLSemanticAnalyzer.java | 159 +-
.../apache/hadoop/hive/ql/parse/EximUtil.java | 173 +-
.../hive/ql/parse/ExportSemanticAnalyzer.java | 121 +-
.../hadoop/hive/ql/parse/GenTezProcContext.java | 5 +-
.../hadoop/hive/ql/parse/GenTezUtils.java | 21 +-
.../apache/hadoop/hive/ql/parse/GenTezWork.java | 40 +-
.../hadoop/hive/ql/parse/GenTezWorkWalker.java | 6 +
.../org/apache/hadoop/hive/ql/parse/HiveLexer.g | 2 +
.../apache/hadoop/hive/ql/parse/HiveParser.g | 51 +-
.../hadoop/hive/ql/parse/IdentifiersParser.g | 8 +-
.../hive/ql/parse/ImportSemanticAnalyzer.java | 768 +-
.../hive/ql/parse/MetaDataExportListener.java | 2 +-
.../hive/ql/parse/OptimizeTezProcContext.java | 14 +
.../hadoop/hive/ql/parse/PTFTranslator.java | 14 +-
.../apache/hadoop/hive/ql/parse/ParseUtils.java | 42 +-
.../org/apache/hadoop/hive/ql/parse/QB.java | 10 +-
.../hadoop/hive/ql/parse/ReplicationSpec.java | 314 +
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 168 +-
.../hive/ql/parse/TableAccessAnalyzer.java | 6 +-
.../hadoop/hive/ql/parse/TezCompiler.java | 19 +-
.../hive/ql/parse/TypeCheckProcFactory.java | 6 +-
.../hadoop/hive/ql/parse/UnparseTranslator.java | 2 +-
.../ql/parse/UpdateDeleteSemanticAnalyzer.java | 13 +-
.../hadoop/hive/ql/parse/WindowingSpec.java | 9 +-
.../hive/ql/parse/spark/GenSparkUtils.java | 33 +-
.../hadoop/hive/ql/plan/AddPartitionDesc.java | 15 +
.../apache/hadoop/hive/ql/plan/BaseWork.java | 1 +
.../hadoop/hive/ql/plan/CreateTableDesc.java | 14 +
.../hadoop/hive/ql/plan/DropTableDesc.java | 23 +-
.../hive/ql/plan/ExprNodeConstantDesc.java | 15 +-
.../hadoop/hive/ql/plan/ExprNodeDescUtils.java | 39 +
.../hadoop/hive/ql/plan/ExprNodeNullDesc.java | 69 -
.../apache/hadoop/hive/ql/plan/JoinDesc.java | 19 +
.../apache/hadoop/hive/ql/plan/MapJoinDesc.java | 29 +-
.../org/apache/hadoop/hive/ql/plan/MapWork.java | 12 +-
.../hadoop/hive/ql/plan/PartitionDesc.java | 4 +-
.../apache/hadoop/hive/ql/plan/PlanUtils.java | 50 +-
.../hadoop/hive/ql/plan/ReduceSinkDesc.java | 12 +
.../apache/hadoop/hive/ql/plan/ReduceWork.java | 5 +-
.../apache/hadoop/hive/ql/plan/SparkWork.java | 11 +-
.../apache/hadoop/hive/ql/plan/TableDesc.java | 5 +-
.../hadoop/hive/ql/plan/VectorMapJoinDesc.java | 107 +
.../hadoop/hive/ql/plan/ptf/BoundaryDef.java | 42 +-
.../hadoop/hive/ql/plan/ptf/CurrentRowDef.java | 12 +-
.../hive/ql/plan/ptf/RangeBoundaryDef.java | 38 +-
.../hive/ql/plan/ptf/ValueBoundaryDef.java | 50 +-
.../hadoop/hive/ql/plan/ptf/WindowFrameDef.java | 31 +-
.../AuthorizationPreEventListener.java | 68 +-
...MultiPartitionAuthorizationProviderBase.java | 43 +
.../MetaStoreAuthzAPIAuthorizerEmbedOnly.java | 3 +-
.../StorageBasedAuthorizationProvider.java | 11 +-
.../hadoop/hive/ql/session/SessionState.java | 33 +-
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 12 +-
.../hadoop/hive/ql/txn/compactor/Cleaner.java | 38 +-
.../hive/ql/txn/compactor/CompactorThread.java | 12 +-
.../hadoop/hive/ql/txn/compactor/Initiator.java | 11 +-
.../hadoop/hive/ql/txn/compactor/Worker.java | 42 +-
.../org/apache/hadoop/hive/ql/udf/UDFCrc32.java | 75 +
.../org/apache/hadoop/hive/ql/udf/UDFMd5.java | 88 +
.../apache/hadoop/hive/ql/udf/UDFRegExp.java | 76 -
.../org/apache/hadoop/hive/ql/udf/UDFSha1.java | 88 +
.../hive/ql/udf/generic/GenericUDAFAverage.java | 43 +-
.../ql/udf/generic/GenericUDAFFirstValue.java | 20 +-
.../ql/udf/generic/GenericUDAFLastValue.java | 18 +-
.../hive/ql/udf/generic/GenericUDAFMax.java | 20 +-
.../generic/GenericUDAFStreamingEvaluator.java | 97 +-
.../hive/ql/udf/generic/GenericUDAFSum.java | 41 +-
.../hadoop/hive/ql/udf/generic/GenericUDF.java | 15 +-
.../ql/udf/generic/GenericUDFAddMonths.java | 5 +-
.../hive/ql/udf/generic/GenericUDFArray.java | 5 +-
.../hive/ql/udf/generic/GenericUDFBaseDTI.java | 18 +
.../hive/ql/udf/generic/GenericUDFCbrt.java | 3 +-
.../hive/ql/udf/generic/GenericUDFCoalesce.java | 2 +-
.../ql/udf/generic/GenericUDFDateFormat.java | 125 +
.../hive/ql/udf/generic/GenericUDFDecode.java | 26 +-
.../hive/ql/udf/generic/GenericUDFEncode.java | 2 +-
.../ql/udf/generic/GenericUDFFactorial.java | 3 +-
.../hive/ql/udf/generic/GenericUDFGreatest.java | 1 +
.../hive/ql/udf/generic/GenericUDFInstr.java | 2 +-
.../hive/ql/udf/generic/GenericUDFLastDay.java | 4 +-
.../ql/udf/generic/GenericUDFLevenshtein.java | 5 +-
.../hive/ql/udf/generic/GenericUDFLocate.java | 2 +-
.../hive/ql/udf/generic/GenericUDFMap.java | 20 +-
.../hive/ql/udf/generic/GenericUDFNextDay.java | 11 +-
.../ql/udf/generic/GenericUDFParamUtils.java | 71 +
.../hive/ql/udf/generic/GenericUDFPrintf.java | 3 +-
.../hive/ql/udf/generic/GenericUDFQuarter.java | 85 +
.../hive/ql/udf/generic/GenericUDFRegExp.java | 133 +
.../hive/ql/udf/generic/GenericUDFSha2.java | 137 +
.../hive/ql/udf/generic/GenericUDFSoundex.java | 6 +-
.../ql/udf/generic/GenericUDFTimestamp.java | 24 +
.../ql/udf/generic/GenericUDFTranslate.java | 8 +-
.../hive/ql/udf/generic/GenericUDFTrunc.java | 11 +-
.../hive/ql/udf/generic/GenericUDFUtils.java | 6 +-
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 77 +-
.../session/TestSparkSessionManagerImpl.java | 5 +
.../exec/vector/mapjoin/TestDebugDisplay.java | 63 +
.../mapjoin/fast/CommonFastHashTable.java | 128 +
.../mapjoin/fast/RandomByteArrayStream.java | 92 +
.../vector/mapjoin/fast/RandomLongStream.java | 49 +
.../fast/TestVectorMapJoinFastLongHashMap.java | 219 +
.../TestVectorMapJoinFastMultiKeyHashMap.java | 231 +
.../TestVectorMapJoinRowBytesContainer.java | 74 +
.../hive/ql/io/filters/TestBloomFilter.java | 458 -
.../hadoop/hive/ql/io/filters/TestMurmur3.java | 189 -
.../hive/ql/io/orc/TestColumnStatistics.java | 20 +-
.../hadoop/hive/ql/io/orc/TestFileDump.java | 2 +-
.../hadoop/hive/ql/io/orc/TestJsonFileDump.java | 138 +
.../hadoop/hive/ql/io/orc/TestOrcFile.java | 105 +-
.../hive/ql/io/orc/TestOrcSerDeStats.java | 12 +-
.../hadoop/hive/ql/io/orc/TestOrcTimezone1.java | 4 +-
.../hadoop/hive/ql/io/orc/TestOrcTimezone2.java | 2 +-
.../hive/ql/io/orc/TestRecordReaderImpl.java | 328 +-
.../io/parquet/AbstractTestParquetDirect.java | 23 +-
.../ql/io/parquet/TestArrayCompatibility.java | 18 +
.../hive/ql/io/parquet/TestMapStructures.java | 18 +
.../hive/ql/io/sarg/TestSearchArgumentImpl.java | 104 +-
.../hive/ql/lockmgr/TestDbTxnManager.java | 2 +
.../hive/ql/lockmgr/TestDbTxnManager2.java | 233 +
.../ql/optimizer/physical/TestVectorizer.java | 2 +-
...tedCharsInColumnNameCreateTableNegative.java | 87 +
.../hadoop/hive/ql/plan/TestViewEntity.java | 108 +
.../hadoop/hive/ql/session/TestAddResource.java | 116 +-
.../hive/ql/txn/compactor/TestCleaner.java | 56 +-
.../hive/ql/txn/compactor/TestInitiator.java | 63 +-
.../hive/ql/txn/compactor/TestWorker.java | 45 +
.../hadoop/hive/ql/udaf/TestStreamingSum.java | 16 +-
.../hadoop/hive/ql/udf/TestGenericUDFDate.java | 92 -
.../hive/ql/udf/TestGenericUDFDateAdd.java | 145 -
.../hive/ql/udf/TestGenericUDFDateDiff.java | 116 -
.../hive/ql/udf/TestGenericUDFDateSub.java | 143 -
.../hadoop/hive/ql/udf/TestGenericUDFUtils.java | 58 -
.../apache/hadoop/hive/ql/udf/TestUDFCrc32.java | 74 +
.../apache/hadoop/hive/ql/udf/TestUDFJson.java | 18 +
.../apache/hadoop/hive/ql/udf/TestUDFMd5.java | 57 +
.../apache/hadoop/hive/ql/udf/TestUDFSha1.java | 57 +
.../hive/ql/udf/generic/TestGenericUDFDate.java | 92 +
.../ql/udf/generic/TestGenericUDFDateAdd.java | 143 +
.../ql/udf/generic/TestGenericUDFDateDiff.java | 116 +
.../udf/generic/TestGenericUDFDateFormat.java | 173 +
.../ql/udf/generic/TestGenericUDFDateSub.java | 143 +
.../udf/generic/TestGenericUDFLevenshtein.java | 4 +-
.../ql/udf/generic/TestGenericUDFNextDay.java | 4 +-
.../ql/udf/generic/TestGenericUDFQuarter.java | 182 +
.../ql/udf/generic/TestGenericUDFRegexp.java | 135 +
.../hive/ql/udf/generic/TestGenericUDFSha2.java | 271 +
.../ql/udf/generic/TestGenericUDFUtils.java | 57 +
.../authorization_set_nonexistent_conf.q | 7 +
.../queries/clientnegative/insertsel_fail.q | 1 +
.../clientnegative/protectmode_part_no_drop2.q | 11 +
.../clientpositive/alter_change_db_location.q | 5 +
.../clientpositive/alter_partition_coltype.q | 1 +
.../clientpositive/annotate_stats_part.q | 1 +
.../clientpositive/auto_sortmerge_join_13.q | 2 +
.../test/queries/clientpositive/bucket_many.q | 16 +
ql/src/test/queries/clientpositive/cbo_join.q | 1 +
ql/src/test/queries/clientpositive/cbo_limit.q | 2 +-
.../queries/clientpositive/cbo_rp_auto_join0.q | 24 +
.../queries/clientpositive/cbo_rp_auto_join1.q | 274 +
.../queries/clientpositive/cbo_simple_select.q | 3 +-
.../test/queries/clientpositive/cbo_subq_in.q | 8 +-
.../queries/clientpositive/cbo_subq_not_in.q | 4 +-
.../test/queries/clientpositive/cbo_udf_udaf.q | 12 +-
.../queries/clientpositive/decimal_precision2.q | 15 +
.../dynamic_partition_pruning_2.q | 17 +
.../clientpositive/encryption_move_tbl.q | 18 +
.../test/queries/clientpositive/explainuser_2.q | 23 +
.../extrapolate_part_stats_partial.q | 2 +
.../extrapolate_part_stats_partial_ndv.q | 2 +
ql/src/test/queries/clientpositive/fold_case.q | 12 +
.../clientpositive/fold_eq_with_case_when.q | 21 +
ql/src/test/queries/clientpositive/fold_when.q | 31 +
.../clientpositive/hybridgrace_hashjoin_1.q | 258 +
.../clientpositive/hybridgrace_hashjoin_2.q | 152 +
.../queries/clientpositive/hybridhashjoin.q | 250 -
.../clientpositive/insert_overwrite_directory.q | 141 +
.../queries/clientpositive/join_on_varchar.q | 12 +
.../clientpositive/json_serde_qualified_types.q | 12 +
.../queries/clientpositive/limit_pushdown.q | 4 +
.../queries/clientpositive/mapjoin_mapjoin.q | 1 +
ql/src/test/queries/clientpositive/mergejoin.q | 124 +
.../queries/clientpositive/optimize_nullscan.q | 4 +
.../clientpositive/orc_int_type_promotion.q | 79 +
.../clientpositive/partition_coltype_literals.q | 75 +
.../test/queries/clientpositive/repl_1_drop.q | 84 +
.../queries/clientpositive/repl_2_exim_basic.q | 79 +
.../clientpositive/repl_3_exim_metadata.q | 40 +
.../runtime_skewjoin_mapjoin_spark.q | 24 +
.../test/queries/clientpositive/tez_join_hash.q | 2 +
ql/src/test/queries/clientpositive/tez_smb_1.q | 1 +
.../test/queries/clientpositive/tez_smb_main.q | 7 +
ql/src/test/queries/clientpositive/tez_union.q | 18 +
.../clientpositive/tez_union_multiinsert.q | 120 +
.../clientpositive/timestamp_ints_casts.q | 74 +
.../clientpositive/udaf_percentile_approx_23.q | 2 +-
ql/src/test/queries/clientpositive/udf_crc32.q | 13 +
.../queries/clientpositive/udf_date_format.q | 60 +
ql/src/test/queries/clientpositive/udf_md5.q | 13 +
.../test/queries/clientpositive/udf_quarter.q | 100 +
ql/src/test/queries/clientpositive/udf_sha1.q | 13 +
ql/src/test/queries/clientpositive/udf_sha2.q | 41 +
ql/src/test/queries/clientpositive/union12.q | 2 +
ql/src/test/queries/clientpositive/union17.q | 2 +-
ql/src/test/queries/clientpositive/union20.q | 2 +-
ql/src/test/queries/clientpositive/union21.q | 2 +-
ql/src/test/queries/clientpositive/union27.q | 2 +-
.../queries/clientpositive/union_remove_22.q | 2 +
.../clientpositive/union_remove_6_subq.q | 2 +
.../queries/clientpositive/vector_aggregate_9.q | 2 +
.../clientpositive/vector_binary_join_groupby.q | 55 +
.../clientpositive/vector_char_mapjoin1.q | 9 +
.../clientpositive/vector_decimal_mapjoin.q | 43 +-
.../queries/clientpositive/vector_inner_join.q | 61 +
.../test/queries/clientpositive/vector_join30.q | 160 +
.../clientpositive/vector_join_filters.q | 38 +
.../queries/clientpositive/vector_join_nulls.q | 33 +
.../clientpositive/vector_left_outer_join.q | 1 +
.../clientpositive/vector_left_outer_join2.q | 64 +
.../clientpositive/vector_leftsemi_mapjoin.q | 403 +
.../vector_mr_diff_schema_alias.q | 115 +
.../clientpositive/vector_nullsafe_join.q | 72 +
.../queries/clientpositive/vector_outer_join0.q | 25 +
.../queries/clientpositive/vector_outer_join1.q | 64 +
.../queries/clientpositive/vector_outer_join2.q | 39 +
.../queries/clientpositive/vector_outer_join3.q | 80 +
.../queries/clientpositive/vector_outer_join4.q | 66 +
.../queries/clientpositive/vector_outer_join5.q | 173 +
.../queries/clientpositive/vectorized_casts.q | 2 +-
.../queries/clientpositive/vectorized_context.q | 1 +
.../clientpositive/vectorized_timestamp_funcs.q | 36 +-
.../vectorized_timestamp_ints_casts.q | 76 +
.../queries/clientpositive/windowing_navfn.q | 4 +-
.../clientpositive/windowing_windowspec2.q | 24 +
ql/src/test/resources/orc-file-dump.json | 1354 ++
ql/src/test/resources/orc-file-has-null.out | 42 +-
.../alter_rename_partition_failure3.q.out | 2 +-
.../alter_table_add_partition.q.out | 2 +-
.../clientnegative/alter_view_failure5.q.out | 2 +-
.../clientnegative/alter_view_failure7.q.out | 2 +-
.../clientnegative/archive_partspec1.q.out | 2 +-
.../clientnegative/archive_partspec5.q.out | 8 +-
.../authorization_set_nonexistent_conf.q.out | 9 +
.../results/clientnegative/insertsel_fail.q.out | 1 +
.../clientnegative/join_nonexistent_part.q.out | 2 +-
.../protectmode_part_no_drop2.q.out | 51 +
ql/src/test/results/clientnegative/touch2.q.out | 2 +-
.../truncate_partition_column.q.out | 2 +-
.../clientnegative/udf_add_months_error_1.q.out | 2 +-
.../clientnegative/udf_last_day_error_1.q.out | 2 +-
.../clientnegative/udf_next_day_error_1.q.out | 2 +-
.../clientnegative/udf_next_day_error_2.q.out | 2 +-
.../alter_change_db_location.q.out | 35 +
.../clientpositive/annotate_stats_filter.q.out | 4 +-
.../clientpositive/annotate_stats_join.q.out | 32 +-
.../clientpositive/annotate_stats_select.q.out | 2 +-
.../results/clientpositive/auto_join32.q.out | 30 +
.../clientpositive/auto_sortmerge_join_1.q.out | 3 +
.../clientpositive/auto_sortmerge_join_2.q.out | 2 +
.../clientpositive/auto_sortmerge_join_3.q.out | 3 +
.../clientpositive/auto_sortmerge_join_4.q.out | 3 +
.../clientpositive/auto_sortmerge_join_5.q.out | 3 +
.../clientpositive/auto_sortmerge_join_7.q.out | 3 +
.../clientpositive/auto_sortmerge_join_8.q.out | 3 +
.../results/clientpositive/bucket_many.q.out | 230 +
.../clientpositive/bucketcontext_1.q.out | 1 +
.../clientpositive/bucketcontext_2.q.out | 1 +
.../clientpositive/bucketcontext_3.q.out | 1 +
.../clientpositive/bucketcontext_4.q.out | 1 +
.../clientpositive/bucketcontext_5.q.out | 1 +
.../clientpositive/bucketcontext_6.q.out | 1 +
.../clientpositive/bucketcontext_7.q.out | 1 +
.../clientpositive/bucketcontext_8.q.out | 1 +
.../results/clientpositive/bucketmapjoin1.q.out | 88 +
.../test/results/clientpositive/cbo_join.q.out | 350 +-
.../test/results/clientpositive/cbo_limit.q.out | 4 +-
.../clientpositive/cbo_rp_auto_join0.q.out | 258 +
.../clientpositive/cbo_rp_auto_join1.q.out | 1512 ++
.../clientpositive/cbo_simple_select.q.out | 10 +
.../results/clientpositive/cbo_subq_in.q.out | 38 +-
.../clientpositive/cbo_subq_not_in.q.out | 32 +-
.../results/clientpositive/cbo_udf_udaf.q.out | 34 +-
.../clientpositive/correlationoptimizer12.q.out | 8 +-
.../clientpositive/correlationoptimizer3.q.out | 60 +-
.../clientpositive/correlationoptimizer6.q.out | 32 +-
.../results/clientpositive/ctas_colname.q.out | 8 +-
.../clientpositive/decimal_precision2.q.out | 163 +
.../results/clientpositive/decimal_udf.q.out | 2 +-
.../encryption_insert_partition_static.q.out | 14 +-
.../encrypted/encryption_move_tbl.q.out | 43 +
.../test/results/clientpositive/fold_case.q.out | 301 +
.../clientpositive/fold_eq_with_case_when.q.out | 231 +
.../test/results/clientpositive/fold_when.q.out | 480 +
.../groupby_grouping_window.q.out | 4 +-
.../clientpositive/groupby_resolution.q.out | 4 +-
.../results/clientpositive/groupby_sort_6.q.out | 54 +
.../test/results/clientpositive/input23.q.out | 22 +-
.../test/results/clientpositive/input26.q.out | 18 +
ql/src/test/results/clientpositive/input6.q.out | 2 +-
ql/src/test/results/clientpositive/input8.q.out | 2 +-
ql/src/test/results/clientpositive/input9.q.out | 8 +-
.../insert_overwrite_directory.q.out | 1813 +++
.../join_cond_pushdown_unqual2.q.out | 24 +-
.../join_cond_pushdown_unqual4.q.out | 24 +-
.../results/clientpositive/join_nullsafe.q.out | 10 +-
.../clientpositive/join_on_varchar.q.out | 146 +
.../test/results/clientpositive/join_view.q.out | 25 +
.../json_serde_qualified_types.q.out | 33 +
.../results/clientpositive/limit_pushdown.q.out | 88 +
.../clientpositive/literal_decimal.q.out | 6 +-
.../clientpositive/load_dyn_part14.q.out | 24 +-
.../test/results/clientpositive/mergejoin.q.out | 3157 ++++
.../results/clientpositive/metadataonly1.q.out | 20 +
.../results/clientpositive/nullgroup5.q.out | 19 +
.../clientpositive/num_op_type_conv.q.out | 6 +-
.../clientpositive/optimize_nullscan.q.out | 269 +-
.../clientpositive/orc_int_type_promotion.q.out | 377 +
.../clientpositive/partition_boolexpr.q.out | 32 +
.../partition_coltype_literals.q.out | 647 +
.../clientpositive/partition_timestamp.q.out | 116 +-
.../clientpositive/partition_timestamp2.q.out | 250 +-
.../clientpositive/ppd_constant_expr.q.out | 4 +-
.../results/clientpositive/ppd_union_view.q.out | 80 +
ql/src/test/results/clientpositive/ptf.q.out | 160 +-
.../results/clientpositive/ptf_streaming.q.out | 84 +-
.../ql_rewrite_gbtoidx_cbo_2.q.out | 14 +-
.../results/clientpositive/quotedid_basic.q.out | 8 +-
.../clientpositive/reduce_deduplicate.q.out | 34 +
.../results/clientpositive/repl_1_drop.q.out | 345 +
.../clientpositive/repl_2_exim_basic.q.out | 494 +
.../clientpositive/repl_3_exim_metadata.q.out | 222 +
.../runtime_skewjoin_mapjoin_spark.q.out | 669 +
.../test/results/clientpositive/sample6.q.out | 20 +
.../results/clientpositive/show_functions.q.out | 10 +
.../results/clientpositive/smb_mapjoin9.q.out | 82 +
.../results/clientpositive/smb_mapjoin_11.q.out | 1 +
.../results/clientpositive/smb_mapjoin_12.q.out | 2 +
.../results/clientpositive/smb_mapjoin_13.q.out | 1 +
.../results/clientpositive/smb_mapjoin_15.q.out | 3 +
.../clientpositive/sort_merge_join_desc_5.q.out | 1 +
.../clientpositive/spark/auto_join0.q.out | 2 +-
.../clientpositive/spark/auto_join15.q.out | 2 +-
.../clientpositive/spark/auto_join20.q.out | 4 +-
.../clientpositive/spark/auto_join21.q.out | 2 +-
.../clientpositive/spark/auto_join23.q.out | 2 +-
.../clientpositive/spark/auto_join28.q.out | 8 +-
.../clientpositive/spark/auto_join29.q.out | 18 +-
.../clientpositive/spark/auto_join30.q.out | 42 +-
.../clientpositive/spark/auto_join31.q.out | 6 +-
.../clientpositive/spark/auto_join32.q.out | 26 +
.../clientpositive/spark/auto_join8.q.out | 2 +-
.../spark/auto_sortmerge_join_1.q.out | 2 +
.../spark/auto_sortmerge_join_2.q.out | 1 +
.../spark/auto_sortmerge_join_3.q.out | 2 +
.../spark/auto_sortmerge_join_4.q.out | 2 +
.../spark/auto_sortmerge_join_5.q.out | 2 +
.../spark/auto_sortmerge_join_7.q.out | 2 +
.../spark/auto_sortmerge_join_8.q.out | 2 +
.../results/clientpositive/spark/bucket5.q.out | 2 +-
.../clientpositive/spark/bucketmapjoin1.q.out | 120 +
.../results/clientpositive/spark/cbo_gby.q.out | 120 +
.../clientpositive/spark/cbo_limit.q.out | 90 +
.../clientpositive/spark/cbo_semijoin.q.out | 440 +
.../spark/cbo_simple_select.q.out | 755 +
.../clientpositive/spark/cbo_stats.q.out | 14 +
.../clientpositive/spark/cbo_subq_in.q.out | 149 +
.../clientpositive/spark/cbo_subq_not_in.q.out | 365 +
.../clientpositive/spark/cbo_udf_udaf.q.out | 121 +
.../clientpositive/spark/cbo_union.q.out | 920 ++
.../results/clientpositive/spark/ctas.q.out | 20 +-
.../spark/escape_clusterby1.q.out | 4 +-
.../clientpositive/spark/escape_sortby1.q.out | 4 +-
.../clientpositive/spark/groupby10.q.out | 2 +-
.../groupby7_map_multi_single_reducer.q.out | 2 +-
.../groupby7_noskew_multi_single_reducer.q.out | 4 +-
.../clientpositive/spark/groupby8_map.q.out | 2 +-
.../clientpositive/spark/groupby8_noskew.q.out | 2 +-
...pby_complex_types_multi_single_reducer.q.out | 38 +-
.../spark/groupby_multi_single_reducer3.q.out | 8 +-
.../spark/identity_project_remove_skip.q.out | 4 +-
.../results/clientpositive/spark/input14.q.out | 2 +-
.../results/clientpositive/spark/input17.q.out | 2 +-
.../results/clientpositive/spark/input18.q.out | 2 +-
.../clientpositive/spark/join0.q.java1.7.out | 2 +-
.../clientpositive/spark/join0.q.java1.8.out | 238 +
.../results/clientpositive/spark/join15.q.out | 2 +-
.../results/clientpositive/spark/join20.q.out | 4 +-
.../results/clientpositive/spark/join21.q.out | 2 +-
.../results/clientpositive/spark/join23.q.out | 2 +-
.../results/clientpositive/spark/join40.q.out | 4 +-
.../results/clientpositive/spark/join8.q.out | 2 +-
.../clientpositive/spark/join_nullsafe.q.out | 10 +-
.../clientpositive/spark/join_view.q.out | 26 +
.../spark/lateral_view_explode2.q.out | 4 +-
.../clientpositive/spark/limit_pushdown.q.out | 94 +
.../spark/list_bucket_dml_2.q.java1.8.out | 665 +
.../clientpositive/spark/load_dyn_part14.q.out | 8 +-
.../clientpositive/spark/mapjoin_decimal.q.out | 8 +-
.../spark/mapjoin_filter_on_outerjoin.q.out | 4 +-
.../spark/mapjoin_test_outer.q.out | 4 +-
.../clientpositive/spark/multi_insert.q.out | 8 +-
.../clientpositive/spark/multi_insert_gby.q.out | 4 +-
.../spark/multi_insert_gby3.q.out | 6 +-
.../spark/multi_insert_lateral_view.q.out | 2 +-
...i_insert_move_tasks_share_dependencies.q.out | 72 +-
.../spark/multigroupby_singlemr.q.out | 2 +-
.../spark/optimize_nullscan.q.out | 192 +
.../results/clientpositive/spark/parallel.q.out | 2 +-
.../clientpositive/spark/parallel_join0.q.out | 2 +-
.../clientpositive/spark/ppd_join4.q.out | 2 +-
.../clientpositive/spark/ppd_transform.q.out | 4 +-
.../test/results/clientpositive/spark/ptf.q.out | 194 +-
.../clientpositive/spark/ptf_streaming.q.out | 100 +-
.../spark/reduce_deduplicate.q.out | 36 +-
.../spark/reduce_deduplicate_exclude_join.q.out | 2 +-
.../spark/runtime_skewjoin_mapjoin_spark.q.out | 314 +
.../results/clientpositive/spark/sample6.q.out | 20 +
.../results/clientpositive/spark/semijoin.q.out | 40 +-
.../clientpositive/spark/smb_mapjoin_11.q.out | 1 +
.../clientpositive/spark/smb_mapjoin_12.q.out | 2 +
.../clientpositive/spark/smb_mapjoin_13.q.out | 1 +
.../clientpositive/spark/smb_mapjoin_15.q.out | 3 +
.../results/clientpositive/spark/sort.q.out | 2 +-
.../spark/sort_merge_join_desc_5.q.out | 43 +-
.../clientpositive/spark/subquery_in.q.out | 8 +-
.../spark/subquery_multiinsert.q.java1.8.out | 890 ++
.../clientpositive/spark/transform_ppr1.q.out | 2 +-
.../clientpositive/spark/transform_ppr2.q.out | 2 +-
.../results/clientpositive/spark/union12.q.out | 200 +
.../results/clientpositive/spark/union17.q.out | 855 +
.../results/clientpositive/spark/union20.q.out | 200 +
.../results/clientpositive/spark/union21.q.out | 724 +
.../results/clientpositive/spark/union22.q.out | 1602 ++
.../results/clientpositive/spark/union24.q.out | 1649 ++
.../results/clientpositive/spark/union26.q.out | 1249 ++
.../results/clientpositive/spark/union27.q.out | 136 +
.../results/clientpositive/spark/union3.q.out | 2 +-
.../results/clientpositive/spark/union31.q.out | 815 +
.../results/clientpositive/spark/union32.q.out | 669 +
.../results/clientpositive/spark/union34.q.out | 406 +
.../clientpositive/spark/union_date.q.out | 142 +
.../clientpositive/spark/union_date_trim.q.out | 54 +
.../spark/union_lateralview.q.out | 251 +
.../clientpositive/spark/union_ppr.q.out | 2 +-
.../clientpositive/spark/union_remove_12.q.out | 281 +
.../clientpositive/spark/union_remove_13.q.out | 306 +
.../clientpositive/spark/union_remove_14.q.out | 283 +
.../clientpositive/spark/union_remove_22.q.out | 401 +
.../clientpositive/spark/union_remove_23.q.out | 265 +
.../clientpositive/spark/union_remove_25.q.out | 2 +-
.../spark/union_remove_6_subq.q.out | 1181 ++
.../clientpositive/spark/union_script.q.out | 1524 ++
.../clientpositive/spark/union_top_level.q.out | 1012 ++
.../clientpositive/spark/union_view.q.out | 264 +
.../spark/vector_between_in.q.out | 6088 +-------
.../spark/vector_cast_constant.q.java1.7.out | 16 +-
.../spark/vector_cast_constant.q.java1.8.out | 16 +-
.../spark/vector_decimal_mapjoin.q.out | 366 +-
.../clientpositive/spark/vector_elt.q.out | 2 +-
.../spark/vectorization_decimal_date.q.out | 20 +-
.../spark/vectorization_short_regress.q.out | 8 +-
.../clientpositive/spark/vectorized_ptf.q.out | 194 +-
.../spark/vectorized_timestamp_funcs.q.out | 106 +-
.../results/clientpositive/subquery_in.q.out | 8 +-
.../subquery_in_explain_rewrite.q.out | 4 +-
.../clientpositive/subquery_in_having.q.out | 6 +-
.../results/clientpositive/subquery_notin.q.out | 24 +-
.../subquery_notin_having.q.java1.7.out | 2 +-
.../subquery_unqualcolumnrefs.q.out | 16 +-
.../tez/acid_vectorization_partition.q.out | 20 +-
.../results/clientpositive/tez/auto_join0.q.out | 1 +
.../results/clientpositive/tez/auto_join1.q.out | 1 +
.../clientpositive/tez/auto_join29.q.out | 503 +
.../clientpositive/tez/auto_join30.q.out | 4 +
.../tez/auto_sortmerge_join_1.q.out | 3 +
.../tez/auto_sortmerge_join_10.q.out | 3 +
.../tez/auto_sortmerge_join_11.q.out | 4 +
.../tez/auto_sortmerge_join_12.q.out | 3 +
.../tez/auto_sortmerge_join_13.q.out | 3 +
.../tez/auto_sortmerge_join_14.q.out | 2 +
.../tez/auto_sortmerge_join_15.q.out | 2 +
.../tez/auto_sortmerge_join_2.q.out | 2 +
.../tez/auto_sortmerge_join_3.q.out | 3 +
.../tez/auto_sortmerge_join_4.q.out | 3 +
.../tez/auto_sortmerge_join_5.q.out | 1 +
.../tez/auto_sortmerge_join_7.q.out | 3 +
.../tez/auto_sortmerge_join_8.q.out | 3 +
.../tez/auto_sortmerge_join_9.q.out | 29 +
.../tez/bucket_map_join_tez1.q.out | 16 +
.../tez/bucket_map_join_tez2.q.out | 8 +
.../results/clientpositive/tez/cbo_join.q.out | 350 +-
.../results/clientpositive/tez/cbo_limit.q.out | 4 +-
.../clientpositive/tez/cbo_simple_select.q.out | 10 +
.../clientpositive/tez/cbo_subq_in.q.out | 38 +-
.../clientpositive/tez/cbo_subq_not_in.q.out | 32 +-
.../clientpositive/tez/cbo_udf_udaf.q.out | 34 +-
.../tez/correlationoptimizer1.q.out | 1 +
.../tez/cross_product_check_2.q.out | 9 +
.../tez/dynamic_partition_pruning.q.out | 51 +
.../tez/dynamic_partition_pruning_2.q.out | 123 +
.../clientpositive/tez/explainuser_1.q.out | 6 +-
.../clientpositive/tez/explainuser_2.q.out | 3056 ++--
.../tez/hybridgrace_hashjoin_1.q.out | 1587 ++
.../tez/hybridgrace_hashjoin_2.q.out | 1417 ++
.../clientpositive/tez/hybridhashjoin.q.out | 1560 --
.../clientpositive/tez/join_nullsafe.q.out | 14 +-
.../clientpositive/tez/limit_pushdown.q.out | 125 +-
.../clientpositive/tez/lvj_mapjoin.q.out | 2 +
.../clientpositive/tez/mapjoin_decimal.q.out | 10 +-
.../clientpositive/tez/mapjoin_mapjoin.q.out | 6 +
.../results/clientpositive/tez/mergejoin.q.out | 3118 ++++
.../clientpositive/tez/metadataonly1.q.out | 20 +
.../test/results/clientpositive/tez/mrr.q.out | 54 +-
.../clientpositive/tez/optimize_nullscan.q.out | 192 +
.../test/results/clientpositive/tez/ptf.q.out | 160 +-
.../clientpositive/tez/ptf_streaming.q.out | 84 +-
.../clientpositive/tez/subquery_in.q.out | 8 +-
.../tez/tez_bmj_schema_evolution.q.out | 1 +
.../clientpositive/tez/tez_join_hash.q.out | 2 +
.../clientpositive/tez/tez_smb_main.q.out | 12 +
.../results/clientpositive/tez/tez_union.q.out | 79 +
.../clientpositive/tez/tez_union_group_by.q.out | 67 +
.../tez/tez_union_multiinsert.q.out | 4293 +++++
.../clientpositive/tez/unionDistinct_1.q.out | 10 +
.../results/clientpositive/tez/union_view.q.out | 1004 ++
.../clientpositive/tez/vector_aggregate_9.q.out | 8 +-
.../clientpositive/tez/vector_between_in.q.out | 6088 +-------
.../tez/vector_binary_join_groupby.q.out | 303 +
.../tez/vector_char_mapjoin1.q.out | 43 +-
.../clientpositive/tez/vector_coalesce.q.out | 26 +-
.../tez/vector_count_distinct.q.out | 28 +-
.../clientpositive/tez/vector_decimal_2.q.out | 4 +
.../tez/vector_decimal_mapjoin.q.out | 371 +-
.../clientpositive/tez/vector_decimal_udf.q.out | 3 +-
.../results/clientpositive/tez/vector_elt.q.out | 2 +-
.../clientpositive/tez/vector_inner_join.q.out | 806 +
.../clientpositive/tez/vector_join30.q.out | 1367 ++
.../tez/vector_join_filters.q.out | 222 +
.../clientpositive/tez/vector_join_nulls.q.out | 195 +
.../tez/vector_left_outer_join2.q.out | 555 +
.../tez/vector_left_outer_join3.q.out | 222 +
.../tez/vector_leftsemi_mapjoin.q.out | 13807 +++++++++++++++++
.../tez/vector_mapjoin_reduce.q.out | 4 +
.../tez/vector_mr_diff_schema_alias.q.out | 381 +
.../tez/vector_nullsafe_join.q.out | 1210 ++
.../clientpositive/tez/vector_outer_join.q.out | 2204 +++
.../clientpositive/tez/vector_outer_join0.q.out | 232 +
.../clientpositive/tez/vector_outer_join1.q.out | 541 +
.../clientpositive/tez/vector_outer_join2.q.out | 238 +
.../clientpositive/tez/vector_outer_join3.q.out | 527 +
.../clientpositive/tez/vector_outer_join4.q.out | 864 ++
.../clientpositive/tez/vector_outer_join5.q.out | 1328 ++
.../tez/vector_varchar_mapjoin1.q.out | 5 +-
.../tez/vectorization_decimal_date.q.out | 20 +-
.../tez/vectorization_limit.q.out | 31 +-
.../tez/vectorization_short_regress.q.out | 8 +-
.../clientpositive/tez/vectorized_casts.q.out | 52 +-
.../tez/vectorized_distinct_gby.q.out | 51 +-
.../vectorized_dynamic_partition_pruning.q.out | 51 +
.../clientpositive/tez/vectorized_mapjoin.q.out | 1 +
.../tez/vectorized_nested_mapjoin.q.out | 2 +
.../tez/vectorized_parquet_types.q.out | 347 +
.../clientpositive/tez/vectorized_ptf.q.out | 160 +-
.../tez/vectorized_timestamp_funcs.q.out | 108 +-
.../tez/vectorized_timestamp_ints_casts.q.out | 234 +
.../clientpositive/timestamp_ints_casts.q.out | 264 +
.../udaf_percentile_approx_23.q.out | 6 +-
ql/src/test/results/clientpositive/udf4.q.out | 2 +-
ql/src/test/results/clientpositive/udf6.q.out | 30 +-
ql/src/test/results/clientpositive/udf7.q.out | 2 +-
.../test/results/clientpositive/udf_case.q.out | 2 +-
.../results/clientpositive/udf_coalesce.q.out | 4 +-
.../test/results/clientpositive/udf_crc32.q.out | 60 +
.../clientpositive/udf_date_format.q.out | 172 +
.../results/clientpositive/udf_decode.q.out | 2 +-
.../test/results/clientpositive/udf_elt.q.out | 4 +-
.../results/clientpositive/udf_greatest.q.out | 4 +-
.../test/results/clientpositive/udf_hour.q.out | 2 +-
ql/src/test/results/clientpositive/udf_if.q.out | 4 +-
.../test/results/clientpositive/udf_instr.q.out | 4 +-
.../clientpositive/udf_isnull_isnotnull.q.out | 2 +-
.../test/results/clientpositive/udf_least.q.out | 4 +-
.../results/clientpositive/udf_locate.q.out | 4 +-
.../test/results/clientpositive/udf_md5.q.out | 61 +
.../results/clientpositive/udf_minute.q.out | 2 +-
.../test/results/clientpositive/udf_nvl.q.out | 2 +-
.../results/clientpositive/udf_parse_url.q.out | 2 +-
.../results/clientpositive/udf_quarter.q.out | 246 +
.../results/clientpositive/udf_second.q.out | 2 +-
.../test/results/clientpositive/udf_sha1.q.out | 61 +
.../test/results/clientpositive/udf_sha2.q.out | 134 +
.../test/results/clientpositive/udf_size.q.out | 2 +-
.../test/results/clientpositive/udf_trunc.q.out | 64 +-
.../test/results/clientpositive/udf_when.q.out | 4 +-
.../test/results/clientpositive/union12.q.out | 8 +-
.../test/results/clientpositive/union17.q.out | 4 +-
.../test/results/clientpositive/union20.q.out | 22 +-
.../test/results/clientpositive/union21.q.out | 6 +-
.../test/results/clientpositive/union27.q.out | 4 +-
.../test/results/clientpositive/union30.q.out | 42 +-
.../clientpositive/union_lateralview.q.out | 16 +-
.../clientpositive/union_remove_22.q.out | 16 +-
.../clientpositive/union_remove_6_subq.q.out | 12 +-
.../results/clientpositive/union_view.q.out | 368 +
.../clientpositive/vector_aggregate_9.q.out | 8 +-
.../clientpositive/vector_between_in.q.out | 6088 +-------
.../vector_binary_join_groupby.q.out | 293 +
.../clientpositive/vector_char_mapjoin1.q.out | 36 +-
.../clientpositive/vector_coalesce.q.out | 6 +-
.../clientpositive/vector_decimal_2.q.out | 4 +
.../clientpositive/vector_decimal_mapjoin.q.out | 370 +-
.../clientpositive/vector_decimal_udf.q.out | 3 +-
.../results/clientpositive/vector_elt.q.out | 2 +-
.../clientpositive/vector_inner_join.q.out | 799 +
.../results/clientpositive/vector_join30.q.out | 2194 +++
.../clientpositive/vector_join_filters.q.out | 222 +
.../clientpositive/vector_join_nulls.q.out | 195 +
.../vector_left_outer_join2.q.out | 572 +
.../vector_leftsemi_mapjoin.q.out | 13572 ++++++++++++++++
.../vector_mr_diff_schema_alias.q.out | 396 +
.../clientpositive/vector_nullsafe_join.q.out | 1208 ++
.../clientpositive/vector_outer_join0.q.out | 230 +
.../clientpositive/vector_outer_join1.q.out | 534 +
.../clientpositive/vector_outer_join2.q.out | 232 +
.../clientpositive/vector_outer_join3.q.out | 509 +
.../clientpositive/vector_outer_join4.q.out | 857 +
.../clientpositive/vector_outer_join5.q.out | 1300 ++
.../vector_varchar_mapjoin1.q.out | 4 +-
.../vectorization_decimal_date.q.out | 20 +-
.../vectorization_short_regress.q.out | 8 +-
.../clientpositive/vectorized_casts.q.out | 52 +-
.../results/clientpositive/vectorized_ptf.q.out | 167 +-
.../clientpositive/vectorized_shufflejoin.q.out | 1 -
.../vectorized_timestamp_funcs.q.out | 108 +-
.../vectorized_timestamp_ints_casts.q.out | 266 +
.../clientpositive/windowing_navfn.q.out | 10 +
.../clientpositive/windowing_streaming.q.out | 16 +-
.../clientpositive/windowing_windowspec2.q.out | 478 +
ql/src/test/templates/TestCliDriver.vm | 18 +-
ql/src/test/templates/TestCompareCliDriver.vm | 21 +-
ql/src/test/templates/TestNegativeCliDriver.vm | 18 +-
ql/src/test/templates/TestParseNegative.vm | 17 +-
serde/pom.xml | 2 +-
.../hadoop/hive/ql/io/sarg/PredicateLeaf.java | 19 +-
.../apache/hadoop/hive/serde2/SerDeUtils.java | 1 +
.../apache/hadoop/hive/serde2/WriteBuffers.java | 25 +-
.../binarysortable/BinarySortableSerDe.java | 157 +-
.../fast/BinarySortableSerializeWrite.java | 102 +-
.../hadoop/hive/serde2/fast/SerializeWrite.java | 12 +-
.../hive/serde2/io/HiveDecimalWritable.java | 5 +-
.../hive/serde2/io/HiveVarcharWritable.java | 7 +-
.../hive/serde2/io/TimestampWritable.java | 11 +-
.../hive/serde2/lazy/LazySerDeParameters.java | 18 +
.../lazy/fast/LazySimpleSerializeWrite.java | 13 +-
.../LazyObjectInspectorFactory.java | 24 +-
.../LazyPrimitiveObjectInspectorFactory.java | 41 +-
.../hive/serde2/lazybinary/LazyBinaryUtils.java | 19 +-
.../fast/LazyBinarySerializeWrite.java | 15 +-
.../LazyBinaryObjectInspectorFactory.java | 24 +-
.../MetadataListStructObjectInspector.java | 18 +-
.../ObjectInspectorConverters.java | 5 +-
.../objectinspector/ObjectInspectorUtils.java | 4 +-
.../AbstractPrimitiveObjectInspector.java | 1 +
.../PrimitiveObjectInspectorConverter.java | 7 +-
.../PrimitiveObjectInspectorFactory.java | 26 +-
.../PrimitiveObjectInspectorUtils.java | 42 +-
.../primitive/WritableVoidObjectInspector.java | 5 +
.../hive/serde2/typeinfo/CharTypeInfo.java | 5 +-
.../hive/serde2/typeinfo/DecimalTypeInfo.java | 7 +-
.../hive/serde2/typeinfo/PrimitiveTypeInfo.java | 5 +-
.../hive/serde2/typeinfo/TypeInfoFactory.java | 31 +-
.../hive/serde2/typeinfo/TypeInfoUtils.java | 27 +-
.../hive/serde2/typeinfo/VarcharTypeInfo.java | 5 +-
.../apache/hadoop/hive/serde2/VerifyFast.java | 10 +-
.../hive/serde2/binarysortable/MyTestClass.java | 105 +
.../binarysortable/MyTestPrimitiveClass.java | 20 +
.../binarysortable/TestBinarySortableFast.java | 13 +-
.../binarysortable/TestBinarySortableSerDe.java | 10 +-
.../hive/serde2/lazy/TestLazySimpleFast.java | 22 +-
.../TestPrimitiveObjectInspectorUtils.java | 95 +-
service/pom.xml | 7 +-
.../org/apache/hive/service/CookieSigner.java | 108 +
.../apache/hive/service/auth/HttpAuthUtils.java | 72 +
.../auth/LdapAuthenticationProviderImpl.java | 2 +-
.../apache/hive/service/cli/ColumnValue.java | 2 +
.../service/cli/session/HiveSessionImpl.java | 14 +-
.../service/cli/session/SessionManager.java | 49 +-
.../thrift/EmbeddedThriftBinaryCLIService.java | 5 +
.../cli/thrift/ThriftBinaryCLIService.java | 1 +
.../service/cli/thrift/ThriftCLIService.java | 79 +-
.../service/cli/thrift/ThriftHttpServlet.java | 210 +-
.../apache/hive/service/server/HiveServer2.java | 138 +-
.../apache/hive/service/TestCookieSigner.java | 59 +
.../TestLdapAuthenticationProviderImpl.java | 18 +
.../service/cli/session/TestSessionHooks.java | 3 +-
.../thrift/ThriftCliServiceTestWithCookie.java | 221 +
shims/0.20S/pom.xml | 2 +-
.../hadoop/hive/shims/Hadoop20SShims.java | 35 +
shims/0.23/pom.xml | 2 +-
.../apache/hadoop/hive/shims/Hadoop23Shims.java | 50 +-
shims/aggregator/pom.xml | 2 +-
shims/common/pom.xml | 2 +-
.../apache/hadoop/hive/shims/HadoopShims.java | 22 +
.../hadoop/hive/shims/HadoopShimsSecure.java | 4 +
shims/pom.xml | 2 +-
shims/scheduler/pom.xml | 2 +-
spark-client/pom.xml | 4 +-
.../apache/hive/spark/client/JobContext.java | 6 +
.../hive/spark/client/JobContextImpl.java | 10 +-
.../apache/hive/spark/client/RemoteDriver.java | 34 +-
.../hive/spark/client/SparkClientImpl.java | 34 +-
.../hive/spark/client/SparkClientUtilities.java | 23 +-
.../apache/hive/spark/client/rpc/RpcServer.java | 11 +-
.../apache/hive/spark/client/rpc/TestRpc.java | 32 +-
testutils/metastore/dbs/derby/execute.sh | 37 +
testutils/metastore/dbs/derby/prepare.sh | 63 +
testutils/metastore/dbs/postgres/execute.sh | 29 +
testutils/metastore/dbs/postgres/prepare.sh | 72 +
testutils/metastore/execute-test-on-lxc.sh | 20 +-
testutils/metastore/metastore-upgrade-test.sh | 5 +-
testutils/pom.xml | 2 +-
.../org/apache/hive/ptest/execution/PTest.java | 12 +-
.../ptest/execution/conf/TestConfiguration.java | 12 +-
.../ptest2/src/main/resources/source-prep.vm | 11 +
.../hive/ptest/execution/TestScripts.java | 23 +
.../TestScripts.testPrepGit.approved.txt | 2 +-
.../TestScripts.testPrepHadoop1.approved.txt | 111 +
.../TestScripts.testPrepNone.approved.txt | 2 +-
.../TestScripts.testPrepSvn.approved.txt | 2 +-
.../execution/conf/TestTestConfiguration.java | 38 +-
.../resources/test-configuration.properties | 2 +
1226 files changed, 154032 insertions(+), 31056 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
index 000eb38,f4f0f07..f7adb39
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/MapredParquetInputFormat.java
@@@ -20,7 -20,8 +20,8 @@@ import org.apache.hadoop.hive.ql.exec.U
import org.apache.hadoop.hive.ql.exec.vector.VectorizedInputFormatInterface;
import org.apache.hadoop.hive.ql.io.parquet.read.DataWritableReadSupport;
import org.apache.hadoop.hive.ql.io.parquet.read.ParquetRecordReaderWrapper;
-import org.apache.hadoop.io.ArrayWritable;
+import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
+ import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.mapred.FileInputFormat;
import org.apache.hadoop.mapred.RecordReader;
@@@ -34,7 -35,7 +35,7 @@@ import parquet.hadoop.ParquetInputForma
* NOTE: With HIVE-9235 we removed "implements VectorizedParquetInputFormat" since all data types
* are not currently supported. Removing the interface turns off vectorization.
*/
- public class MapredParquetInputFormat extends FileInputFormat<Void, ObjectArrayWritable> {
-public class MapredParquetInputFormat extends FileInputFormat<NullWritable, ArrayWritable> {
++public class MapredParquetInputFormat extends FileInputFormat<NullWritable, ObjectArrayWritable> {
private static final Log LOG = LogFactory.getLog(MapredParquetInputFormat.class);
@@@ -53,7 -54,7 +54,7 @@@
@SuppressWarnings({ "unchecked", "rawtypes" })
@Override
- public org.apache.hadoop.mapred.RecordReader<Void, ObjectArrayWritable> getRecordReader(
- public org.apache.hadoop.mapred.RecordReader<NullWritable, ArrayWritable> getRecordReader(
++ public org.apache.hadoop.mapred.RecordReader<NullWritable, ObjectArrayWritable> getRecordReader(
final org.apache.hadoop.mapred.InputSplit split,
final org.apache.hadoop.mapred.JobConf job,
final org.apache.hadoop.mapred.Reporter reporter
@@@ -69,7 -70,7 +70,7 @@@
if (LOG.isDebugEnabled()) {
LOG.debug("Using row-mode record reader");
}
- return (RecordReader<Void, ObjectArrayWritable>)
- return (RecordReader<NullWritable, ArrayWritable>)
++ return (RecordReader<NullWritable, ObjectArrayWritable>)
new ParquetRecordReaderWrapper(realInput, split, job, reporter);
}
} catch (final InterruptedException e) {
http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java
index 4e1820c,843e079..c557963
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/VectorizedParquetInputFormat.java
@@@ -51,8 -52,8 +51,8 @@@ public class VectorizedParquetInputForm
private final ParquetRecordReaderWrapper internalReader;
private VectorizedRowBatchCtx rbCtx;
- private ArrayWritable internalValues;
+ private ObjectArrayWritable internalValues;
- private Void internalKey;
+ private NullWritable internalKey;
private VectorColumnAssign[] assigners;
public VectorizedParquetRecordReader(
http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java
----------------------------------------------------------------------
diff --cc ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java
index 0a5edbb,5c36564..f513572
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/read/ParquetRecordReaderWrapper.java
@@@ -33,7 -33,9 +33,8 @@@ import org.apache.hadoop.hive.ql.io.par
import org.apache.hadoop.hive.ql.io.sarg.SearchArgumentFactory;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
import org.apache.hadoop.hive.serde2.ColumnProjectionUtils;
-import org.apache.hadoop.io.ArrayWritable;
+import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
+ import org.apache.hadoop.io.NullWritable;
-import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
import org.apache.hadoop.mapred.JobConf;
@@@ -58,7 -60,7 +59,7 @@@ import parquet.schema.MessageTypeParser
import com.google.common.base.Strings;
- public class ParquetRecordReaderWrapper implements RecordReader<Void, ObjectArrayWritable> {
-public class ParquetRecordReaderWrapper implements RecordReader<NullWritable, ArrayWritable> {
++public class ParquetRecordReaderWrapper implements RecordReader<NullWritable, ObjectArrayWritable> {
public static final Log LOG = LogFactory.getLog(ParquetRecordReaderWrapper.class);
private final long splitLen; // for getPos()
@@@ -194,7 -197,7 +195,7 @@@
}
@Override
- public boolean next(final Void key, final ObjectArrayWritable value) throws IOException {
- public boolean next(final NullWritable key, final ArrayWritable value) throws IOException {
++ public boolean next(final NullWritable key, final ObjectArrayWritable value) throws IOException {
if (eof) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java
----------------------------------------------------------------------
diff --cc ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java
index 5f7f597,94a780d..9c4cf5c
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java
@@@ -14,7 -32,8 +32,8 @@@ import org.apache.hadoop.fs.FileSystem
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe;
import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.io.ArrayWritable;
+import org.apache.hadoop.hive.serde2.io.ObjectArrayWritable;
+ import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.JobConf;
@@@ -119,16 -138,16 +138,16 @@@ public abstract class AbstractTestParqu
Assert.assertEquals(message, toString(expected), toString(actual));
}
- public static List<ArrayWritable> read(Path parquetFile) throws IOException {
- List<ArrayWritable> records = new ArrayList<ArrayWritable>();
+ public static List<ObjectArrayWritable> read(Path parquetFile) throws IOException {
+ List<ObjectArrayWritable> records = new ArrayList<ObjectArrayWritable>();
- RecordReader<Void, ObjectArrayWritable> reader = new MapredParquetInputFormat().
- RecordReader<NullWritable, ArrayWritable> reader = new MapredParquetInputFormat().
++ RecordReader<NullWritable, ObjectArrayWritable> reader = new MapredParquetInputFormat().
getRecordReader(new FileSplit(
parquetFile, 0, fileLength(parquetFile), (String[]) null),
new JobConf(), null);
- Void alwaysNull = reader.createKey();
+ NullWritable alwaysNull = reader.createKey();
- ArrayWritable record = reader.createValue();
+ ObjectArrayWritable record = reader.createValue();
while (reader.next(alwaysNull, record)) {
records.add(record);
record = reader.createValue(); // a new value so the last isn't clobbered
http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5a0ae3c2/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
----------------------------------------------------------------------
[36/50] [abbrv] hive git commit: HIVE-10696 : TestAddResource tests
are non-portable (Hari Subramaniyan reviewed by Alexander Pivovarov,
Thejas Nair)
Posted by sp...@apache.org.
HIVE-10696 : TestAddResource tests are non-portable (Hari Subramaniyan reviewed by Alexander Pivovarov, Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/5818a4a2
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/5818a4a2
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/5818a4a2
Branch: refs/heads/parquet
Commit: 5818a4a2054bafd78ecbfda0798ece1949d0d25d
Parents: 1b91dba
Author: Hari Subramaniyan <ha...@apache.org>
Authored: Mon May 18 13:12:23 2015 -0700
Committer: Hari Subramaniyan <ha...@apache.org>
Committed: Mon May 18 13:12:23 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/session/TestAddResource.java | 98 ++++++++++++--------
1 file changed, 57 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/5818a4a2/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java b/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
index 6176b2d..1f07235 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
@@ -31,6 +31,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.junit.After;
import org.junit.Before;
@@ -38,6 +39,7 @@ import org.junit.Test;
import org.mockito.Mockito;
import org.apache.hadoop.hive.ql.session.SessionState.ResourceType;
import org.apache.hadoop.hive.ql.session.SessionState;
+import org.apache.hadoop.util.Shell;
import java.io.BufferedWriter;
import java.io.FileWriter;
@@ -45,7 +47,7 @@ import java.io.FileWriter;
public class TestAddResource {
- private static final String TEST_JAR_DIR = System.getProperty("test.tmp.dir", ".") + "/";
+ private static final String TEST_JAR_DIR = System.getProperty("test.tmp.dir", ".") + File.pathSeparator;
private HiveConf conf;
private ResourceType t;
@@ -74,11 +76,11 @@ public class TestAddResource {
// add all the dependencies to a list
List<URI> list = new LinkedList<URI>();
List<String> addList = new LinkedList<String>();
- list.add(new URI(TEST_JAR_DIR + "testjar1.jar"));
- list.add(new URI(TEST_JAR_DIR + "testjar2.jar"));
- list.add(new URI(TEST_JAR_DIR + "testjar3.jar"));
- list.add(new URI(TEST_JAR_DIR + "testjar4.jar"));
- list.add(new URI(TEST_JAR_DIR + "testjar5.jar"));
+ list.add(createURI(TEST_JAR_DIR + "testjar1.jar"));
+ list.add(createURI(TEST_JAR_DIR + "testjar2.jar"));
+ list.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
+ list.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
+ list.add(createURI(TEST_JAR_DIR + "testjar5.jar"));
//return all the dependency urls
Mockito.when(ss.resolveAndDownload(t, query, false)).thenReturn(list);
@@ -87,7 +89,7 @@ public class TestAddResource {
Set<String> dependencies = ss.list_resource(t, null);
LinkedList<URI> actual = new LinkedList<URI>();
for (String dependency : dependencies) {
- actual.add(new URI(dependency));
+ actual.add(createURI(dependency));
}
// sort both the lists
@@ -109,11 +111,11 @@ public class TestAddResource {
List<URI> list = new LinkedList<URI>();
List<String> addList = new LinkedList<String>();
- list.add(new URI(TEST_JAR_DIR + "testjar1.jar"));
- list.add(new URI(TEST_JAR_DIR + "testjar2.jar"));
- list.add(new URI(TEST_JAR_DIR + "testjar3.jar"));
- list.add(new URI(TEST_JAR_DIR + "testjar4.jar"));
- list.add(new URI(TEST_JAR_DIR + "testjar5.jar"));
+ list.add(createURI(TEST_JAR_DIR + "testjar1.jar"));
+ list.add(createURI(TEST_JAR_DIR + "testjar2.jar"));
+ list.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
+ list.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
+ list.add(createURI(TEST_JAR_DIR + "testjar5.jar"));
Collections.sort(list);
@@ -125,7 +127,7 @@ public class TestAddResource {
Set<String> dependencies = ss.list_resource(t, null);
LinkedList<URI> actual = new LinkedList<URI>();
for (String dependency : dependencies) {
- actual.add(new URI(dependency));
+ actual.add(createURI(dependency));
}
Collections.sort(actual);
@@ -147,13 +149,13 @@ public class TestAddResource {
// add dependencies for the jars
List<URI> list1 = new LinkedList<URI>();
List<URI> list2 = new LinkedList<URI>();
- list1.add(new URI(TEST_JAR_DIR + "testjar1.jar"));
- list1.add(new URI(TEST_JAR_DIR + "testjar2.jar"));
- list1.add(new URI(TEST_JAR_DIR + "testjar3.jar"));
- list1.add(new URI(TEST_JAR_DIR + "testjar4.jar"));
- list2.add(new URI(TEST_JAR_DIR + "testjar5.jar"));
- list2.add(new URI(TEST_JAR_DIR + "testjar3.jar"));
- list2.add(new URI(TEST_JAR_DIR + "testjar4.jar"));
+ list1.add(createURI(TEST_JAR_DIR + "testjar1.jar"));
+ list1.add(createURI(TEST_JAR_DIR + "testjar2.jar"));
+ list1.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
+ list1.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
+ list2.add(createURI(TEST_JAR_DIR + "testjar5.jar"));
+ list2.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
+ list2.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
Mockito.when(ss.resolveAndDownload(t, query1, false)).thenReturn(list1);
Mockito.when(ss.resolveAndDownload(t, query2, false)).thenReturn(list2);
@@ -164,7 +166,7 @@ public class TestAddResource {
Set<String> dependencies = ss.list_resource(t, null);
LinkedList<URI> actual = new LinkedList<URI>();
for (String dependency : dependencies) {
- actual.add(new URI(dependency));
+ actual.add(createURI(dependency));
}
List<URI> expected = union(list1, list2);
@@ -176,6 +178,20 @@ public class TestAddResource {
}
+ /**
+ * @param path
+ * @return URI corresponding to the path.
+ */
+ private static URI createURI(String path) throws URISyntaxException {
+ if (!Shell.WINDOWS) {
+ // If this is not windows shell, path better follow unix convention.
+ // Else, the below call will throw an URISyntaxException
+ return new URI(path);
+ } else {
+ return new Path(path).toUri();
+ }
+ }
+
// Test when two jars are added with shared dependencies and one jar is deleted, the shared dependencies should not be deleted
@Test
public void testDeleteJar() throws URISyntaxException, IOException {
@@ -187,13 +203,13 @@ public class TestAddResource {
List<URI> list1 = new LinkedList<URI>();
List<URI> list2 = new LinkedList<URI>();
List<String> addList = new LinkedList<String>();
- list1.add(new URI(TEST_JAR_DIR + "testjar1.jar"));
- list1.add(new URI(TEST_JAR_DIR + "testjar2.jar"));
- list1.add(new URI(TEST_JAR_DIR + "testjar3.jar"));
- list1.add(new URI(TEST_JAR_DIR + "testjar4.jar"));
- list2.add(new URI(TEST_JAR_DIR + "testjar5.jar"));
- list2.add(new URI(TEST_JAR_DIR + "testjar3.jar"));
- list2.add(new URI(TEST_JAR_DIR + "testjar4.jar"));
+ list1.add(createURI(TEST_JAR_DIR + "testjar1.jar"));
+ list1.add(createURI(TEST_JAR_DIR + "testjar2.jar"));
+ list1.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
+ list1.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
+ list2.add(createURI(TEST_JAR_DIR + "testjar5.jar"));
+ list2.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
+ list2.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
Collections.sort(list1);
Collections.sort(list2);
@@ -211,7 +227,7 @@ public class TestAddResource {
Set<String> dependencies = ss.list_resource(t, null);
LinkedList<URI> actual = new LinkedList<URI>();
for (String dependency : dependencies) {
- actual.add(new URI(dependency));
+ actual.add(createURI(dependency));
}
List<URI> expected = list2;
Collections.sort(expected);
@@ -242,16 +258,16 @@ public class TestAddResource {
List<URI> list2 = new LinkedList<URI>();
List<URI> list3 = new LinkedList<URI>();
List<String> addList = new LinkedList<String>();
- list1.add(new URI(TEST_JAR_DIR + "testjar1.jar"));
- list1.add(new URI(TEST_JAR_DIR + "testjar2.jar"));
- list1.add(new URI(TEST_JAR_DIR + "testjar3.jar"));
- list1.add(new URI(TEST_JAR_DIR + "testjar4.jar"));
- list2.add(new URI(TEST_JAR_DIR + "testjar5.jar"));
- list2.add(new URI(TEST_JAR_DIR + "testjar3.jar"));
- list2.add(new URI(TEST_JAR_DIR + "testjar4.jar"));
- list3.add(new URI(TEST_JAR_DIR + "testjar4.jar"));
- list3.add(new URI(TEST_JAR_DIR + "testjar2.jar"));
- list3.add(new URI(TEST_JAR_DIR + "testjar5.jar"));
+ list1.add(createURI(TEST_JAR_DIR + "testjar1.jar"));
+ list1.add(createURI(TEST_JAR_DIR + "testjar2.jar"));
+ list1.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
+ list1.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
+ list2.add(createURI(TEST_JAR_DIR + "testjar5.jar"));
+ list2.add(createURI(TEST_JAR_DIR + "testjar3.jar"));
+ list2.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
+ list3.add(createURI(TEST_JAR_DIR + "testjar4.jar"));
+ list3.add(createURI(TEST_JAR_DIR + "testjar2.jar"));
+ list3.add(createURI(TEST_JAR_DIR + "testjar5.jar"));
Collections.sort(list1);
Collections.sort(list2);
@@ -273,7 +289,7 @@ public class TestAddResource {
Set<String> dependencies = ss.list_resource(t, null);
LinkedList<URI> actual = new LinkedList<URI>();
for (String dependency : dependencies) {
- actual.add(new URI(dependency));
+ actual.add(createURI(dependency));
}
List<URI> expected = union(list2, list3);
Collections.sort(expected);
@@ -290,7 +306,7 @@ public class TestAddResource {
dependencies = ss.list_resource(t, null);
actual = new LinkedList<URI>();
for (String dependency : dependencies) {
- actual.add(new URI(dependency));
+ actual.add(createURI(dependency));
}
expected = new LinkedList<URI>(list3);
Collections.sort(expected);
[22/50] [abbrv] hive git commit: HIVE-10559:
IndexOutOfBoundsException with RemoveDynamicPruningBySize (Wei Zhang via
Gunther Hagleitner)
Posted by sp...@apache.org.
HIVE-10559: IndexOutOfBoundsException with RemoveDynamicPruningBySize (Wei Zhang via Gunther Hagleitner)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/6a299391
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/6a299391
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/6a299391
Branch: refs/heads/parquet
Commit: 6a2993912730702edb68dd37acee7ec0af4e6d19
Parents: 2b9f2f5
Author: Gunther Hagleitner <gu...@apache.org>
Authored: Thu May 14 15:48:34 2015 -0700
Committer: Gunther Hagleitner <gu...@apache.org>
Committed: Thu May 14 15:48:34 2015 -0700
----------------------------------------------------------------------
.../hive/ql/optimizer/ConvertJoinMapJoin.java | 25 ++--
.../optimizer/RemoveDynamicPruningBySize.java | 19 ++-
.../hadoop/hive/ql/parse/GenTezUtils.java | 16 +++
.../hive/ql/parse/OptimizeTezProcContext.java | 14 +++
.../hadoop/hive/ql/parse/TezCompiler.java | 19 ++-
.../dynamic_partition_pruning_2.q | 17 +++
.../tez/dynamic_partition_pruning_2.q.out | 116 +++++++++++++++++++
7 files changed, 196 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/6a299391/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
index d42b643..bcffdbc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConvertJoinMapJoin.java
@@ -636,13 +636,26 @@ public class ConvertJoinMapJoin implements NodeProcessor {
// we might have generated a dynamic partition operator chain. Since
// we're removing the reduce sink we need do remove that too.
Set<Operator<?>> dynamicPartitionOperators = new HashSet<Operator<?>>();
+ Map<Operator<?>, AppMasterEventOperator> opEventPairs = new HashMap<>();
for (Operator<?> c : p.getChildOperators()) {
- if (hasDynamicPartitionBroadcast(c)) {
+ AppMasterEventOperator event = findDynamicPartitionBroadcast(c);
+ if (event != null) {
dynamicPartitionOperators.add(c);
+ opEventPairs.put(c, event);
}
}
for (Operator<?> c : dynamicPartitionOperators) {
- p.removeChild(c);
+ if (context.pruningOpsRemovedByPriorOpt.isEmpty() ||
+ !context.pruningOpsRemovedByPriorOpt.contains(opEventPairs.get(c))) {
+ p.removeChild(c);
+ // at this point we've found the fork in the op pipeline that has the pruning as a child plan.
+ LOG.info("Disabling dynamic pruning for: "
+ + ((DynamicPruningEventDesc) opEventPairs.get(c).getConf()).getTableScan().getName()
+ + ". Need to be removed together with reduce sink");
+ }
+ }
+ for (Operator<?> op : dynamicPartitionOperators) {
+ context.pruningOpsRemovedByPriorOpt.add(opEventPairs.get(op));
}
}
mapJoinOp.getParentOperators().remove(bigTablePosition);
@@ -662,15 +675,13 @@ public class ConvertJoinMapJoin implements NodeProcessor {
return mapJoinOp;
}
- private boolean hasDynamicPartitionBroadcast(Operator<?> parent) {
- boolean hasDynamicPartitionPruning = false;
+ private AppMasterEventOperator findDynamicPartitionBroadcast(Operator<?> parent) {
for (Operator<?> op : parent.getChildOperators()) {
while (op != null) {
if (op instanceof AppMasterEventOperator && op.getConf() instanceof DynamicPruningEventDesc) {
// found dynamic partition pruning operator
- hasDynamicPartitionPruning = true;
- break;
+ return (AppMasterEventOperator)op;
}
if (op instanceof ReduceSinkOperator || op instanceof FileSinkOperator) {
// crossing reduce sink or file sink means the pruning isn't for this parent.
@@ -686,6 +697,6 @@ public class ConvertJoinMapJoin implements NodeProcessor {
}
}
- return hasDynamicPartitionPruning;
+ return null;
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6a299391/ql/src/java/org/apache/hadoop/hive/ql/optimizer/RemoveDynamicPruningBySize.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/RemoveDynamicPruningBySize.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/RemoveDynamicPruningBySize.java
index 4803959..5d01311 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/RemoveDynamicPruningBySize.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/RemoveDynamicPruningBySize.java
@@ -24,10 +24,10 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.conf.HiveConf.ConfVars;
import org.apache.hadoop.hive.ql.exec.AppMasterEventOperator;
-import org.apache.hadoop.hive.ql.exec.Operator;
import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.lib.NodeProcessor;
import org.apache.hadoop.hive.ql.lib.NodeProcessorCtx;
+import org.apache.hadoop.hive.ql.parse.GenTezUtils;
import org.apache.hadoop.hive.ql.parse.OptimizeTezProcContext;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.plan.AppMasterEventDesc;
@@ -52,20 +52,15 @@ public class RemoveDynamicPruningBySize implements NodeProcessor {
AppMasterEventDesc desc = event.getConf();
if (desc.getStatistics().getDataSize() > context.conf
- .getLongVar(ConfVars.TEZ_DYNAMIC_PARTITION_PRUNING_MAX_DATA_SIZE)) {
- Operator<?> child = event;
- Operator<?> curr = event;
-
- while (curr.getChildOperators().size() <= 1) {
- child = curr;
- curr = curr.getParentOperators().get(0);
- }
- // at this point we've found the fork in the op pipeline that has the
- // pruning as a child plan.
+ .getLongVar(ConfVars.TEZ_DYNAMIC_PARTITION_PRUNING_MAX_DATA_SIZE) &&
+ (context.pruningOpsRemovedByPriorOpt.isEmpty() ||
+ !context.pruningOpsRemovedByPriorOpt.contains(event))) {
+ context.pruningOpsRemovedByPriorOpt.add(event);
+ GenTezUtils.getUtils().removeBranch(event);
+ // at this point we've found the fork in the op pipeline that has the pruning as a child plan.
LOG.info("Disabling dynamic pruning for: "
+ ((DynamicPruningEventDesc) desc).getTableScan().getName()
+ ". Expected data size is too big: " + desc.getStatistics().getDataSize());
- curr.removeChild(child);
}
return false;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6a299391/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
index 241e9d7..0edfc5d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/GenTezUtils.java
@@ -461,4 +461,20 @@ public class GenTezUtils {
findRoots(p, ops);
}
}
+
+ /**
+ * Remove an operator branch. When we see a fork, we know it's time to do the removal.
+ * @param event the leaf node of which branch to be removed
+ */
+ public void removeBranch(AppMasterEventOperator event) {
+ Operator<?> child = event;
+ Operator<?> curr = event;
+
+ while (curr.getChildOperators().size() <= 1) {
+ child = curr;
+ curr = curr.getParentOperators().get(0);
+ }
+
+ curr.removeChild(child);
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/6a299391/ql/src/java/org/apache/hadoop/hive/ql/parse/OptimizeTezProcContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/OptimizeTezProcContext.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/OptimizeTezProcContext.java
index ee71971..e58eb66 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/OptimizeTezProcContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/OptimizeTezProcContext.java
@@ -49,6 +49,19 @@ public class OptimizeTezProcContext implements NodeProcessorCtx{
public final Set<ReadEntity> inputs;
public final Set<WriteEntity> outputs;
+ /* Two of the optimization rules, ConvertJoinMapJoin and RemoveDynamicPruningBySize, are put into
+ stats dependent optimizations and run together in TezCompiler. There's no guarantee which one
+ runs first, but in either case, the prior one may have removed a chain which the latter one is
+ not aware of. So we need to remember the leaf node(s) of that chain so it can be skipped.
+
+ For example, as ConvertJoinMapJoin is removing the reduce sink, it may also have removed a
+ dynamic partition pruning operator chain. However, RemoveDynamicPruningBySize doesn't know this
+ and still tries to traverse that removed chain which will cause NPE.
+
+ This may also happen when RemoveDynamicPruningBySize happens first.
+ */
+ public HashSet<AppMasterEventOperator> pruningOpsRemovedByPriorOpt;
+
public final Set<ReduceSinkOperator> visitedReduceSinks
= new HashSet<ReduceSinkOperator>();
@@ -66,6 +79,7 @@ public class OptimizeTezProcContext implements NodeProcessorCtx{
this.parseContext = parseContext;
this.inputs = inputs;
this.outputs = outputs;
+ this.pruningOpsRemovedByPriorOpt = new HashSet<AppMasterEventOperator>();
}
public void setRootOperators(Deque<Operator<? extends OperatorDesc>> roots) {
http://git-wip-us.apache.org/repos/asf/hive/blob/6a299391/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
index ea12990..56707af 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TezCompiler.java
@@ -149,7 +149,7 @@ public class TezCompiler extends TaskCompiler {
if (component.size() != 1) {
LOG.info("Found cycle in operator plan...");
cycleFree = false;
- removeEventOperator(component);
+ removeEventOperator(component, procCtx);
break;
}
}
@@ -157,7 +157,7 @@ public class TezCompiler extends TaskCompiler {
}
}
- private void removeEventOperator(Set<Operator<?>> component) {
+ private void removeEventOperator(Set<Operator<?>> component, OptimizeTezProcContext context) {
AppMasterEventOperator victim = null;
for (Operator<?> o : component) {
if (o instanceof AppMasterEventOperator) {
@@ -169,20 +169,17 @@ public class TezCompiler extends TaskCompiler {
}
}
- Operator<?> child = victim;
- Operator<?> curr = victim;
-
- while (curr.getChildOperators().size() <= 1) {
- child = curr;
- curr = curr.getParentOperators().get(0);
+ if (victim == null ||
+ (!context.pruningOpsRemovedByPriorOpt.isEmpty() &&
+ context.pruningOpsRemovedByPriorOpt.contains(victim))) {
+ return;
}
- // at this point we've found the fork in the op pipeline that has the
- // pruning as a child plan.
+ GenTezUtils.getUtils().removeBranch(victim);
+ // at this point we've found the fork in the op pipeline that has the pruning as a child plan.
LOG.info("Disabling dynamic pruning for: "
+ ((DynamicPruningEventDesc) victim.getConf()).getTableScan().toString()
+ ". Needed to break cyclic dependency");
- curr.removeChild(child);
}
// Tarjan's algo
http://git-wip-us.apache.org/repos/asf/hive/blob/6a299391/ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q b/ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q
index 5a7f113..a4e84b1 100644
--- a/ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q
+++ b/ql/src/test/queries/clientpositive/dynamic_partition_pruning_2.q
@@ -118,3 +118,20 @@ SELECT amount FROM agg_01, dim_shops WHERE dim_shops_id = id AND label = 'bar';
SELECT amount FROM agg_01, dim_shops WHERE dim_shops_id = id AND label = 'foo'
UNION ALL
SELECT amount FROM agg_01, dim_shops WHERE dim_shops_id = id AND label = 'bar';
+
+set hive.tez.dynamic.partition.pruning.max.event.size=1000000;
+set hive.tez.dynamic.partition.pruning.max.data.size=10000;
+-- Dynamic partition pruning will be removed as data size exceeds the limit;
+-- and for self join on partitioning column, it should not fail (HIVE-10559).
+explain
+select count(*)
+from srcpart s1,
+ srcpart s2
+where s1.ds = s2.ds
+;
+
+select count(*)
+from srcpart s1,
+ srcpart s2
+where s1.ds = s2.ds
+;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/6a299391/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out
index 8c8531c..8b0b81d 100644
--- a/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out
+++ b/ql/src/test/results/clientpositive/tez/dynamic_partition_pruning_2.q.out
@@ -963,3 +963,119 @@ POSTHOOK: Input: default@dim_shops
4
5
6
+PREHOOK: query: -- Dynamic partition pruning will be removed as data size exceeds the limit;
+-- and for self join on partitioning column, it should not fail (HIVE-10559).
+explain
+select count(*)
+from srcpart s1,
+ srcpart s2
+where s1.ds = s2.ds
+PREHOOK: type: QUERY
+POSTHOOK: query: -- Dynamic partition pruning will be removed as data size exceeds the limit;
+-- and for self join on partitioning column, it should not fail (HIVE-10559).
+explain
+select count(*)
+from srcpart s1,
+ srcpart s2
+where s1.ds = s2.ds
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 3 (BROADCAST_EDGE)
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: s1
+ filterExpr: ds is not null (type: boolean)
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ds (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ input vertices:
+ 1 Map 3
+ Statistics: Num rows: 2200 Data size: 23372 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: s1
+ filterExpr: ds is not null (type: boolean)
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ds (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 2000 Data size: 21248 Basic stats: COMPLETE Column stats: NONE
+ Reducer 2
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*)
+from srcpart s1,
+ srcpart s2
+where s1.ds = s2.ds
+PREHOOK: type: QUERY
+PREHOOK: Input: default@srcpart
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+PREHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*)
+from srcpart s1,
+ srcpart s2
+where s1.ds = s2.ds
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@srcpart
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-08/hr=12
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=11
+POSTHOOK: Input: default@srcpart@ds=2008-04-09/hr=12
+#### A masked pattern was here ####
+2000000
[24/50] [abbrv] hive git commit: HIVE-10657 Remove copyBytes
operation from MD5 UDF (Alexander Pivovarov, reviewed by Jason Dere)
Posted by sp...@apache.org.
HIVE-10657 Remove copyBytes operation from MD5 UDF (Alexander Pivovarov, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bf7810ab
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bf7810ab
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bf7810ab
Branch: refs/heads/parquet
Commit: bf7810ab64caabc47d4370d04f80c0c1f182426a
Parents: ad62e2e
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Fri May 8 11:36:47 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Fri May 15 10:20:32 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/ql/udf/UDFMd5.java | 33 +++++++++++++-------
1 file changed, 21 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bf7810ab/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMd5.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMd5.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMd5.java
index 62c16c2..14928ef 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMd5.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMd5.java
@@ -18,7 +18,10 @@
package org.apache.hadoop.hive.ql.udf;
-import org.apache.commons.codec.digest.DigestUtils;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import org.apache.commons.codec.binary.Hex;
import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDF;
import org.apache.hadoop.io.BytesWritable;
@@ -39,6 +42,15 @@ import org.apache.hadoop.io.Text;
public class UDFMd5 extends UDF {
private final Text result = new Text();
+ private final MessageDigest digest;
+
+ public UDFMd5() {
+ try {
+ digest = MessageDigest.getInstance("MD5");
+ } catch (NoSuchAlgorithmException e) {
+ throw new RuntimeException(e);
+ }
+ }
/**
* Convert String to md5
@@ -48,8 +60,10 @@ public class UDFMd5 extends UDF {
return null;
}
- String str = n.toString();
- String md5Hex = DigestUtils.md5Hex(str);
+ digest.reset();
+ digest.update(n.getBytes(), 0, n.getLength());
+ byte[] md5Bytes = digest.digest();
+ String md5Hex = Hex.encodeHexString(md5Bytes);
result.set(md5Hex);
return result;
@@ -63,17 +77,12 @@ public class UDFMd5 extends UDF {
return null;
}
- byte[] bytes = copyBytes(b);
- String md5Hex = DigestUtils.md5Hex(bytes);
+ digest.reset();
+ digest.update(b.getBytes(), 0, b.getLength());
+ byte[] md5Bytes = digest.digest();
+ String md5Hex = Hex.encodeHexString(md5Bytes);
result.set(md5Hex);
return result;
}
-
- protected byte[] copyBytes(BytesWritable b) {
- int size = b.getLength();
- byte[] result = new byte[size];
- System.arraycopy(b.getBytes(), 0, result, 0, size);
- return result;
- }
}
[49/50] [abbrv] hive git commit: HIVE-10747: Enable the cleanup of
side effect for the Encryption related qfile test (Ferdinand Xu,
reviewed by Sergio Pena)
Posted by sp...@apache.org.
HIVE-10747: Enable the cleanup of side effect for the Encryption related qfile test (Ferdinand Xu, reviewed by Sergio Pena)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1253a8db
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1253a8db
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1253a8db
Branch: refs/heads/parquet
Commit: 1253a8db49a313edb325731379b8d15effd2ff01
Parents: d703c22
Author: Sergio Pena <se...@cloudera.com>
Authored: Wed May 20 10:03:01 2015 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Wed May 20 10:03:01 2015 -0500
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/QTestUtil.java | 14 ++++++--------
1 file changed, 6 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1253a8db/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index f357c14..2b4bd85 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -733,14 +733,12 @@ public class QTestUtil {
clearTablesCreatedDuringTests();
clearKeysCreatedInTests();
- if (clusterType != MiniClusterType.encrypted) {
- // allocate and initialize a new conf since a test can
- // modify conf by using 'set' commands
- conf = new HiveConf (Driver.class);
- initConf();
- // renew the metastore since the cluster type is unencrypted
- db = Hive.get(conf); // propagate new conf to meta store
- }
+ // allocate and initialize a new conf since a test can
+ // modify conf by using 'set' commands
+ conf = new HiveConf(Driver.class);
+ initConf();
+ // renew the metastore since the cluster type is unencrypted
+ db = Hive.get(conf); // propagate new conf to meta store
setup.preTest(conf);
}
[11/50] [abbrv] hive git commit: HIVE-10636 : CASE comparison
operator rotation optimization (Ashutosh Chauhan via Gopal V)
Posted by sp...@apache.org.
HIVE-10636 : CASE comparison operator rotation optimization (Ashutosh Chauhan via Gopal V)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/05a3d2ab
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/05a3d2ab
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/05a3d2ab
Branch: refs/heads/parquet
Commit: 05a3d2ab77f11110a357532b1959b4a3bc9798e9
Parents: 3138334
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Tue May 12 14:58:06 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Thu May 14 08:33:12 2015 -0700
----------------------------------------------------------------------
.../hive/ql/exec/ExprNodeEvaluatorFactory.java | 7 -
.../ql/exec/vector/VectorizationContext.java | 13 +-
.../optimizer/ConstantPropagateProcFactory.java | 86 +++++--
.../hive/ql/optimizer/GroupByOptimizer.java | 7 +-
.../PrunerExpressionOperatorFactory.java | 3 -
.../hive/ql/optimizer/SimpleFetchOptimizer.java | 2 -
.../calcite/translator/RexNodeConverter.java | 5 +-
.../ql/optimizer/lineage/ExprProcFactory.java | 3 +-
.../ql/optimizer/pcr/PcrExprProcFactory.java | 3 +-
.../hive/ql/parse/TableAccessAnalyzer.java | 6 +-
.../hive/ql/parse/TypeCheckProcFactory.java | 4 +-
.../hive/ql/plan/ExprNodeConstantDesc.java | 14 +-
.../hadoop/hive/ql/plan/ExprNodeNullDesc.java | 69 ------
.../apache/hadoop/hive/ql/stats/StatsUtils.java | 12 +-
.../hive/ql/udf/generic/GenericUDFCoalesce.java | 2 +-
.../hive/ql/udf/generic/GenericUDFGreatest.java | 1 +
.../hive/ql/udf/generic/GenericUDFInstr.java | 2 +-
.../hive/ql/udf/generic/GenericUDFLocate.java | 2 +-
.../hive/ql/udf/generic/GenericUDFPrintf.java | 3 +-
.../ql/udf/generic/GenericUDFTranslate.java | 8 +-
.../hive/ql/udf/generic/GenericUDFUtils.java | 6 +-
.../clientpositive/fold_eq_with_case_when.q | 21 ++
.../clientpositive/annotate_stats_filter.q.out | 4 +-
.../clientpositive/fold_eq_with_case_when.q.out | 231 +++++++++++++++++++
.../test/results/clientpositive/fold_when.q.out | 2 +-
ql/src/test/results/clientpositive/input6.q.out | 2 +-
.../results/clientpositive/join_nullsafe.q.out | 10 +-
.../clientpositive/spark/join_nullsafe.q.out | 10 +-
.../subquery_notin_having.q.java1.7.out | 2 +-
.../clientpositive/tez/join_nullsafe.q.out | 14 +-
.../clientpositive/tez/vector_coalesce.q.out | 26 +--
.../clientpositive/vector_coalesce.q.out | 6 +-
.../hive/serde2/io/HiveVarcharWritable.java | 7 +-
.../ObjectInspectorConverters.java | 5 +-
.../objectinspector/ObjectInspectorUtils.java | 4 +-
.../AbstractPrimitiveObjectInspector.java | 1 +
.../primitive/WritableVoidObjectInspector.java | 5 +
37 files changed, 396 insertions(+), 212 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java
index a149571..f08321c 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.NullWritable;
@@ -57,12 +56,6 @@ public final class ExprNodeEvaluatorFactory {
if (desc instanceof ExprNodeFieldDesc) {
return new ExprNodeFieldEvaluator((ExprNodeFieldDesc) desc);
}
- // Null node, a constant node with value NULL and no type information
- if (desc instanceof ExprNodeNullDesc) {
- return new ExprNodeConstantEvaluator(new ExprNodeConstantDesc(TypeInfoFactory
- .getPrimitiveTypeInfoFromPrimitiveWritable(NullWritable.class), null));
- }
-
throw new RuntimeException(
"Cannot find ExprNodeEvaluator for the exprNodeDesc = " + desc);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index df39218..48f34a9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -92,7 +92,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.udf.SettableUDF;
import org.apache.hadoop.hive.ql.udf.UDFConv;
@@ -441,7 +440,7 @@ public class VectorizationContext {
ve = getGenericUdfVectorExpression(expr.getGenericUDF(),
childExpressions, mode, exprDesc.getTypeInfo());
}
- } else if (exprDesc instanceof ExprNodeNullDesc) {
+ } else if (exprDesc instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc)exprDesc).getValue()) {
ve = getConstantVectorExpression(null, exprDesc.getTypeInfo(), mode);
} else if (exprDesc instanceof ExprNodeConstantDesc) {
ve = getConstantVectorExpression(((ExprNodeConstantDesc) exprDesc).getValue(), exprDesc.getTypeInfo(),
@@ -1450,7 +1449,7 @@ public class VectorizationContext {
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
HiveDecimal decimalValue = castConstantToDecimal(constantValue, child.getTypeInfo());
return getConstantVectorExpression(decimalValue, returnType, Mode.PROJECTION);
- } else if (child instanceof ExprNodeNullDesc) {
+ } else if (child instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc)child).getValue()) {
return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
}
if (isIntFamily(inputType)) {
@@ -1477,7 +1476,7 @@ public class VectorizationContext {
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
String strValue = castConstantToString(constantValue, child.getTypeInfo());
return getConstantVectorExpression(strValue, returnType, Mode.PROJECTION);
- } else if (child instanceof ExprNodeNullDesc) {
+ } else if (child instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc)child).getValue()) {
return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
}
if (inputType.equals("boolean")) {
@@ -1564,7 +1563,7 @@ public class VectorizationContext {
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
Double doubleValue = castConstantToDouble(constantValue, child.getTypeInfo());
return getConstantVectorExpression(doubleValue, returnType, Mode.PROJECTION);
- } else if (child instanceof ExprNodeNullDesc) {
+ } else if (child instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc)child).getValue()) {
return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
}
if (isIntFamily(inputType)) {
@@ -1590,7 +1589,7 @@ public class VectorizationContext {
// Don't do constant folding here. Wait until the optimizer is changed to do it.
// Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
return null;
- } else if (child instanceof ExprNodeNullDesc) {
+ } else if (child instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc)child).getValue()) {
return getConstantVectorExpression(null, TypeInfoFactory.booleanTypeInfo, Mode.PROJECTION);
}
// Long and double are handled using descriptors, string needs to be specially handled.
@@ -1620,7 +1619,7 @@ public class VectorizationContext {
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
Long longValue = castConstantToLong(constantValue, child.getTypeInfo());
return getConstantVectorExpression(longValue, TypeInfoFactory.longTypeInfo, Mode.PROJECTION);
- } else if (child instanceof ExprNodeNullDesc) {
+ } else if (child instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc)child).getValue()) {
return getConstantVectorExpression(null, TypeInfoFactory.longTypeInfo, Mode.PROJECTION);
}
// Float family, timestamp are handled via descriptor based lookup, int family needs
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
index f536ef6..209f717 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ConstantPropagateProcFactory.java
@@ -54,7 +54,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.plan.FileSinkDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
@@ -84,13 +83,12 @@ import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.WritableConstantBooleanObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.io.BooleanWritable;
import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Lists;
/**
* Factory for generating the different node processors used by ConstantPropagate.
@@ -140,7 +138,7 @@ public final class ConstantPropagateProcFactory {
* @return cast constant, or null if the type cast failed.
*/
private static ExprNodeConstantDesc typeCast(ExprNodeDesc desc, TypeInfo ti) {
- if (desc instanceof ExprNodeNullDesc) {
+ if (desc instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc)desc).getValue()) {
return null;
}
if (!(ti instanceof PrimitiveTypeInfo) || !(desc.getTypeInfo() instanceof PrimitiveTypeInfo)) {
@@ -351,7 +349,7 @@ public final class ConstantPropagateProcFactory {
ExprNodeColumnDesc c = (ExprNodeColumnDesc) operand;
ColumnInfo ci = resolveColumn(rs, c);
if (ci != null) {
- constants.put(ci, new ExprNodeNullDesc());
+ constants.put(ci, new ExprNodeConstantDesc(ci.getType(), null));
}
}
}
@@ -365,6 +363,57 @@ public final class ConstantPropagateProcFactory {
}
private static ExprNodeDesc shortcutFunction(GenericUDF udf, List<ExprNodeDesc> newExprs) throws UDFArgumentException {
+
+ if (udf instanceof GenericUDFOPEqual) {
+ assert newExprs.size() == 2;
+ boolean foundUDFInFirst = false;
+ ExprNodeGenericFuncDesc caseOrWhenexpr = null;
+ if (newExprs.get(0) instanceof ExprNodeGenericFuncDesc) {
+ caseOrWhenexpr = (ExprNodeGenericFuncDesc) newExprs.get(0);
+ if (caseOrWhenexpr.getGenericUDF() instanceof GenericUDFWhen || caseOrWhenexpr.getGenericUDF() instanceof GenericUDFCase) {
+ foundUDFInFirst = true;
+ }
+ }
+ if (!foundUDFInFirst && newExprs.get(1) instanceof ExprNodeGenericFuncDesc) {
+ caseOrWhenexpr = (ExprNodeGenericFuncDesc) newExprs.get(1);
+ if (!(caseOrWhenexpr.getGenericUDF() instanceof GenericUDFWhen || caseOrWhenexpr.getGenericUDF() instanceof GenericUDFCase)) {
+ return null;
+ }
+ }
+ if (null == caseOrWhenexpr) {
+ // we didn't find case or when udf
+ return null;
+ }
+ GenericUDF childUDF = caseOrWhenexpr.getGenericUDF();
+ List<ExprNodeDesc> children = caseOrWhenexpr.getChildren();
+ int i;
+ if (childUDF instanceof GenericUDFWhen) {
+ for (i = 1; i < children.size(); i+=2) {
+ children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(),
+ Lists.newArrayList(children.get(i),newExprs.get(foundUDFInFirst ? 1 : 0))));
+ }
+ if(children.size() % 2 == 1) {
+ i = children.size()-1;
+ children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(),
+ Lists.newArrayList(children.get(i),newExprs.get(foundUDFInFirst ? 1 : 0))));
+ }
+ return caseOrWhenexpr;
+ } else if (childUDF instanceof GenericUDFCase) {
+ for (i = 2; i < children.size(); i+=2) {
+ children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(),
+ Lists.newArrayList(children.get(i),newExprs.get(foundUDFInFirst ? 1 : 0))));
+ }
+ if(children.size() % 2 == 0) {
+ i = children.size()-1;
+ children.set(i, ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPEqual(),
+ Lists.newArrayList(children.get(i),newExprs.get(foundUDFInFirst ? 1 : 0))));
+ }
+ return caseOrWhenexpr;
+ } else {
+ // cant happen
+ return null;
+ }
+ }
if (udf instanceof GenericUDFOPAnd) {
for (int i = 0; i < 2; i++) {
ExprNodeDesc childExpr = newExprs.get(i);
@@ -422,11 +471,8 @@ public final class ConstantPropagateProcFactory {
return null;
}
ExprNodeDesc thenExpr = newExprs.get(1);
- if (thenExpr instanceof ExprNodeNullDesc && (newExprs.size() == 2 || newExprs.get(2) instanceof ExprNodeNullDesc)) {
- return thenExpr;
- }
ExprNodeDesc elseExpr = newExprs.size() == 3 ? newExprs.get(2) :
- new ExprNodeConstantDesc(newExprs.get(2).getTypeInfo(),null);
+ new ExprNodeConstantDesc(newExprs.get(1).getTypeInfo(),null);
ExprNodeDesc whenExpr = newExprs.get(0);
if (whenExpr instanceof ExprNodeConstantDesc) {
@@ -444,7 +490,7 @@ public final class ConstantPropagateProcFactory {
} else if(thenVal.equals(elseVal)){
return thenExpr;
} else if (thenVal instanceof Boolean && elseVal instanceof Boolean) {
- return Boolean.TRUE.equals(thenVal) ? newExprs.get(0) :
+ return Boolean.TRUE.equals(thenVal) ? whenExpr :
ExprNodeGenericFuncDesc.newInstance(new GenericUDFOPNot(), newExprs.subList(0, 1));
} else {
return null;
@@ -461,10 +507,6 @@ public final class ConstantPropagateProcFactory {
return null;
}
ExprNodeDesc thenExpr = newExprs.get(2);
- if (thenExpr instanceof ExprNodeNullDesc && (newExprs.size() == 3 || newExprs.get(3) instanceof ExprNodeNullDesc)) {
- return thenExpr;
- }
-
ExprNodeDesc elseExpr = newExprs.size() == 4 ? newExprs.get(3) :
new ExprNodeConstantDesc(newExprs.get(2).getTypeInfo(),null);
@@ -559,16 +601,14 @@ public final class ConstantPropagateProcFactory {
}
Object value = constant.getValue();
PrimitiveTypeInfo pti = (PrimitiveTypeInfo) constant.getTypeInfo();
- Object writableValue =
+ Object writableValue = null == value ? value :
PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(pti)
.getPrimitiveWritableObject(value);
arguments[i] = new DeferredJavaObject(writableValue);
argois[i] =
ObjectInspectorUtils.getConstantObjectInspector(constant.getWritableObjectInspector(),
writableValue);
- } else if (desc instanceof ExprNodeNullDesc) {
- argois[i] = desc.getWritableObjectInspector();
- arguments[i] = new DeferredJavaObject(((ExprNodeNullDesc) desc).getValue());
+
} else if (desc instanceof ExprNodeGenericFuncDesc) {
ExprNodeDesc evaluatedFn = foldExpr((ExprNodeGenericFuncDesc)desc);
if (null == evaluatedFn || !(evaluatedFn instanceof ExprNodeConstantDesc)) {
@@ -589,11 +629,7 @@ public final class ConstantPropagateProcFactory {
Object o = udf.evaluate(arguments);
LOG.debug(udf.getClass().getName() + "(" + exprs + ")=" + o);
if (o == null) {
- if (oi instanceof PrimitiveObjectInspector) {
-
- return new ExprNodeConstantDesc(((PrimitiveObjectInspector) oi).getTypeInfo(), o);
- }
- return new ExprNodeNullDesc();
+ return new ExprNodeConstantDesc(TypeInfoUtils.getTypeInfoFromObjectInspector(oi), o);
}
Class<?> clz = o.getClass();
if (PrimitiveObjectInspectorUtils.isPrimitiveWritableClass(clz)) {
@@ -686,7 +722,7 @@ public final class ConstantPropagateProcFactory {
LOG.warn("Filter expression " + condn + " holds false!");
}
}
- if (newCondn instanceof ExprNodeNullDesc || (newCondn instanceof ExprNodeConstantDesc && ((ExprNodeConstantDesc)newCondn).getValue() == null)) {
+ if (newCondn instanceof ExprNodeConstantDesc && ((ExprNodeConstantDesc)newCondn).getValue() == null) {
// where null is same as where false
newCondn = new ExprNodeConstantDesc(Boolean.FALSE);
}
@@ -1027,7 +1063,7 @@ public final class ConstantPropagateProcFactory {
List<ExprNodeDesc> newExprs = new ArrayList<ExprNodeDesc>();
for (ExprNodeDesc expr : exprs) {
ExprNodeDesc newExpr = foldExpr(expr, constants, cppCtx, op, tag, false);
- if (newExpr instanceof ExprNodeConstantDesc || newExpr instanceof ExprNodeNullDesc) {
+ if (newExpr instanceof ExprNodeConstantDesc) {
LOG.info("expr " + newExpr + " fold from " + expr + " is removed.");
continue;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
index 1e47fcb..af54286 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/GroupByOptimizer.java
@@ -55,7 +55,6 @@ import org.apache.hadoop.hive.ql.plan.AggregationDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.plan.GroupByDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.SelectDesc;
@@ -340,9 +339,6 @@ public class GroupByOptimizer implements Transform {
}
else {
tableColsMapping.remove(outputColumnName);
- if (selectCol instanceof ExprNodeNullDesc) {
- newConstantCols.add(outputColumnName);
- }
if (selectCol instanceof ExprNodeConstantDesc) {
// Lets see if this constant was folded because of optimization.
String origCol = ((ExprNodeConstantDesc) selectCol).getFoldedFromCol();
@@ -380,8 +376,7 @@ public class GroupByOptimizer implements Transform {
}
}
// Constants and nulls are OK
- else if ((expr instanceof ExprNodeConstantDesc) ||
- (expr instanceof ExprNodeNullDesc)) {
+ else if (expr instanceof ExprNodeConstantDesc) {
continue;
} else {
return GroupByOptimizerSortMatch.NO_MATCH;
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerExpressionOperatorFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerExpressionOperatorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerExpressionOperatorFactory.java
index e633fdc..306e714 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerExpressionOperatorFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerExpressionOperatorFactory.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
/**
* Expression processor factory for pruning. Each processor tries to
@@ -182,8 +181,6 @@ public abstract class PrunerExpressionOperatorFactory {
Object... nodeOutputs) throws SemanticException {
if (nd instanceof ExprNodeConstantDesc) {
return ((ExprNodeConstantDesc) nd).clone();
- } else if (nd instanceof ExprNodeNullDesc) {
- return ((ExprNodeNullDesc) nd).clone();
}
return new ExprNodeConstantDesc(((ExprNodeDesc)nd).getTypeInfo(), null);
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
index 0328007..317454d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
@@ -64,7 +64,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.plan.FetchWork;
import org.apache.hadoop.hive.ql.plan.ListSinkDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
@@ -262,7 +261,6 @@ public class SimpleFetchOptimizer implements Transform {
private boolean checkExpression(ExprNodeDesc expr) {
if (expr instanceof ExprNodeConstantDesc ||
- expr instanceof ExprNodeNullDesc||
expr instanceof ExprNodeColumnDesc) {
return true;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
index abd7afd..3d05161 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/RexNodeConverter.java
@@ -63,7 +63,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDescUtils;
import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseBinary;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
@@ -124,9 +123,7 @@ public class RexNodeConverter {
}
public RexNode convert(ExprNodeDesc expr) throws SemanticException {
- if (expr instanceof ExprNodeNullDesc) {
- return createNullLiteral(expr);
- } else if (expr instanceof ExprNodeGenericFuncDesc) {
+ if (expr instanceof ExprNodeGenericFuncDesc) {
return convert((ExprNodeGenericFuncDesc) expr);
} else if (expr instanceof ExprNodeConstantDesc) {
return convert((ExprNodeConstantDesc) expr);
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java
index 86d221d..c930b80 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/ExprProcFactory.java
@@ -49,7 +49,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
/**
@@ -136,7 +135,7 @@ public class ExprProcFactory {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
- assert (nd instanceof ExprNodeConstantDesc || nd instanceof ExprNodeNullDesc);
+ assert (nd instanceof ExprNodeConstantDesc);
// Create a dependency that has no basecols
Dependency dep = new Dependency();
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java
index cbd4e6c..d5102bc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/pcr/PcrExprProcFactory.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.serde2.SerDeException;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -392,7 +391,7 @@ public final class PcrExprProcFactory {
@Override
public Object process(Node nd, Stack<Node> stack, NodeProcessorCtx procCtx,
Object... nodeOutputs) throws SemanticException {
- if (nd instanceof ExprNodeConstantDesc || nd instanceof ExprNodeNullDesc) {
+ if (nd instanceof ExprNodeConstantDesc) {
return new NodeInfoWrapper(WalkState.CONSTANT, null,
(ExprNodeDesc) nd);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java
index 01398f0..cc0a7d1 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TableAccessAnalyzer.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.plan.ExprNodeColumnDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.SelectDesc;
@@ -291,8 +290,7 @@ public class TableAccessAnalyzer {
continue;
}
- if ((colExpr instanceof ExprNodeConstantDesc) ||
- (colExpr instanceof ExprNodeNullDesc)) {
+ if (colExpr instanceof ExprNodeConstantDesc) {
currColNames.remove(outputColName);
continue;
} else if (colExpr instanceof ExprNodeColumnDesc) {
@@ -317,7 +315,7 @@ public class TableAccessAnalyzer {
if (expr instanceof ExprNodeColumnDesc) {
ExprNodeColumnDesc colExpr = (ExprNodeColumnDesc)expr;
colList.add(colExpr.getColumn());
- } else if (expr instanceof ExprNodeConstantDesc || expr instanceof ExprNodeNullDesc) {
+ } else if (expr instanceof ExprNodeConstantDesc) {
continue;
} else {
return null;
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
index a38511a..0e97530 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
@@ -59,7 +59,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.udf.SettableUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFBaseCompare;
@@ -78,6 +77,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.hive.serde2.typeinfo.VarcharTypeInfo;
+import org.apache.hadoop.io.NullWritable;
import org.apache.hive.common.util.DateUtils;
import com.google.common.collect.Lists;
@@ -240,7 +240,7 @@ public class TypeCheckProcFactory {
return desc;
}
- return new ExprNodeNullDesc();
+ return new ExprNodeConstantDesc(TypeInfoFactory.getPrimitiveTypeInfoFromPrimitiveWritable(NullWritable.class), null);
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
index 89a175e..2674fe3 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeConstantDesc.java
@@ -23,12 +23,11 @@ import java.io.Serializable;
import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
/**
* A constant expression.
@@ -83,13 +82,8 @@ public class ExprNodeConstantDesc extends ExprNodeDesc implements Serializable {
@Override
public ConstantObjectInspector getWritableObjectInspector() {
- PrimitiveTypeInfo pti = (PrimitiveTypeInfo) getTypeInfo();
- // Convert from Java to Writable
- Object writableValue = PrimitiveObjectInspectorFactory
- .getPrimitiveJavaObjectInspector(pti).getPrimitiveWritableObject(
- getValue());
- return PrimitiveObjectInspectorFactory
- .getPrimitiveWritableConstantObjectInspector((PrimitiveTypeInfo) getTypeInfo(), writableValue);
+ return ObjectInspectorUtils.getConstantObjectInspector(
+ TypeInfoUtils.getStandardJavaObjectInspectorFromTypeInfo(typeInfo), value);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeNullDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeNullDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeNullDesc.java
deleted file mode 100644
index 25b16da..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ExprNodeNullDesc.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.plan;
-
-import java.io.Serializable;
-
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.io.NullWritable;
-
-/**
- * ExprNodeNullDesc.
- *
- */
-public class ExprNodeNullDesc extends ExprNodeDesc implements Serializable {
-
- private static final long serialVersionUID = 1L;
-
- public ExprNodeNullDesc() {
- super(TypeInfoFactory
- .getPrimitiveTypeInfoFromPrimitiveWritable(NullWritable.class));
- }
-
- public Object getValue() {
- return null;
- }
-
- @Override
- public String toString() {
- return "null";
- }
-
- @Override
- public String getExprString() {
- return "null";
- }
-
- @Override
- public ExprNodeDesc clone() {
- return new ExprNodeNullDesc();
- }
-
- @Override
- public boolean isSame(Object o) {
- if (!(o instanceof ExprNodeNullDesc)) {
- return false;
- }
- if (!typeInfo.equals(((ExprNodeNullDesc) o).getTypeInfo())) {
- return false;
- }
-
- return true;
- }
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
index 508d880..10871e4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java
@@ -50,7 +50,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.ql.plan.ExprNodeNullDesc;
import org.apache.hadoop.hive.ql.plan.Statistics;
import org.apache.hadoop.hive.ql.plan.Statistics.State;
import org.apache.hadoop.hive.ql.util.JavaDataModel;
@@ -1151,13 +1150,6 @@ public class StatsUtils {
colType = engfd.getTypeString();
countDistincts = numRows;
oi = engfd.getWritableObjectInspector();
- } else if (end instanceof ExprNodeNullDesc) {
-
- // null projection
- ExprNodeNullDesc ennd = (ExprNodeNullDesc) end;
- colName = ennd.getName();
- colType = "null";
- numNulls = numRows;
} else if (end instanceof ExprNodeColumnListDesc) {
// column list
@@ -1473,7 +1465,7 @@ public class StatsUtils {
double result = a * b;
return (result > Long.MAX_VALUE) ? Long.MAX_VALUE : (long)result;
}
-
+
/** Bounded addition - overflows become MAX_VALUE */
public static long safeAdd(long a, long b) {
try {
@@ -1482,7 +1474,7 @@ public class StatsUtils {
return Long.MAX_VALUE;
}
}
-
+
/** Bounded multiplication - overflows become MAX_VALUE */
public static long safeMult(long a, long b) {
try {
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCoalesce.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCoalesce.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCoalesce.java
index 8890e69..aa708ce 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCoalesce.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFCoalesce.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
/**
* GenericUDF Class for SQL construct "COALESCE(a, b, c)".
- *
+ *
* NOTES: 1. a, b and c should have the same TypeInfo, or an exception will be
* thrown.
*/
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFGreatest.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFGreatest.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFGreatest.java
index e919345..e1eab89 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFGreatest.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFGreatest.java
@@ -97,6 +97,7 @@ public class GenericUDFGreatest extends GenericUDF {
return getStandardDisplayString(getFuncName(), children, ",");
}
+ @Override
protected String getFuncName() {
return "greatest";
}
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInstr.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInstr.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInstr.java
index 0a13ac9..0f7d4d6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInstr.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFInstr.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.io.Text;
* Generic UDF for string function <code>INSTR(str,substr)</code>. This mimcs
* the function from MySQL
* http://dev.mysql.com/doc/refman/5.1/en/string-functions.html#function_instr
- *
+ *
* <pre>
* usage:
* INSTR(str, substr)
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLocate.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLocate.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLocate.java
index 094f280..137eb3e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLocate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFLocate.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.io.Text;
* Generic UDF for string function <code>LOCATE(substr, str)</code>,
* <code>LOCATE(substr, str, start)</code>. This mimcs the function from MySQL
* http://dev.mysql.com/doc/refman/5.1/en/string-functions.html#function_locate
- *
+ *
* <pre>
* usage:
* LOCATE(substr, str)
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPrintf.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPrintf.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPrintf.java
index cb6dd62..e52e431 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPrintf.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFPrintf.java
@@ -133,8 +133,9 @@ public class GenericUDFPrintf extends GenericUDF {
}
}
formatter.format(pattern.toString(), argumentList.toArray());
-
resultText.set(sb.toString());
+ formatter.close();
+
return resultText;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTranslate.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTranslate.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTranslate.java
index 4ac542f..2717f00 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTranslate.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFTranslate.java
@@ -41,7 +41,7 @@ import org.apache.hadoop.io.Text;
/**
* TRANSLATE(string input, string from, string to) is an equivalent function to translate in
* PostGresSQL. See explain extended annotation below to read more about how this UDF works
- *
+ *
*/
@UDFType(deterministic = true)
//@formatter:off
@@ -188,7 +188,7 @@ public class GenericUDFTranslate extends GenericUDF {
/**
* Pre-processes the from and to strings by calling {@link #populateMappings(Text, Text)} if
* necessary.
- *
+ *
* @param from
* from string to be used for translation
* @param to
@@ -215,7 +215,7 @@ public class GenericUDFTranslate extends GenericUDF {
/**
* Pre-process the from and to strings populate {@link #replacementMap} and {@link #deletionSet}.
- *
+ *
* @param from
* from string to be used for translation
* @param to
@@ -255,7 +255,7 @@ public class GenericUDFTranslate extends GenericUDF {
/**
* Translates the input string based on {@link #replacementMap} and {@link #deletionSet} and
* returns the translated string.
- *
+ *
* @param input
* input string to perform the translation on
* @return translated string
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
index 09d2d1f..222e0e0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFUtils.java
@@ -175,7 +175,7 @@ public final class GenericUDFUtils {
* that.
*/
if (commonTypeInfo instanceof DecimalTypeInfo) {
- if ((!FunctionRegistry.isExactNumericType((PrimitiveTypeInfo) oiTypeInfo)) ||
+ if ((!FunctionRegistry.isExactNumericType((PrimitiveTypeInfo) oiTypeInfo)) ||
(!FunctionRegistry.isExactNumericType((PrimitiveTypeInfo) rTypeInfo))) {
commonTypeInfo = TypeInfoFactory.doubleTypeInfo;
}
@@ -204,8 +204,8 @@ public final class GenericUDFUtils {
/**
* Convert the return Object if necessary (when the ObjectInspectors of
- * different possibilities are not all the same). If reuse is true,
- * the result Object will be the same object as the last invocation
+ * different possibilities are not all the same). If reuse is true,
+ * the result Object will be the same object as the last invocation
* (as long as the oi is the same)
*/
public Object convertIfNecessary(Object o, ObjectInspector oi, boolean reuse) {
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/test/queries/clientpositive/fold_eq_with_case_when.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/fold_eq_with_case_when.q b/ql/src/test/queries/clientpositive/fold_eq_with_case_when.q
new file mode 100644
index 0000000..b6b54b4
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/fold_eq_with_case_when.q
@@ -0,0 +1,21 @@
+explain
+SELECT
+SUM((CASE WHEN 1000000 = 0 THEN NULL ELSE l_partkey / 1000000 END)),
+SUM(1) AS `sum_number_of_records_ok` FROM lineitem
+WHERE
+(((CASE WHEN ('N' = l_returnflag) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('MAIL' = l_shipmode) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('O' = l_linestatus) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('NONE' = l_shipinstruct) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('All' = (CASE WHEN (l_shipmode = 'TRUCK') THEN 'East' WHEN (l_shipmode = 'MAIL') THEN 'West' WHEN (l_shipmode = 'REG AIR') THEN 'BizDev' ELSE 'Other' END)) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('AIR' = l_shipmode) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('1996-03-30' = TO_DATE(l_shipdate)) THEN 1 ELSE NULL END) = 1) AND
+((CASE WHEN ('RAIL' = l_shipmode) THEN 1 ELSE NULL END) = 1) AND (1 = 1) AND
+((CASE WHEN (1 = l_linenumber) THEN 1 ELSE 1 END) = 1) AND (1 = 1))
+GROUP BY l_orderkey;
+
+
+explain select key from src where (case key when '238' then 1 else 2 end) = 1;
+explain select key from src where (case key when '238' then 1 when '94' then 1 else 3 end) = cast('1' as int);
+explain select key from src where (case key when '238' then 1 else 2 end) = (case when key != '238' then 1 else 1 end);
+explain select key from src where (case key when '238' then 1 end) = (case when key != '238' then 1 when key = '23' then 1 end);
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/annotate_stats_filter.q.out b/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
index e8cd06d..aa66bc6 100644
--- a/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
+++ b/ql/src/test/results/clientpositive/annotate_stats_filter.q.out
@@ -262,7 +262,7 @@ STAGE PLANS:
predicate: zip is null (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: state (type: string), locid (type: int), null (type: void), year (type: int)
+ expressions: state (type: string), locid (type: int), null (type: bigint), year (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 94 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
@@ -721,7 +721,7 @@ STAGE PLANS:
predicate: ((year = 2001) and year is null) (type: boolean)
Statistics: Num rows: 1 Data size: 102 Basic stats: COMPLETE Column stats: COMPLETE
Select Operator
- expressions: state (type: string), locid (type: int), zip (type: bigint), null (type: void)
+ expressions: state (type: string), locid (type: int), zip (type: bigint), null (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 98 Basic stats: COMPLETE Column stats: COMPLETE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out b/ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out
new file mode 100644
index 0000000..45a0cb1
--- /dev/null
+++ b/ql/src/test/results/clientpositive/fold_eq_with_case_when.q.out
@@ -0,0 +1,231 @@
+PREHOOK: query: explain
+SELECT
+SUM((CASE WHEN 1000000 = 0 THEN NULL ELSE l_partkey / 1000000 END)),
+SUM(1) AS `sum_number_of_records_ok` FROM lineitem
+WHERE
+(((CASE WHEN ('N' = l_returnflag) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('MAIL' = l_shipmode) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('O' = l_linestatus) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('NONE' = l_shipinstruct) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('All' = (CASE WHEN (l_shipmode = 'TRUCK') THEN 'East' WHEN (l_shipmode = 'MAIL') THEN 'West' WHEN (l_shipmode = 'REG AIR') THEN 'BizDev' ELSE 'Other' END)) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('AIR' = l_shipmode) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('1996-03-30' = TO_DATE(l_shipdate)) THEN 1 ELSE NULL END) = 1) AND
+((CASE WHEN ('RAIL' = l_shipmode) THEN 1 ELSE NULL END) = 1) AND (1 = 1) AND
+((CASE WHEN (1 = l_linenumber) THEN 1 ELSE 1 END) = 1) AND (1 = 1))
+GROUP BY l_orderkey
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+SELECT
+SUM((CASE WHEN 1000000 = 0 THEN NULL ELSE l_partkey / 1000000 END)),
+SUM(1) AS `sum_number_of_records_ok` FROM lineitem
+WHERE
+(((CASE WHEN ('N' = l_returnflag) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('MAIL' = l_shipmode) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('O' = l_linestatus) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('NONE' = l_shipinstruct) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('All' = (CASE WHEN (l_shipmode = 'TRUCK') THEN 'East' WHEN (l_shipmode = 'MAIL') THEN 'West' WHEN (l_shipmode = 'REG AIR') THEN 'BizDev' ELSE 'Other' END)) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('AIR' = l_shipmode) THEN 1 ELSE 1 END) = 1) AND
+((CASE WHEN ('1996-03-30' = TO_DATE(l_shipdate)) THEN 1 ELSE NULL END) = 1) AND
+((CASE WHEN ('RAIL' = l_shipmode) THEN 1 ELSE NULL END) = 1) AND (1 = 1) AND
+((CASE WHEN (1 = l_linenumber) THEN 1 ELSE 1 END) = 1) AND (1 = 1))
+GROUP BY l_orderkey
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: lineitem
+ Statistics: Num rows: 100 Data size: 11999 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (CASE WHEN (('1996-03-30' = to_date(l_shipdate))) THEN (true) ELSE (null) END and CASE WHEN (('RAIL' = l_shipmode)) THEN (true) ELSE (null) END) (type: boolean)
+ Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: l_orderkey (type: int), (UDFToDouble(l_partkey) / 1000000.0) (type: double)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(_col1), sum(1)
+ keys: _col0 (type: int)
+ mode: hash
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: int)
+ Statistics: Num rows: 25 Data size: 2999 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: double), _col2 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0), sum(VALUE._col1)
+ keys: KEY._col0 (type: int)
+ mode: mergepartial
+ outputColumnNames: _col0, _col1, _col2
+ Statistics: Num rows: 12 Data size: 1439 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col1 (type: double), _col2 (type: bigint)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 12 Data size: 1439 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 12 Data size: 1439 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain select key from src where (case key when '238' then 1 else 2 end) = 1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select key from src where (case key when '238' then 1 else 2 end) = 1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key = '238') (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain select key from src where (case key when '238' then 1 when '94' then 1 else 3 end) = cast('1' as int)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select key from src where (case key when '238' then 1 when '94' then 1 else 3 end) = cast('1' as int)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: CASE (key) WHEN ('238') THEN (true) WHEN ('94') THEN (true) ELSE (false) END (type: int)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain select key from src where (case key when '238' then 1 else 2 end) = (case when key != '238' then 1 else 1 end)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select key from src where (case key when '238' then 1 else 2 end) = (case when key != '238' then 1 else 1 end)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: (key = '238') (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: explain select key from src where (case key when '238' then 1 end) = (case when key != '238' then 1 when key = '23' then 1 end)
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select key from src where (case key when '238' then 1 end) = (case when key != '238' then 1 when key = '23' then 1 end)
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: src
+ Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: CASE (key) WHEN ('238') THEN (CASE WHEN ((key <> '238')) THEN ((1 = 1)) WHEN ((key = '23')) THEN ((1 = 1)) END) END (type: int)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/test/results/clientpositive/fold_when.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/fold_when.q.out b/ql/src/test/results/clientpositive/fold_when.q.out
index 37803e0..51d4767 100644
--- a/ql/src/test/results/clientpositive/fold_when.q.out
+++ b/ql/src/test/results/clientpositive/fold_when.q.out
@@ -153,7 +153,7 @@ STAGE PLANS:
alias: src
Statistics: Num rows: 500 Data size: 5312 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (CASE WHEN ((key = '238')) THEN (1) ELSE (null) END = 1) (type: boolean)
+ predicate: CASE WHEN ((key = '238')) THEN (true) ELSE (null) END (type: int)
Statistics: Num rows: 250 Data size: 2656 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: key (type: string)
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/test/results/clientpositive/input6.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/input6.q.out b/ql/src/test/results/clientpositive/input6.q.out
index 38c9fe1..5ed2767 100644
--- a/ql/src/test/results/clientpositive/input6.q.out
+++ b/ql/src/test/results/clientpositive/input6.q.out
@@ -35,7 +35,7 @@ STAGE PLANS:
predicate: key is null (type: boolean)
Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: null (type: void), value (type: string)
+ expressions: null (type: string), value (type: string)
outputColumnNames: _col0, _col1
Statistics: Num rows: 12 Data size: 91 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/test/results/clientpositive/join_nullsafe.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/join_nullsafe.q.out b/ql/src/test/results/clientpositive/join_nullsafe.q.out
index 9bdfcbd..27ceae5 100644
--- a/ql/src/test/results/clientpositive/join_nullsafe.q.out
+++ b/ql/src/test/results/clientpositive/join_nullsafe.q.out
@@ -1523,9 +1523,9 @@ STAGE PLANS:
predicate: key is null (type: boolean)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: null (type: void)
+ key expressions: null (type: int)
sort order: +
- Map-reduce partition columns: null (type: void)
+ Map-reduce partition columns: null (type: int)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: value (type: int)
TableScan
@@ -1535,9 +1535,9 @@ STAGE PLANS:
predicate: value is null (type: boolean)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: null (type: void)
+ key expressions: null (type: int)
sort order: +
- Map-reduce partition columns: null (type: void)
+ Map-reduce partition columns: null (type: int)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: key (type: int)
Reduce Operator Tree:
@@ -1551,7 +1551,7 @@ STAGE PLANS:
outputColumnNames: _col1, _col5
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: null (type: void), _col1 (type: int), _col5 (type: int), null (type: void)
+ expressions: null (type: int), _col1 (type: int), _col5 (type: int), null (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/test/results/clientpositive/spark/join_nullsafe.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/join_nullsafe.q.out b/ql/src/test/results/clientpositive/spark/join_nullsafe.q.out
index 96db301..70459f7 100644
--- a/ql/src/test/results/clientpositive/spark/join_nullsafe.q.out
+++ b/ql/src/test/results/clientpositive/spark/join_nullsafe.q.out
@@ -1576,9 +1576,9 @@ STAGE PLANS:
predicate: key is null (type: boolean)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: null (type: void)
+ key expressions: null (type: int)
sort order: +
- Map-reduce partition columns: null (type: void)
+ Map-reduce partition columns: null (type: int)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: value (type: int)
Map 3
@@ -1590,9 +1590,9 @@ STAGE PLANS:
predicate: value is null (type: boolean)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: null (type: void)
+ key expressions: null (type: int)
sort order: +
- Map-reduce partition columns: null (type: void)
+ Map-reduce partition columns: null (type: int)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: key (type: int)
Reducer 2
@@ -1607,7 +1607,7 @@ STAGE PLANS:
outputColumnNames: _col1, _col5
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: null (type: void), _col1 (type: int), _col5 (type: int), null (type: void)
+ expressions: null (type: int), _col1 (type: int), _col5 (type: int), null (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out b/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
index 2d5b486..c1bbf0e 100644
--- a/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
+++ b/ql/src/test/results/clientpositive/subquery_notin_having.q.java1.7.out
@@ -658,7 +658,7 @@ STAGE PLANS:
predicate: p_mfgr is null (type: boolean)
Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: null (type: void), p_retailprice (type: double)
+ expressions: null (type: string), p_retailprice (type: double)
outputColumnNames: _col0, _col1
Statistics: Num rows: 13 Data size: 1573 Basic stats: COMPLETE Column stats: NONE
Group By Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/test/results/clientpositive/tez/join_nullsafe.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/join_nullsafe.q.out b/ql/src/test/results/clientpositive/tez/join_nullsafe.q.out
index 787f1f5..6fadf5a 100644
--- a/ql/src/test/results/clientpositive/tez/join_nullsafe.q.out
+++ b/ql/src/test/results/clientpositive/tez/join_nullsafe.q.out
@@ -1576,9 +1576,9 @@ STAGE PLANS:
predicate: key is null (type: boolean)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: null (type: void)
+ key expressions: null (type: int)
sort order: +
- Map-reduce partition columns: null (type: void)
+ Map-reduce partition columns: null (type: int)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: value (type: int)
Map 3
@@ -1590,9 +1590,9 @@ STAGE PLANS:
predicate: value is null (type: boolean)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Reduce Output Operator
- key expressions: null (type: void)
+ key expressions: null (type: int)
sort order: +
- Map-reduce partition columns: null (type: void)
+ Map-reduce partition columns: null (type: int)
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
value expressions: key (type: int)
Reducer 2
@@ -1601,13 +1601,13 @@ STAGE PLANS:
condition map:
Inner Join 0 to 1
keys:
- 0 null (type: void)
- 1 null (type: void)
+ 0 null (type: int)
+ 1 null (type: int)
nullSafes: [true]
outputColumnNames: _col1, _col5
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: null (type: void), _col1 (type: int), _col5 (type: int), null (type: void)
+ expressions: null (type: int), _col1 (type: int), _col5 (type: int), null (type: int)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
File Output Operator
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/test/results/clientpositive/tez/vector_coalesce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_coalesce.q.out b/ql/src/test/results/clientpositive/tez/vector_coalesce.q.out
index db5e4f8..c787ce9 100644
--- a/ql/src/test/results/clientpositive/tez/vector_coalesce.q.out
+++ b/ql/src/test/results/clientpositive/tez/vector_coalesce.q.out
@@ -19,7 +19,7 @@ STAGE PLANS:
Filter Operator
predicate: cdouble is null (type: boolean)
Select Operator
- expressions: null (type: void), cstring1 (type: string), cint (type: int), cfloat (type: float), csmallint (type: smallint), COALESCE(null,cstring1,cint,cfloat,csmallint) (type: string)
+ expressions: null (type: double), cstring1 (type: string), cint (type: int), cfloat (type: float), csmallint (type: smallint), COALESCE(null,cstring1,cint,cfloat,csmallint) (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Limit
Number of rows: 10
@@ -68,7 +68,7 @@ STAGE PLANS:
Filter Operator
predicate: ctinyint is null (type: boolean)
Select Operator
- expressions: null (type: void), cdouble (type: double), cint (type: int), COALESCE(null,(cdouble + log2(cint)),0) (type: double)
+ expressions: null (type: tinyint), cdouble (type: double), cint (type: int), COALESCE(null,(cdouble + log2(cint)),0) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
Limit
Number of rows: 10
@@ -117,7 +117,7 @@ STAGE PLANS:
Filter Operator
predicate: (cfloat is null and cbigint is null) (type: boolean)
Select Operator
- expressions: null (type: void), null (type: void), 0 (type: int)
+ expressions: null (type: float), null (type: bigint), 0.0 (type: float)
outputColumnNames: _col0, _col1, _col2
Limit
Number of rows: 10
@@ -135,16 +135,16 @@ WHERE (cfloat IS NULL AND cbigint IS NULL) LIMIT 10
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc
#### A masked pattern was here ####
-NULL NULL 0
-NULL NULL 0
-NULL NULL 0
-NULL NULL 0
-NULL NULL 0
-NULL NULL 0
-NULL NULL 0
-NULL NULL 0
-NULL NULL 0
-NULL NULL 0
+NULL NULL 0.0
+NULL NULL 0.0
+NULL NULL 0.0
+NULL NULL 0.0
+NULL NULL 0.0
+NULL NULL 0.0
+NULL NULL 0.0
+NULL NULL 0.0
+NULL NULL 0.0
+NULL NULL 0.0
PREHOOK: query: EXPLAIN SELECT ctimestamp1, ctimestamp2, coalesce(ctimestamp1, ctimestamp2)
FROM alltypesorc
WHERE ctimestamp1 IS NOT NULL OR ctimestamp2 IS NOT NULL LIMIT 10
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/ql/src/test/results/clientpositive/vector_coalesce.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_coalesce.q.out b/ql/src/test/results/clientpositive/vector_coalesce.q.out
index c63f2d1..02a4e62 100644
--- a/ql/src/test/results/clientpositive/vector_coalesce.q.out
+++ b/ql/src/test/results/clientpositive/vector_coalesce.q.out
@@ -21,7 +21,7 @@ STAGE PLANS:
predicate: cdouble is null (type: boolean)
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: null (type: void), cstring1 (type: string), cint (type: int), cfloat (type: float), csmallint (type: smallint), COALESCE(null,cstring1,cint,cfloat,csmallint) (type: string)
+ expressions: null (type: double), cstring1 (type: string), cint (type: int), cfloat (type: float), csmallint (type: smallint), COALESCE(null,cstring1,cint,cfloat,csmallint) (type: string)
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Limit
@@ -87,7 +87,7 @@ STAGE PLANS:
predicate: ctinyint is null (type: boolean)
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: null (type: void), cdouble (type: double), cint (type: int), COALESCE(null,(cdouble + log2(cint)),0) (type: double)
+ expressions: null (type: tinyint), cdouble (type: double), cint (type: int), COALESCE(null,(cdouble + log2(cint)),0) (type: double)
outputColumnNames: _col0, _col1, _col2, _col3
Statistics: Num rows: 6144 Data size: 1320982 Basic stats: COMPLETE Column stats: NONE
Limit
@@ -153,7 +153,7 @@ STAGE PLANS:
predicate: (cfloat is null and cbigint is null) (type: boolean)
Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
Select Operator
- expressions: null (type: void), null (type: void), 0 (type: int)
+ expressions: null (type: float), null (type: bigint), 0.0 (type: float)
outputColumnNames: _col0, _col1, _col2
Statistics: Num rows: 3072 Data size: 660491 Basic stats: COMPLETE Column stats: NONE
Limit
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java b/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java
index a165b84..2e24730 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/io/HiveVarcharWritable.java
@@ -17,14 +17,8 @@
*/
package org.apache.hadoop.hive.serde2.io;
-import java.io.DataInput;
-import java.io.DataOutput;
-import java.io.IOException;
-
import org.apache.hadoop.hive.common.type.HiveBaseChar;
import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.hive.shims.ShimLoader;
-import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.WritableComparable;
public class HiveVarcharWritable extends HiveBaseCharWritable
@@ -74,6 +68,7 @@ public class HiveVarcharWritable extends HiveBaseCharWritable
set(getHiveVarchar(), maxLength);
}
+ @Override
public int compareTo(HiveVarcharWritable rhs) {
return value.compareTo(rhs.value);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
index 8a7c4a5..8ef8ce1 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorConverters.java
@@ -62,6 +62,7 @@ public final class ObjectInspectorConverters {
*
*/
public static class IdentityConverter implements Converter {
+ @Override
public Object convert(Object input) {
return input;
}
@@ -136,7 +137,7 @@ public final class ObjectInspectorConverters {
(SettableBinaryObjectInspector)outputOI);
case DECIMAL:
return new PrimitiveObjectInspectorConverter.HiveDecimalConverter(
- (PrimitiveObjectInspector) inputOI,
+ inputOI,
(SettableHiveDecimalObjectInspector) outputOI);
default:
throw new RuntimeException("Hive internal error: conversion of "
@@ -239,7 +240,7 @@ public final class ObjectInspectorConverters {
// Create a writable object inspector for primitive type and return it.
PrimitiveObjectInspector primOutputOI = (PrimitiveObjectInspector) outputOI;
return PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(
- (PrimitiveTypeInfo)primOutputOI.getTypeInfo());
+ primOutputOI.getTypeInfo());
case STRUCT:
StructObjectInspector structOutputOI = (StructObjectInspector) outputOI;
// create a standard settable struct object inspector.
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
index 15778af..f3f7d95 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/ObjectInspectorUtils.java
@@ -1029,10 +1029,10 @@ public final class ObjectInspectorUtils {
public static ConstantObjectInspector getConstantObjectInspector(ObjectInspector oi, Object value) {
if (oi instanceof ConstantObjectInspector) {
- return (ConstantObjectInspector) oi;
+ return (ConstantObjectInspector) oi;
}
ObjectInspector writableOI = getStandardObjectInspector(oi, ObjectInspectorCopyOption.WRITABLE);
- Object writableValue =
+ Object writableValue = value == null ? value :
ObjectInspectorConverters.getConverter(oi, writableOI).convert(value);
switch (writableOI.getCategory()) {
case PRIMITIVE:
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java
index baa4a94..0cbd30e 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/AbstractPrimitiveObjectInspector.java
@@ -83,6 +83,7 @@ public abstract class AbstractPrimitiveObjectInspector implements
return typeInfo.getTypeName();
}
+ @Override
public PrimitiveTypeInfo getTypeInfo() {
return this.typeInfo;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/05a3d2ab/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableVoidObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableVoidObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableVoidObjectInspector.java
index f3f4838..8ffc91d 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableVoidObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/WritableVoidObjectInspector.java
@@ -46,4 +46,9 @@ public class WritableVoidObjectInspector extends
public Object getPrimitiveJavaObject(Object o) {
return null;
}
+
+ @Override
+ public boolean equals(Object obj) {
+ return null != obj && obj instanceof WritableVoidObjectInspector;
+ }
}
[28/50] [abbrv] hive git commit: HIVE-10670: Duplicate declaration of
curator-recipes at pom.xml (Hari via Xuefu)
Posted by sp...@apache.org.
HIVE-10670: Duplicate declaration of curator-recipes at pom.xml (Hari via Xuefu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f106730b
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f106730b
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f106730b
Branch: refs/heads/parquet
Commit: f106730ba8d6d65265ad25690e4e8056a7053155
Parents: 3056e7a
Author: Xuefu Zhang <xz...@Cloudera.com>
Authored: Fri May 15 15:13:15 2015 -0700
Committer: Xuefu Zhang <xz...@Cloudera.com>
Committed: Fri May 15 15:13:15 2015 -0700
----------------------------------------------------------------------
pom.xml | 5 -----
1 file changed, 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f106730b/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 2e4ca36..08d6dc3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -502,11 +502,6 @@
<version>${curator.version}</version>
</dependency>
<dependency>
- <groupId>org.apache.curator</groupId>
- <artifactId>curator-recipes</artifactId>
- <version>${curator.version}</version>
- </dependency>
- <dependency>
<groupId>org.codehaus.groovy</groupId>
<artifactId>groovy-all</artifactId>
<version>${groovy.version}</version>
[38/50] [abbrv] hive git commit: HIVE-10719: Hive metastore failure
when alter table rename is attempted. (Vikram Dixit K, reviewed by Thejas Nair,
Sergey Shelukhin)
Posted by sp...@apache.org.
HIVE-10719: Hive metastore failure when alter table rename is attempted. (Vikram Dixit K, reviewed by Thejas Nair, Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/325a592f
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/325a592f
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/325a592f
Branch: refs/heads/parquet
Commit: 325a592f5f670bef6d6921161ef2d511d21c07d8
Parents: 129c496
Author: vikram <vi...@hortonworks.com>
Authored: Mon May 18 12:46:49 2015 -0700
Committer: vikram <vi...@hortonworks.com>
Committed: Mon May 18 13:52:46 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/metastore/HiveAlterHandler.java | 4 +--
.../clientpositive/alter_change_db_location.q | 5 +++
.../alter_change_db_location.q.out | 35 ++++++++++++++++++++
3 files changed, 42 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/325a592f/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index a3f2359..5391171 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -175,8 +175,8 @@ public class HiveAlterHandler implements AlterHandler {
// that means user is asking metastore to move data to new location
// corresponding to the new name
// get new location
- Path databasePath = constructRenamedPath(
- wh.getDefaultDatabasePath(newt.getDbName()), srcPath);
+ Database db = msdb.getDatabase(newt.getDbName());
+ Path databasePath = constructRenamedPath(wh.getDatabasePath(db), srcPath);
destPath = new Path(databasePath, newt.getTableName());
destFs = wh.getFs(destPath);
http://git-wip-us.apache.org/repos/asf/hive/blob/325a592f/ql/src/test/queries/clientpositive/alter_change_db_location.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/alter_change_db_location.q b/ql/src/test/queries/clientpositive/alter_change_db_location.q
new file mode 100644
index 0000000..a74ba2a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/alter_change_db_location.q
@@ -0,0 +1,5 @@
+create database newDB location "/tmp/";
+describe database extended newDB;
+use newDB;
+create table tab (name string);
+alter table tab rename to newName;
http://git-wip-us.apache.org/repos/asf/hive/blob/325a592f/ql/src/test/results/clientpositive/alter_change_db_location.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/alter_change_db_location.q.out b/ql/src/test/results/clientpositive/alter_change_db_location.q.out
new file mode 100644
index 0000000..64bd680
--- /dev/null
+++ b/ql/src/test/results/clientpositive/alter_change_db_location.q.out
@@ -0,0 +1,35 @@
+#### A masked pattern was here ####
+PREHOOK: type: CREATEDATABASE
+PREHOOK: Output: database:newDB
+#### A masked pattern was here ####
+POSTHOOK: type: CREATEDATABASE
+POSTHOOK: Output: database:newDB
+#### A masked pattern was here ####
+PREHOOK: query: describe database extended newDB
+PREHOOK: type: DESCDATABASE
+POSTHOOK: query: describe database extended newDB
+POSTHOOK: type: DESCDATABASE
+newdb location/in/test hive_test_user USER
+PREHOOK: query: use newDB
+PREHOOK: type: SWITCHDATABASE
+PREHOOK: Input: database:newdb
+POSTHOOK: query: use newDB
+POSTHOOK: type: SWITCHDATABASE
+POSTHOOK: Input: database:newdb
+PREHOOK: query: create table tab (name string)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:newdb
+PREHOOK: Output: newDB@tab
+POSTHOOK: query: create table tab (name string)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:newdb
+POSTHOOK: Output: newDB@tab
+PREHOOK: query: alter table tab rename to newName
+PREHOOK: type: ALTERTABLE_RENAME
+PREHOOK: Input: newdb@tab
+PREHOOK: Output: newdb@tab
+POSTHOOK: query: alter table tab rename to newName
+POSTHOOK: type: ALTERTABLE_RENAME
+POSTHOOK: Input: newdb@tab
+POSTHOOK: Output: newDB@newName
+POSTHOOK: Output: newdb@tab
[16/50] [abbrv] hive git commit: HIVE-10565: Native Vector Map Join
doesn't handle filtering and matching on LEFT OUTER JOIN repeated key
correctly (Matt McCline via Gunther Hagleitner)
Posted by sp...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/results/clientpositive/vector_join30.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_join30.q.out b/ql/src/test/results/clientpositive/vector_join30.q.out
new file mode 100644
index 0000000..57f9aeb
--- /dev/null
+++ b/ql/src/test/results/clientpositive/vector_join30.q.out
@@ -0,0 +1,2194 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE orcsrc STORED AS ORC AS SELECT * FROM src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orcsrc
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE orcsrc STORED AS ORC AS SELECT * FROM src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orcsrc
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1, Stage-4 , consists of Stage-8, Stage-9, Stage-2
+ Stage-8 has a backup stage: Stage-2
+ Stage-5 depends on stages: Stage-8
+ Stage-3 depends on stages: Stage-2, Stage-5, Stage-6
+ Stage-9 has a backup stage: Stage-2
+ Stage-6 depends on stages: Stage-9
+ Stage-2
+ Stage-4 is a root stage
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME1
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+
+ Stage: Stage-5
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-9
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+
+ Stage: Stage-6
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Statistics: Num rows: 275 Data size: 48400 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+103231310608
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-6 depends on stages: Stage-1, Stage-4 , consists of Stage-7, Stage-2
+ Stage-7 has a backup stage: Stage-2
+ Stage-5 depends on stages: Stage-7
+ Stage-3 depends on stages: Stage-2, Stage-5
+ Stage-2
+ Stage-4 is a root stage
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-6
+ Conditional Operator
+
+ Stage: Stage-7
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME1
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+
+ Stage: Stage-5
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+103231310608
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-6 depends on stages: Stage-1, Stage-4 , consists of Stage-7, Stage-2
+ Stage-7 has a backup stage: Stage-2
+ Stage-5 depends on stages: Stage-7
+ Stage-3 depends on stages: Stage-2, Stage-5
+ Stage-2
+ Stage-4 is a root stage
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-6
+ Conditional Operator
+
+ Stage: Stage-7
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+
+ Stage: Stage-5
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+103231310608
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-9 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-10, Stage-11, Stage-12, Stage-2
+ Stage-10 has a backup stage: Stage-2
+ Stage-6 depends on stages: Stage-10
+ Stage-3 depends on stages: Stage-2, Stage-6, Stage-7, Stage-8
+ Stage-11 has a backup stage: Stage-2
+ Stage-7 depends on stages: Stage-11
+ Stage-12 has a backup stage: Stage-2
+ Stage-8 depends on stages: Stage-12
+ Stage-2
+ Stage-4 is a root stage
+ Stage-5 is a root stage
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-9
+ Conditional Operator
+
+ Stage: Stage-10
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME1
+ Fetch Operator
+ limit: -1
+ $INTNAME2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME1
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ $INTNAME2
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+
+ Stage: Stage-6
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-11
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME
+ Fetch Operator
+ limit: -1
+ $INTNAME2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ $INTNAME2
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+
+ Stage: Stage-7
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-12
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME
+ Fetch Operator
+ limit: -1
+ $INTNAME1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ $INTNAME1
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+
+ Stage: Stage-8
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-5
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+348019368476
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-8 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-9, Stage-10, Stage-2
+ Stage-9 has a backup stage: Stage-2
+ Stage-6 depends on stages: Stage-9
+ Stage-3 depends on stages: Stage-2, Stage-6, Stage-7
+ Stage-10 has a backup stage: Stage-2
+ Stage-7 depends on stages: Stage-10
+ Stage-2
+ Stage-4 is a root stage
+ Stage-5 is a root stage
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-8
+ Conditional Operator
+
+ Stage: Stage-9
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME1
+ Fetch Operator
+ limit: -1
+ $INTNAME2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME1
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ $INTNAME2
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+
+ Stage: Stage-6
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Left Outer Join0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-10
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME
+ Fetch Operator
+ limit: -1
+ $INTNAME2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ $INTNAME2
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+
+ Stage: Stage-7
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Left Outer Join0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Left Outer Join0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Statistics: Num rows: 1100 Data size: 193600 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-5
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+348019368476
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-8, Stage-2
+ Stage-8 has a backup stage: Stage-2
+ Stage-6 depends on stages: Stage-8
+ Stage-3 depends on stages: Stage-2, Stage-6
+ Stage-2
+ Stage-4 is a root stage
+ Stage-5 is a root stage
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME1
+ Fetch Operator
+ limit: -1
+ $INTNAME2
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME1
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ $INTNAME2
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+
+ Stage: Stage-6
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ Left Outer Join0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ Left Outer Join0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Statistics: Num rows: 1100 Data size: 193600 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-5
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+348019368476
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-8, Stage-2
+ Stage-8 has a backup stage: Stage-2
+ Stage-6 depends on stages: Stage-8
+ Stage-3 depends on stages: Stage-2, Stage-6
+ Stage-2
+ Stage-4 is a root stage
+ Stage-5 is a root stage
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME
+ Fetch Operator
+ limit: -1
+ $INTNAME1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ $INTNAME1
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+
+ Stage: Stage-6
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ Right Outer Join0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ Right Outer Join0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Statistics: Num rows: 1100 Data size: 193600 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-5
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+348019368476
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-7 depends on stages: Stage-1, Stage-4, Stage-5 , consists of Stage-8, Stage-2
+ Stage-8 has a backup stage: Stage-2
+ Stage-6 depends on stages: Stage-8
+ Stage-3 depends on stages: Stage-2, Stage-6
+ Stage-2
+ Stage-4 is a root stage
+ Stage-5 is a root stage
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-7
+ Conditional Operator
+
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $INTNAME
+ Fetch Operator
+ limit: -1
+ $INTNAME1
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $INTNAME
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ $INTNAME1
+ TableScan
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+
+ Stage: Stage-6
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ Right Outer Join0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+ Local Work:
+ Map Reduce Local Work
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ TableScan
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Operator Tree:
+ Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ Right Outer Join0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Statistics: Num rows: 1100 Data size: 193600 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-4
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-5
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ table:
+ input format: org.apache.hadoop.mapred.SequenceFileInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveSequenceFileOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+348019368476
[35/50] [abbrv] hive git commit: Updating RELEASE_NOTES after 1.2.0
release
Posted by sp...@apache.org.
Updating RELEASE_NOTES after 1.2.0 release
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/1b91dba0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/1b91dba0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/1b91dba0
Branch: refs/heads/parquet
Commit: 1b91dba0d13319c0da4eb3cffc85bf1bf69a1976
Parents: f4923ce
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Mon May 18 12:37:01 2015 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Mon May 18 12:37:01 2015 -0700
----------------------------------------------------------------------
RELEASE_NOTES.txt | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/1b91dba0/RELEASE_NOTES.txt
----------------------------------------------------------------------
diff --git a/RELEASE_NOTES.txt b/RELEASE_NOTES.txt
index 28272a9..7889919 100644
--- a/RELEASE_NOTES.txt
+++ b/RELEASE_NOTES.txt
@@ -336,6 +336,7 @@ Release Notes - Hive - Version 1.2.0
* [HIVE-10542] - Full outer joins in tez produce incorrect results in certain cases
* [HIVE-10543] - improve error message in MetaStoreAuthzAPIAuthorizerEmbedOnly
* [HIVE-10544] - Beeline/Hive JDBC Driver fails in HTTP mode on Windows with java.lang.NoSuchFieldError: INSTANCE
+ * [HIVE-10548] - Remove dependency to s3 repository in root pom
* [HIVE-10564] - webhcat should use webhcat-site.xml properties for controller job submission
* [HIVE-10572] - Improve Hive service test to check empty string
* [HIVE-10576] - add jar command does not work with Windows OS
@@ -345,10 +346,12 @@ Release Notes - Hive - Version 1.2.0
* [HIVE-10603] - increase default permgen space for HS2 on windows
* [HIVE-10604] - update webhcat-default.xml with 1.2 version numbers
* [HIVE-10607] - Combination of ReducesinkDedup + TopN optimization yields incorrect result if there are multiple GBY in reducer
+ * [HIVE-10609] - Vectorization : Q64 fails with ClassCastException
* [HIVE-10610] - hive command fails to get hadoop version
* [HIVE-10612] - HIVE-10578 broke TestSQLStdHiveAccessControllerHS2 tests
* [HIVE-10614] - schemaTool upgrade from 0.14.0 to 1.3.0 causes failure
* [HIVE-10638] - HIVE-9736 introduces issues with Hadoop23Shims.checkFileAccess
+ * [HIVE-10715] - RAT failures - many files do not have ASF licenses
** Improvement
@@ -444,6 +447,7 @@ Release Notes - Hive - Version 1.2.0
* [HIVE-10019] - Configure jenkins precommit jobs to run HMS upgrade tests
* [HIVE-10222] - Upgrade Calcite dependency to newest version
* [HIVE-10268] - Merge cbo branch into trunk
+ * [HIVE-10676] - Update Hive's README to mention spark, and to remove jdk1.6
** Test
[09/50] [abbrv] hive git commit: HIVE-10619 Fix ConcurrentHashMap.get
in MetadataListStructObjectInspector.getInstance (52) (Alexander Pivovarov,
reviewed by Szehon Ho)
Posted by sp...@apache.org.
HIVE-10619 Fix ConcurrentHashMap.get in MetadataListStructObjectInspector.getInstance (52) (Alexander Pivovarov, reviewed by Szehon Ho)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/03eca770
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/03eca770
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/03eca770
Branch: refs/heads/parquet
Commit: 03eca7708ef2b1d9bdf3a4265639dd3d5541ca74
Parents: b09fcfb
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Tue May 5 20:57:42 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Wed May 13 13:13:33 2015 -0700
----------------------------------------------------------------------
.../serde2/objectinspector/MetadataListStructObjectInspector.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/03eca770/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java
index e68325f..5a2beb9 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/MetadataListStructObjectInspector.java
@@ -49,7 +49,7 @@ public class MetadataListStructObjectInspector extends
List<String> columnNames) {
ArrayList<List<String>> key = new ArrayList<List<String>>(1);
key.add(columnNames);
- MetadataListStructObjectInspector result = cached.get(columnNames);
+ MetadataListStructObjectInspector result = cached.get(key);
if (result == null) {
result = new MetadataListStructObjectInspector(columnNames);
MetadataListStructObjectInspector prev = cached.putIfAbsent(key, result);
[31/50] [abbrv] hive git commit: HIVE-10190: CBO: AST mode checks for
TABLESAMPLE with AST.toString().contains("TOK_TABLESPLITSAMPLE")
Posted by sp...@apache.org.
HIVE-10190: CBO: AST mode checks for TABLESAMPLE with AST.toString().contains("TOK_TABLESPLITSAMPLE")
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/037559fd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/037559fd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/037559fd
Branch: refs/heads/parquet
Commit: 037559fd74c1a2d3ab6eee094ebce7934f3b9f39
Parents: b28d394
Author: Sergio Pena <se...@cloudera.com>
Authored: Mon May 18 10:06:38 2015 -0500
Committer: Sergio Pena <se...@cloudera.com>
Committed: Mon May 18 10:06:38 2015 -0500
----------------------------------------------------------------------
.../ql/optimizer/calcite/HiveCalciteUtil.java | 14 +++----
.../apache/hadoop/hive/ql/parse/ParseUtils.java | 42 ++++++++++++++++++--
2 files changed, 45 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/037559fd/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
index 372c93d..f4e7c45 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveCalciteUtil.java
@@ -58,6 +58,8 @@ import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ExprNodeConverter;
import org.apache.hadoop.hive.ql.parse.ASTNode;
+import org.apache.hadoop.hive.ql.parse.HiveParser;
+import org.apache.hadoop.hive.ql.parse.ParseUtils;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import com.google.common.base.Function;
@@ -95,15 +97,11 @@ public class HiveCalciteUtil {
}
public static boolean validateASTForUnsupportedTokens(ASTNode ast) {
- String astTree = ast.toStringTree();
- // if any of following tokens are present in AST, bail out
- String[] tokens = { "TOK_CHARSETLITERAL", "TOK_TABLESPLITSAMPLE" };
- for (String token : tokens) {
- if (astTree.contains(token)) {
- return false;
- }
+ if (ParseUtils.containsTokenOfType(ast, HiveParser.TOK_CHARSETLITERAL, HiveParser.TOK_TABLESPLITSAMPLE)) {
+ return false;
+ } else {
+ return true;
}
- return true;
}
public static List<RexNode> getProjsFromBelowAsInputRef(final RelNode rel) {
http://git-wip-us.apache.org/repos/asf/hive/blob/037559fd/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
index 373429c..18f0180 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ParseUtils.java
@@ -18,15 +18,15 @@
package org.apache.hadoop.hive.ql.parse;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
+import java.util.*;
import org.apache.hadoop.hive.common.JavaUtils;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.ql.ErrorMsg;
+import org.apache.hadoop.hive.ql.exec.PTFUtils;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.lib.Node;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.serde2.typeinfo.CharTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.DecimalTypeInfo;
@@ -229,5 +229,41 @@ public final class ParseUtils {
return className;
}
+ public static boolean containsTokenOfType(ASTNode root, Integer ... tokens) {
+ final Set<Integer> tokensToMatch = new HashSet<Integer>();
+ for (Integer tokenTypeToMatch : tokens) {
+ tokensToMatch.add(tokenTypeToMatch);
+ }
+
+ return ParseUtils.containsTokenOfType(root, new PTFUtils.Predicate<ASTNode>() {
+ @Override
+ public boolean apply(ASTNode node) {
+ return tokensToMatch.contains(node.getType());
+ }
+ });
+ }
+
+ public static boolean containsTokenOfType(ASTNode root, PTFUtils.Predicate<ASTNode> predicate) {
+ Queue<ASTNode> queue = new ArrayDeque<ASTNode>();
+
+ // BFS
+ queue.add(root);
+ while (!queue.isEmpty()) {
+ ASTNode current = queue.remove();
+ // If the predicate matches, then return true.
+ // Otherwise visit the next set of nodes that haven't been seen.
+ if (predicate.apply(current)) {
+ return true;
+ } else {
+ // Guard because ASTNode.getChildren.iterator returns null if no children available (bug).
+ if (current.getChildCount() > 0) {
+ for (Node child : current.getChildren()) {
+ queue.add((ASTNode) child);
+ }
+ }
+ }
+ }
+ return false;
+ }
}
[44/50] [abbrv] hive git commit: HIVE-10569 : Hive CLI gets stuck
when hive.exec.parallel=true;
and some exception happens during SessionState.start (Rohit Agarwal via
Ashutosh Chauhan)
Posted by sp...@apache.org.
HIVE-10569 : Hive CLI gets stuck when hive.exec.parallel=true; and some exception happens during SessionState.start (Rohit Agarwal via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b70a3352
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b70a3352
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b70a3352
Branch: refs/heads/parquet
Commit: b70a3352822b661bcd7c944105767c95c034f568
Parents: 4a2a2bc
Author: Rohit Agarwal <mi...@gmail.com>
Authored: Thu Apr 30 18:44:00 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Tue May 19 21:45:09 2015 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b70a3352/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
index 94ce0d0..ad0abde 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/TaskRunner.java
@@ -75,6 +75,7 @@ public class TaskRunner extends Thread {
runSequential();
} finally {
runner = null;
+ result.setRunning(false);
}
}
[45/50] [abbrv] hive git commit: HIVE-10745 : Better null handling by
Vectorizer (Ashutosh Chauhan via Hari Sankar Sivarama Subramaniyan)
Posted by sp...@apache.org.
HIVE-10745 : Better null handling by Vectorizer (Ashutosh Chauhan via Hari Sankar Sivarama Subramaniyan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d1176128
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d1176128
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d1176128
Branch: refs/heads/parquet
Commit: d11761287daeedf4ca8dc8a5977667046722096c
Parents: b70a335
Author: Ashutosh Chauhan <ha...@apache.org>
Authored: Mon May 18 16:52:55 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Tue May 19 22:17:55 2015 -0700
----------------------------------------------------------------------
.../hive/ql/exec/ExprNodeEvaluatorFactory.java | 2 -
.../ql/exec/vector/VectorizationContext.java | 50 ++++++++++++--------
2 files changed, 30 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d1176128/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java
index f08321c..5a532c4 100755
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ExprNodeEvaluatorFactory.java
@@ -27,8 +27,6 @@ import org.apache.hadoop.hive.ql.plan.ExprNodeConstantDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeFieldDesc;
import org.apache.hadoop.hive.ql.plan.ExprNodeGenericFuncDesc;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.hadoop.io.NullWritable;
/**
* ExprNodeEvaluatorFactory.
http://git-wip-us.apache.org/repos/asf/hive/blob/d1176128/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
index 48f34a9..61d2972 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizationContext.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.hive.ql.exec.vector;
import java.lang.reflect.Constructor;
-import java.math.BigDecimal;
import java.sql.Date;
import java.sql.Timestamp;
import java.util.ArrayList;
@@ -37,7 +36,6 @@ import java.util.regex.Pattern;
import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.serde2.io.HiveDecimalWritable;
import org.apache.hadoop.hive.common.type.HiveChar;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.HiveIntervalDayTime;
@@ -129,8 +127,8 @@ public class VectorizationContext {
private static final Log LOG = LogFactory.getLog(
VectorizationContext.class.getName());
- private String contextName;
- private int level;
+ private final String contextName;
+ private final int level;
VectorExpressionDescriptor vMap;
@@ -359,7 +357,7 @@ public class VectorizationContext {
}
public int[] currentScratchColumns() {
- TreeSet<Integer> treeSet = new TreeSet();
+ TreeSet<Integer> treeSet = new TreeSet<Integer>();
for (Integer col : usedOutputColumns) {
treeSet.add(initialOutputCol + col);
}
@@ -440,8 +438,6 @@ public class VectorizationContext {
ve = getGenericUdfVectorExpression(expr.getGenericUDF(),
childExpressions, mode, exprDesc.getTypeInfo());
}
- } else if (exprDesc instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc)exprDesc).getValue()) {
- ve = getConstantVectorExpression(null, exprDesc.getTypeInfo(), mode);
} else if (exprDesc instanceof ExprNodeConstantDesc) {
ve = getConstantVectorExpression(((ExprNodeConstantDesc) exprDesc).getValue(), exprDesc.getTypeInfo(),
mode);
@@ -691,6 +687,10 @@ public class VectorizationContext {
case DECIMAL:
genericUdf = new GenericUDFToDecimal();
break;
+ case VOID:
+ case UNKNOWN:
+ // fall-through to throw exception, its not expected for execution to reach here.
+ break;
}
if (genericUdf == null) {
if (udfClass == null) {
@@ -1345,8 +1345,11 @@ public class VectorizationContext {
}
private HiveDecimal castConstantToDecimal(Object scalar, TypeInfo type) throws HiveException {
+
+ if (null == scalar) {
+ return null;
+ }
PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
- int scale = HiveDecimalUtils.getScaleForType(ptinfo);
String typename = type.getTypeName();
HiveDecimal rawDecimal;
switch (ptinfo.getPrimitiveCategory()) {
@@ -1384,6 +1387,9 @@ public class VectorizationContext {
}
private String castConstantToString(Object scalar, TypeInfo type) throws HiveException {
+ if (null == scalar) {
+ return null;
+ }
PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
String typename = type.getTypeName();
switch (ptinfo.getPrimitiveCategory()) {
@@ -1403,6 +1409,9 @@ public class VectorizationContext {
}
private Double castConstantToDouble(Object scalar, TypeInfo type) throws HiveException {
+ if (null == scalar) {
+ return null;
+ }
PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
String typename = type.getTypeName();
switch (ptinfo.getPrimitiveCategory()) {
@@ -1422,6 +1431,9 @@ public class VectorizationContext {
}
private Long castConstantToLong(Object scalar, TypeInfo type) throws HiveException {
+ if (null == scalar) {
+ return null;
+ }
PrimitiveTypeInfo ptinfo = (PrimitiveTypeInfo) type;
String typename = type.getTypeName();
switch (ptinfo.getPrimitiveCategory()) {
@@ -1449,8 +1461,6 @@ public class VectorizationContext {
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
HiveDecimal decimalValue = castConstantToDecimal(constantValue, child.getTypeInfo());
return getConstantVectorExpression(decimalValue, returnType, Mode.PROJECTION);
- } else if (child instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc)child).getValue()) {
- return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
}
if (isIntFamily(inputType)) {
return createVectorExpression(CastLongToDecimal.class, childExpr, Mode.PROJECTION, returnType);
@@ -1476,8 +1486,6 @@ public class VectorizationContext {
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
String strValue = castConstantToString(constantValue, child.getTypeInfo());
return getConstantVectorExpression(strValue, returnType, Mode.PROJECTION);
- } else if (child instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc)child).getValue()) {
- return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
}
if (inputType.equals("boolean")) {
// Boolean must come before the integer family. It's a special case.
@@ -1563,8 +1571,6 @@ public class VectorizationContext {
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
Double doubleValue = castConstantToDouble(constantValue, child.getTypeInfo());
return getConstantVectorExpression(doubleValue, returnType, Mode.PROJECTION);
- } else if (child instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc)child).getValue()) {
- return getConstantVectorExpression(null, returnType, Mode.PROJECTION);
}
if (isIntFamily(inputType)) {
return createVectorExpression(CastLongToDouble.class, childExpr, Mode.PROJECTION, returnType);
@@ -1586,11 +1592,12 @@ public class VectorizationContext {
ExprNodeDesc child = childExpr.get(0);
String inputType = childExpr.get(0).getTypeString();
if (child instanceof ExprNodeConstantDesc) {
+ if (null == ((ExprNodeConstantDesc)child).getValue()) {
+ return getConstantVectorExpression(null, TypeInfoFactory.booleanTypeInfo, Mode.PROJECTION);
+ }
// Don't do constant folding here. Wait until the optimizer is changed to do it.
// Family of related JIRAs: HIVE-7421, HIVE-7422, and HIVE-7424.
return null;
- } else if (child instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc)child).getValue()) {
- return getConstantVectorExpression(null, TypeInfoFactory.booleanTypeInfo, Mode.PROJECTION);
}
// Long and double are handled using descriptors, string needs to be specially handled.
if (isStringFamily(inputType)) {
@@ -1619,8 +1626,6 @@ public class VectorizationContext {
Object constantValue = ((ExprNodeConstantDesc) child).getValue();
Long longValue = castConstantToLong(constantValue, child.getTypeInfo());
return getConstantVectorExpression(longValue, TypeInfoFactory.longTypeInfo, Mode.PROJECTION);
- } else if (child instanceof ExprNodeConstantDesc && null == ((ExprNodeConstantDesc)child).getValue()) {
- return getConstantVectorExpression(null, TypeInfoFactory.longTypeInfo, Mode.PROJECTION);
}
// Float family, timestamp are handled via descriptor based lookup, int family needs
// special handling.
@@ -1770,11 +1775,16 @@ public class VectorizationContext {
variableArgPositions.add(i);
argDescs[i].setVariable(getInputColumnIndex(((ExprNodeColumnDesc) child).getColumn()));
} else if (child instanceof ExprNodeConstantDesc) {
-
+ if (((ExprNodeConstantDesc) child).getValue() == null) {
+ // cannot handle constant null at the moment
+ throw new HiveException("Unable to vectorize custom UDF. Custom udf containing "
+ + "constant null argument cannot be currently vectorized.");
+ }
// this is a constant
argDescs[i].setConstant((ExprNodeConstantDesc) child);
} else {
- throw new HiveException("Unable to vectorize Custom UDF");
+ throw new HiveException("Unable to vectorize custom UDF. Encountered unsupported expr desc : "
+ + child);
}
}
[39/50] [abbrv] hive git commit: HIVE-10675 : Provide option to skip
Accumulo related Hive tests in itests directory (Hari Subramaniyan,
reviewed by Sushanth Sowmyan)
Posted by sp...@apache.org.
HIVE-10675 : Provide option to skip Accumulo related Hive tests in itests directory (Hari Subramaniyan, reviewed by Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/46b4cc9c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/46b4cc9c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/46b4cc9c
Branch: refs/heads/parquet
Commit: 46b4cc9c5aee9152d99f3613e4d9d031aec1947c
Parents: 325a592
Author: Hari Subramaniyan <ha...@apache.org>
Authored: Mon May 18 14:48:27 2015 -0700
Committer: Hari Subramaniyan <ha...@apache.org>
Committed: Mon May 18 14:48:27 2015 -0700
----------------------------------------------------------------------
itests/qtest/pom.xml | 58 +++++++++++++++++++++++++++++++----------------
1 file changed, 39 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/46b4cc9c/itests/qtest/pom.xml
----------------------------------------------------------------------
diff --git a/itests/qtest/pom.xml b/itests/qtest/pom.xml
index 90411df..e90f6f4 100644
--- a/itests/qtest/pom.xml
+++ b/itests/qtest/pom.xml
@@ -36,17 +36,15 @@
<run_disabled>false</run_disabled>
<clustermode></clustermode>
<execute.beeline.tests>false</execute.beeline.tests>
+ <!-- Profile activation clause for accumulo-tests will flip skip.accumulo.tests to false
+ as long as -DskipAccumuloTests is not specified -->
+ <skip.accumulo.tests>true</skip.accumulo.tests>
</properties>
<dependencies>
<!-- dependencies are always listed in sorted order by groupId, artifectId -->
<!-- test intra-project -->
<dependency>
- <groupId>org.apache.accumulo</groupId>
- <artifactId>accumulo-minicluster</artifactId>
- <scope>test</scope>
- </dependency>
- <dependency>
<groupId>org.apache.hive</groupId>
<artifactId>hive-ant</artifactId>
<version>${project.version}</version>
@@ -362,6 +360,24 @@
</dependency>
</dependencies>
</profile>
+ <profile>
+ <id>accumulo-tests</id>
+ <activation>
+ <property>
+ <name>!skipAccumuloTests</name>
+ </property>
+ </activation>
+ <properties>
+ <skip.accumulo.tests>false</skip.accumulo.tests>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.accumulo</groupId>
+ <artifactId>accumulo-minicluster</artifactId>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </profile>
</profiles>
<build>
@@ -407,7 +423,6 @@
<mkdir dir="${project.build.directory}/qfile-results/hbase-handler/positive/" />
<mkdir dir="${project.build.directory}/qfile-results/hbase-handler/negative/" />
<mkdir dir="${project.build.directory}/qfile-results/hbase-handler/minimrpositive/" />
- <mkdir dir="${project.build.directory}/qfile-results/accumulo-handler/positive/" />
<mkdir dir="${project.build.directory}/qfile-results/contribpositive"/>
<mkdir dir="${project.build.directory}/qfile-results/contribnegative"/>
@@ -599,19 +614,24 @@
cleanupScript="q_test_cleanup.sql"/>
<!-- Accumulo Positive -->
- <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
- outputDirectory="${project.build.directory}/generated-test-sources/java/org/apache/hadoop/hive/cli/"
- templatePath="${basedir}/${hive.path.to.root}/accumulo-handler/src/test/templates/" template="TestAccumuloCliDriver.vm"
- queryDirectory="${basedir}/${hive.path.to.root}/accumulo-handler/src/test/queries/positive/"
- queryFile="${qfile}"
- runDisabled="${run_disabled}"
- clusterMode="${clustermode}"
- resultsDirectory="${basedir}/${hive.path.to.root}/accumulo-handler/src/test/results/positive/" className="TestAccumuloCliDriver"
- logFile="${project.build.directory}/testaccumuloclidrivergen.log"
- logDirectory="${project.build.directory}/qfile-results/accumulo-handler/positive/"
- initScript="q_test_init.sql"
- cleanupScript="q_test_cleanup.sql"/>
-
+ <if>
+ <equals arg1="${skip.accumulo.tests}" arg2="false" />
+ <then>
+ <mkdir dir="${project.build.directory}/qfile-results/accumulo-handler/positive/" />
+ <qtestgen hiveRootDirectory="${basedir}/${hive.path.to.root}/"
+ outputDirectory="${project.build.directory}/generated-test-sources/java/org/apache/hadoop/hive/cli/"
+ templatePath="${basedir}/${hive.path.to.root}/accumulo-handler/src/test/templates/" template="TestAccumuloCliDriver.vm"
+ queryDirectory="${basedir}/${hive.path.to.root}/accumulo-handler/src/test/queries/positive/"
+ queryFile="${qfile}"
+ runDisabled="${run_disabled}"
+ clusterMode="${clustermode}"
+ resultsDirectory="${basedir}/${hive.path.to.root}/accumulo-handler/src/test/results/positive/" className="TestAccumuloCliDriver"
+ logFile="${project.build.directory}/testaccumuloclidrivergen.log"
+ logDirectory="${project.build.directory}/qfile-results/accumulo-handler/positive/"
+ initScript="q_test_init.sql"
+ cleanupScript="q_test_cleanup.sql"/>
+ </then>
+ </if>
<!-- Beeline -->
<if>
[23/50] [abbrv] hive git commit: HIVE-10650 : Improve sum() function
over windowing to support additional range formats (Aihua Xu via Ashutosh
Chauhan)
Posted by sp...@apache.org.
HIVE-10650 : Improve sum() function over windowing to support additional range formats (Aihua Xu via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ad62e2ed
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ad62e2ed
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ad62e2ed
Branch: refs/heads/parquet
Commit: ad62e2ede556558eb5eb0158c5f2fe069234a821
Parents: 6a29939
Author: Aihua Xu <ai...@gmail.com>
Authored: Tue May 12 14:45:00 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Thu May 14 19:31:14 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/PTFTranslator.java | 9 +-
.../hadoop/hive/ql/parse/WindowingSpec.java | 2 +-
.../hadoop/hive/ql/plan/ptf/BoundaryDef.java | 37 +-
.../hadoop/hive/ql/plan/ptf/CurrentRowDef.java | 12 +-
.../hive/ql/plan/ptf/RangeBoundaryDef.java | 38 +-
.../hive/ql/plan/ptf/ValueBoundaryDef.java | 50 +-
.../hadoop/hive/ql/plan/ptf/WindowFrameDef.java | 13 +-
.../generic/GenericUDAFStreamingEvaluator.java | 40 +-
.../hadoop/hive/ql/udaf/TestStreamingSum.java | 10 +-
.../clientpositive/windowing_windowspec.q | 2 -
.../clientpositive/windowing_windowspec2.q | 24 +
.../clientpositive/windowing_windowspec.q.out | 108 -----
.../clientpositive/windowing_windowspec2.q.out | 478 +++++++++++++++++++
13 files changed, 661 insertions(+), 162 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ad62e2ed/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
index d7f1c7f..c1b8a1d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
@@ -545,9 +545,7 @@ public class PTFTranslator {
throws SemanticException {
if (bndSpec instanceof ValueBoundarySpec) {
ValueBoundarySpec vBndSpec = (ValueBoundarySpec) bndSpec;
- ValueBoundaryDef vbDef = new ValueBoundaryDef();
- vbDef.setAmt(vBndSpec.getAmt());
- vbDef.setDirection(vBndSpec.getDirection());
+ ValueBoundaryDef vbDef = new ValueBoundaryDef(vBndSpec.getDirection(), vBndSpec.getAmt());
PTFTranslator.validateNoLeadLagInValueBoundarySpec(vBndSpec.getExpression());
PTFExpressionDef exprDef = null;
try {
@@ -561,10 +559,7 @@ public class PTFTranslator {
}
else if (bndSpec instanceof RangeBoundarySpec) {
RangeBoundarySpec rBndSpec = (RangeBoundarySpec) bndSpec;
- RangeBoundaryDef rbDef = new RangeBoundaryDef();
- rbDef.setAmt(rBndSpec.getAmt());
- rbDef.setDirection(rBndSpec.getDirection());
- return rbDef;
+ return new RangeBoundaryDef(rBndSpec.getDirection(), rBndSpec.getAmt());
} else if (bndSpec instanceof CurrentRowSpec) {
CurrentRowDef cbDef = new CurrentRowDef();
return cbDef;
http://git-wip-us.apache.org/repos/asf/hive/blob/ad62e2ed/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
index 6dfa214..50f86df 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/WindowingSpec.java
@@ -274,7 +274,7 @@ public class WindowingSpec {
}
if ( end.getDirection() == Direction.PRECEDING &&
- start.getAmt() == BoundarySpec.UNBOUNDED_AMOUNT ) {
+ end.getAmt() == BoundarySpec.UNBOUNDED_AMOUNT ) {
throw new SemanticException("End of a WindowFrame cannot be UNBOUNDED PRECEDING");
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ad62e2ed/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java
index eeb094c..c6518dc 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/BoundaryDef.java
@@ -28,11 +28,42 @@ public abstract class BoundaryDef {
return direction;
}
- public void setDirection(Direction direction) {
- this.direction = direction;
+ /**
+ * Returns if the bound is PRECEDING.
+ * @return if the bound is PRECEDING
+ */
+ public boolean isPreceding() {
+ return false;
}
+ /**
+ * Returns if the bound is FOLLOWING.
+ * @return if the bound is FOLLOWING
+ */
+ public boolean isFollowing() {
+ return false;
+ }
+
+ /**
+ * Returns if the bound is CURRENT ROW.
+ * @return if the bound is CURRENT ROW
+ */
+ public boolean isCurrentRow() {
+ return false;
+ }
+
+ /**
+ * Returns offset from XX PRECEDING/FOLLOWING.
+ *
+ * @return offset from XX PRECEDING/FOLLOWING
+ */
public abstract int getAmt();
+ /**
+ * Returns signed offset from XX PRECEDING/FOLLOWING. Nagative for preceding.
+ *
+ * @return signed offset from XX PRECEDING/FOLLOWING
+ */
+ public abstract int getRelativeOffset();
public boolean isUnbounded() {
return this.getAmt() == BoundarySpec.UNBOUNDED_AMOUNT;
@@ -43,4 +74,4 @@ public abstract class BoundaryDef {
return direction == null ? "" :
direction + "(" + (getAmt() == Integer.MAX_VALUE ? "MAX" : getAmt()) + ")";
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ad62e2ed/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/CurrentRowDef.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/CurrentRowDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/CurrentRowDef.java
index 768fae0..f5e2848 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/CurrentRowDef.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/CurrentRowDef.java
@@ -35,4 +35,14 @@ public class CurrentRowDef extends BoundaryDef {
public int getAmt() {
return 0;
}
-}
\ No newline at end of file
+
+ @Override
+ public boolean isCurrentRow() {
+ return true;
+ }
+
+ @Override
+ public int getRelativeOffset() {
+ return 0;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ad62e2ed/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/RangeBoundaryDef.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/RangeBoundaryDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/RangeBoundaryDef.java
index ce57625..f93399d 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/RangeBoundaryDef.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/RangeBoundaryDef.java
@@ -18,9 +18,28 @@
package org.apache.hadoop.hive.ql.plan.ptf;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction;
public class RangeBoundaryDef extends BoundaryDef {
private int amt;
+ private final int relativeOffset;
+
+ public RangeBoundaryDef(Direction direction, int amt) {
+ this.direction = direction;
+ this.amt = amt;
+
+ // Calculate relative offset
+ switch(this.direction) {
+ case PRECEDING:
+ relativeOffset = -amt;
+ break;
+ case FOLLOWING:
+ relativeOffset = amt;
+ break;
+ default:
+ relativeOffset = 0;
+ }
+ }
public int compareTo(BoundaryDef other) {
int c = getDirection().compareTo(other.getDirection());
@@ -28,7 +47,7 @@ public class RangeBoundaryDef extends BoundaryDef {
return c;
}
RangeBoundaryDef rb = (RangeBoundaryDef) other;
- return getAmt() - rb.getAmt();
+ return this.direction == Direction.PRECEDING ? rb.amt - this.amt : this.amt - rb.amt;
}
@Override
@@ -36,7 +55,18 @@ public class RangeBoundaryDef extends BoundaryDef {
return amt;
}
- public void setAmt(int amt) {
- this.amt = amt;
+ @Override
+ public boolean isPreceding() {
+ return this.direction == Direction.PRECEDING;
+ }
+
+ @Override
+ public boolean isFollowing() {
+ return this.direction == Direction.FOLLOWING;
+ }
+
+ @Override
+ public int getRelativeOffset() {
+ return relativeOffset;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ad62e2ed/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ValueBoundaryDef.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ValueBoundaryDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ValueBoundaryDef.java
index e89578c..3725ac8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ValueBoundaryDef.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/ValueBoundaryDef.java
@@ -18,13 +18,30 @@
package org.apache.hadoop.hive.ql.plan.ptf;
-import org.apache.hadoop.hive.ql.exec.ExprNodeEvaluator;
-import org.apache.hadoop.hive.ql.plan.ExprNodeDesc;
+import org.apache.hadoop.hive.ql.parse.WindowingSpec.Direction;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
public class ValueBoundaryDef extends BoundaryDef {
private PTFExpressionDef expressionDef;
- private int amt;
+ private final int amt;
+ private final int relativeOffset;
+
+ public ValueBoundaryDef(Direction direction, int amt) {
+ this.direction = direction;
+ this.amt = amt;
+
+ // Calculate relative offset
+ switch(this.direction) {
+ case PRECEDING:
+ relativeOffset = -amt;
+ break;
+ case FOLLOWING:
+ relativeOffset = amt;
+ break;
+ default:
+ relativeOffset = 0;
+ }
+ }
public int compareTo(BoundaryDef other) {
int c = getDirection().compareTo(other.getDirection());
@@ -32,7 +49,7 @@ public class ValueBoundaryDef extends BoundaryDef {
return c;
}
ValueBoundaryDef vb = (ValueBoundaryDef) other;
- return getAmt() - vb.getAmt();
+ return this.direction == Direction.PRECEDING ? vb.amt - this.amt : this.amt - vb.amt;
}
public PTFExpressionDef getExpressionDef() {
@@ -43,14 +60,6 @@ public class ValueBoundaryDef extends BoundaryDef {
this.expressionDef = expressionDef;
}
- public ExprNodeDesc getExprNode() {
- return expressionDef == null ? null : expressionDef.getExprNode();
- }
-
- public ExprNodeEvaluator getExprEvaluator() {
- return expressionDef == null ? null : expressionDef.getExprEvaluator();
- }
-
public ObjectInspector getOI() {
return expressionDef == null ? null : expressionDef.getOI();
}
@@ -60,7 +69,18 @@ public class ValueBoundaryDef extends BoundaryDef {
return amt;
}
- public void setAmt(int amt) {
- this.amt = amt;
+ @Override
+ public int getRelativeOffset() {
+ return relativeOffset;
+ }
+
+ @Override
+ public boolean isPreceding() {
+ return this.direction == Direction.PRECEDING;
+ }
+
+ @Override
+ public boolean isFollowing() {
+ return this.direction == Direction.FOLLOWING;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ad62e2ed/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java
index d153b08..5a85f69 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/ptf/WindowFrameDef.java
@@ -22,11 +22,20 @@ package org.apache.hadoop.hive.ql.plan.ptf;
public class WindowFrameDef {
private BoundaryDef start;
private BoundaryDef end;
+ private final int windowSize;
public WindowFrameDef(BoundaryDef start, BoundaryDef end) {
this.start = start;
this.end = end;
+
+ // Calculate window size
+ if (start.getDirection() == end.getDirection()) {
+ windowSize = Math.abs(end.getAmt() - start.getAmt()) + 1;
+ } else {
+ windowSize = end.getAmt() + start.getAmt() + 1;
+ }
}
+
public BoundaryDef getStart() {
return start;
}
@@ -44,11 +53,11 @@ public class WindowFrameDef {
}
public int getWindowSize() {
- return end.getAmt() + start.getAmt() + 1;
+ return windowSize;
}
@Override
public String toString() {
return start + "~" + end;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/ad62e2ed/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStreamingEvaluator.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStreamingEvaluator.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStreamingEvaluator.java
index 578c356..f27d066 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStreamingEvaluator.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDAFStreamingEvaluator.java
@@ -65,7 +65,7 @@ public abstract class GenericUDAFStreamingEvaluator<T1> extends
class StreamingState extends AbstractAggregationBuffer {
final AggregationBuffer wrappedBuf;
- final List<T1> results;
+ final List<T1> results; // Hold the aggregation results for each row in the partition
int numRows; // Number of rows processed in the partition.
StreamingState(AggregationBuffer buf) {
@@ -127,7 +127,7 @@ public abstract class GenericUDAFStreamingEvaluator<T1> extends
class SumAvgStreamingState extends StreamingState {
- final List<T2> intermediateVals;
+ final List<T2> intermediateVals; // Keep track of S[0..x]
SumAvgStreamingState(AggregationBuffer buf) {
super(buf);
@@ -163,20 +163,23 @@ public abstract class GenericUDAFStreamingEvaluator<T1> extends
}
/**
- * After the number of rows processed is more than the size of FOLLOWING window,
- * we can generate a PTF result for a previous row when a new row gets processed.
+ * For the cases "X preceding and Y preceding" or the number of processed rows
+ * is more than the size of FOLLOWING window, we are able to generate a PTF result
+ * for a previous row.
* @return
*/
public boolean hasResultReady() {
- return this.numRows >= wFrameDef.getEnd().getAmt();
+ return this.numRows >= wFrameDef.getEnd().getRelativeOffset();
}
/**
- * Retrieve the next stored intermediate result to generate the result for next available row
+ * Retrieve the next stored intermediate result, i.e.,
+ * Get S[x-1] in the computation of S[x..y] = S[y] - S[x-1].
*/
public T2 retrieveNextIntermediateValue() {
if (!wFrameDef.getStart().isUnbounded()
- && (this.numRows - wFrameDef.getEnd().getAmt()) >= (wFrameDef.getStart().getAmt() + 1)) {
+ && !this.intermediateVals.isEmpty()
+ && this.numRows >= wFrameDef.getWindowSize()) {
return this.intermediateVals.remove(0);
}
@@ -196,11 +199,20 @@ public abstract class GenericUDAFStreamingEvaluator<T1> extends
SumAvgStreamingState ss = (SumAvgStreamingState) agg;
wrappedEval.iterate(ss.wrappedBuf, parameters);
- // Generate the result for a previous row, of whose window all the rows have been processed.
+
+ // We need to insert 'null' before processing first row for the case: X preceding and y preceding
+ if (ss.numRows == 0) {
+ for (int i = wFrameDef.getEnd().getRelativeOffset(); i < 0; i++) {
+ ss.results.add(null);
+ }
+ }
+
+ // Generate the result for the windowing ending at the current row
if (ss.hasResultReady()) {
ss.results.add(getNextResult(ss));
}
- if (!wFrameDef.isStartUnbounded()) {
+ if (!wFrameDef.isStartUnbounded()
+ && ss.numRows + 1 >= wFrameDef.getStart().getRelativeOffset()) {
ss.intermediateVals.add(getCurrentIntermediateResult(ss));
}
@@ -212,11 +224,17 @@ public abstract class GenericUDAFStreamingEvaluator<T1> extends
SumAvgStreamingState ss = (SumAvgStreamingState) agg;
Object o = wrappedEval.terminate(ss.wrappedBuf);
- // After all the rows are processed, continue to generate results for the rows that results haven't generate
- for (int i = 0; i < wFrameDef.getEnd().getAmt(); i++) {
+ // After all the rows are processed, continue to generate results for the rows that results haven't generated.
+ // For the case: X following and Y following, process first Y-X results and then insert X nulls.
+ // For the case X preceding and Y following, process Y results.
+ for (int i = Math.max(0, wFrameDef.getStart().getRelativeOffset()); i < wFrameDef.getEnd().getRelativeOffset(); i++) {
ss.results.add(getNextResult(ss));
ss.numRows++;
}
+ for (int i = 0; i < wFrameDef.getStart().getRelativeOffset(); i++) {
+ ss.results.add(null);
+ ss.numRows++;
+ }
return o;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ad62e2ed/ql/src/test/org/apache/hadoop/hive/ql/udaf/TestStreamingSum.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udaf/TestStreamingSum.java b/ql/src/test/org/apache/hadoop/hive/ql/udaf/TestStreamingSum.java
index 88cafc0..c26de3f 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/udaf/TestStreamingSum.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udaf/TestStreamingSum.java
@@ -54,19 +54,13 @@ public class TestStreamingSum {
if (p == 0) {
start = new CurrentRowDef();
} else {
- RangeBoundaryDef startR = new RangeBoundaryDef();
- startR.setDirection(Direction.PRECEDING);
- startR.setAmt(p);
- start = startR;
+ start = new RangeBoundaryDef(Direction.PRECEDING, p);
}
if (f == 0) {
end = new CurrentRowDef();
} else {
- RangeBoundaryDef endR = new RangeBoundaryDef();
- endR.setDirection(Direction.FOLLOWING);
- endR.setAmt(f);
- end = endR;
+ end = new RangeBoundaryDef(Direction.FOLLOWING, f);
}
return new WindowFrameDef(start, end);
http://git-wip-us.apache.org/repos/asf/hive/blob/ad62e2ed/ql/src/test/queries/clientpositive/windowing_windowspec.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/windowing_windowspec.q b/ql/src/test/queries/clientpositive/windowing_windowspec.q
index 202eb74..63f97b7 100644
--- a/ql/src/test/queries/clientpositive/windowing_windowspec.q
+++ b/ql/src/test/queries/clientpositive/windowing_windowspec.q
@@ -31,8 +31,6 @@ select s, sum(i) over(partition by ts order by s) from over10k limit 100;
select f, sum(f) over (partition by ts order by f range between unbounded preceding and current row) from over10k limit 100;
-select f, sum(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100;
-
select s, i, round(avg(d) over (partition by s order by i) / 10.0 , 2) from over10k limit 7;
select s, i, round((avg(d) over w1 + 10.0) - (avg(d) over w1 - 10.0),2) from over10k window w1 as (partition by s order by i) limit 7;
http://git-wip-us.apache.org/repos/asf/hive/blob/ad62e2ed/ql/src/test/queries/clientpositive/windowing_windowspec2.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/windowing_windowspec2.q b/ql/src/test/queries/clientpositive/windowing_windowspec2.q
new file mode 100644
index 0000000..3e8aa93
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/windowing_windowspec2.q
@@ -0,0 +1,24 @@
+drop table over10k;
+
+create table over10k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ bo boolean,
+ s string,
+ ts timestamp,
+ dec decimal,
+ bin binary)
+ row format delimited
+ fields terminated by '|';
+
+load data local inpath '../../data/files/over10k' into table over10k;
+
+select ts, f, sum(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100;
+select ts, f, sum(f) over (partition by ts order by f rows between unbounded preceding and 1 preceding) from over10k limit 100;
+select ts, f, sum(f) over (partition by ts order by f rows between 1 following and 2 following) from over10k limit 100;
+select ts, f, sum(f) over (partition by ts order by f rows between unbounded preceding and 1 following) from over10k limit 100;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/ad62e2ed/ql/src/test/results/clientpositive/windowing_windowspec.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/windowing_windowspec.q.out b/ql/src/test/results/clientpositive/windowing_windowspec.q.out
index 66b0b52..8d78c22 100644
--- a/ql/src/test/results/clientpositive/windowing_windowspec.q.out
+++ b/ql/src/test/results/clientpositive/windowing_windowspec.q.out
@@ -800,114 +800,6 @@ POSTHOOK: Input: default@over10k
71.68 722.6499947607517
79.46 802.1099938452244
80.02 882.1299904882908
-PREHOOK: query: select f, sum(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100
-PREHOOK: type: QUERY
-PREHOOK: Input: default@over10k
-#### A masked pattern was here ####
-POSTHOOK: query: select f, sum(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100
-POSTHOOK: type: QUERY
-POSTHOOK: Input: default@over10k
-#### A masked pattern was here ####
-3.17 14.0600004196167
-10.89 28.600000381469727
-14.54 43.38000011444092
-14.78 58.0600004196167
-17.85 67.78000068664551
-20.61 81.9300012588501
-28.69 96.3700008392334
-29.22 109.69000053405762
-31.17 127.42999839782715
-38.35 137.3499984741211
-38.61 147.60999870300293
-39.48 156.97999954223633
-40.54 160.22999954223633
-41.6 167.70000076293945
-46.08 182.5800018310547
-54.36 198.97999954223633
-56.94 222.3400001525879
-64.96 249.7799949645996
-73.52 273.99999618530273
-78.58 298.4700012207031
-81.41 318.2200012207031
-84.71 332.1300048828125
-87.43 344.9100036621094
-91.36 356.45999908447266
-92.96 366.79000091552734
-95.04 279.36000061035156
-0.83 2.8199999928474426
-1.99 6.550000011920929
-3.73 15.409999668598175
-8.86 25.199999570846558
-10.62 34.52999925613403
-11.32 43.6299991607666
-12.83 49.46999931335449
-14.7 53.80999946594238
-14.96 60.06999969482422
-17.58 66.34000015258789
-19.1 72.65000057220459
-21.01 84.64000129699707
-26.95 94.29000091552734
-27.23 104.26000022888184
-29.07 112.95999908447266
-29.71 117.8499984741211
-31.84 122.55999946594238
-31.94 128.80999946594238
-35.32 136.42000007629395
-37.32 143.07999992370605
-38.5 153.22000122070312
-42.08 162.20000076293945
-44.3 169.54000091552734
-44.66 177.88000106811523
-46.84 184.68999862670898
-48.89 190.02999877929688
-49.64 195.64999771118164
-50.28 200.89999771118164
-52.09 205.2699966430664
-53.26 209.71999740600586
-54.09 215.88999938964844
-56.45 220.55999755859375
-56.76 228.70999908447266
-61.41 236.5
-61.88 243.07999801635742
-63.03 250.87000274658203
-64.55 258.08000564575195
-68.62 272.3300018310547
-76.13 288.3500061035156
-79.05 304.2300033569336
-80.43 317.02000427246094
-81.41 323.74000549316406
-82.85 328.67000579833984
-83.98 332.4500045776367
-84.21 336.59000396728516
-85.55 341.67000579833984
-87.93 346.62000274658203
-88.93 356.6800003051758
-94.27 370.57999420166016
-99.45 282.6499938964844
-0.36 0.8400000035762787
-0.48 1.6300000250339508
-0.79 2.9000000059604645
-1.27 7.020000010728836
-4.48 15.540000021457672
-9.0 38.02000045776367
-23.27 61.87999963760376
-25.13 82.73999977111816
-25.34 99.64999961853027
-25.91 105.38999938964844
-29.01 110.72999954223633
-30.47 123.34000015258789
-37.95 136.72999954223633
-39.3 153.6299991607666
-45.91 175.5999984741211
-52.44 191.74999618530273
-54.1 209.14999771118164
-56.7 222.0099983215332
-58.77 231.6599998474121
-62.09 245.7599983215332
-68.2 260.73999786376953
-71.68 281.4299964904785
-79.46 299.35999298095703
-80.02 312.4499969482422
PREHOOK: query: select s, i, round(avg(d) over (partition by s order by i) / 10.0 , 2) from over10k limit 7
PREHOOK: type: QUERY
PREHOOK: Input: default@over10k
http://git-wip-us.apache.org/repos/asf/hive/blob/ad62e2ed/ql/src/test/results/clientpositive/windowing_windowspec2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/windowing_windowspec2.q.out b/ql/src/test/results/clientpositive/windowing_windowspec2.q.out
new file mode 100644
index 0000000..0879344
--- /dev/null
+++ b/ql/src/test/results/clientpositive/windowing_windowspec2.q.out
@@ -0,0 +1,478 @@
+PREHOOK: query: drop table over10k
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table over10k
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table over10k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ bo boolean,
+ s string,
+ ts timestamp,
+ dec decimal,
+ bin binary)
+ row format delimited
+ fields terminated by '|'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@over10k
+POSTHOOK: query: create table over10k(
+ t tinyint,
+ si smallint,
+ i int,
+ b bigint,
+ f float,
+ d double,
+ bo boolean,
+ s string,
+ ts timestamp,
+ dec decimal,
+ bin binary)
+ row format delimited
+ fields terminated by '|'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@over10k
+PREHOOK: query: load data local inpath '../../data/files/over10k' into table over10k
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@over10k
+POSTHOOK: query: load data local inpath '../../data/files/over10k' into table over10k
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@over10k
+PREHOOK: query: select ts, f, sum(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select ts, f, sum(f) over (partition by ts order by f rows between 2 preceding and 1 preceding) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+2013-03-01 09:11:58.70307 3.17 NULL
+2013-03-01 09:11:58.70307 10.89 3.1700000762939453
+2013-03-01 09:11:58.70307 14.54 14.0600004196167
+2013-03-01 09:11:58.70307 14.78 25.43000030517578
+2013-03-01 09:11:58.70307 17.85 29.31999969482422
+2013-03-01 09:11:58.70307 20.61 32.63000011444092
+2013-03-01 09:11:58.70307 28.69 38.46000099182129
+2013-03-01 09:11:58.70307 29.22 49.30000114440918
+2013-03-01 09:11:58.70307 31.17 57.90999984741211
+2013-03-01 09:11:58.70307 38.35 60.38999938964844
+2013-03-01 09:11:58.70307 38.61 69.51999855041504
+2013-03-01 09:11:58.70307 39.48 76.95999908447266
+2013-03-01 09:11:58.70307 40.54 78.09000015258789
+2013-03-01 09:11:58.70307 41.6 80.02000045776367
+2013-03-01 09:11:58.70307 46.08 82.13999938964844
+2013-03-01 09:11:58.70307 54.36 87.68000030517578
+2013-03-01 09:11:58.70307 56.94 100.44000244140625
+2013-03-01 09:11:58.70307 64.96 111.29999923706055
+2013-03-01 09:11:58.70307 73.52 121.89999771118164
+2013-03-01 09:11:58.70307 78.58 138.47999572753906
+2013-03-01 09:11:58.70307 81.41 152.0999984741211
+2013-03-01 09:11:58.70307 84.71 159.99000549316406
+2013-03-01 09:11:58.70307 87.43 166.12000274658203
+2013-03-01 09:11:58.70307 91.36 172.13999938964844
+2013-03-01 09:11:58.70307 92.96 178.79000091552734
+2013-03-01 09:11:58.70307 95.04 184.31999969482422
+2013-03-01 09:11:58.703071 0.83 NULL
+2013-03-01 09:11:58.703071 1.99 0.8299999833106995
+2013-03-01 09:11:58.703071 3.73 2.8199999928474426
+2013-03-01 09:11:58.703071 8.86 5.7200000286102295
+2013-03-01 09:11:58.703071 10.62 12.589999675750732
+2013-03-01 09:11:58.703071 11.32 19.479999542236328
+2013-03-01 09:11:58.703071 12.83 21.9399995803833
+2013-03-01 09:11:58.703071 14.7 24.149999618530273
+2013-03-01 09:11:58.703071 14.96 27.52999973297119
+2013-03-01 09:11:58.703071 17.58 29.65999984741211
+2013-03-01 09:11:58.703071 19.1 32.53999996185303
+2013-03-01 09:11:58.703071 21.01 36.68000030517578
+2013-03-01 09:11:58.703071 26.95 40.11000061035156
+2013-03-01 09:11:58.703071 27.23 47.96000099182129
+2013-03-01 09:11:58.703071 29.07 54.18000030517578
+2013-03-01 09:11:58.703071 29.71 56.29999923706055
+2013-03-01 09:11:58.703071 31.84 58.779998779296875
+2013-03-01 09:11:58.703071 31.94 61.54999923706055
+2013-03-01 09:11:58.703071 35.32 63.78000068664551
+2013-03-01 09:11:58.703071 37.32 67.26000022888184
+2013-03-01 09:11:58.703071 38.5 72.63999938964844
+2013-03-01 09:11:58.703071 42.08 75.81999969482422
+2013-03-01 09:11:58.703071 44.3 80.58000183105469
+2013-03-01 09:11:58.703071 44.66 86.38000106811523
+2013-03-01 09:11:58.703071 46.84 88.95999908447266
+2013-03-01 09:11:58.703071 48.89 91.5
+2013-03-01 09:11:58.703071 49.64 95.72999954223633
+2013-03-01 09:11:58.703071 50.28 98.52999877929688
+2013-03-01 09:11:58.703071 52.09 99.91999816894531
+2013-03-01 09:11:58.703071 53.26 102.36999893188477
+2013-03-01 09:11:58.703071 54.09 105.3499984741211
+2013-03-01 09:11:58.703071 56.45 107.3499984741211
+2013-03-01 09:11:58.703071 56.76 110.54000091552734
+2013-03-01 09:11:58.703071 61.41 113.20999908447266
+2013-03-01 09:11:58.703071 61.88 118.16999816894531
+2013-03-01 09:11:58.703071 63.03 123.29000091552734
+2013-03-01 09:11:58.703071 64.55 124.90999984741211
+2013-03-01 09:11:58.703071 68.62 127.58000183105469
+2013-03-01 09:11:58.703071 76.13 133.17000579833984
+2013-03-01 09:11:58.703071 79.05 144.75
+2013-03-01 09:11:58.703071 80.43 155.18000030517578
+2013-03-01 09:11:58.703071 81.41 159.4800033569336
+2013-03-01 09:11:58.703071 82.85 161.84000396728516
+2013-03-01 09:11:58.703071 83.98 164.26000213623047
+2013-03-01 09:11:58.703071 84.21 166.8300018310547
+2013-03-01 09:11:58.703071 85.55 168.19000244140625
+2013-03-01 09:11:58.703071 87.93 169.76000213623047
+2013-03-01 09:11:58.703071 88.93 173.4800033569336
+2013-03-01 09:11:58.703071 94.27 176.86000061035156
+2013-03-01 09:11:58.703071 99.45 183.1999969482422
+2013-03-01 09:11:58.703072 0.36 NULL
+2013-03-01 09:11:58.703072 0.48 0.36000001430511475
+2013-03-01 09:11:58.703072 0.79 0.8400000035762787
+2013-03-01 09:11:58.703072 1.27 1.270000010728836
+2013-03-01 09:11:58.703072 4.48 2.060000002384186
+2013-03-01 09:11:58.703072 9.0 5.75
+2013-03-01 09:11:58.703072 23.27 13.480000019073486
+2013-03-01 09:11:58.703072 25.13 32.27000045776367
+2013-03-01 09:11:58.703072 25.34 48.39999961853027
+2013-03-01 09:11:58.703072 25.91 50.46999931335449
+2013-03-01 09:11:58.703072 29.01 51.25
+2013-03-01 09:11:58.703072 30.47 54.920000076293945
+2013-03-01 09:11:58.703072 37.95 59.47999954223633
+2013-03-01 09:11:58.703072 39.3 68.42000007629395
+2013-03-01 09:11:58.703072 45.91 77.25
+2013-03-01 09:11:58.703072 52.44 85.20999908447266
+2013-03-01 09:11:58.703072 54.1 98.3499984741211
+2013-03-01 09:11:58.703072 56.7 106.53999710083008
+2013-03-01 09:11:58.703072 58.77 110.79999923706055
+2013-03-01 09:11:58.703072 62.09 115.47000122070312
+2013-03-01 09:11:58.703072 68.2 120.86000061035156
+2013-03-01 09:11:58.703072 71.68 130.28999710083008
+2013-03-01 09:11:58.703072 79.46 139.87999725341797
+2013-03-01 09:11:58.703072 80.02 151.13999938964844
+PREHOOK: query: select ts, f, sum(f) over (partition by ts order by f rows between unbounded preceding and 1 preceding) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select ts, f, sum(f) over (partition by ts order by f rows between unbounded preceding and 1 preceding) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+2013-03-01 09:11:58.70307 3.17 NULL
+2013-03-01 09:11:58.70307 10.89 3.1700000762939453
+2013-03-01 09:11:58.70307 14.54 14.0600004196167
+2013-03-01 09:11:58.70307 14.78 28.600000381469727
+2013-03-01 09:11:58.70307 17.85 43.38000011444092
+2013-03-01 09:11:58.70307 20.61 61.230000495910645
+2013-03-01 09:11:58.70307 28.69 81.8400011062622
+2013-03-01 09:11:58.70307 29.22 110.53000164031982
+2013-03-01 09:11:58.70307 31.17 139.75000095367432
+2013-03-01 09:11:58.70307 38.35 170.92000102996826
+2013-03-01 09:11:58.70307 38.61 209.26999950408936
+2013-03-01 09:11:58.70307 39.48 247.88000011444092
+2013-03-01 09:11:58.70307 40.54 287.35999965667725
+2013-03-01 09:11:58.70307 41.6 327.9000005722046
+2013-03-01 09:11:58.70307 46.08 369.4999990463257
+2013-03-01 09:11:58.70307 54.36 415.58000087738037
+2013-03-01 09:11:58.70307 56.94 469.94000148773193
+2013-03-01 09:11:58.70307 64.96 526.8800001144409
+2013-03-01 09:11:58.70307 73.52 591.8399991989136
+2013-03-01 09:11:58.70307 78.58 665.35999584198
+2013-03-01 09:11:58.70307 81.41 743.9399976730347
+2013-03-01 09:11:58.70307 84.71 825.350001335144
+2013-03-01 09:11:58.70307 87.43 910.0600004196167
+2013-03-01 09:11:58.70307 91.36 997.4900007247925
+2013-03-01 09:11:58.70307 92.96 1088.850001335144
+2013-03-01 09:11:58.70307 95.04 1181.8100004196167
+2013-03-01 09:11:58.703071 0.83 NULL
+2013-03-01 09:11:58.703071 1.99 0.8299999833106995
+2013-03-01 09:11:58.703071 3.73 2.8199999928474426
+2013-03-01 09:11:58.703071 8.86 6.550000011920929
+2013-03-01 09:11:58.703071 10.62 15.409999668598175
+2013-03-01 09:11:58.703071 11.32 26.029999554157257
+2013-03-01 09:11:58.703071 12.83 37.349999248981476
+2013-03-01 09:11:58.703071 14.7 50.17999917268753
+2013-03-01 09:11:58.703071 14.96 64.87999898195267
+2013-03-01 09:11:58.703071 17.58 79.83999902009964
+2013-03-01 09:11:58.703071 19.1 97.4199989438057
+2013-03-01 09:11:58.703071 21.01 116.51999932527542
+2013-03-01 09:11:58.703071 26.95 137.52999955415726
+2013-03-01 09:11:58.703071 27.23 164.4800003170967
+2013-03-01 09:11:58.703071 29.07 191.70999985933304
+2013-03-01 09:11:58.703071 29.71 220.77999955415726
+2013-03-01 09:11:58.703071 31.84 250.4899986386299
+2013-03-01 09:11:58.703071 31.94 282.3299987912178
+2013-03-01 09:11:58.703071 35.32 314.2699993252754
+2013-03-01 09:11:58.703071 37.32 349.58999902009964
+2013-03-01 09:11:58.703071 38.5 386.90999871492386
+2013-03-01 09:11:58.703071 42.08 425.40999871492386
+2013-03-01 09:11:58.703071 44.3 467.49000054597855
+2013-03-01 09:11:58.703071 44.66 511.7899997830391
+2013-03-01 09:11:58.703071 46.84 556.4499996304512
+2013-03-01 09:11:58.703071 48.89 603.2899997830391
+2013-03-01 09:11:58.703071 49.64 652.1799991726875
+2013-03-01 09:11:58.703071 50.28 701.819998562336
+2013-03-01 09:11:58.703071 52.09 752.0999973416328
+2013-03-01 09:11:58.703071 53.26 804.1899974942207
+2013-03-01 09:11:58.703071 54.09 857.4499958157539
+2013-03-01 09:11:58.703071 56.45 911.5399959683418
+2013-03-01 09:11:58.703071 56.76 967.9899967312813
+2013-03-01 09:11:58.703071 61.41 1024.7499950528145
+2013-03-01 09:11:58.703071 61.88 1086.1599949002266
+2013-03-01 09:11:58.703071 63.03 1148.0399959683418
+2013-03-01 09:11:58.703071 64.55 1211.0699947476387
+2013-03-01 09:11:58.703071 68.62 1275.6199977993965
+2013-03-01 09:11:58.703071 76.13 1344.2400005459785
+2013-03-01 09:11:58.703071 79.05 1420.3699977993965
+2013-03-01 09:11:58.703071 80.43 1499.4200008511543
+2013-03-01 09:11:58.703071 81.41 1579.85000115633
+2013-03-01 09:11:58.703071 82.85 1661.2600048184395
+2013-03-01 09:11:58.703071 83.98 1744.1100032925606
+2013-03-01 09:11:58.703071 84.21 1828.0900066494942
+2013-03-01 09:11:58.703071 85.55 1912.3000057339668
+2013-03-01 09:11:58.703071 87.93 1997.8500087857246
+2013-03-01 09:11:58.703071 88.93 2085.7800090909004
+2013-03-01 09:11:58.703071 94.27 2174.710009396076
+2013-03-01 09:11:58.703071 99.45 2268.9800060391426
+2013-03-01 09:11:58.703072 0.36 NULL
+2013-03-01 09:11:58.703072 0.48 0.36000001430511475
+2013-03-01 09:11:58.703072 0.79 0.8400000035762787
+2013-03-01 09:11:58.703072 1.27 1.6300000250339508
+2013-03-01 09:11:58.703072 4.48 2.9000000059604645
+2013-03-01 09:11:58.703072 9.0 7.380000025033951
+2013-03-01 09:11:58.703072 23.27 16.38000002503395
+2013-03-01 09:11:58.703072 25.13 39.65000048279762
+2013-03-01 09:11:58.703072 25.34 64.77999964356422
+2013-03-01 09:11:58.703072 25.91 90.11999979615211
+2013-03-01 09:11:58.703072 29.01 116.02999964356422
+2013-03-01 09:11:58.703072 30.47 145.03999987244606
+2013-03-01 09:11:58.703072 37.95 175.50999918580055
+2013-03-01 09:11:58.703072 39.3 213.45999994874
+2013-03-01 09:11:58.703072 45.91 252.75999918580055
+2013-03-01 09:11:58.703072 52.44 298.66999903321266
+2013-03-01 09:11:58.703072 54.1 351.10999765992165
+2013-03-01 09:11:58.703072 56.7 405.20999613404274
+2013-03-01 09:11:58.703072 58.77 461.9099968969822
+2013-03-01 09:11:58.703072 62.09 520.6799973547459
+2013-03-01 09:11:58.703072 68.2 582.7699975073338
+2013-03-01 09:11:58.703072 71.68 650.9699944555759
+2013-03-01 09:11:58.703072 79.46 722.6499947607517
+2013-03-01 09:11:58.703072 80.02 802.1099938452244
+PREHOOK: query: select ts, f, sum(f) over (partition by ts order by f rows between 1 following and 2 following) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select ts, f, sum(f) over (partition by ts order by f rows between 1 following and 2 following) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+2013-03-01 09:11:58.70307 3.17 25.43000030517578
+2013-03-01 09:11:58.70307 10.89 29.31999969482422
+2013-03-01 09:11:58.70307 14.54 32.63000011444092
+2013-03-01 09:11:58.70307 14.78 38.46000099182129
+2013-03-01 09:11:58.70307 17.85 49.30000114440918
+2013-03-01 09:11:58.70307 20.61 57.90999984741211
+2013-03-01 09:11:58.70307 28.69 60.38999938964844
+2013-03-01 09:11:58.70307 29.22 69.51999855041504
+2013-03-01 09:11:58.70307 31.17 76.95999908447266
+2013-03-01 09:11:58.70307 38.35 78.09000015258789
+2013-03-01 09:11:58.70307 38.61 80.02000045776367
+2013-03-01 09:11:58.70307 39.48 82.13999938964844
+2013-03-01 09:11:58.70307 40.54 87.68000030517578
+2013-03-01 09:11:58.70307 41.6 100.44000244140625
+2013-03-01 09:11:58.70307 46.08 111.29999923706055
+2013-03-01 09:11:58.70307 54.36 121.89999771118164
+2013-03-01 09:11:58.70307 56.94 138.47999572753906
+2013-03-01 09:11:58.70307 64.96 152.0999984741211
+2013-03-01 09:11:58.70307 73.52 159.99000549316406
+2013-03-01 09:11:58.70307 78.58 166.12000274658203
+2013-03-01 09:11:58.70307 81.41 172.13999938964844
+2013-03-01 09:11:58.70307 84.71 178.79000091552734
+2013-03-01 09:11:58.70307 87.43 184.31999969482422
+2013-03-01 09:11:58.70307 91.36 188.0
+2013-03-01 09:11:58.70307 92.96 95.04000091552734
+2013-03-01 09:11:58.70307 95.04 NULL
+2013-03-01 09:11:58.703071 0.83 5.7200000286102295
+2013-03-01 09:11:58.703071 1.99 12.589999675750732
+2013-03-01 09:11:58.703071 3.73 19.479999542236328
+2013-03-01 09:11:58.703071 8.86 21.9399995803833
+2013-03-01 09:11:58.703071 10.62 24.149999618530273
+2013-03-01 09:11:58.703071 11.32 27.52999973297119
+2013-03-01 09:11:58.703071 12.83 29.65999984741211
+2013-03-01 09:11:58.703071 14.7 32.53999996185303
+2013-03-01 09:11:58.703071 14.96 36.68000030517578
+2013-03-01 09:11:58.703071 17.58 40.11000061035156
+2013-03-01 09:11:58.703071 19.1 47.96000099182129
+2013-03-01 09:11:58.703071 21.01 54.18000030517578
+2013-03-01 09:11:58.703071 26.95 56.29999923706055
+2013-03-01 09:11:58.703071 27.23 58.779998779296875
+2013-03-01 09:11:58.703071 29.07 61.54999923706055
+2013-03-01 09:11:58.703071 29.71 63.78000068664551
+2013-03-01 09:11:58.703071 31.84 67.26000022888184
+2013-03-01 09:11:58.703071 31.94 72.63999938964844
+2013-03-01 09:11:58.703071 35.32 75.81999969482422
+2013-03-01 09:11:58.703071 37.32 80.58000183105469
+2013-03-01 09:11:58.703071 38.5 86.38000106811523
+2013-03-01 09:11:58.703071 42.08 88.95999908447266
+2013-03-01 09:11:58.703071 44.3 91.5
+2013-03-01 09:11:58.703071 44.66 95.72999954223633
+2013-03-01 09:11:58.703071 46.84 98.52999877929688
+2013-03-01 09:11:58.703071 48.89 99.91999816894531
+2013-03-01 09:11:58.703071 49.64 102.36999893188477
+2013-03-01 09:11:58.703071 50.28 105.3499984741211
+2013-03-01 09:11:58.703071 52.09 107.3499984741211
+2013-03-01 09:11:58.703071 53.26 110.54000091552734
+2013-03-01 09:11:58.703071 54.09 113.20999908447266
+2013-03-01 09:11:58.703071 56.45 118.16999816894531
+2013-03-01 09:11:58.703071 56.76 123.29000091552734
+2013-03-01 09:11:58.703071 61.41 124.90999984741211
+2013-03-01 09:11:58.703071 61.88 127.58000183105469
+2013-03-01 09:11:58.703071 63.03 133.17000579833984
+2013-03-01 09:11:58.703071 64.55 144.75
+2013-03-01 09:11:58.703071 68.62 155.18000030517578
+2013-03-01 09:11:58.703071 76.13 159.4800033569336
+2013-03-01 09:11:58.703071 79.05 161.84000396728516
+2013-03-01 09:11:58.703071 80.43 164.26000213623047
+2013-03-01 09:11:58.703071 81.41 166.8300018310547
+2013-03-01 09:11:58.703071 82.85 168.19000244140625
+2013-03-01 09:11:58.703071 83.98 169.76000213623047
+2013-03-01 09:11:58.703071 84.21 173.4800033569336
+2013-03-01 09:11:58.703071 85.55 176.86000061035156
+2013-03-01 09:11:58.703071 87.93 183.1999969482422
+2013-03-01 09:11:58.703071 88.93 193.7199935913086
+2013-03-01 09:11:58.703071 94.27 99.44999694824219
+2013-03-01 09:11:58.703071 99.45 NULL
+2013-03-01 09:11:58.703072 0.36 1.270000010728836
+2013-03-01 09:11:58.703072 0.48 2.060000002384186
+2013-03-01 09:11:58.703072 0.79 5.75
+2013-03-01 09:11:58.703072 1.27 13.480000019073486
+2013-03-01 09:11:58.703072 4.48 32.27000045776367
+2013-03-01 09:11:58.703072 9.0 48.39999961853027
+2013-03-01 09:11:58.703072 23.27 50.46999931335449
+2013-03-01 09:11:58.703072 25.13 51.25
+2013-03-01 09:11:58.703072 25.34 54.920000076293945
+2013-03-01 09:11:58.703072 25.91 59.47999954223633
+2013-03-01 09:11:58.703072 29.01 68.42000007629395
+2013-03-01 09:11:58.703072 30.47 77.25
+2013-03-01 09:11:58.703072 37.95 85.20999908447266
+2013-03-01 09:11:58.703072 39.3 98.3499984741211
+2013-03-01 09:11:58.703072 45.91 106.53999710083008
+2013-03-01 09:11:58.703072 52.44 110.79999923706055
+2013-03-01 09:11:58.703072 54.1 115.47000122070312
+2013-03-01 09:11:58.703072 56.7 120.86000061035156
+2013-03-01 09:11:58.703072 58.77 130.28999710083008
+2013-03-01 09:11:58.703072 62.09 139.87999725341797
+2013-03-01 09:11:58.703072 68.2 151.13999938964844
+2013-03-01 09:11:58.703072 71.68 159.47999572753906
+2013-03-01 09:11:58.703072 79.46 161.30999755859375
+2013-03-01 09:11:58.703072 80.02 168.27000427246094
+PREHOOK: query: select ts, f, sum(f) over (partition by ts order by f rows between unbounded preceding and 1 following) from over10k limit 100
+PREHOOK: type: QUERY
+PREHOOK: Input: default@over10k
+#### A masked pattern was here ####
+POSTHOOK: query: select ts, f, sum(f) over (partition by ts order by f rows between unbounded preceding and 1 following) from over10k limit 100
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@over10k
+#### A masked pattern was here ####
+2013-03-01 09:11:58.70307 3.17 14.0600004196167
+2013-03-01 09:11:58.70307 10.89 28.600000381469727
+2013-03-01 09:11:58.70307 14.54 43.38000011444092
+2013-03-01 09:11:58.70307 14.78 61.230000495910645
+2013-03-01 09:11:58.70307 17.85 81.8400011062622
+2013-03-01 09:11:58.70307 20.61 110.53000164031982
+2013-03-01 09:11:58.70307 28.69 139.75000095367432
+2013-03-01 09:11:58.70307 29.22 170.92000102996826
+2013-03-01 09:11:58.70307 31.17 209.26999950408936
+2013-03-01 09:11:58.70307 38.35 247.88000011444092
+2013-03-01 09:11:58.70307 38.61 287.35999965667725
+2013-03-01 09:11:58.70307 39.48 327.9000005722046
+2013-03-01 09:11:58.70307 40.54 369.4999990463257
+2013-03-01 09:11:58.70307 41.6 415.58000087738037
+2013-03-01 09:11:58.70307 46.08 469.94000148773193
+2013-03-01 09:11:58.70307 54.36 526.8800001144409
+2013-03-01 09:11:58.70307 56.94 591.8399991989136
+2013-03-01 09:11:58.70307 64.96 665.35999584198
+2013-03-01 09:11:58.70307 73.52 743.9399976730347
+2013-03-01 09:11:58.70307 78.58 825.350001335144
+2013-03-01 09:11:58.70307 81.41 910.0600004196167
+2013-03-01 09:11:58.70307 84.71 997.4900007247925
+2013-03-01 09:11:58.70307 87.43 1088.850001335144
+2013-03-01 09:11:58.70307 91.36 1181.8100004196167
+2013-03-01 09:11:58.70307 92.96 1276.850001335144
+2013-03-01 09:11:58.70307 95.04 1276.850001335144
+2013-03-01 09:11:58.703071 0.83 2.8199999928474426
+2013-03-01 09:11:58.703071 1.99 6.550000011920929
+2013-03-01 09:11:58.703071 3.73 15.409999668598175
+2013-03-01 09:11:58.703071 8.86 26.029999554157257
+2013-03-01 09:11:58.703071 10.62 37.349999248981476
+2013-03-01 09:11:58.703071 11.32 50.17999917268753
+2013-03-01 09:11:58.703071 12.83 64.87999898195267
+2013-03-01 09:11:58.703071 14.7 79.83999902009964
+2013-03-01 09:11:58.703071 14.96 97.4199989438057
+2013-03-01 09:11:58.703071 17.58 116.51999932527542
+2013-03-01 09:11:58.703071 19.1 137.52999955415726
+2013-03-01 09:11:58.703071 21.01 164.4800003170967
+2013-03-01 09:11:58.703071 26.95 191.70999985933304
+2013-03-01 09:11:58.703071 27.23 220.77999955415726
+2013-03-01 09:11:58.703071 29.07 250.4899986386299
+2013-03-01 09:11:58.703071 29.71 282.3299987912178
+2013-03-01 09:11:58.703071 31.84 314.2699993252754
+2013-03-01 09:11:58.703071 31.94 349.58999902009964
+2013-03-01 09:11:58.703071 35.32 386.90999871492386
+2013-03-01 09:11:58.703071 37.32 425.40999871492386
+2013-03-01 09:11:58.703071 38.5 467.49000054597855
+2013-03-01 09:11:58.703071 42.08 511.7899997830391
+2013-03-01 09:11:58.703071 44.3 556.4499996304512
+2013-03-01 09:11:58.703071 44.66 603.2899997830391
+2013-03-01 09:11:58.703071 46.84 652.1799991726875
+2013-03-01 09:11:58.703071 48.89 701.819998562336
+2013-03-01 09:11:58.703071 49.64 752.0999973416328
+2013-03-01 09:11:58.703071 50.28 804.1899974942207
+2013-03-01 09:11:58.703071 52.09 857.4499958157539
+2013-03-01 09:11:58.703071 53.26 911.5399959683418
+2013-03-01 09:11:58.703071 54.09 967.9899967312813
+2013-03-01 09:11:58.703071 56.45 1024.7499950528145
+2013-03-01 09:11:58.703071 56.76 1086.1599949002266
+2013-03-01 09:11:58.703071 61.41 1148.0399959683418
+2013-03-01 09:11:58.703071 61.88 1211.0699947476387
+2013-03-01 09:11:58.703071 63.03 1275.6199977993965
+2013-03-01 09:11:58.703071 64.55 1344.2400005459785
+2013-03-01 09:11:58.703071 68.62 1420.3699977993965
+2013-03-01 09:11:58.703071 76.13 1499.4200008511543
+2013-03-01 09:11:58.703071 79.05 1579.85000115633
+2013-03-01 09:11:58.703071 80.43 1661.2600048184395
+2013-03-01 09:11:58.703071 81.41 1744.1100032925606
+2013-03-01 09:11:58.703071 82.85 1828.0900066494942
+2013-03-01 09:11:58.703071 83.98 1912.3000057339668
+2013-03-01 09:11:58.703071 84.21 1997.8500087857246
+2013-03-01 09:11:58.703071 85.55 2085.7800090909004
+2013-03-01 09:11:58.703071 87.93 2174.710009396076
+2013-03-01 09:11:58.703071 88.93 2268.9800060391426
+2013-03-01 09:11:58.703071 94.27 2368.430002987385
+2013-03-01 09:11:58.703071 99.45 2368.430002987385
+2013-03-01 09:11:58.703072 0.36 0.8400000035762787
+2013-03-01 09:11:58.703072 0.48 1.6300000250339508
+2013-03-01 09:11:58.703072 0.79 2.9000000059604645
+2013-03-01 09:11:58.703072 1.27 7.380000025033951
+2013-03-01 09:11:58.703072 4.48 16.38000002503395
+2013-03-01 09:11:58.703072 9.0 39.65000048279762
+2013-03-01 09:11:58.703072 23.27 64.77999964356422
+2013-03-01 09:11:58.703072 25.13 90.11999979615211
+2013-03-01 09:11:58.703072 25.34 116.02999964356422
+2013-03-01 09:11:58.703072 25.91 145.03999987244606
+2013-03-01 09:11:58.703072 29.01 175.50999918580055
+2013-03-01 09:11:58.703072 30.47 213.45999994874
+2013-03-01 09:11:58.703072 37.95 252.75999918580055
+2013-03-01 09:11:58.703072 39.3 298.66999903321266
+2013-03-01 09:11:58.703072 45.91 351.10999765992165
+2013-03-01 09:11:58.703072 52.44 405.20999613404274
+2013-03-01 09:11:58.703072 54.1 461.9099968969822
+2013-03-01 09:11:58.703072 56.7 520.6799973547459
+2013-03-01 09:11:58.703072 58.77 582.7699975073338
+2013-03-01 09:11:58.703072 62.09 650.9699944555759
+2013-03-01 09:11:58.703072 68.2 722.6499947607517
+2013-03-01 09:11:58.703072 71.68 802.1099938452244
+2013-03-01 09:11:58.703072 79.46 882.1299904882908
+2013-03-01 09:11:58.703072 80.02 963.4199914038181
[43/50] [abbrv] hive git commit: HIVE-10628: Incorrect result when
vectorized native mapjoin is enabled using null safe operators <=> (Matt
McCline reviewed by Jason Dere)
Posted by sp...@apache.org.
HIVE-10628: Incorrect result when vectorized native mapjoin is enabled using null safe operators <=> (Matt McCline reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/4a2a2bcd
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/4a2a2bcd
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/4a2a2bcd
Branch: refs/heads/parquet
Commit: 4a2a2bcd0202e2b4d3ee5eb2548d288b9c5b6881
Parents: 312711b
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Tue May 19 17:12:09 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Tue May 19 17:12:09 2015 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 1 +
.../hive/ql/optimizer/physical/Vectorizer.java | 12 +-
.../clientpositive/vector_nullsafe_join.q | 72 ++
.../tez/vector_nullsafe_join.q.out | 1210 ++++++++++++++++++
.../clientpositive/vector_nullsafe_join.q.out | 1208 +++++++++++++++++
5 files changed, 2502 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/4a2a2bcd/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index c79c36c..a485408 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -233,6 +233,7 @@ minitez.query.files.shared=alter_merge_2_orc.q,\
vector_mr_diff_schema_alias.q,\
vector_multi_insert.q,\
vector_non_string_partition.q,\
+ vector_nullsafe_join.q,\
vector_orderby_5.q,\
vector_outer_join0.q,\
vector_outer_join1.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/4a2a2bcd/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 705b185..b429c56 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -1552,6 +1552,16 @@ public class Vectorizer implements PhysicalPlanResolver {
return vectorOp;
}
+ private boolean onExpressionHasNullSafes(MapJoinDesc desc) {
+ boolean[] nullSafes = desc.getNullSafes();
+ for (boolean nullSafe : nullSafes) {
+ if (nullSafe) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private boolean canSpecializeMapJoin(Operator<? extends OperatorDesc> op, MapJoinDesc desc,
boolean isTez) {
@@ -1562,7 +1572,7 @@ public class Vectorizer implements PhysicalPlanResolver {
HiveConf.ConfVars.HIVE_VECTORIZATION_MAPJOIN_NATIVE_ENABLED)) {
// Currently, only under Tez and non-N-way joins.
- if (isTez && desc.getConds().length == 1) {
+ if (isTez && desc.getConds().length == 1 && !onExpressionHasNullSafes(desc)) {
// Ok, all basic restrictions satisfied so far...
specialize = true;
http://git-wip-us.apache.org/repos/asf/hive/blob/4a2a2bcd/ql/src/test/queries/clientpositive/vector_nullsafe_join.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_nullsafe_join.q b/ql/src/test/queries/clientpositive/vector_nullsafe_join.q
new file mode 100644
index 0000000..316b2a6
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_nullsafe_join.q
@@ -0,0 +1,72 @@
+SET hive.vectorized.execution.enabled=true;
+SET hive.auto.convert.join=true;
+
+-- SORT_QUERY_RESULTS
+--
+-- We currently do not support null safes (i.e the <=> operator) in native vector map join.
+-- The explain output will show vectorized execution for both. We verify the query
+-- results are the same (HIVE-10628 shows native will produce the wrong results
+-- otherwise).
+--
+-- This query for "HIVE-3315 join predicate transitive" triggers HIVE-10640-
+-- explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL;
+-- select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL;
+--
+CREATE TABLE myinput1_txt(key int, value int);
+LOAD DATA LOCAL INPATH '../../data/files/in8.txt' INTO TABLE myinput1_txt;
+CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt;
+
+SET hive.vectorized.execution.mapjoin.native.enabled=false;
+
+-- merging
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value;
+select * from myinput1 a join myinput1 b on a.key<=>b.value;
+
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key;
+select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key;
+
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key;
+select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key;
+
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value;
+select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value;
+
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value;
+select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value;
+
+-- outer joins
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key<=>b.value;
+SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key<=>b.value;
+SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key<=>b.value;
+
+-- map joins
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value;
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value;
+
+SET hive.vectorized.execution.mapjoin.native.enabled=true;
+
+-- merging
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value;
+select * from myinput1 a join myinput1 b on a.key<=>b.value;
+
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key;
+select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key;
+
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key;
+select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key;
+
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value;
+select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value;
+
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value;
+select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value;
+
+-- outer joins
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key<=>b.value;
+SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key<=>b.value;
+SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key<=>b.value;
+
+-- map joins
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value;
+SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/4a2a2bcd/ql/src/test/results/clientpositive/tez/vector_nullsafe_join.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_nullsafe_join.q.out b/ql/src/test/results/clientpositive/tez/vector_nullsafe_join.q.out
new file mode 100644
index 0000000..db170a9
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vector_nullsafe_join.q.out
@@ -0,0 +1,1210 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+--
+-- We currently do not support null safes (i.e the <=> operator) in native vector map join.
+-- The explain output will show vectorized execution for both. We verify the query
+-- results are the same (HIVE-10628 shows native will produce the wrong results
+-- otherwise).
+--
+-- This query for "HIVE-3315 join predicate transitive" triggers HIVE-10640-
+-- explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL;
+-- select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL;
+--
+CREATE TABLE myinput1_txt(key int, value int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@myinput1_txt
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+--
+-- We currently do not support null safes (i.e the <=> operator) in native vector map join.
+-- The explain output will show vectorized execution for both. We verify the query
+-- results are the same (HIVE-10628 shows native will produce the wrong results
+-- otherwise).
+--
+-- This query for "HIVE-3315 join predicate transitive" triggers HIVE-10640-
+-- explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL;
+-- select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.key is NULL;
+--
+CREATE TABLE myinput1_txt(key int, value int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@myinput1_txt
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in8.txt' INTO TABLE myinput1_txt
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@myinput1_txt
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in8.txt' INTO TABLE myinput1_txt
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@myinput1_txt
+PREHOOK: query: CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@myinput1_txt
+PREHOOK: Output: database:default
+PREHOOK: Output: default@myinput1
+POSTHOOK: query: CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@myinput1_txt
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@myinput1
+PREHOOK: query: -- merging
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value
+PREHOOK: type: QUERY
+POSTHOOK: query: -- merging
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ nullSafes: [true]
+ outputColumnNames: _col0, _col1, _col5, _col6
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: value (type: int)
+ sort order: +
+ Map-reduce partition columns: value (type: int)
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ value expressions: key (type: int)
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ input vertices:
+ 1 Map 2
+ 2 Map 3
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: value (type: int)
+ sort order: +
+ Map-reduce partition columns: value (type: int)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ value expressions: key (type: int)
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: key (type: int)
+ sort order: +
+ Map-reduce partition columns: key (type: int)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ value expressions: value (type: int)
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10 10 NULL
+100 100 100 100 100 100
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+ nullSafes: [true]
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ input vertices:
+ 1 Map 2
+ 2 Map 3
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: value (type: int)
+ sort order: +
+ Map-reduce partition columns: value (type: int)
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ value expressions: key (type: int)
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: key (type: int)
+ sort order: +
+ Map-reduce partition columns: key (type: int)
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ value expressions: value (type: int)
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10 10 NULL
+100 100 100 100 100 100
+NULL 10 10 NULL NULL 10
+NULL 10 10 NULL NULL 35
+NULL 10 10 NULL NULL NULL
+NULL 10 48 NULL NULL 10
+NULL 10 48 NULL NULL 35
+NULL 10 48 NULL NULL NULL
+NULL 10 NULL NULL NULL 10
+NULL 10 NULL NULL NULL 35
+NULL 10 NULL NULL NULL NULL
+NULL 35 10 NULL NULL 10
+NULL 35 10 NULL NULL 35
+NULL 35 10 NULL NULL NULL
+NULL 35 48 NULL NULL 10
+NULL 35 48 NULL NULL 35
+NULL 35 48 NULL NULL NULL
+NULL 35 NULL NULL NULL 10
+NULL 35 NULL NULL NULL 35
+NULL 35 NULL NULL NULL NULL
+NULL NULL 10 NULL NULL 10
+NULL NULL 10 NULL NULL 35
+NULL NULL 10 NULL NULL NULL
+NULL NULL 48 NULL NULL 10
+NULL NULL 48 NULL NULL 35
+NULL NULL 48 NULL NULL NULL
+NULL NULL NULL NULL NULL 10
+NULL NULL NULL NULL NULL 35
+NULL NULL NULL NULL NULL NULL
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+ nullSafes: [true, false]
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ input vertices:
+ 1 Map 2
+ 2 Map 3
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: value (type: int), key (type: int)
+ sort order: ++
+ Map-reduce partition columns: value (type: int), key (type: int)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: key (type: int), value (type: int)
+ sort order: ++
+ Map-reduce partition columns: key (type: int), value (type: int)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+100 100 100 100 100 100
+NULL 10 10 NULL NULL 10
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+ nullSafes: [true, true]
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ input vertices:
+ 1 Map 2
+ 2 Map 3
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: value (type: int), key (type: int)
+ sort order: ++
+ Map-reduce partition columns: value (type: int), key (type: int)
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: key (type: int), value (type: int)
+ sort order: ++
+ Map-reduce partition columns: key (type: int), value (type: int)
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10 10 NULL
+100 100 100 100 100 100
+NULL 10 10 NULL NULL 10
+NULL NULL NULL NULL NULL NULL
+PREHOOK: query: -- outer joins
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: -- outer joins
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+48 NULL NULL NULL
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
+PREHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL 35
+NULL NULL NULL NULL
+PREHOOK: query: SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+48 NULL NULL NULL
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL 35
+NULL NULL NULL NULL
+PREHOOK: query: -- map joins
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: -- map joins
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
+PREHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
+PREHOOK: query: -- merging
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value
+PREHOOK: type: QUERY
+POSTHOOK: query: -- merging
+explain select * from myinput1 a join myinput1 b on a.key<=>b.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ nullSafes: [true]
+ outputColumnNames: _col0, _col1, _col5, _col6
+ input vertices:
+ 1 Map 2
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: value (type: int)
+ sort order: +
+ Map-reduce partition columns: value (type: int)
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ value expressions: key (type: int)
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ input vertices:
+ 1 Map 2
+ 2 Map 3
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: value (type: int)
+ sort order: +
+ Map-reduce partition columns: value (type: int)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ value expressions: key (type: int)
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: key (type: int)
+ sort order: +
+ Map-reduce partition columns: key (type: int)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ value expressions: value (type: int)
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key=c.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10 10 NULL
+100 100 100 100 100 100
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int)
+ 1 value (type: int)
+ 2 key (type: int)
+ nullSafes: [true]
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ input vertices:
+ 1 Map 2
+ 2 Map 3
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: value (type: int)
+ sort order: +
+ Map-reduce partition columns: value (type: int)
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ value expressions: key (type: int)
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: key (type: int)
+ sort order: +
+ Map-reduce partition columns: key (type: int)
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ value expressions: value (type: int)
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value join myinput1 c on a.key<=>c.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10 10 NULL
+100 100 100 100 100 100
+NULL 10 10 NULL NULL 10
+NULL 10 10 NULL NULL 35
+NULL 10 10 NULL NULL NULL
+NULL 10 48 NULL NULL 10
+NULL 10 48 NULL NULL 35
+NULL 10 48 NULL NULL NULL
+NULL 10 NULL NULL NULL 10
+NULL 10 NULL NULL NULL 35
+NULL 10 NULL NULL NULL NULL
+NULL 35 10 NULL NULL 10
+NULL 35 10 NULL NULL 35
+NULL 35 10 NULL NULL NULL
+NULL 35 48 NULL NULL 10
+NULL 35 48 NULL NULL 35
+NULL 35 48 NULL NULL NULL
+NULL 35 NULL NULL NULL 10
+NULL 35 NULL NULL NULL 35
+NULL 35 NULL NULL NULL NULL
+NULL NULL 10 NULL NULL 10
+NULL NULL 10 NULL NULL 35
+NULL NULL 10 NULL NULL NULL
+NULL NULL 48 NULL NULL 10
+NULL NULL 48 NULL NULL 35
+NULL NULL 48 NULL NULL NULL
+NULL NULL NULL NULL NULL 10
+NULL NULL NULL NULL NULL 35
+NULL NULL NULL NULL NULL NULL
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+ nullSafes: [true, false]
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ input vertices:
+ 1 Map 2
+ 2 Map 3
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 6 Data size: 26 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: value (type: int), key (type: int)
+ sort order: ++
+ Map-reduce partition columns: value (type: int), key (type: int)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: value is not null (type: boolean)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: key (type: int), value (type: int)
+ sort order: ++
+ Map-reduce partition columns: key (type: int), value (type: int)
+ Statistics: Num rows: 3 Data size: 12 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value=b.key join myinput1 c on a.key<=>c.key AND a.value=c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+100 100 100 100 100 100
+NULL 10 10 NULL NULL 10
+PREHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 1 <- Map 2 (BROADCAST_EDGE), Map 3 (BROADCAST_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: a
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 key (type: int), value (type: int)
+ 1 value (type: int), key (type: int)
+ 2 key (type: int), value (type: int)
+ nullSafes: [true, true]
+ outputColumnNames: _col0, _col1, _col5, _col6, _col10, _col11
+ input vertices:
+ 1 Map 2
+ 2 Map 3
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: int), _col5 (type: int), _col6 (type: int), _col10 (type: int), _col11 (type: int)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 13 Data size: 52 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: b
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: value (type: int), key (type: int)
+ sort order: ++
+ Map-reduce partition columns: value (type: int), key (type: int)
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: c
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: key (type: int), value (type: int)
+ sort order: ++
+ Map-reduce partition columns: key (type: int), value (type: int)
+ Statistics: Num rows: 6 Data size: 24 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from myinput1 a join myinput1 b on a.key<=>b.value AND a.value<=>b.key join myinput1 c on a.key<=>c.key AND a.value<=>c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10 10 NULL
+100 100 100 100 100 100
+NULL 10 10 NULL NULL 10
+NULL NULL NULL NULL NULL NULL
+PREHOOK: query: -- outer joins
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: -- outer joins
+SELECT * FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+48 NULL NULL NULL
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
+PREHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL 35
+NULL NULL NULL NULL
+PREHOOK: query: SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT * FROM myinput1 a FULL OUTER JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+48 NULL NULL NULL
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL 35
+NULL NULL NULL NULL
+PREHOOK: query: -- map joins
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: -- map joins
+SELECT /*+ MAPJOIN(a) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
+PREHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT /*+ MAPJOIN(b) */ * FROM myinput1 a JOIN myinput1 b ON a.key<=>b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+10 NULL NULL 10
+100 100 100 100
+NULL 10 10 NULL
+NULL 10 48 NULL
+NULL 10 NULL NULL
+NULL 35 10 NULL
+NULL 35 48 NULL
+NULL 35 NULL NULL
+NULL NULL 10 NULL
+NULL NULL 48 NULL
+NULL NULL NULL NULL
[13/50] [abbrv] hive git commit: HIVE-10590 fix potential NPE in
HiveMetaStore.equals (Alexander Pivovarov, reviewed by Ashutosh Chauhan)
Posted by sp...@apache.org.
HIVE-10590 fix potential NPE in HiveMetaStore.equals (Alexander Pivovarov, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2d87c12d
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2d87c12d
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2d87c12d
Branch: refs/heads/parquet
Commit: 2d87c12d914044540a4f5ed7fe500e3c245fbead
Parents: bc0138c
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Sun May 3 18:22:12 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Thu May 14 14:42:12 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hive/metastore/HiveMetaStore.java | 10 ++++++++--
1 file changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/2d87c12d/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
index 986579a..e796048 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java
@@ -2024,8 +2024,14 @@ public class HiveMetaStore extends ThriftHiveMetastore {
if (!p1.isSetValues() || !p2.isSetValues()) return p1.isSetValues() == p2.isSetValues();
if (p1.getValues().size() != p2.getValues().size()) return false;
for (int i = 0; i < p1.getValues().size(); ++i) {
- String v1 = p1.getValues().get(i), v2 = p2.getValues().get(i);
- if ((v1 == null && v2 != null) || !v1.equals(v2)) return false;
+ String v1 = p1.getValues().get(i);
+ String v2 = p2.getValues().get(i);
+ if (v1 == null && v2 == null) {
+ continue;
+ }
+ if (v1 == null || !v1.equals(v2)) {
+ return false;
+ }
}
return true;
}
[15/50] [abbrv] hive git commit: HIVE-10565: Native Vector Map Join
doesn't handle filtering and matching on LEFT OUTER JOIN repeated key
correctly (Matt McCline via Gunther Hagleitner)
Posted by sp...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/results/clientpositive/vector_join_filters.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_join_filters.q.out b/ql/src/test/results/clientpositive/vector_join_filters.q.out
new file mode 100644
index 0000000..48fc072
--- /dev/null
+++ b/ql/src/test/results/clientpositive/vector_join_filters.q.out
@@ -0,0 +1,222 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE myinput1_txt(key int, value int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@myinput1_txt
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE myinput1_txt(key int, value int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@myinput1_txt
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE myinput1_txt
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@myinput1_txt
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE myinput1_txt
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@myinput1_txt
+PREHOOK: query: CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@myinput1_txt
+PREHOOK: Output: database:default
+PREHOOK: Output: default@myinput1
+POSTHOOK: query: CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@myinput1_txt
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@myinput1
+Warning: Map Join MAPJOIN[21][bigTable=?] in task 'Stage-2:MAPRED' is a cross product
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+Warning: Map Join MAPJOIN[17][bigTable=a] in task 'Stage-2:MAPRED' is a cross product
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4937935
+Warning: Map Join MAPJOIN[17][bigTable=b] in task 'Stage-2:MAPRED' is a cross product
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4937935
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4937935
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4937935
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4937935
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/results/clientpositive/vector_join_nulls.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_join_nulls.q.out b/ql/src/test/results/clientpositive/vector_join_nulls.q.out
new file mode 100644
index 0000000..c1516f2
--- /dev/null
+++ b/ql/src/test/results/clientpositive/vector_join_nulls.q.out
@@ -0,0 +1,195 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE myinput1_txt(key int, value int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@myinput1_txt
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE myinput1_txt(key int, value int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@myinput1_txt
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE myinput1_txt
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@myinput1_txt
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in1.txt' INTO TABLE myinput1_txt
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@myinput1_txt
+PREHOOK: query: CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@myinput1_txt
+PREHOOK: Output: database:default
+PREHOOK: Output: default@myinput1
+POSTHOOK: query: CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@myinput1_txt
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@myinput1
+Warning: Map Join MAPJOIN[18][bigTable=?] in task 'Stage-2:MAPRED' is a cross product
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+13630578
+Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-2:MAPRED' is a cross product
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+13630578
+Warning: Map Join MAPJOIN[17][bigTable=?] in task 'Stage-2:MAPRED' is a cross product
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+13630578
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4509856
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3112070
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4542003
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4542038
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4543491
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4542003
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3079923
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4509891
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3113558
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3079923
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3112070
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3113558
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3112070
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/results/clientpositive/vector_left_outer_join2.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_left_outer_join2.q.out b/ql/src/test/results/clientpositive/vector_left_outer_join2.q.out
index 23d3f32..f9077c8 100644
--- a/ql/src/test/results/clientpositive/vector_left_outer_join2.q.out
+++ b/ql/src/test/results/clientpositive/vector_left_outer_join2.q.out
@@ -1,6 +1,10 @@
-PREHOOK: query: drop table if exists TJOIN1
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+drop table if exists TJOIN1
PREHOOK: type: DROPTABLE
-POSTHOOK: query: drop table if exists TJOIN1
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+drop table if exists TJOIN1
POSTHOOK: type: DROPTABLE
PREHOOK: query: drop table if exists TJOIN2
PREHOOK: type: DROPTABLE
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/results/clientpositive/vector_outer_join5.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vector_outer_join5.q.out b/ql/src/test/results/clientpositive/vector_outer_join5.q.out
new file mode 100644
index 0000000..bbe8ba1
--- /dev/null
+++ b/ql/src/test/results/clientpositive/vector_outer_join5.q.out
@@ -0,0 +1,1300 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+create table sorted_mod_4 stored as orc
+as select ctinyint, pmod(cint, 4) as cmodint from alltypesorc
+where cint is not null and ctinyint is not null
+order by ctinyint
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: database:default
+PREHOOK: Output: default@sorted_mod_4
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+create table sorted_mod_4 stored as orc
+as select ctinyint, pmod(cint, 4) as cmodint from alltypesorc
+where cint is not null and ctinyint is not null
+order by ctinyint
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@sorted_mod_4
+PREHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sorted_mod_4
+PREHOOK: Output: default@sorted_mod_4
+POSTHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@sorted_mod_4
+POSTHOOK: Output: default@sorted_mod_4
+PREHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS FOR COLUMNS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+POSTHOOK: query: ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS FOR COLUMNS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+PREHOOK: query: create table small_table stored
+as orc as select ctinyint, cbigint from alltypesorc limit 100
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: database:default
+PREHOOK: Output: default@small_table
+POSTHOOK: query: create table small_table stored
+as orc as select ctinyint, cbigint from alltypesorc limit 100
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@small_table
+PREHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table
+PREHOOK: Output: default@small_table
+POSTHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table
+POSTHOOK: Output: default@small_table
+PREHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS FOR COLUMNS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table
+#### A masked pattern was here ####
+POSTHOOK: query: ANALYZE TABLE small_table COMPUTE STATISTICS FOR COLUMNS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table
+#### A masked pattern was here ####
+PREHOOK: query: explain
+select count(*) from (select s.*, st.*
+from sorted_mod_4 s
+left outer join small_table st
+on s.ctinyint = st.ctinyint
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.*, st.*
+from sorted_mod_4 s
+left outer join small_table st
+on s.ctinyint = st.ctinyint
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-2 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $hdt$_0:$hdt$_1:st
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $hdt$_0:$hdt$_1:st
+ TableScan
+ alias: st
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.*, st.*
+from sorted_mod_4 s
+left outer join small_table st
+on s.ctinyint = st.ctinyint
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table
+PREHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.*, st.*
+from sorted_mod_4 s
+left outer join small_table st
+on s.ctinyint = st.ctinyint
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table
+POSTHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+6876
+PREHOOK: query: explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.cmodint = 2
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.cmodint = 2
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-2 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $hdt$_0:$hdt$_1:sm
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $hdt$_0:$hdt$_1:sm
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ filter predicates:
+ 0 {(_col1 = 2)}
+ 1
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint), cmodint (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {(_col1 = 2)}
+ 1
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.cmodint = 2
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table
+PREHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.cmodint = 2
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table
+POSTHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+6058
+PREHOOK: query: explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and pmod(s.ctinyint, 4) = s.cmodint
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and pmod(s.ctinyint, 4) = s.cmodint
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-2 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $hdt$_0:$hdt$_1:sm
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $hdt$_0:$hdt$_1:sm
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ filter predicates:
+ 0 {((UDFToInteger(_col0) pmod 4) = _col1)}
+ 1
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint), cmodint (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {((UDFToInteger(_col0) pmod 4) = _col1)}
+ 1
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and pmod(s.ctinyint, 4) = s.cmodint
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table
+PREHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and pmod(s.ctinyint, 4) = s.cmodint
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table
+POSTHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+6248
+PREHOOK: query: explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.ctinyint < 100
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.ctinyint < 100
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-2 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $hdt$_0:$hdt$_1:sm
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $hdt$_0:$hdt$_1:sm
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ filter predicates:
+ 0 {(_col0 < 100)}
+ 1
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {(_col0 < 100)}
+ 1
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.ctinyint < 100
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table
+PREHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.ctinyint < 100
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table
+POSTHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+6876
+PREHOOK: query: explain
+select count(*) from (select s.*, sm.*, s2.*
+from sorted_mod_4 s
+left outer join small_table sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join sorted_mod_4 s2
+ on s2.ctinyint = s.ctinyint
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.*, sm.*, s2.*
+from sorted_mod_4 s
+left outer join small_table sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join sorted_mod_4 s2
+ on s2.ctinyint = s.ctinyint
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-8 is a root stage
+ Stage-3 depends on stages: Stage-8
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $hdt$_0:$hdt$_1:sm
+ Fetch Operator
+ limit: -1
+ $hdt$_0:$hdt$_2:s
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $hdt$_0:$hdt$_1:sm
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cbigint (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 UDFToLong(_col1) (type: bigint)
+ 1 (_col0 pmod UDFToLong(8)) (type: bigint)
+ $hdt$_0:$hdt$_2:s
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: ctinyint (type: tinyint), cmodint (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 UDFToLong(_col1) (type: bigint)
+ 1 (_col0 pmod UDFToLong(8)) (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: tinyint)
+ 1 _col0 (type: tinyint)
+ Statistics: Num rows: 7329 Data size: 58641 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.*, sm.*, s2.*
+from sorted_mod_4 s
+left outer join small_table sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join sorted_mod_4 s2
+ on s2.ctinyint = s.ctinyint
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table
+PREHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.*, sm.*, s2.*
+from sorted_mod_4 s
+left outer join small_table sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join sorted_mod_4 s2
+ on s2.ctinyint = s.ctinyint
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table
+POSTHOOK: Input: default@sorted_mod_4
+#### A masked pattern was here ####
+3268334
+PREHOOK: query: create table mod_8_mod_4 stored as orc
+as select pmod(ctinyint, 8) as cmodtinyint, pmod(cint, 4) as cmodint from alltypesorc
+where cint is not null and ctinyint is not null
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: database:default
+PREHOOK: Output: default@mod_8_mod_4
+POSTHOOK: query: create table mod_8_mod_4 stored as orc
+as select pmod(ctinyint, 8) as cmodtinyint, pmod(cint, 4) as cmodint from alltypesorc
+where cint is not null and ctinyint is not null
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@mod_8_mod_4
+PREHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mod_8_mod_4
+PREHOOK: Output: default@mod_8_mod_4
+POSTHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mod_8_mod_4
+POSTHOOK: Output: default@mod_8_mod_4
+PREHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS FOR COLUMNS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mod_8_mod_4
+#### A masked pattern was here ####
+POSTHOOK: query: ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS FOR COLUMNS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mod_8_mod_4
+#### A masked pattern was here ####
+PREHOOK: query: create table small_table2 stored
+as orc as select pmod(ctinyint, 16) as cmodtinyint, cbigint from alltypesorc limit 100
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@alltypesorc
+PREHOOK: Output: database:default
+PREHOOK: Output: default@small_table2
+POSTHOOK: query: create table small_table2 stored
+as orc as select pmod(ctinyint, 16) as cmodtinyint, cbigint from alltypesorc limit 100
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@alltypesorc
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@small_table2
+PREHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table2
+PREHOOK: Output: default@small_table2
+POSTHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table2
+POSTHOOK: Output: default@small_table2
+PREHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS FOR COLUMNS
+PREHOOK: type: QUERY
+PREHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+POSTHOOK: query: ANALYZE TABLE small_table2 COMPUTE STATISTICS FOR COLUMNS
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+PREHOOK: query: explain
+select count(*) from (select s.*, st.*
+from mod_8_mod_4 s
+left outer join small_table2 st
+on s.cmodtinyint = st.cmodtinyint
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.*, st.*
+from mod_8_mod_4 s
+left outer join small_table2 st
+on s.cmodtinyint = st.cmodtinyint
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-2 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $hdt$_0:$hdt$_1:st
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $hdt$_0:$hdt$_1:st
+ TableScan
+ alias: st
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.*, st.*
+from mod_8_mod_4 s
+left outer join small_table2 st
+on s.cmodtinyint = st.cmodtinyint
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mod_8_mod_4
+PREHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.*, st.*
+from mod_8_mod_4 s
+left outer join small_table2 st
+on s.cmodtinyint = st.cmodtinyint
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mod_8_mod_4
+POSTHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+39112
+PREHOOK: query: explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodint = 2
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodint = 2
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-2 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $hdt$_0:$hdt$_1:sm
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $hdt$_0:$hdt$_1:sm
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ filter predicates:
+ 0 {(_col1 = 2)}
+ 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int), cmodint (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {(_col1 = 2)}
+ 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodint = 2
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mod_8_mod_4
+PREHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodint = 2
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mod_8_mod_4
+POSTHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+11171
+PREHOOK: query: explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and pmod(s.cmodtinyint, 4) = s.cmodint
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and pmod(s.cmodtinyint, 4) = s.cmodint
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-2 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $hdt$_0:$hdt$_1:sm
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $hdt$_0:$hdt$_1:sm
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ filter predicates:
+ 0 {((_col0 pmod 4) = _col1)}
+ 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int), cmodint (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {((_col0 pmod 4) = _col1)}
+ 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and pmod(s.cmodtinyint, 4) = s.cmodint
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mod_8_mod_4
+PREHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and pmod(s.cmodtinyint, 4) = s.cmodint
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mod_8_mod_4
+POSTHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+14371
+PREHOOK: query: explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodtinyint < 3
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodtinyint < 3
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-5 is a root stage
+ Stage-2 depends on stages: Stage-5
+ Stage-0 depends on stages: Stage-2
+
+STAGE PLANS:
+ Stage: Stage-5
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $hdt$_0:$hdt$_1:sm
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $hdt$_0:$hdt$_1:sm
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ filter predicates:
+ 0 {(_col0 < 3)}
+ 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+
+ Stage: Stage-2
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ filter predicates:
+ 0 {(_col0 < 3)}
+ 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodtinyint < 3
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mod_8_mod_4
+PREHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodtinyint < 3
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mod_8_mod_4
+POSTHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+17792
+PREHOOK: query: explain
+select count(*) from (select s.*, sm.*, s2.*
+from mod_8_mod_4 s
+left outer join small_table2 sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join mod_8_mod_4 s2
+ on s2.cmodtinyint = s.cmodtinyint
+) t1
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select count(*) from (select s.*, sm.*, s2.*
+from mod_8_mod_4 s
+left outer join small_table2 sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join mod_8_mod_4 s2
+ on s2.cmodtinyint = s.cmodtinyint
+) t1
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-8 is a root stage
+ Stage-3 depends on stages: Stage-8
+ Stage-0 depends on stages: Stage-3
+
+STAGE PLANS:
+ Stage: Stage-8
+ Map Reduce Local Work
+ Alias -> Map Local Tables:
+ $hdt$_0:$hdt$_1:sm
+ Fetch Operator
+ limit: -1
+ $hdt$_0:$hdt$_2:s
+ Fetch Operator
+ limit: -1
+ Alias -> Map Local Operator Tree:
+ $hdt$_0:$hdt$_1:sm
+ TableScan
+ alias: sm
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cbigint (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 100 Data size: 392 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 UDFToLong(_col1) (type: bigint)
+ 1 (_col0 pmod UDFToLong(8)) (type: bigint)
+ $hdt$_0:$hdt$_2:s
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ HashTable Sink Operator
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+
+ Stage: Stage-3
+ Map Reduce
+ Map Operator Tree:
+ TableScan
+ alias: s
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: cmodtinyint (type: int), cmodint (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 6058 Data size: 48464 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 UDFToLong(_col1) (type: bigint)
+ 1 (_col0 pmod UDFToLong(8)) (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 6663 Data size: 53310 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: int)
+ 1 _col0 (type: int)
+ Statistics: Num rows: 7329 Data size: 58641 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: count()
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Local Work:
+ Map Reduce Local Work
+ Execution mode: vectorized
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: count(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select count(*) from (select s.*, sm.*, s2.*
+from mod_8_mod_4 s
+left outer join small_table2 sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join mod_8_mod_4 s2
+ on s2.cmodtinyint = s.cmodtinyint
+) t1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@mod_8_mod_4
+PREHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+POSTHOOK: query: select count(*) from (select s.*, sm.*, s2.*
+from mod_8_mod_4 s
+left outer join small_table2 sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join mod_8_mod_4 s2
+ on s2.cmodtinyint = s.cmodtinyint
+) t1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@mod_8_mod_4
+POSTHOOK: Input: default@small_table2
+#### A masked pattern was here ####
+6524438
[17/50] [abbrv] hive git commit: HIVE-10565: Native Vector Map Join
doesn't handle filtering and matching on LEFT OUTER JOIN repeated key
correctly (Matt McCline via Gunther Hagleitner)
Posted by sp...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/results/clientpositive/tez/vectorized_timestamp_ints_casts.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorized_timestamp_ints_casts.q.out b/ql/src/test/results/clientpositive/tez/vectorized_timestamp_ints_casts.q.out
new file mode 100644
index 0000000..1e74446
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vectorized_timestamp_ints_casts.q.out
@@ -0,0 +1,234 @@
+PREHOOK: query: explain
+select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: alltypesorc
+ Filter Operator
+ predicate: ((cbigint % 250) = 0) (type: boolean)
+ Select Operator
+ expressions: CAST( ctinyint AS TIMESTAMP) (type: timestamp), CAST( csmallint AS TIMESTAMP) (type: timestamp), CAST( cint AS TIMESTAMP) (type: timestamp), CAST( cbigint AS TIMESTAMP) (type: timestamp), CAST( cfloat AS TIMESTAMP) (type: timestamp), CAST( cdouble AS TIMESTAMP) (type: timestamp), CAST( cboolean1 AS TIMESTAMP) (type: timestamp), CAST( (cbigint * 0) AS TIMESTAMP) (type: timestamp), ctimestamp1 (type: timestamp), CAST( cstring1 AS TIMESTAMP) (type: timestamp), CAST( substr(cstring1, 1, 1) AS TIMESTAMP) (type: timestamp)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
+ ListSink
+
+PREHOOK: query: select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1969-12-08 10:43:03.25 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.748 NULL NULL
+1969-12-31 15:59:59.964 1969-12-31 15:59:59.8 NULL 1970-01-19 04:24:39 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.817 NULL NULL
+1969-12-31 15:59:59.97 1969-12-31 15:59:59.8 NULL 1970-01-17 05:10:52.25 1969-12-31 15:59:30 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 16:00:12.935 NULL NULL
+1969-12-31 15:59:59.949 NULL 1970-01-09 14:53:20.971 1970-01-12 20:45:23.25 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL
+1969-12-31 15:59:59.949 NULL 1970-01-09 07:39:13.882 1969-12-09 07:45:32.75 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL
+1969-12-31 16:00:00.02 1969-12-31 16:00:15.601 NULL 1969-12-27 11:19:26.75 1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.129 NULL NULL
+1969-12-31 15:59:59.962 1969-12-31 16:00:15.601 NULL 1969-12-10 03:41:51 1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:58.614 NULL NULL
+1969-12-31 15:59:59.995 1969-12-31 16:00:15.601 NULL 1970-01-07 18:06:56 1969-12-31 15:59:55 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 16:00:04.679 NULL NULL
+1969-12-31 16:00:00.048 1969-12-31 16:00:15.601 NULL 1969-12-22 11:03:59 1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:50.235 NULL NULL
+1969-12-31 16:00:00.008 NULL 1969-12-24 00:12:58.862 1969-12-20 21:16:47.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL
+1969-12-31 16:00:00.008 NULL 1969-12-30 11:24:23.566 1969-12-16 11:20:17.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL
+1969-12-31 16:00:00.008 NULL 1970-01-09 23:39:39.664 1970-01-10 17:09:21.5 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL
+1969-12-31 16:00:00.008 NULL 1969-12-23 21:59:27.689 1970-01-19 01:16:31.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL
+1969-12-31 16:00:00.008 NULL 1970-01-10 23:29:48.972 1969-12-10 02:41:39 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL
+1969-12-31 16:00:00.008 NULL 1970-01-11 10:34:27.246 1970-01-14 14:49:59.25 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL
+1969-12-31 15:59:59.941 1969-12-31 15:59:52.804 NULL 1969-12-13 02:11:50 1969-12-31 15:59:01 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:13.15 NULL NULL
+1969-12-31 15:59:59.979 1969-12-31 15:59:52.804 NULL 1970-01-18 12:27:09 1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:55.9 NULL NULL
+1969-12-31 15:59:59.94 1969-12-31 15:59:52.804 NULL 1970-01-18 05:11:54.75 1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:52.408 NULL NULL
+1969-12-31 15:59:59.986 1969-12-31 15:59:52.804 NULL 1969-12-13 16:50:00.5 1969-12-31 15:59:46 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:11.065 NULL NULL
+1969-12-31 16:00:00.059 1969-12-31 15:59:52.804 NULL 1969-12-18 11:57:25.5 1969-12-31 16:00:59 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.956 NULL NULL
+1969-12-31 15:59:59.992 1969-12-31 15:59:52.804 NULL 1969-12-10 06:06:48.5 1969-12-31 15:59:52 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:03.136 NULL NULL
+1969-12-31 16:00:00.005 1969-12-31 15:59:52.804 NULL 1969-12-19 21:53:12.5 1969-12-31 16:00:05 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.973 NULL NULL
+1969-12-31 15:59:59.976 1969-12-31 15:59:52.804 NULL 1970-01-10 06:18:31 1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.145 NULL NULL
+1969-12-31 15:59:59.95 1969-12-31 15:59:52.804 NULL 1969-12-19 17:33:32.75 1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:54.733 NULL NULL
+1969-12-31 16:00:00.011 NULL 1969-12-30 22:03:04.018 1970-01-21 12:50:53.75 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL
+1969-12-31 16:00:00.011 NULL 1969-12-27 18:49:09.583 1970-01-14 22:35:27 1969-12-31 16:00:11 NULL 1969-12-31 16:00:00.001 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL
+PREHOOK: query: explain
+select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: alltypesorc
+ Filter Operator
+ predicate: ((cbigint % 250) = 0) (type: boolean)
+ Select Operator
+ expressions: CAST( ctinyint AS TIMESTAMP) (type: timestamp), CAST( csmallint AS TIMESTAMP) (type: timestamp), CAST( cint AS TIMESTAMP) (type: timestamp), CAST( cbigint AS TIMESTAMP) (type: timestamp), CAST( cfloat AS TIMESTAMP) (type: timestamp), CAST( cdouble AS TIMESTAMP) (type: timestamp), CAST( cboolean1 AS TIMESTAMP) (type: timestamp), CAST( (cbigint * 0) AS TIMESTAMP) (type: timestamp), ctimestamp1 (type: timestamp), CAST( cstring1 AS TIMESTAMP) (type: timestamp), CAST( substr(cstring1, 1, 1) AS TIMESTAMP) (type: timestamp)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7, _col8, _col9, _col10
+ ListSink
+
+PREHOOK: query: select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0
+PREHOOK: type: QUERY
+PREHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+POSTHOOK: query: select
+-- to timestamp
+ cast (ctinyint as timestamp)
+ ,cast (csmallint as timestamp)
+ ,cast (cint as timestamp)
+ ,cast (cbigint as timestamp)
+ ,cast (cfloat as timestamp)
+ ,cast (cdouble as timestamp)
+ ,cast (cboolean1 as timestamp)
+ ,cast (cbigint * 0 as timestamp)
+ ,cast (ctimestamp1 as timestamp)
+ ,cast (cstring1 as timestamp)
+ ,cast (substr(cstring1, 1, 1) as timestamp)
+from alltypesorc
+-- limit output to a reasonably small number of rows
+where cbigint % 250 = 0
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@alltypesorc
+#### A masked pattern was here ####
+1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1906-06-05 13:34:10 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.748 NULL NULL
+1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 2020-09-11 19:50:00 1969-12-31 15:59:24 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.817 NULL NULL
+1969-12-31 15:59:30 1969-12-31 15:56:40 NULL 2015-04-23 22:10:50 1969-12-31 15:59:30 1969-12-31 15:56:40 NULL 1969-12-31 16:00:00 1969-12-31 16:00:12.935 NULL NULL
+1969-12-31 15:59:09 NULL 1994-07-07 10:09:31 2003-05-25 21:27:30 1969-12-31 15:59:09 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL
+1969-12-31 15:59:09 NULL 1993-09-08 22:51:22 1908-10-29 07:05:50 1969-12-31 15:59:09 NULL 1969-12-31 16:00:01 1969-12-31 16:00:00 1969-12-31 16:00:08.451 NULL NULL
+1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1958-07-07 21:05:50 1969-12-31 16:00:20 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:45.129 NULL NULL
+1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1911-02-07 01:30:00 1969-12-31 15:59:22 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:58.614 NULL NULL
+1969-12-31 15:59:55 1969-12-31 20:20:01 NULL 1989-05-28 20:33:20 1969-12-31 15:59:55 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 16:00:04.679 NULL NULL
+1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1944-10-18 03:23:20 1969-12-31 16:00:48 1969-12-31 20:20:01 NULL 1969-12-31 16:00:00 1969-12-31 15:59:50.235 NULL NULL
+1969-12-31 16:00:08 NULL 1949-01-13 00:21:02 1940-06-26 15:47:30 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL
+1969-12-31 16:00:08 NULL 1966-09-27 07:32:46 1928-05-26 10:07:30 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL
+1969-12-31 16:00:08 NULL 1995-07-07 22:01:04 1997-07-05 20:58:20 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL
+1969-12-31 16:00:08 NULL 1948-10-12 08:01:29 2020-05-04 04:20:50 1969-12-31 16:00:08 NULL 1969-12-31 16:00:00 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL
+1969-12-31 16:00:08 NULL 1998-03-27 00:56:12 1910-12-27 06:10:00 1969-12-31 16:00:08 NULL 1969-12-31 16:00:01 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL
+1969-12-31 16:00:08 NULL 1999-07-01 15:14:06 2008-03-13 02:07:30 1969-12-31 16:00:08 NULL 1969-12-31 16:00:01 1969-12-31 16:00:00 1969-12-31 16:00:15.892 NULL NULL
+1969-12-31 15:59:01 1969-12-31 14:00:04 NULL 1919-02-22 13:13:20 1969-12-31 15:59:01 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:13.15 NULL NULL
+1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 2018-11-16 20:30:00 1969-12-31 15:59:39 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:55.9 NULL NULL
+1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 2018-01-18 14:32:30 1969-12-31 15:59:00 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:52.408 NULL NULL
+1969-12-31 15:59:46 1969-12-31 14:00:04 NULL 1920-10-24 09:28:20 1969-12-31 15:59:46 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:11.065 NULL NULL
+1969-12-31 16:00:59 1969-12-31 14:00:04 NULL 1933-12-12 05:05:00 1969-12-31 16:00:59 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.956 NULL NULL
+1969-12-31 15:59:52 1969-12-31 14:00:04 NULL 1911-05-18 17:28:20 1969-12-31 15:59:52 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:03.136 NULL NULL
+1969-12-31 16:00:05 1969-12-31 14:00:04 NULL 1937-10-25 22:48:20 1969-12-31 16:00:05 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 16:00:10.973 NULL NULL
+1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1996-04-09 21:36:40 1969-12-31 15:59:36 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:53.145 NULL NULL
+1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1937-04-28 15:05:50 1969-12-31 15:59:10 1969-12-31 14:00:04 NULL 1969-12-31 16:00:00 1969-12-31 15:59:54.733 NULL NULL
+1969-12-31 16:00:11 NULL 1967-12-14 19:06:58 2027-02-19 08:15:50 1969-12-31 16:00:11 NULL 1969-12-31 16:00:01 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL
+1969-12-31 16:00:11 NULL 1959-05-16 04:19:43 2009-01-30 06:50:00 1969-12-31 16:00:11 NULL 1969-12-31 16:00:01 1969-12-31 16:00:00 1969-12-31 16:00:02.351 NULL NULL
[40/50] [abbrv] hive git commit: HIVE-10727 : Import throws error
message "org.apache.thrift.protocol.TProtocolException: Required field
'filesAdded' is unset!" (Sushanth Sowmyan, reviewed by Alan Gates)
Posted by sp...@apache.org.
HIVE-10727 : Import throws error message "org.apache.thrift.protocol.TProtocolException: Required field 'filesAdded' is unset!" (Sushanth Sowmyan, reviewed by Alan Gates)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c567a77c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c567a77c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c567a77c
Branch: refs/heads/parquet
Commit: c567a77c214e1f97fffda8614cb05db581c6e151
Parents: 46b4cc9
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Tue May 19 14:11:38 2015 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Tue May 19 14:13:18 2015 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java | 2 ++
1 file changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c567a77c/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 1bb9b15..8c948a9 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -1930,6 +1930,8 @@ private void constructOneLBLocationMap(FileStatus fSta,
for (Path p : newFiles) {
insertData.addToFilesAdded(p.toString());
}
+ } else {
+ insertData.setFilesAdded(new ArrayList<String>());
}
FireEventRequest rqst = new FireEventRequest(true, data);
rqst.setDbName(tbl.getDbName());
[08/50] [abbrv] hive git commit: HIVE-10582 : variable typo in
HiveOpConverter (714) and SemanticAnalyzer (7496) (Alexander Pivovarov via
Ashutosh Chauhan)
Posted by sp...@apache.org.
HIVE-10582 : variable typo in HiveOpConverter (714) and SemanticAnalyzer (7496) (Alexander Pivovarov via Ashutosh Chauhan)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/b09fcfb5
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/b09fcfb5
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/b09fcfb5
Branch: refs/heads/parquet
Commit: b09fcfb558c6cdb451a76924ea2c14f64282c61b
Parents: fbdd7da
Author: Alexander Pivovarov <(apivovarov@gmail.com)>
Authored: Wed May 6 17:51:00 2015 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Wed May 13 11:59:01 2015 -0700
----------------------------------------------------------------------
.../hive/ql/optimizer/calcite/translator/HiveOpConverter.java | 4 +---
.../java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +-
2 files changed, 2 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/b09fcfb5/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
index b8ae604..efc2542 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/HiveOpConverter.java
@@ -27,14 +27,12 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import org.apache.calcite.rel.RelCollation;
import org.apache.calcite.rel.RelCollations;
import org.apache.calcite.rel.RelDistribution;
import org.apache.calcite.rel.RelDistribution.Type;
import org.apache.calcite.rel.RelFieldCollation;
import org.apache.calcite.rel.RelNode;
import org.apache.calcite.rel.core.SemiJoin;
-import org.apache.calcite.rel.core.SortExchange;
import org.apache.calcite.rel.rules.MultiJoin;
import org.apache.calcite.rex.RexInputRef;
import org.apache.calcite.rex.RexLiteral;
@@ -711,7 +709,7 @@ public class HiveOpConverter {
continue;
}
int vindex = exprBack == null ? -1 : ExprNodeDescUtils.indexOf(exprBack, reduceValuesBack);
- if (kindex >= 0) {
+ if (vindex >= 0) {
index[i] = -vindex - 1;
continue;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/b09fcfb5/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 2993539..30c87ad 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -7496,7 +7496,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
continue;
}
int vindex = exprBack == null ? -1 : ExprNodeDescUtils.indexOf(exprBack, reduceValuesBack);
- if (kindex >= 0) {
+ if (vindex >= 0) {
index[i] = -vindex - 1;
continue;
}
[27/50] [abbrv] hive git commit: HIVE-10714: Bloom filter column
names specification should be case insensitive (Prasanth Jayachandran
reviewed by Gopal V)
Posted by sp...@apache.org.
HIVE-10714: Bloom filter column names specification should be case insensitive (Prasanth Jayachandran reviewed by Gopal V)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3056e7aa
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3056e7aa
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3056e7aa
Branch: refs/heads/parquet
Commit: 3056e7aa856a6b591cf5c19743579f0991a477af
Parents: f82c0c2
Author: Prasanth Jayachandran <j....@gmail.com>
Authored: Fri May 15 11:22:51 2015 -0700
Committer: Prasanth Jayachandran <j....@gmail.com>
Committed: Fri May 15 11:22:51 2015 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java | 2 +-
ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3056e7aa/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
index ba59b35..0776018 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcUtils.java
@@ -58,7 +58,7 @@ public class OrcUtils {
int numFlattenedCols = getFlattenedColumnsCount(inspector);
boolean[] results = new boolean[numFlattenedCols];
if (selectedColumns != null && !selectedColumns.isEmpty()) {
- includeColumnsImpl(results, selectedColumns, allColumns, inspector);
+ includeColumnsImpl(results, selectedColumns.toLowerCase(), allColumns, inspector);
}
return results;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/3056e7aa/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
index b56c1ca..5b200f1 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java
@@ -316,7 +316,7 @@ public class TestFileDump {
.compress(CompressionKind.ZLIB)
.bufferSize(10000)
.rowIndexStride(1000)
- .bloomFilterColumns("s");
+ .bloomFilterColumns("S");
Writer writer = OrcFile.createWriter(testFilePath, options);
Random r1 = new Random(1);
String[] words = new String[]{"It", "was", "the", "best", "of", "times,",
[33/50] [abbrv] hive git commit: HIVE-10715 : RAT failures - many
files do not have ASF licenses (Sushanth Sowmyan, reviewed by Vikram Dixit)
Posted by sp...@apache.org.
HIVE-10715 : RAT failures - many files do not have ASF licenses (Sushanth Sowmyan, reviewed by Vikram Dixit)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/c640a386
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/c640a386
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/c640a386
Branch: refs/heads/parquet
Commit: c640a3864f602a17a643494278fe06e2faa8561b
Parents: 65d04bc
Author: Sushanth Sowmyan <kh...@gmail.com>
Authored: Thu May 14 17:40:00 2015 -0700
Committer: Sushanth Sowmyan <kh...@gmail.com>
Committed: Mon May 18 12:07:59 2015 -0700
----------------------------------------------------------------------
.../hive/hbase/HBaseTestStructSerializer.java | 20 +++++++++++++++++++-
.../src/test/e2e/templeton/inpdir/xmlmapper.py | 15 +++++++++++++++
.../src/test/e2e/templeton/inpdir/xmlreducer.py | 16 ++++++++++++++++
.../hive/jdbc/HttpRequestInterceptorBase.java | 18 ++++++++++++++++++
pom.xml | 2 ++
.../convert/HiveCollectionConverter.java | 18 ++++++++++++++++++
.../hive/ql/io/parquet/convert/Repeated.java | 18 ++++++++++++++++++
.../ql/optimizer/calcite/HiveRelCollation.java | 18 ++++++++++++++++++
.../calcite/reloperators/HiveGroupingID.java | 17 +++++++++++++++++
.../calcite/reloperators/HiveSortExchange.java | 18 ++++++++++++++++++
.../hive/ql/udf/generic/GenericUDFBaseDTI.java | 18 ++++++++++++++++++
.../apache/hadoop/hive/ql/TestTxnCommands2.java | 18 ++++++++++++++++++
.../io/parquet/AbstractTestParquetDirect.java | 18 ++++++++++++++++++
.../ql/io/parquet/TestArrayCompatibility.java | 18 ++++++++++++++++++
.../hive/ql/io/parquet/TestMapStructures.java | 18 ++++++++++++++++++
.../hadoop/hive/ql/session/TestAddResource.java | 18 ++++++++++++++++++
.../apache/hadoop/hive/ql/udf/TestUDFJson.java | 18 ++++++++++++++++++
.../hive/serde2/lazy/LazySerDeParameters.java | 18 ++++++++++++++++++
.../org/apache/hive/service/CookieSigner.java | 18 ++++++++++++++++++
.../TestLdapAuthenticationProviderImpl.java | 18 ++++++++++++++++++
20 files changed, 339 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestStructSerializer.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestStructSerializer.java b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestStructSerializer.java
index 73d1903..dfdaa17 100644
--- a/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestStructSerializer.java
+++ b/hbase-handler/src/test/org/apache/hadoop/hive/hbase/HBaseTestStructSerializer.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.hbase;
import java.util.Properties;
@@ -72,4 +90,4 @@ public class HBaseTestStructSerializer extends HBaseStructValue {
return lazyObject;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/hcatalog/src/test/e2e/templeton/inpdir/xmlmapper.py
----------------------------------------------------------------------
diff --git a/hcatalog/src/test/e2e/templeton/inpdir/xmlmapper.py b/hcatalog/src/test/e2e/templeton/inpdir/xmlmapper.py
index 935fb9b..99f507a 100644
--- a/hcatalog/src/test/e2e/templeton/inpdir/xmlmapper.py
+++ b/hcatalog/src/test/e2e/templeton/inpdir/xmlmapper.py
@@ -1,5 +1,20 @@
#!/usr/bin/env python
# dftmapper.py
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
import sys
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/hcatalog/src/test/e2e/templeton/inpdir/xmlreducer.py
----------------------------------------------------------------------
diff --git a/hcatalog/src/test/e2e/templeton/inpdir/xmlreducer.py b/hcatalog/src/test/e2e/templeton/inpdir/xmlreducer.py
index 484f02c..09ed646 100644
--- a/hcatalog/src/test/e2e/templeton/inpdir/xmlreducer.py
+++ b/hcatalog/src/test/e2e/templeton/inpdir/xmlreducer.py
@@ -1,4 +1,20 @@
#!/usr/bin/env python
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
import sys
for line in sys.stdin:
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/jdbc/src/java/org/apache/hive/jdbc/HttpRequestInterceptorBase.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HttpRequestInterceptorBase.java b/jdbc/src/java/org/apache/hive/jdbc/HttpRequestInterceptorBase.java
index 62f5369..68b0ff1 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HttpRequestInterceptorBase.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HttpRequestInterceptorBase.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hive.jdbc;
import java.io.IOException;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 08d6dc3..920f7a5 100644
--- a/pom.xml
+++ b/pom.xml
@@ -889,6 +889,8 @@
<exclude>bin/**</exclude>
<exclude>itests/**</exclude>
<exclude>docs/**</exclude>
+ <exclude>**/README.md</exclude>
+ <exclude>**/*.iml</exclude>
<exclude>**/*.txt</exclude>
<exclude>**/*.log</exclude>
<exclude>**/*.arcconfig</exclude>
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
index 6621a87..0fd538e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/HiveCollectionConverter.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.io.parquet.convert;
import com.google.common.base.Preconditions;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
index fdea782..0130aef 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/parquet/convert/Repeated.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.io.parquet.convert;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelCollation.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelCollation.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelCollation.java
index a40135f..d63f996 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelCollation.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/HiveRelCollation.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.optimizer.calcite;
import org.apache.calcite.rel.RelCollationImpl;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java
index e3cdb48..2fb9508 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveGroupingID.java
@@ -1,3 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.hadoop.hive.ql.optimizer.calcite.reloperators;
import org.apache.calcite.sql.SqlAggFunction;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java
index 3961636..d969b59 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/reloperators/HiveSortExchange.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.optimizer.calcite.reloperators;
import org.apache.calcite.plan.RelOptCluster;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseDTI.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseDTI.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseDTI.java
index dcdbd53..4c399b0 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseDTI.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFBaseDTI.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.udf.generic;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
index 3c987dd..33ca998 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/TestTxnCommands2.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql;
import org.apache.commons.io.FileUtils;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java
index 127a093..94a780d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/AbstractTestParquetDirect.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.io.parquet;
import com.google.common.base.Joiner;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
index f7f3e57..f349aa0 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestArrayCompatibility.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.io.parquet;
import java.util.List;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
index ca48050..6fc83e0 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/io/parquet/TestMapStructures.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.io.parquet;
import java.util.Arrays;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java b/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
index c63498d..6176b2d 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/session/TestAddResource.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.session;
import static org.junit.Assert.assertEquals;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFJson.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFJson.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFJson.java
index 39a3e20..bafaf1c 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFJson.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFJson.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.ql.udf;
import org.apache.hadoop.hive.ql.metadata.HiveException;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
index 7baf7a7..60d11a2 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazySerDeParameters.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hadoop.hive.serde2.lazy;
import java.util.ArrayList;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/service/src/java/org/apache/hive/service/CookieSigner.java
----------------------------------------------------------------------
diff --git a/service/src/java/org/apache/hive/service/CookieSigner.java b/service/src/java/org/apache/hive/service/CookieSigner.java
index 0b468fe..ee51c24 100644
--- a/service/src/java/org/apache/hive/service/CookieSigner.java
+++ b/service/src/java/org/apache/hive/service/CookieSigner.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hive.service;
import org.apache.commons.codec.binary.Base64;
http://git-wip-us.apache.org/repos/asf/hive/blob/c640a386/service/src/test/org/apache/hive/service/auth/TestLdapAuthenticationProviderImpl.java
----------------------------------------------------------------------
diff --git a/service/src/test/org/apache/hive/service/auth/TestLdapAuthenticationProviderImpl.java b/service/src/test/org/apache/hive/service/auth/TestLdapAuthenticationProviderImpl.java
index c7db3a9..5e91a0e 100644
--- a/service/src/test/org/apache/hive/service/auth/TestLdapAuthenticationProviderImpl.java
+++ b/service/src/test/org/apache/hive/service/auth/TestLdapAuthenticationProviderImpl.java
@@ -1,3 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
package org.apache.hive.service.auth;
import javax.security.sasl.AuthenticationException;
[12/50] [abbrv] hive git commit: HIVE-10630:Renaming tables across
encryption zones renames table even though the operation throws error (Eugene
Koifman, reviewed by Sergio Pena and Ashutosh Chauhan)
Posted by sp...@apache.org.
HIVE-10630:Renaming tables across encryption zones renames table even though the operation throws error (Eugene Koifman, reviewed by Sergio Pena and Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/bc0138c4
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/bc0138c4
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/bc0138c4
Branch: refs/heads/parquet
Commit: bc0138c436add2335d2045b6c7bf86bc6a15cc27
Parents: 05a3d2a
Author: Eugene Koifman <ek...@hortonworks.com>
Authored: Thu May 14 10:29:00 2015 -0700
Committer: Eugene Koifman <ek...@hortonworks.com>
Committed: Thu May 14 10:29:00 2015 -0700
----------------------------------------------------------------------
.../test/resources/testconfiguration.properties | 3 +-
.../org/apache/hadoop/hive/ql/QTestUtil.java | 10 ++++-
.../hadoop/hive/metastore/HiveAlterHandler.java | 28 ++++++++++---
.../hadoop/hive/metastore/ObjectStore.java | 2 +-
.../clientpositive/encryption_move_tbl.q | 18 ++++++++
.../encrypted/encryption_move_tbl.q.out | 43 ++++++++++++++++++++
6 files changed, 96 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/itests/src/test/resources/testconfiguration.properties
----------------------------------------------------------------------
diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties
index eeb46cc..f9c9351 100644
--- a/itests/src/test/resources/testconfiguration.properties
+++ b/itests/src/test/resources/testconfiguration.properties
@@ -337,7 +337,8 @@ encrypted.query.files=encryption_join_unencrypted_tbl.q,\
encryption_select_read_only_encrypted_tbl.q,\
encryption_select_read_only_unencrypted_tbl.q,\
encryption_load_data_to_encrypted_tables.q, \
- encryption_unencrypted_nonhdfs_external_tables.q
+ encryption_unencrypted_nonhdfs_external_tables.q \
+ encryption_move_tbl.q
beeline.positive.exclude=add_part_exist.q,\
alter1.q,\
http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
----------------------------------------------------------------------
diff --git a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
index 3e29d3c..f357c14 100644
--- a/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
+++ b/itests/util/src/main/java/org/apache/hadoop/hive/ql/QTestUtil.java
@@ -1029,7 +1029,7 @@ public class QTestUtil {
rc = cliDriver.processLine(command);
}
- if (rc != 0) {
+ if (rc != 0 && !ignoreErrors()) {
break;
}
command = "";
@@ -1040,6 +1040,14 @@ public class QTestUtil {
return rc;
}
+ /**
+ * This allows a .q file to continue executing after a statement runs into an error which is convenient
+ * if you want to use another hive cmd after the failure to sanity check the state of the system.
+ */
+ private boolean ignoreErrors() {
+ return conf.getBoolVar(HiveConf.ConfVars.CLIIGNOREERRORS);
+ }
+
private boolean isHiveCommand(String command) {
String[] cmd = command.trim().split("\\s+");
if (HiveCommand.find(cmd) != null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
index 50ec1e8..a3f2359 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hive.metastore.api.NoSuchObjectException;
import org.apache.hadoop.hive.metastore.api.Partition;
import org.apache.hadoop.hive.metastore.api.Table;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hive.common.util.HiveStringUtils;
import com.google.common.collect.Lists;
@@ -260,17 +261,18 @@ public class HiveAlterHandler implements AlterHandler {
// rename the src to destination
try {
if (srcFs.exists(srcPath) && !srcFs.rename(srcPath, destPath)) {
- throw new IOException("Renaming " + srcPath + " to " + destPath + " is failed");
+ throw new IOException("Renaming " + srcPath + " to " + destPath + " failed");
}
} catch (IOException e) {
+ LOG.error("Alter Table operation for " + dbname + "." + name + " failed.", e);
boolean revertMetaDataTransaction = false;
try {
msdb.openTransaction();
- msdb.alterTable(dbname, newt.getTableName(), oldt);
+ msdb.alterTable(newt.getDbName(), newt.getTableName(), oldt);
for (ObjectPair<Partition, String> pair : altps) {
Partition part = pair.getFirst();
part.getSd().setLocation(pair.getSecond());
- msdb.alterPartition(dbname, name, part.getValues(), part);
+ msdb.alterPartition(newt.getDbName(), name, part.getValues(), part);
}
revertMetaDataTransaction = msdb.commitTransaction();
} catch (Exception e1) {
@@ -288,8 +290,8 @@ public class HiveAlterHandler implements AlterHandler {
msdb.rollbackTransaction();
}
}
- throw new InvalidOperationException("Unable to access old location "
- + srcPath + " for table " + dbname + "." + name);
+ throw new InvalidOperationException("Alter Table operation for " + dbname + "." + name +
+ " failed to move data due to: '" + getSimpleMessage(e) + "' See hive log file for details.");
}
}
}
@@ -298,6 +300,22 @@ public class HiveAlterHandler implements AlterHandler {
}
}
+ /**
+ * RemoteExceptionS from hadoop RPC wrap the stack trace into e.getMessage() which makes
+ * logs/stack traces confusing.
+ * @param ex
+ * @return
+ */
+ String getSimpleMessage(IOException ex) {
+ if(ex instanceof RemoteException) {
+ String msg = ex.getMessage();
+ if(msg == null || !msg.contains("\n")) {
+ return msg;
+ }
+ return msg.substring(0, msg.indexOf('\n'));
+ }
+ return ex.getMessage();
+ }
public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname,
final String name, final List<String> part_vals, final Partition new_part)
throws InvalidOperationException, InvalidObjectException, AlreadyExistsException,
http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 583e23f..fd61333 100644
--- a/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -2758,7 +2758,7 @@ public class ObjectStore implements RawStore, Configurable {
MTable oldt = getMTable(dbname, name);
if (oldt == null) {
- throw new MetaException("table " + name + " doesn't exist");
+ throw new MetaException("table " + dbname + "." + name + " doesn't exist");
}
// For now only alter name, owner, parameters, cols, bucketcols are allowed
http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/ql/src/test/queries/clientpositive/encryption_move_tbl.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/encryption_move_tbl.q b/ql/src/test/queries/clientpositive/encryption_move_tbl.q
new file mode 100644
index 0000000..1168c6c
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/encryption_move_tbl.q
@@ -0,0 +1,18 @@
+-- SORT_QUERY_RESULTS;
+
+-- we're setting this so that TestNegaiveCliDriver.vm doesn't stop processing after ALTER TABLE fails;
+
+set hive.cli.errors.ignore=true;
+
+DROP TABLE IF EXISTS encrypted_table;
+CREATE TABLE encrypted_table (key INT, value STRING) LOCATION '${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table';
+CRYPTO CREATE_KEY --keyName key_128 --bitLength 128;
+CRYPTO CREATE_ZONE --keyName key_128 --path ${hiveconf:hive.metastore.warehouse.dir}/default/encrypted_table;
+
+INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src;
+SHOW TABLES;
+ALTER TABLE default.encrypted_table RENAME TO default.plain_table;
+SHOW TABLES;
+
+CRYPTO DELETE_KEY --keyName key_128;
+
http://git-wip-us.apache.org/repos/asf/hive/blob/bc0138c4/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out b/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
new file mode 100644
index 0000000..d18827e
--- /dev/null
+++ b/ql/src/test/results/clientpositive/encrypted/encryption_move_tbl.q.out
@@ -0,0 +1,43 @@
+PREHOOK: query: DROP TABLE IF EXISTS encrypted_table
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS encrypted_table
+POSTHOOK: type: DROPTABLE
+#### A masked pattern was here ####
+PREHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+PREHOOK: Output: database:default
+PREHOOK: Output: default@encrypted_table
+#### A masked pattern was here ####
+POSTHOOK: type: CREATETABLE
+#### A masked pattern was here ####
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@encrypted_table
+Encryption key created: 'key_128'
+Encryption zone created: '/build/ql/test/data/warehouse/default/encrypted_table' using key: 'key_128'
+PREHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@encrypted_table
+POSTHOOK: query: INSERT OVERWRITE TABLE encrypted_table SELECT * FROM src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@encrypted_table
+POSTHOOK: Lineage: encrypted_table.key EXPRESSION [(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: encrypted_table.value SIMPLE [(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+encrypted_table
+src
+PREHOOK: query: ALTER TABLE default.encrypted_table RENAME TO default.plain_table
+PREHOOK: type: ALTERTABLE_RENAME
+PREHOOK: Input: default@encrypted_table
+PREHOOK: Output: default@encrypted_table
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Unable to alter table. Alter Table operation for default.encrypted_table failed to move data due to: '/build/ql/test/data/warehouse/default/encrypted_table can't be moved from an encryption zone.' See hive log file for details.
+PREHOOK: query: SHOW TABLES
+PREHOOK: type: SHOWTABLES
+POSTHOOK: query: SHOW TABLES
+POSTHOOK: type: SHOWTABLES
+encrypted_table
+src
[29/50] [abbrv] hive git commit: HIVE-10531 Implement isClosed() to
HiveQueryResultSet (Yun-young LEE via Alexander Pivovarov)
Posted by sp...@apache.org.
HIVE-10531 Implement isClosed() to HiveQueryResultSet (Yun-young LEE via Alexander Pivovarov)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/52308ab6
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/52308ab6
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/52308ab6
Branch: refs/heads/parquet
Commit: 52308ab6ca14cc7ac7519a561f5b94e91db28087
Parents: f106730
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Thu May 14 18:32:03 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Sat May 16 20:42:06 2015 -0700
----------------------------------------------------------------------
jdbc/src/java/org/apache/hive/jdbc/HiveQueryResultSet.java | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/52308ab6/jdbc/src/java/org/apache/hive/jdbc/HiveQueryResultSet.java
----------------------------------------------------------------------
diff --git a/jdbc/src/java/org/apache/hive/jdbc/HiveQueryResultSet.java b/jdbc/src/java/org/apache/hive/jdbc/HiveQueryResultSet.java
index 7bc09cd..0cef509 100644
--- a/jdbc/src/java/org/apache/hive/jdbc/HiveQueryResultSet.java
+++ b/jdbc/src/java/org/apache/hive/jdbc/HiveQueryResultSet.java
@@ -468,4 +468,9 @@ public class HiveQueryResultSet extends HiveBaseResultSet {
public int getRow() throws SQLException {
return rowsFetched;
}
+
+ @Override
+ public boolean isClosed() {
+ return isClosed;
+ }
}
[03/50] [abbrv] hive git commit: HIVE-10581 fix comparison of String
objects using == in Hive line 785
Posted by sp...@apache.org.
HIVE-10581 fix comparison of String objects using == in Hive line 785
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/500f298c
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/500f298c
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/500f298c
Branch: refs/heads/parquet
Commit: 500f298c8c14044c879fa19585cab7129d2a4e29
Parents: 5333bd3
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Sat May 2 22:31:15 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Tue May 12 16:21:46 2015 -0700
----------------------------------------------------------------------
ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/500f298c/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 39c48b5..1bb9b15 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -782,7 +782,7 @@ public class Hive {
}
org.apache.hadoop.hive.metastore.api.Table baseTbl = getTable(tableName).getTTable();
- if (baseTbl.getTableType() == TableType.VIRTUAL_VIEW.toString()) {
+ if (TableType.VIRTUAL_VIEW.toString().equals(baseTbl.getTableType())) {
throw new HiveException("tableName="+ tableName +" is a VIRTUAL VIEW. Index on VIRTUAL VIEW is not supported.");
}
if (baseTbl.isTemporary()) {
[26/50] [abbrv] hive git commit: HIVE-10641 create CRC32 UDF
(Alexander Pivovarov, reviewed by Jason Dere)
Posted by sp...@apache.org.
HIVE-10641 create CRC32 UDF (Alexander Pivovarov, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f82c0c20
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f82c0c20
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f82c0c20
Branch: refs/heads/parquet
Commit: f82c0c20c5038f54b4a24561c943ca646a272d18
Parents: 84a2235
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Wed May 6 23:14:29 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Fri May 15 10:27:26 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/FunctionRegistry.java | 2 +
.../org/apache/hadoop/hive/ql/udf/UDFCrc32.java | 75 ++++++++++++++++++++
.../apache/hadoop/hive/ql/udf/TestUDFCrc32.java | 74 +++++++++++++++++++
ql/src/test/queries/clientpositive/udf_crc32.q | 13 ++++
.../results/clientpositive/show_functions.q.out | 2 +
.../test/results/clientpositive/udf_crc32.q.out | 60 ++++++++++++++++
6 files changed, 226 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f82c0c20/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index 606185c..7ce0a1c 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hive.ql.udf.UDFBase64;
import org.apache.hadoop.hive.ql.udf.UDFBin;
import org.apache.hadoop.hive.ql.udf.UDFConv;
import org.apache.hadoop.hive.ql.udf.UDFCos;
+import org.apache.hadoop.hive.ql.udf.UDFCrc32;
import org.apache.hadoop.hive.ql.udf.UDFDayOfMonth;
import org.apache.hadoop.hive.ql.udf.UDFDegrees;
import org.apache.hadoop.hive.ql.udf.UDFE;
@@ -219,6 +220,7 @@ public final class FunctionRegistry {
system.registerUDF("tan", UDFTan.class, false);
system.registerUDF("e", UDFE.class, false);
system.registerGenericUDF("factorial", GenericUDFFactorial.class);
+ system.registerUDF("crc32", UDFCrc32.class, false);
system.registerUDF("conv", UDFConv.class, false);
system.registerUDF("bin", UDFBin.class, false);
http://git-wip-us.apache.org/repos/asf/hive/blob/f82c0c20/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCrc32.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCrc32.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCrc32.java
new file mode 100644
index 0000000..c1f0e38
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFCrc32.java
@@ -0,0 +1,75 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf;
+
+import java.util.zip.CRC32;
+
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDF;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+
+/**
+ * UDFCrc32.
+ *
+ */
+@Description(name = "crc32",
+ value = "_FUNC_(str or bin) - Computes a cyclic redundancy check value "
+ + "for string or binary argument and returns bigint value.",
+ extended = "Example:\n"
+ + " > SELECT _FUNC_('ABC');\n"
+ + " 2743272264\n"
+ + " > SELECT _FUNC_(binary('ABC'));\n"
+ + " 2743272264")
+public class UDFCrc32 extends UDF {
+
+ private final LongWritable result = new LongWritable();
+ private final CRC32 crc32 = new CRC32();
+
+ /**
+ * CRC32 for string
+ */
+ public LongWritable evaluate(Text n) {
+ if (n == null) {
+ return null;
+ }
+
+ crc32.reset();
+ crc32.update(n.getBytes(), 0, n.getLength());
+
+ result.set(crc32.getValue());
+ return result;
+ }
+
+ /**
+ * CRC32 for binary
+ */
+ public LongWritable evaluate(BytesWritable b) {
+ if (b == null) {
+ return null;
+ }
+
+ crc32.reset();
+ crc32.update(b.getBytes(), 0, b.getLength());
+
+ result.set(crc32.getValue());
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/f82c0c20/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFCrc32.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFCrc32.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFCrc32.java
new file mode 100644
index 0000000..97a8672
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/TestUDFCrc32.java
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.LongWritable;
+import org.apache.hadoop.io.Text;
+
+public class TestUDFCrc32 extends TestCase {
+
+ public void testCrc32Str() throws HiveException {
+ UDFCrc32 udf = new UDFCrc32();
+
+ runAndVerifyStr("ABC", 2743272264L, udf);
+ runAndVerifyStr("", 0L, udf);
+ // repeat again
+ runAndVerifyStr("ABC", 2743272264L, udf);
+ runAndVerifyStr("", 0L, udf);
+ // null
+ runAndVerifyStr(null, null, udf);
+ }
+
+ public void testCrc32Bin() throws HiveException {
+ UDFCrc32 udf = new UDFCrc32();
+
+ runAndVerifyBin(new byte[] { 65, 66, 67 }, 2743272264L, udf);
+ runAndVerifyBin(new byte[0], 0L, udf);
+ // repeat again
+ runAndVerifyBin(new byte[] { 65, 66, 67 }, 2743272264L, udf);
+ runAndVerifyBin(new byte[0], 0L, udf);
+ // null
+ runAndVerifyBin(null, null, udf);
+ }
+
+ private void runAndVerifyStr(String str, Long expResult, UDFCrc32 udf) throws HiveException {
+ Text t = str != null ? new Text(str) : null;
+ LongWritable output = (LongWritable) udf.evaluate(t);
+ if (expResult == null) {
+ assertNull(output);
+ } else {
+ assertNotNull(output);
+ assertEquals("crc32() test ", expResult.longValue(), output.get());
+ }
+ }
+
+ private void runAndVerifyBin(byte[] binV, Long expResult, UDFCrc32 udf) throws HiveException {
+ BytesWritable binWr = binV != null ? new BytesWritable(binV) : null;
+ LongWritable output = (LongWritable) udf.evaluate(binWr);
+ if (expResult == null) {
+ assertNull(output);
+ } else {
+ assertNotNull(output);
+ assertEquals("crc32() test ", expResult.longValue(), output.get());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/f82c0c20/ql/src/test/queries/clientpositive/udf_crc32.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udf_crc32.q b/ql/src/test/queries/clientpositive/udf_crc32.q
new file mode 100644
index 0000000..650205e
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/udf_crc32.q
@@ -0,0 +1,13 @@
+DESCRIBE FUNCTION crc32;
+DESC FUNCTION EXTENDED crc32;
+
+explain select crc32('ABC');
+
+select
+crc32('ABC'),
+crc32(''),
+crc32(binary('ABC')),
+crc32(binary('')),
+crc32(cast(null as string)),
+crc32(cast(null as binary)),
+crc32(null);
http://git-wip-us.apache.org/repos/asf/hive/blob/f82c0c20/ql/src/test/results/clientpositive/show_functions.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/show_functions.q.out b/ql/src/test/results/clientpositive/show_functions.q.out
index 0948154..2c138a3 100644
--- a/ql/src/test/results/clientpositive/show_functions.q.out
+++ b/ql/src/test/results/clientpositive/show_functions.q.out
@@ -50,6 +50,7 @@ cos
count
covar_pop
covar_samp
+crc32
create_union
cume_dist
current_database
@@ -244,6 +245,7 @@ cos
count
covar_pop
covar_samp
+crc32
create_union
cume_dist
current_database
http://git-wip-us.apache.org/repos/asf/hive/blob/f82c0c20/ql/src/test/results/clientpositive/udf_crc32.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udf_crc32.q.out b/ql/src/test/results/clientpositive/udf_crc32.q.out
new file mode 100644
index 0000000..8280210
--- /dev/null
+++ b/ql/src/test/results/clientpositive/udf_crc32.q.out
@@ -0,0 +1,60 @@
+PREHOOK: query: DESCRIBE FUNCTION crc32
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESCRIBE FUNCTION crc32
+POSTHOOK: type: DESCFUNCTION
+crc32(str or bin) - Computes a cyclic redundancy check value for string or binary argument and returns bigint value.
+PREHOOK: query: DESC FUNCTION EXTENDED crc32
+PREHOOK: type: DESCFUNCTION
+POSTHOOK: query: DESC FUNCTION EXTENDED crc32
+POSTHOOK: type: DESCFUNCTION
+crc32(str or bin) - Computes a cyclic redundancy check value for string or binary argument and returns bigint value.
+Example:
+ > SELECT crc32('ABC');
+ 2743272264
+ > SELECT crc32(binary('ABC'));
+ 2743272264
+PREHOOK: query: explain select crc32('ABC')
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select crc32('ABC')
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ TableScan
+ alias: _dummy_table
+ Row Limit Per Split: 1
+ Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+ Select Operator
+ expressions: 2743272264 (type: bigint)
+ outputColumnNames: _col0
+ Statistics: Num rows: 0 Data size: 1 Basic stats: PARTIAL Column stats: COMPLETE
+ ListSink
+
+PREHOOK: query: select
+crc32('ABC'),
+crc32(''),
+crc32(binary('ABC')),
+crc32(binary('')),
+crc32(cast(null as string)),
+crc32(cast(null as binary)),
+crc32(null)
+PREHOOK: type: QUERY
+PREHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+POSTHOOK: query: select
+crc32('ABC'),
+crc32(''),
+crc32(binary('ABC')),
+crc32(binary('')),
+crc32(cast(null as string)),
+crc32(cast(null as binary)),
+crc32(null)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: _dummy_database@_dummy_table
+#### A masked pattern was here ####
+2743272264 0 2743272264 0 NULL NULL NULL
[47/50] [abbrv] hive git commit: HIVE-10706 Make
vectorized_timestamp_funcs test more stable (Alexander Pivovarov,
reviewed by Jason Dere)
Posted by sp...@apache.org.
HIVE-10706 Make vectorized_timestamp_funcs test more stable (Alexander Pivovarov, reviewed by Jason Dere)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/0d0757b7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/0d0757b7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/0d0757b7
Branch: refs/heads/parquet
Commit: 0d0757b73e20465d3d5178524f41d93c39450481
Parents: 3d95e10
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Wed May 13 23:27:02 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Tue May 19 23:18:31 2015 -0700
----------------------------------------------------------------------
.../clientpositive/vectorized_timestamp_funcs.q | 36 +++----
.../spark/vectorized_timestamp_funcs.q.out | 108 ++++++++++---------
.../tez/vectorized_timestamp_funcs.q.out | 108 ++++++++++---------
.../vectorized_timestamp_funcs.q.out | 108 ++++++++++---------
4 files changed, 192 insertions(+), 168 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0757b7/ql/src/test/queries/clientpositive/vectorized_timestamp_funcs.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vectorized_timestamp_funcs.q b/ql/src/test/queries/clientpositive/vectorized_timestamp_funcs.q
index 8a2d5aa..a6b7761 100644
--- a/ql/src/test/queries/clientpositive/vectorized_timestamp_funcs.q
+++ b/ql/src/test/queries/clientpositive/vectorized_timestamp_funcs.q
@@ -142,31 +142,31 @@ FROM alltypesorc_string;
-- SUM of timestamps are not vectorized reduce-side because they produce a double instead of a long (HIVE-8211)...
EXPLAIN SELECT
- sum(ctimestamp1)
+ round(sum(ctimestamp1), 3)
FROM alltypesorc_string;
SELECT
- sum(ctimestamp1)
+ round(sum(ctimestamp1), 3)
FROM alltypesorc_string;
EXPLAIN SELECT
- avg(ctimestamp1),
- variance(ctimestamp1),
- var_pop(ctimestamp1),
- var_samp(ctimestamp1),
- std(ctimestamp1),
- stddev(ctimestamp1),
- stddev_pop(ctimestamp1),
- stddev_samp(ctimestamp1)
+ round(avg(ctimestamp1), 0),
+ variance(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_pop(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_samp(ctimestamp1) between 9.20684592523616E19 and 9.20684592523617E19,
+ round(std(ctimestamp1), 3),
+ round(stddev(ctimestamp1), 3),
+ round(stddev_pop(ctimestamp1), 3),
+ round(stddev_samp(ctimestamp1), 3)
FROM alltypesorc_string;
SELECT
- avg(ctimestamp1),
- variance(ctimestamp1),
- var_pop(ctimestamp1),
- var_samp(ctimestamp1),
- std(ctimestamp1),
- stddev(ctimestamp1),
- stddev_pop(ctimestamp1),
- stddev_samp(ctimestamp1)
+ round(avg(ctimestamp1), 0),
+ variance(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_pop(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_samp(ctimestamp1) between 9.20684592523616E19 and 9.20684592523617E19,
+ round(std(ctimestamp1), 3),
+ round(stddev(ctimestamp1), 3),
+ round(stddev_pop(ctimestamp1), 3),
+ round(stddev_samp(ctimestamp1), 3)
FROM alltypesorc_string;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0757b7/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out b/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out
index 3044582..e9bef38 100644
--- a/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorized_timestamp_funcs.q.out
@@ -697,12 +697,12 @@ POSTHOOK: Input: default@alltypesorc_string
1969-12-31 23:59:44.088 1970-01-01 00:00:15.007 39 40
PREHOOK: query: -- SUM of timestamps are not vectorized reduce-side because they produce a double instead of a long (HIVE-8211)...
EXPLAIN SELECT
- sum(ctimestamp1)
+ round(sum(ctimestamp1), 3)
FROM alltypesorc_string
PREHOOK: type: QUERY
POSTHOOK: query: -- SUM of timestamps are not vectorized reduce-side because they produce a double instead of a long (HIVE-8211)...
EXPLAIN SELECT
- sum(ctimestamp1)
+ round(sum(ctimestamp1), 3)
FROM alltypesorc_string
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -741,13 +741,17 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: round(_col0, 3) (type: double)
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: vectorized
Stage: Stage-0
@@ -757,38 +761,38 @@ STAGE PLANS:
ListSink
PREHOOK: query: SELECT
- sum(ctimestamp1)
+ round(sum(ctimestamp1), 3)
FROM alltypesorc_string
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc_string
#### A masked pattern was here ####
POSTHOOK: query: SELECT
- sum(ctimestamp1)
+ round(sum(ctimestamp1), 3)
FROM alltypesorc_string
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc_string
#### A masked pattern was here ####
-1123143.8569999998
+1123143.857
PREHOOK: query: EXPLAIN SELECT
- avg(ctimestamp1),
- variance(ctimestamp1),
- var_pop(ctimestamp1),
- var_samp(ctimestamp1),
- std(ctimestamp1),
- stddev(ctimestamp1),
- stddev_pop(ctimestamp1),
- stddev_samp(ctimestamp1)
+ round(avg(ctimestamp1), 0),
+ variance(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_pop(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_samp(ctimestamp1) between 9.20684592523616E19 and 9.20684592523617E19,
+ round(std(ctimestamp1), 3),
+ round(stddev(ctimestamp1), 3),
+ round(stddev_pop(ctimestamp1), 3),
+ round(stddev_samp(ctimestamp1), 3)
FROM alltypesorc_string
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN SELECT
- avg(ctimestamp1),
- variance(ctimestamp1),
- var_pop(ctimestamp1),
- var_samp(ctimestamp1),
- std(ctimestamp1),
- stddev(ctimestamp1),
- stddev_pop(ctimestamp1),
- stddev_samp(ctimestamp1)
+ round(avg(ctimestamp1), 0),
+ variance(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_pop(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_samp(ctimestamp1) between 9.20684592523616E19 and 9.20684592523617E19,
+ round(std(ctimestamp1), 3),
+ round(stddev(ctimestamp1), 3),
+ round(stddev_pop(ctimestamp1), 3),
+ round(stddev_samp(ctimestamp1), 3)
FROM alltypesorc_string
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -828,13 +832,17 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: round(_col0, 0) (type: double), _col1 BETWEEN 8.97077295279421E19 AND 8.97077295279422E19 (type: boolean), _col2 BETWEEN 8.97077295279421E19 AND 8.97077295279422E19 (type: boolean), _col3 BETWEEN 9.20684592523616E19 AND 9.20684592523617E19 (type: boolean), round(_col4, 3) (type: double), round(_col5, 3) (type: double), round(_col6, 3) (type: double), round(_col7, 3) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -843,29 +851,29 @@ STAGE PLANS:
ListSink
PREHOOK: query: SELECT
- avg(ctimestamp1),
- variance(ctimestamp1),
- var_pop(ctimestamp1),
- var_samp(ctimestamp1),
- std(ctimestamp1),
- stddev(ctimestamp1),
- stddev_pop(ctimestamp1),
- stddev_samp(ctimestamp1)
+ round(avg(ctimestamp1), 0),
+ variance(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_pop(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_samp(ctimestamp1) between 9.20684592523616E19 and 9.20684592523617E19,
+ round(std(ctimestamp1), 3),
+ round(stddev(ctimestamp1), 3),
+ round(stddev_pop(ctimestamp1), 3),
+ round(stddev_samp(ctimestamp1), 3)
FROM alltypesorc_string
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc_string
#### A masked pattern was here ####
POSTHOOK: query: SELECT
- avg(ctimestamp1),
- variance(ctimestamp1),
- var_pop(ctimestamp1),
- var_samp(ctimestamp1),
- std(ctimestamp1),
- stddev(ctimestamp1),
- stddev_pop(ctimestamp1),
- stddev_samp(ctimestamp1)
+ round(avg(ctimestamp1), 0),
+ variance(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_pop(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_samp(ctimestamp1) between 9.20684592523616E19 and 9.20684592523617E19,
+ round(std(ctimestamp1), 3),
+ round(stddev(ctimestamp1), 3),
+ round(stddev_pop(ctimestamp1), 3),
+ round(stddev_samp(ctimestamp1), 3)
FROM alltypesorc_string
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc_string
#### A masked pattern was here ####
-2.8798560435897438E13 8.970772952794214E19 8.970772952794214E19 9.206845925236167E19 9.471416447815086E9 9.471416447815086E9 9.471416447815086E9 9.595231068211004E9
+2.8798560435897E13 true true true 9.471416447815E9 9.471416447815E9 9.471416447815E9 9.595231068211E9
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0757b7/ql/src/test/results/clientpositive/tez/vectorized_timestamp_funcs.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorized_timestamp_funcs.q.out b/ql/src/test/results/clientpositive/tez/vectorized_timestamp_funcs.q.out
index fa3ed21..76e6686 100644
--- a/ql/src/test/results/clientpositive/tez/vectorized_timestamp_funcs.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorized_timestamp_funcs.q.out
@@ -697,12 +697,12 @@ POSTHOOK: Input: default@alltypesorc_string
1969-12-31 23:59:44.088 1970-01-01 00:00:15.007 39 40
PREHOOK: query: -- SUM of timestamps are not vectorized reduce-side because they produce a double instead of a long (HIVE-8211)...
EXPLAIN SELECT
- sum(ctimestamp1)
+ round(sum(ctimestamp1), 3)
FROM alltypesorc_string
PREHOOK: type: QUERY
POSTHOOK: query: -- SUM of timestamps are not vectorized reduce-side because they produce a double instead of a long (HIVE-8211)...
EXPLAIN SELECT
- sum(ctimestamp1)
+ round(sum(ctimestamp1), 3)
FROM alltypesorc_string
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -741,13 +741,17 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: round(_col0, 3) (type: double)
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Execution mode: vectorized
Stage: Stage-0
@@ -757,38 +761,38 @@ STAGE PLANS:
ListSink
PREHOOK: query: SELECT
- sum(ctimestamp1)
+ round(sum(ctimestamp1), 3)
FROM alltypesorc_string
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc_string
#### A masked pattern was here ####
POSTHOOK: query: SELECT
- sum(ctimestamp1)
+ round(sum(ctimestamp1), 3)
FROM alltypesorc_string
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc_string
#### A masked pattern was here ####
-1123143.8569999998
+1123143.857
PREHOOK: query: EXPLAIN SELECT
- avg(ctimestamp1),
- variance(ctimestamp1),
- var_pop(ctimestamp1),
- var_samp(ctimestamp1),
- std(ctimestamp1),
- stddev(ctimestamp1),
- stddev_pop(ctimestamp1),
- stddev_samp(ctimestamp1)
+ round(avg(ctimestamp1), 0),
+ variance(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_pop(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_samp(ctimestamp1) between 9.20684592523616E19 and 9.20684592523617E19,
+ round(std(ctimestamp1), 3),
+ round(stddev(ctimestamp1), 3),
+ round(stddev_pop(ctimestamp1), 3),
+ round(stddev_samp(ctimestamp1), 3)
FROM alltypesorc_string
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN SELECT
- avg(ctimestamp1),
- variance(ctimestamp1),
- var_pop(ctimestamp1),
- var_samp(ctimestamp1),
- std(ctimestamp1),
- stddev(ctimestamp1),
- stddev_pop(ctimestamp1),
- stddev_samp(ctimestamp1)
+ round(avg(ctimestamp1), 0),
+ variance(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_pop(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_samp(ctimestamp1) between 9.20684592523616E19 and 9.20684592523617E19,
+ round(std(ctimestamp1), 3),
+ round(stddev(ctimestamp1), 3),
+ round(stddev_pop(ctimestamp1), 3),
+ round(stddev_samp(ctimestamp1), 3)
FROM alltypesorc_string
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -828,13 +832,17 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: round(_col0, 0) (type: double), _col1 BETWEEN 8.97077295279421E19 AND 8.97077295279422E19 (type: boolean), _col2 BETWEEN 8.97077295279421E19 AND 8.97077295279422E19 (type: boolean), _col3 BETWEEN 9.20684592523616E19 AND 9.20684592523617E19 (type: boolean), round(_col4, 3) (type: double), round(_col5, 3) (type: double), round(_col6, 3) (type: double), round(_col7, 3) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -843,29 +851,29 @@ STAGE PLANS:
ListSink
PREHOOK: query: SELECT
- avg(ctimestamp1),
- variance(ctimestamp1),
- var_pop(ctimestamp1),
- var_samp(ctimestamp1),
- std(ctimestamp1),
- stddev(ctimestamp1),
- stddev_pop(ctimestamp1),
- stddev_samp(ctimestamp1)
+ round(avg(ctimestamp1), 0),
+ variance(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_pop(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_samp(ctimestamp1) between 9.20684592523616E19 and 9.20684592523617E19,
+ round(std(ctimestamp1), 3),
+ round(stddev(ctimestamp1), 3),
+ round(stddev_pop(ctimestamp1), 3),
+ round(stddev_samp(ctimestamp1), 3)
FROM alltypesorc_string
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc_string
#### A masked pattern was here ####
POSTHOOK: query: SELECT
- avg(ctimestamp1),
- variance(ctimestamp1),
- var_pop(ctimestamp1),
- var_samp(ctimestamp1),
- std(ctimestamp1),
- stddev(ctimestamp1),
- stddev_pop(ctimestamp1),
- stddev_samp(ctimestamp1)
+ round(avg(ctimestamp1), 0),
+ variance(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_pop(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_samp(ctimestamp1) between 9.20684592523616E19 and 9.20684592523617E19,
+ round(std(ctimestamp1), 3),
+ round(stddev(ctimestamp1), 3),
+ round(stddev_pop(ctimestamp1), 3),
+ round(stddev_samp(ctimestamp1), 3)
FROM alltypesorc_string
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc_string
#### A masked pattern was here ####
-2.8798560435897438E13 8.970772952794212E19 8.970772952794212E19 9.206845925236166E19 9.471416447815084E9 9.471416447815084E9 9.471416447815084E9 9.595231068211002E9
+2.8798560435897E13 true true true 9.471416447815E9 9.471416447815E9 9.471416447815E9 9.595231068211E9
http://git-wip-us.apache.org/repos/asf/hive/blob/0d0757b7/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out b/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out
index 31a96c6..b3bb908 100644
--- a/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out
+++ b/ql/src/test/results/clientpositive/vectorized_timestamp_funcs.q.out
@@ -662,12 +662,12 @@ POSTHOOK: Input: default@alltypesorc_string
1969-12-31 23:59:44.088 1970-01-01 00:00:15.007 39 40
PREHOOK: query: -- SUM of timestamps are not vectorized reduce-side because they produce a double instead of a long (HIVE-8211)...
EXPLAIN SELECT
- sum(ctimestamp1)
+ round(sum(ctimestamp1), 3)
FROM alltypesorc_string
PREHOOK: type: QUERY
POSTHOOK: query: -- SUM of timestamps are not vectorized reduce-side because they produce a double instead of a long (HIVE-8211)...
EXPLAIN SELECT
- sum(ctimestamp1)
+ round(sum(ctimestamp1), 3)
FROM alltypesorc_string
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -700,13 +700,17 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: round(_col0, 3) (type: double)
+ outputColumnNames: _col0
Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -715,38 +719,38 @@ STAGE PLANS:
ListSink
PREHOOK: query: SELECT
- sum(ctimestamp1)
+ round(sum(ctimestamp1), 3)
FROM alltypesorc_string
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc_string
#### A masked pattern was here ####
POSTHOOK: query: SELECT
- sum(ctimestamp1)
+ round(sum(ctimestamp1), 3)
FROM alltypesorc_string
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc_string
#### A masked pattern was here ####
-1123143.8569999998
+1123143.857
PREHOOK: query: EXPLAIN SELECT
- avg(ctimestamp1),
- variance(ctimestamp1),
- var_pop(ctimestamp1),
- var_samp(ctimestamp1),
- std(ctimestamp1),
- stddev(ctimestamp1),
- stddev_pop(ctimestamp1),
- stddev_samp(ctimestamp1)
+ round(avg(ctimestamp1), 0),
+ variance(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_pop(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_samp(ctimestamp1) between 9.20684592523616E19 and 9.20684592523617E19,
+ round(std(ctimestamp1), 3),
+ round(stddev(ctimestamp1), 3),
+ round(stddev_pop(ctimestamp1), 3),
+ round(stddev_samp(ctimestamp1), 3)
FROM alltypesorc_string
PREHOOK: type: QUERY
POSTHOOK: query: EXPLAIN SELECT
- avg(ctimestamp1),
- variance(ctimestamp1),
- var_pop(ctimestamp1),
- var_samp(ctimestamp1),
- std(ctimestamp1),
- stddev(ctimestamp1),
- stddev_pop(ctimestamp1),
- stddev_samp(ctimestamp1)
+ round(avg(ctimestamp1), 0),
+ variance(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_pop(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_samp(ctimestamp1) between 9.20684592523616E19 and 9.20684592523617E19,
+ round(std(ctimestamp1), 3),
+ round(stddev(ctimestamp1), 3),
+ round(stddev_pop(ctimestamp1), 3),
+ round(stddev_samp(ctimestamp1), 3)
FROM alltypesorc_string
POSTHOOK: type: QUERY
STAGE DEPENDENCIES:
@@ -780,13 +784,17 @@ STAGE PLANS:
mode: mergepartial
outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
- File Output Operator
- compressed: false
+ Select Operator
+ expressions: round(_col0, 0) (type: double), _col1 BETWEEN 8.97077295279421E19 AND 8.97077295279422E19 (type: boolean), _col2 BETWEEN 8.97077295279421E19 AND 8.97077295279422E19 (type: boolean), _col3 BETWEEN 9.20684592523616E19 AND 9.20684592523617E19 (type: boolean), round(_col4, 3) (type: double), round(_col5, 3) (type: double), round(_col6, 3) (type: double), round(_col7, 3) (type: double)
+ outputColumnNames: _col0, _col1, _col2, _col3, _col4, _col5, _col6, _col7
Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
- table:
- input format: org.apache.hadoop.mapred.TextInputFormat
- output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
- serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 64 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
Stage: Stage-0
Fetch Operator
@@ -795,29 +803,29 @@ STAGE PLANS:
ListSink
PREHOOK: query: SELECT
- avg(ctimestamp1),
- variance(ctimestamp1),
- var_pop(ctimestamp1),
- var_samp(ctimestamp1),
- std(ctimestamp1),
- stddev(ctimestamp1),
- stddev_pop(ctimestamp1),
- stddev_samp(ctimestamp1)
+ round(avg(ctimestamp1), 0),
+ variance(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_pop(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_samp(ctimestamp1) between 9.20684592523616E19 and 9.20684592523617E19,
+ round(std(ctimestamp1), 3),
+ round(stddev(ctimestamp1), 3),
+ round(stddev_pop(ctimestamp1), 3),
+ round(stddev_samp(ctimestamp1), 3)
FROM alltypesorc_string
PREHOOK: type: QUERY
PREHOOK: Input: default@alltypesorc_string
#### A masked pattern was here ####
POSTHOOK: query: SELECT
- avg(ctimestamp1),
- variance(ctimestamp1),
- var_pop(ctimestamp1),
- var_samp(ctimestamp1),
- std(ctimestamp1),
- stddev(ctimestamp1),
- stddev_pop(ctimestamp1),
- stddev_samp(ctimestamp1)
+ round(avg(ctimestamp1), 0),
+ variance(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_pop(ctimestamp1) between 8.97077295279421E19 and 8.97077295279422E19,
+ var_samp(ctimestamp1) between 9.20684592523616E19 and 9.20684592523617E19,
+ round(std(ctimestamp1), 3),
+ round(stddev(ctimestamp1), 3),
+ round(stddev_pop(ctimestamp1), 3),
+ round(stddev_samp(ctimestamp1), 3)
FROM alltypesorc_string
POSTHOOK: type: QUERY
POSTHOOK: Input: default@alltypesorc_string
#### A masked pattern was here ####
-2.8798560435897438E13 8.970772952794212E19 8.970772952794212E19 9.206845925236166E19 9.471416447815084E9 9.471416447815084E9 9.471416447815084E9 9.595231068211002E9
+2.8798560435897E13 true true true 9.471416447815E9 9.471416447815E9 9.471416447815E9 9.595231068211E9
[34/50] [abbrv] hive git commit: HIVE-10627: CBO - Queries fail with
Failed to breakup Windowing invocations into Groups (Jesus Camacho Rodriguez
via Laljo John Pullokkaran)
Posted by sp...@apache.org.
HIVE-10627: CBO - Queries fail with Failed to breakup Windowing invocations into Groups (Jesus Camacho Rodriguez via Laljo John Pullokkaran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/f4923cee
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/f4923cee
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/f4923cee
Branch: refs/heads/parquet
Commit: f4923ceeb3ea8ec544ae55236b65620307ce233b
Parents: c640a38
Author: jpullokk <jp...@apache.org>
Authored: Mon May 18 12:34:14 2015 -0700
Committer: jpullokk <jp...@apache.org>
Committed: Mon May 18 12:34:14 2015 -0700
----------------------------------------------------------------------
.../calcite/rules/HiveWindowingFixRule.java | 163 +++++++++++++++++++
.../calcite/translator/ASTConverter.java | 2 -
.../hadoop/hive/ql/parse/CalcitePlanner.java | 11 ++
3 files changed, 174 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/f4923cee/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveWindowingFixRule.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveWindowingFixRule.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveWindowingFixRule.java
new file mode 100644
index 0000000..ff203d3
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/rules/HiveWindowingFixRule.java
@@ -0,0 +1,163 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.optimizer.calcite.rules;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+
+import org.apache.calcite.plan.RelOptRule;
+import org.apache.calcite.plan.RelOptRuleCall;
+import org.apache.calcite.rel.RelNode;
+import org.apache.calcite.rel.core.Aggregate;
+import org.apache.calcite.rel.core.Project;
+import org.apache.calcite.rel.core.RelFactories.ProjectFactory;
+import org.apache.calcite.rex.RexFieldCollation;
+import org.apache.calcite.rex.RexInputRef;
+import org.apache.calcite.rex.RexNode;
+import org.apache.calcite.rex.RexOver;
+import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveProject;
+
+/**
+ * Rule to fix windowing issue when it is done over
+ * aggregation columns (more info in HIVE-10627).
+ *
+ * This rule is applied as a post-processing step after
+ * optimization by Calcite in order to add columns
+ * that may be pruned by RelFieldTrimmer, but are
+ * still needed due to the concrete implementation of
+ * Windowing processing in Hive.
+ */
+public class HiveWindowingFixRule extends RelOptRule {
+
+ public static final HiveWindowingFixRule INSTANCE = new HiveWindowingFixRule();
+
+ private final ProjectFactory projectFactory;
+
+
+ private HiveWindowingFixRule() {
+ super(
+ operand(Project.class,
+ operand(Aggregate.class, any())));
+ this.projectFactory = HiveProject.DEFAULT_PROJECT_FACTORY;
+ }
+
+ @Override
+ public void onMatch(RelOptRuleCall call) {
+ Project project = call.rel(0);
+ Aggregate aggregate = call.rel(1);
+
+ // 1. We go over the expressions in the project operator
+ // and we separate the windowing nodes that are result
+ // of an aggregate expression from the rest of nodes
+ final int groupingFields = aggregate.getGroupCount() + aggregate.getIndicatorCount();
+ Set<String> projectExprsDigest = new HashSet<String>();
+ Map<String, RexNode> windowingExprsDigestToNodes = new HashMap<String,RexNode>();
+ for (RexNode r : project.getChildExps()) {
+ if (r instanceof RexOver) {
+ RexOver rexOverNode = (RexOver) r;
+ // Operands
+ for (RexNode operand : rexOverNode.getOperands()) {
+ if (operand instanceof RexInputRef &&
+ ((RexInputRef)operand).getIndex() >= groupingFields) {
+ windowingExprsDigestToNodes.put(operand.toString(), operand);
+ }
+ }
+ // Partition keys
+ for (RexNode partitionKey : rexOverNode.getWindow().partitionKeys) {
+ if (partitionKey instanceof RexInputRef &&
+ ((RexInputRef)partitionKey).getIndex() >= groupingFields) {
+ windowingExprsDigestToNodes.put(partitionKey.toString(), partitionKey);
+ }
+ }
+ // Order keys
+ for (RexFieldCollation orderKey : rexOverNode.getWindow().orderKeys) {
+ if (orderKey.left instanceof RexInputRef &&
+ ((RexInputRef)orderKey.left).getIndex() >= groupingFields) {
+ windowingExprsDigestToNodes.put(orderKey.left.toString(), orderKey.left);
+ }
+ }
+ } else {
+ projectExprsDigest.add(r.toString());
+ }
+ }
+
+ // 2. We check whether there is a column needed by the
+ // windowing operation that is missing in the
+ // project expressions. For instance, if the windowing
+ // operation is over an aggregation column, Hive expects
+ // that column to be in the Select clause of the query.
+ // The idea is that if there is a column missing, we will
+ // replace the old project operator by two new project
+ // operators:
+ // - a project operator containing the original columns
+ // of the project operator plus all the columns that were
+ // missing
+ // - a project on top of the previous one, that will take
+ // out the columns that were missing and were added by the
+ // previous project
+
+ // These data structures are needed to create the new project
+ // operator (below)
+ final List<RexNode> belowProjectExprs = new ArrayList<RexNode>();
+ final List<String> belowProjectColumnNames = new ArrayList<String>();
+
+ // This data structure is needed to create the new project
+ // operator (top)
+ final List<RexNode> topProjectExprs = new ArrayList<RexNode>();
+
+ final int projectCount = project.getChildExps().size();
+ for (int i = 0; i < projectCount; i++) {
+ belowProjectExprs.add(project.getChildExps().get(i));
+ belowProjectColumnNames.add(project.getRowType().getFieldNames().get(i));
+ topProjectExprs.add(RexInputRef.of(i, project.getRowType()));
+ }
+ boolean windowingFix = false;
+ for (Entry<String, RexNode> windowingExpr : windowingExprsDigestToNodes.entrySet()) {
+ if (!projectExprsDigest.contains(windowingExpr.getKey())) {
+ windowingFix = true;
+ belowProjectExprs.add(windowingExpr.getValue());
+ int colIndex = 0;
+ String alias = "window_col_" + colIndex;
+ while (belowProjectColumnNames.contains(alias)) {
+ alias = "window_col_" + (colIndex++);
+ }
+ belowProjectColumnNames.add(alias);
+ }
+ }
+
+ if (!windowingFix) {
+ // We do not need to do anything, we bail out
+ return;
+ }
+
+ // 3. We need to fix it, we create the two replacement project
+ // operators
+ RelNode newProjectRel = projectFactory.createProject(
+ aggregate, belowProjectExprs, belowProjectColumnNames);
+ RelNode newTopProjectRel = projectFactory.createProject(
+ newProjectRel, topProjectExprs, project.getRowType().getFieldNames());
+
+ call.transformTo(newTopProjectRel);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/f4923cee/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
index 0ada068..95f43d4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/calcite/translator/ASTConverter.java
@@ -54,10 +54,8 @@ import org.apache.calcite.util.ImmutableBitSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
-import org.apache.hadoop.hive.ql.exec.RowSchema;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
import org.apache.hadoop.hive.ql.optimizer.calcite.CalciteSemanticException;
-import org.apache.hadoop.hive.ql.optimizer.calcite.RelOptHiveTable;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveGroupingID;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveSort;
import org.apache.hadoop.hive.ql.optimizer.calcite.reloperators.HiveTableScan;
http://git-wip-us.apache.org/repos/asf/hive/blob/f4923cee/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
index 6e6923c..c412561 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/CalcitePlanner.java
@@ -145,6 +145,7 @@ import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveInsertExchange4Join
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinAddNotNullRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveJoinToMultiJoinRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HivePartitionPruneRule;
+import org.apache.hadoop.hive.ql.optimizer.calcite.rules.HiveWindowingFixRule;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.ASTConverter;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.HiveOpConverter;
import org.apache.hadoop.hive.ql.optimizer.calcite.translator.JoinCondTypeCheckProcFactory;
@@ -855,6 +856,16 @@ public class CalcitePlanner extends SemanticAnalyzer {
calciteOptimizedPlan = hepPlanner.findBestExp();
+ // run rule to fix windowing issue when it is done over
+ // aggregation columns (HIVE-10627)
+ hepPgmBldr = new HepProgramBuilder().addMatchOrder(HepMatchOrder.BOTTOM_UP);
+ hepPgmBldr.addRuleInstance(HiveWindowingFixRule.INSTANCE);
+ hepPlanner = new HepPlanner(hepPgmBldr.build());
+ hepPlanner.registerMetadataProviders(list);
+ cluster.setMetadataProvider(new CachingRelMetadataProvider(chainedProvider, hepPlanner));
+ hepPlanner.setRoot(calciteOptimizedPlan);
+ calciteOptimizedPlan = hepPlanner.findBestExp();
+
if (HiveConf.getBoolVar(conf, ConfVars.HIVE_CBO_RETPATH_HIVEOP)) {
// run rules to aid in translation from Optiq tree -> Hive tree
hepPgmBldr = new HepProgramBuilder().addMatchOrder(HepMatchOrder.BOTTOM_UP);
[19/50] [abbrv] hive git commit: HIVE-10565: Native Vector Map Join
doesn't handle filtering and matching on LEFT OUTER JOIN repeated key
correctly (Matt McCline via Gunther Hagleitner)
Posted by sp...@apache.org.
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/queries/clientpositive/vector_outer_join5.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/vector_outer_join5.q b/ql/src/test/queries/clientpositive/vector_outer_join5.q
new file mode 100644
index 0000000..b7ee4a4
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/vector_outer_join5.q
@@ -0,0 +1,173 @@
+SET hive.vectorized.execution.enabled=true;
+SET hive.vectorized.execution.mapjoin.native.enabled=true;
+set hive.auto.convert.join=true;
+set hive.auto.convert.join.noconditionaltask=true;
+set hive.auto.convert.join.noconditionaltask.size=10000;
+
+-- SORT_QUERY_RESULTS
+
+create table sorted_mod_4 stored as orc
+as select ctinyint, pmod(cint, 4) as cmodint from alltypesorc
+where cint is not null and ctinyint is not null
+order by ctinyint;
+
+ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS;
+ANALYZE TABLE sorted_mod_4 COMPUTE STATISTICS FOR COLUMNS;
+
+create table small_table stored
+as orc as select ctinyint, cbigint from alltypesorc limit 100;
+
+ANALYZE TABLE small_table COMPUTE STATISTICS;
+ANALYZE TABLE small_table COMPUTE STATISTICS FOR COLUMNS;
+
+explain
+select count(*) from (select s.*, st.*
+from sorted_mod_4 s
+left outer join small_table st
+on s.ctinyint = st.ctinyint
+) t1;
+
+select count(*) from (select s.*, st.*
+from sorted_mod_4 s
+left outer join small_table st
+on s.ctinyint = st.ctinyint
+) t1;
+
+explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.cmodint = 2
+) t1;
+
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.cmodint = 2
+) t1;
+
+explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and pmod(s.ctinyint, 4) = s.cmodint
+) t1;
+
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and pmod(s.ctinyint, 4) = s.cmodint
+) t1;
+
+explain
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.ctinyint < 100
+) t1;
+
+select count(*) from (select s.ctinyint, s.cmodint, sm.cbigint
+from sorted_mod_4 s
+left outer join small_table sm
+on s.ctinyint = sm.ctinyint and s.ctinyint < 100
+) t1;
+
+explain
+select count(*) from (select s.*, sm.*, s2.*
+from sorted_mod_4 s
+left outer join small_table sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join sorted_mod_4 s2
+ on s2.ctinyint = s.ctinyint
+) t1;
+
+select count(*) from (select s.*, sm.*, s2.*
+from sorted_mod_4 s
+left outer join small_table sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join sorted_mod_4 s2
+ on s2.ctinyint = s.ctinyint
+) t1;
+
+
+create table mod_8_mod_4 stored as orc
+as select pmod(ctinyint, 8) as cmodtinyint, pmod(cint, 4) as cmodint from alltypesorc
+where cint is not null and ctinyint is not null;
+
+ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS;
+ANALYZE TABLE mod_8_mod_4 COMPUTE STATISTICS FOR COLUMNS;
+
+create table small_table2 stored
+as orc as select pmod(ctinyint, 16) as cmodtinyint, cbigint from alltypesorc limit 100;
+
+ANALYZE TABLE small_table2 COMPUTE STATISTICS;
+ANALYZE TABLE small_table2 COMPUTE STATISTICS FOR COLUMNS;
+
+explain
+select count(*) from (select s.*, st.*
+from mod_8_mod_4 s
+left outer join small_table2 st
+on s.cmodtinyint = st.cmodtinyint
+) t1;
+
+select count(*) from (select s.*, st.*
+from mod_8_mod_4 s
+left outer join small_table2 st
+on s.cmodtinyint = st.cmodtinyint
+) t1;
+
+explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodint = 2
+) t1;
+
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodint = 2
+) t1;
+
+explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and pmod(s.cmodtinyint, 4) = s.cmodint
+) t1;
+
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and pmod(s.cmodtinyint, 4) = s.cmodint
+) t1;
+
+explain
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodtinyint < 3
+) t1;
+
+select count(*) from (select s.cmodtinyint, s.cmodint, sm.cbigint
+from mod_8_mod_4 s
+left outer join small_table2 sm
+on s.cmodtinyint = sm.cmodtinyint and s.cmodtinyint < 3
+) t1;
+
+explain
+select count(*) from (select s.*, sm.*, s2.*
+from mod_8_mod_4 s
+left outer join small_table2 sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join mod_8_mod_4 s2
+ on s2.cmodtinyint = s.cmodtinyint
+) t1;
+
+select count(*) from (select s.*, sm.*, s2.*
+from mod_8_mod_4 s
+left outer join small_table2 sm
+ on pmod(sm.cbigint, 8) = s.cmodint
+left outer join mod_8_mod_4 s2
+ on s2.cmodtinyint = s.cmodtinyint
+) t1;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/results/clientpositive/tez/acid_vectorization_partition.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/acid_vectorization_partition.q.out b/ql/src/test/results/clientpositive/tez/acid_vectorization_partition.q.out
index 3aa0e1a..3b37c72 100644
--- a/ql/src/test/results/clientpositive/tez/acid_vectorization_partition.q.out
+++ b/ql/src/test/results/clientpositive/tez/acid_vectorization_partition.q.out
@@ -38,23 +38,23 @@ POSTHOOK: Input: default@acid_vectorized_part
POSTHOOK: Input: default@acid_vectorized_part@ds=today
POSTHOOK: Input: default@acid_vectorized_part@ds=tomorrow
#### A masked pattern was here ####
--1073279343 oj1YrV5Wa today
-1073279343 oj1YrV5Wa tomorrow
--1073051226 A34p7oRr2WvUJNf today
+-1073279343 oj1YrV5Wa today
-1073051226 A34p7oRr2WvUJNf tomorrow
--1072910839 0iqrc5 today
+-1073051226 A34p7oRr2WvUJNf today
-1072910839 0iqrc5 tomorrow
--1072081801 dPkN74F7 today
+-1072910839 0iqrc5 today
-1072081801 dPkN74F7 tomorrow
--1072076362 2uLyD28144vklju213J1mr tomorrow
+-1072081801 dPkN74F7 today
-1072076362 2uLyD28144vklju213J1mr today
--1071480828 aw724t8c5558x2xneC624 today
+-1072076362 2uLyD28144vklju213J1mr tomorrow
-1071480828 aw724t8c5558x2xneC624 tomorrow
--1071363017 Anj0oF today
+-1071480828 aw724t8c5558x2xneC624 today
-1071363017 Anj0oF tomorrow
--1070883071 0ruyd6Y50JpdGRf6HqD tomorrow
+-1071363017 Anj0oF today
-1070883071 0ruyd6Y50JpdGRf6HqD today
--1070551679 iUR3Q today
+-1070883071 0ruyd6Y50JpdGRf6HqD tomorrow
-1070551679 iUR3Q tomorrow
--1069736047 k17Am8uPHWk02cEf1jet tomorrow
+-1070551679 iUR3Q today
-1069736047 k17Am8uPHWk02cEf1jet today
+-1069736047 k17Am8uPHWk02cEf1jet tomorrow
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/results/clientpositive/tez/vector_join30.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_join30.q.out b/ql/src/test/results/clientpositive/tez/vector_join30.q.out
new file mode 100644
index 0000000..2a14842
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vector_join30.q.out
@@ -0,0 +1,1367 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE orcsrc STORED AS ORC AS SELECT * FROM src
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@src
+PREHOOK: Output: database:default
+PREHOOK: Output: default@orcsrc
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE orcsrc STORED AS ORC AS SELECT * FROM src
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@src
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@orcsrc
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+ Reducer 5 <- Map 4 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ input vertices:
+ 1 Reducer 5
+ Statistics: Num rows: 275 Data size: 48400 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Reducer 5
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+103231310608
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+ Reducer 5 <- Map 4 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ input vertices:
+ 1 Reducer 5
+ Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Reducer 5
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+103231310608
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 4 <- Map 3 (SIMPLE_EDGE), Reducer 2 (BROADCAST_EDGE)
+ Reducer 5 <- Reducer 4 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 3
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 4
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ input vertices:
+ 0 Reducer 2
+ Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 5
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+103231310608
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE), Reducer 5 (BROADCAST_EDGE), Reducer 7 (BROADCAST_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE)
+ Reducer 5 <- Map 4 (SIMPLE_EDGE)
+ Reducer 7 <- Map 6 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 4
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Map 6
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Inner Join 0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ input vertices:
+ 1 Reducer 5
+ 2 Reducer 7
+ Statistics: Num rows: 550 Data size: 96800 Basic stats: COMPLETE Column stats: NONE
+ HybridGraceHashJoin: true
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 3
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Reducer 5
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Execution mode: vectorized
+ Reducer 7
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 250 Data size: 44000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+348019368476
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
+ Reducer 6 <- Map 5 (SIMPLE_EDGE)
+ Reducer 8 <- Map 7 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 3
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Inner Join 0 to 1
+ Left Outer Join0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Statistics: Num rows: 1100 Data size: 193600 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 4
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Reducer 6
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Execution mode: vectorized
+ Reducer 8
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+348019368476
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
+ Reducer 6 <- Map 5 (SIMPLE_EDGE)
+ Reducer 8 <- Map 7 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 3
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ Left Outer Join0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Statistics: Num rows: 1100 Data size: 193600 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 4
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Reducer 6
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Execution mode: vectorized
+ Reducer 8
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+348019368476
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
+ Reducer 6 <- Map 5 (SIMPLE_EDGE)
+ Reducer 8 <- Map 7 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 3
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Left Outer Join0 to 1
+ Right Outer Join0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Statistics: Num rows: 1100 Data size: 193600 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 4
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Reducer 6
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Execution mode: vectorized
+ Reducer 8
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+LEFT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+348019368476
+PREHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+POSTHOOK: query: explain
+FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Reducer 2 <- Map 1 (SIMPLE_EDGE)
+ Reducer 3 <- Reducer 2 (SIMPLE_EDGE), Reducer 6 (SIMPLE_EDGE), Reducer 8 (SIMPLE_EDGE)
+ Reducer 4 <- Reducer 3 (SIMPLE_EDGE)
+ Reducer 6 <- Map 5 (SIMPLE_EDGE)
+ Reducer 8 <- Map 7 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Map 5
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Map 7
+ Map Operator Tree:
+ TableScan
+ alias: orcsrc
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Select Operator
+ expressions: key (type: string), value (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col1 (type: string)
+ sort order: +
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: string)
+ Execution mode: vectorized
+ Reducer 2
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+ Reducer 3
+ Reduce Operator Tree:
+ Merge Join Operator
+ condition map:
+ Right Outer Join0 to 1
+ Right Outer Join0 to 2
+ keys:
+ 0 _col0 (type: string)
+ 1 _col0 (type: string)
+ 2 _col0 (type: string)
+ outputColumnNames: _col2, _col3
+ Statistics: Num rows: 1100 Data size: 193600 Basic stats: COMPLETE Column stats: NONE
+ Group By Operator
+ aggregations: sum(hash(_col2,_col3))
+ mode: hash
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ sort order:
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col0 (type: bigint)
+ Reducer 4
+ Reduce Operator Tree:
+ Group By Operator
+ aggregations: sum(VALUE._col0)
+ mode: mergepartial
+ outputColumnNames: _col0
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 1 Data size: 8 Basic stats: COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+ Execution mode: vectorized
+ Reducer 6
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string), KEY.reducesinkkey0 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Execution mode: vectorized
+ Reducer 8
+ Reduce Operator Tree:
+ Select Operator
+ expressions: VALUE._col0 (type: string)
+ outputColumnNames: _col0
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: string)
+ sort order: +
+ Map-reduce partition columns: _col0 (type: string)
+ Statistics: Num rows: 500 Data size: 88000 Basic stats: COMPLETE Column stats: NONE
+ Execution mode: vectorized
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+PREHOOK: type: QUERY
+PREHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+POSTHOOK: query: FROM
+(SELECT orcsrc.* FROM orcsrc sort by key) x
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Y
+ON (x.key = Y.key)
+RIGHT OUTER JOIN
+(SELECT orcsrc.* FROM orcsrc sort by value) Z
+ON (x.key = Z.key)
+select sum(hash(Y.key,Y.value))
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@orcsrc
+#### A masked pattern was here ####
+348019368476
http://git-wip-us.apache.org/repos/asf/hive/blob/2b9f2f5e/ql/src/test/results/clientpositive/tez/vector_join_filters.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vector_join_filters.q.out b/ql/src/test/results/clientpositive/tez/vector_join_filters.q.out
new file mode 100644
index 0000000..8cc9311
--- /dev/null
+++ b/ql/src/test/results/clientpositive/tez/vector_join_filters.q.out
@@ -0,0 +1,222 @@
+PREHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE myinput1_txt(key int, value int)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@myinput1_txt
+POSTHOOK: query: -- SORT_QUERY_RESULTS
+
+CREATE TABLE myinput1_txt(key int, value int)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@myinput1_txt
+PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE myinput1_txt
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@myinput1_txt
+POSTHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/in3.txt' INTO TABLE myinput1_txt
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@myinput1_txt
+PREHOOK: query: CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt
+PREHOOK: type: CREATETABLE_AS_SELECT
+PREHOOK: Input: default@myinput1_txt
+PREHOOK: Output: database:default
+PREHOOK: Output: default@myinput1
+POSTHOOK: query: CREATE TABLE myinput1 STORED AS ORC AS SELECT * FROM myinput1_txt
+POSTHOOK: type: CREATETABLE_AS_SELECT
+POSTHOOK: Input: default@myinput1_txt
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@myinput1
+Warning: Map Join MAPJOIN[19][bigTable=?] in task 'Map 1' is a cross product
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+Warning: Map Join MAPJOIN[15][bigTable=a] in task 'Map 1' is a cross product
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4937935
+Warning: Map Join MAPJOIN[15][bigTable=b] in task 'Map 2' is a cross product
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b on a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a JOIN myinput1 b ON a.value = b.value and a.key=b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4937935
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4937935
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4937935
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b ON a.key = b.key and a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+4937935
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key = b.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a RIGHT OUTER JOIN myinput1 b ON a.key=b.key and a.value = b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.value=c.value AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.value = c.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a LEFT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) RIGHT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) from myinput1 a RIGHT OUTER JOIN myinput1 b ON (a.value=b.value AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value) LEFT OUTER JOIN myinput1 c ON (b.key=c.key AND c.key > 40 AND c.value > 50 AND c.key = c.value AND b.key > 40 AND b.value > 50 AND b.key = b.value)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3080335
+PREHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+PREHOOK: type: QUERY
+PREHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT sum(hash(a.key,a.value,b.key,b.value)) FROM myinput1 a LEFT OUTER JOIN myinput1 b RIGHT OUTER JOIN myinput1 c ON a.value = b.value and b.key = c.key AND a.key > 40 AND a.value > 50 AND a.key = a.value AND b.key > 40 AND b.value > 50 AND b.key = b.value AND c.key > 40 AND c.value > 50 AND c.key = c.value
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@myinput1
+#### A masked pattern was here ####
+3078400
[14/50] [abbrv] hive git commit: HIVE-10679: JsonSerde ignores
varchar and char size limit specified during table creation (Jason Dere,
reviewed by Hari Subramaniyan)
Posted by sp...@apache.org.
HIVE-10679: JsonSerde ignores varchar and char size limit specified during table creation (Jason Dere, reviewed by Hari Subramaniyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3fa7489e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3fa7489e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3fa7489e
Branch: refs/heads/parquet
Commit: 3fa7489e2d6640d2b16e9c23754f2d5590dfe5e4
Parents: 2d87c12
Author: Hari Subramaniyan <ha...@apache.org>
Authored: Thu May 14 15:19:43 2015 -0700
Committer: Hari Subramaniyan <ha...@apache.org>
Committed: Thu May 14 15:19:43 2015 -0700
----------------------------------------------------------------------
.../data/HCatRecordObjectInspectorFactory.java | 2 +-
.../clientpositive/json_serde_qualified_types.q | 12 +++++++
.../json_serde_qualified_types.q.out | 33 ++++++++++++++++++++
.../PrimitiveObjectInspectorFactory.java | 4 +--
4 files changed, 48 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3fa7489e/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordObjectInspectorFactory.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordObjectInspectorFactory.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordObjectInspectorFactory.java
index 81daa2d..389395e 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordObjectInspectorFactory.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/HCatRecordObjectInspectorFactory.java
@@ -93,7 +93,7 @@ public class HCatRecordObjectInspectorFactory {
switch (typeInfo.getCategory()) {
case PRIMITIVE:
oi = PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(
- ((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory());
+ (PrimitiveTypeInfo) typeInfo);
break;
case STRUCT:
StructTypeInfo structTypeInfo = (StructTypeInfo) typeInfo;
http://git-wip-us.apache.org/repos/asf/hive/blob/3fa7489e/ql/src/test/queries/clientpositive/json_serde_qualified_types.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/json_serde_qualified_types.q b/ql/src/test/queries/clientpositive/json_serde_qualified_types.q
new file mode 100644
index 0000000..8968b9a
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/json_serde_qualified_types.q
@@ -0,0 +1,12 @@
+
+add jar ${system:maven.local.repository}/org/apache/hive/hcatalog/hive-hcatalog-core/${system:hive.version}/hive-hcatalog-core-${system:hive.version}.jar;
+
+create table json_serde_qualified_types (
+ c1 char(10),
+ c2 varchar(20),
+ c3 decimal(10, 5)
+) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe';
+
+describe json_serde_qualified_types;
+
+drop table json_serde_qualified_types;
http://git-wip-us.apache.org/repos/asf/hive/blob/3fa7489e/ql/src/test/results/clientpositive/json_serde_qualified_types.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/json_serde_qualified_types.q.out b/ql/src/test/results/clientpositive/json_serde_qualified_types.q.out
new file mode 100644
index 0000000..0e0ba4e
--- /dev/null
+++ b/ql/src/test/results/clientpositive/json_serde_qualified_types.q.out
@@ -0,0 +1,33 @@
+PREHOOK: query: create table json_serde_qualified_types (
+ c1 char(10),
+ c2 varchar(20),
+ c3 decimal(10, 5)
+) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@json_serde_qualified_types
+POSTHOOK: query: create table json_serde_qualified_types (
+ c1 char(10),
+ c2 varchar(20),
+ c3 decimal(10, 5)
+) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@json_serde_qualified_types
+PREHOOK: query: describe json_serde_qualified_types
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@json_serde_qualified_types
+POSTHOOK: query: describe json_serde_qualified_types
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@json_serde_qualified_types
+c1 char(10) from deserializer
+c2 varchar(20) from deserializer
+c3 decimal(10,5) from deserializer
+PREHOOK: query: drop table json_serde_qualified_types
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@json_serde_qualified_types
+PREHOOK: Output: default@json_serde_qualified_types
+POSTHOOK: query: drop table json_serde_qualified_types
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@json_serde_qualified_types
+POSTHOOK: Output: default@json_serde_qualified_types
http://git-wip-us.apache.org/repos/asf/hive/blob/3fa7489e/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java
----------------------------------------------------------------------
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java
index 7cd50ca..2ed0843 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/objectinspector/primitive/PrimitiveObjectInspectorFactory.java
@@ -391,8 +391,8 @@ public final class PrimitiveObjectInspectorFactory {
case DECIMAL:
result = new JavaHiveDecimalObjectInspector((DecimalTypeInfo)typeInfo);
break;
- default:
- throw new RuntimeException("Failed to create JavaHiveVarcharObjectInspector for " + typeInfo );
+ default:
+ throw new RuntimeException("Failed to create Java ObjectInspector for " + typeInfo);
}
AbstractPrimitiveJavaObjectInspector prev =
[41/50] [abbrv] hive git commit: HIVE-2327 Optimize REGEX UDFs with
constant parameter information (Alexander Pivovarov,
reviewed by Ashutosh Chauhan)
Posted by sp...@apache.org.
HIVE-2327 Optimize REGEX UDFs with constant parameter information (Alexander Pivovarov, reviewed by Ashutosh Chauhan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/312711b7
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/312711b7
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/312711b7
Branch: refs/heads/parquet
Commit: 312711b705b2af950c475572785fc19660ec1c38
Parents: c567a77
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Thu Apr 2 22:35:38 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Tue May 19 15:29:20 2015 -0700
----------------------------------------------------------------------
.../hadoop/hive/ql/exec/FunctionRegistry.java | 5 +-
.../hive/ql/optimizer/physical/Vectorizer.java | 3 +-
.../apache/hadoop/hive/ql/udf/UDFRegExp.java | 76 -----------
.../hive/ql/udf/generic/GenericUDFRegExp.java | 133 ++++++++++++++++++
.../ql/udf/generic/TestGenericUDFRegexp.java | 135 +++++++++++++++++++
.../spark/vectorization_short_regress.q.out | 8 +-
.../tez/vectorization_short_regress.q.out | 8 +-
.../vectorization_short_regress.q.out | 8 +-
8 files changed, 283 insertions(+), 93 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/312711b7/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
index 7ce0a1c..9abe15e 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
@@ -81,7 +81,6 @@ import org.apache.hadoop.hive.ql.udf.UDFPI;
import org.apache.hadoop.hive.ql.udf.UDFParseUrl;
import org.apache.hadoop.hive.ql.udf.UDFRadians;
import org.apache.hadoop.hive.ql.udf.UDFRand;
-import org.apache.hadoop.hive.ql.udf.UDFRegExp;
import org.apache.hadoop.hive.ql.udf.UDFRegExpExtract;
import org.apache.hadoop.hive.ql.udf.UDFRegExpReplace;
import org.apache.hadoop.hive.ql.udf.UDFRepeat;
@@ -249,8 +248,8 @@ public final class FunctionRegistry {
system.registerGenericUDF("initcap", GenericUDFInitCap.class);
system.registerUDF("like", UDFLike.class, true);
- system.registerUDF("rlike", UDFRegExp.class, true);
- system.registerUDF("regexp", UDFRegExp.class, true);
+ system.registerGenericUDF("rlike", GenericUDFRegExp.class);
+ system.registerGenericUDF("regexp", GenericUDFRegExp.class);
system.registerUDF("regexp_replace", UDFRegExpReplace.class, false);
system.registerUDF("regexp_extract", UDFRegExpExtract.class, false);
system.registerUDF("parse_url", UDFParseUrl.class, false);
http://git-wip-us.apache.org/repos/asf/hive/blob/312711b7/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
index 656a5e3..705b185 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/Vectorizer.java
@@ -119,7 +119,6 @@ import org.apache.hadoop.hive.ql.udf.UDFMinute;
import org.apache.hadoop.hive.ql.udf.UDFMonth;
import org.apache.hadoop.hive.ql.udf.UDFRadians;
import org.apache.hadoop.hive.ql.udf.UDFRand;
-import org.apache.hadoop.hive.ql.udf.UDFRegExp;
import org.apache.hadoop.hive.ql.udf.UDFSecond;
import org.apache.hadoop.hive.ql.udf.UDFSign;
import org.apache.hadoop.hive.ql.udf.UDFSin;
@@ -227,7 +226,7 @@ public class Vectorizer implements PhysicalPlanResolver {
supportedGenericUDFs.add(GenericUDFDateDiff.class);
supportedGenericUDFs.add(UDFLike.class);
- supportedGenericUDFs.add(UDFRegExp.class);
+ supportedGenericUDFs.add(GenericUDFRegExp.class);
supportedGenericUDFs.add(UDFSubstr.class);
supportedGenericUDFs.add(GenericUDFLTrim.class);
supportedGenericUDFs.add(GenericUDFRTrim.class);
http://git-wip-us.apache.org/repos/asf/hive/blob/312711b7/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRegExp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRegExp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRegExp.java
deleted file mode 100755
index 76e1d2e..0000000
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFRegExp.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.ql.udf;
-
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hive.ql.exec.Description;
-import org.apache.hadoop.hive.ql.exec.UDF;
-import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions;
-import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterStringColRegExpStringScalar;
-import org.apache.hadoop.io.BooleanWritable;
-import org.apache.hadoop.io.Text;
-
-/**
- * UDFRegExp.
- *
- */
-@Description(name = "rlike,regexp",
- value = "str _FUNC_ regexp - Returns true if str matches regexp and "
- + "false otherwise", extended = "Example:\n"
- + " > SELECT 'fb' _FUNC_ '.*' FROM src LIMIT 1;\n" + " true")
-@VectorizedExpressions({FilterStringColRegExpStringScalar.class})
-public class UDFRegExp extends UDF {
- static final Log LOG = LogFactory.getLog(UDFRegExp.class.getName());
-
- private final Text lastRegex = new Text();
- private Pattern p = null;
- private boolean warned = false;
-
- private final BooleanWritable result = new BooleanWritable();
-
- public UDFRegExp() {
- }
-
- public BooleanWritable evaluate(Text s, Text regex) {
- if (s == null || regex == null) {
- return null;
- }
- if (regex.getLength() == 0) {
- if (!warned) {
- warned = true;
- LOG.warn(getClass().getSimpleName() + " regex is empty. Additional "
- + "warnings for an empty regex will be suppressed.");
- }
- result.set(false);
- return result;
- }
- if (!regex.equals(lastRegex) || p == null) {
- lastRegex.set(regex);
- p = Pattern.compile(regex.toString());
- }
- Matcher m = p.matcher(s.toString());
- result.set(m.find(0));
- return result;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hive/blob/312711b7/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRegExp.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRegExp.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRegExp.java
new file mode 100644
index 0000000..0a9dd7b
--- /dev/null
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFRegExp.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import static org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorUtils.PrimitiveGrouping.STRING_GROUP;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hive.ql.exec.Description;
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedExpressions;
+import org.apache.hadoop.hive.ql.exec.vector.expressions.FilterStringColRegExpStringScalar;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.BooleanWritable;
+
+/**
+ * UDF to extract a specific group identified by a java regex. Note that if a
+ * regexp has a backslash ('\'), then need to specify '\\' For example,
+ * regexp_extract('100-200', '(\\d+)-(\\d+)', 1) will return '100'
+ */
+@Description(name = "rlike,regexp",
+ value = "str _FUNC_ regexp - Returns true if str matches regexp and "
+ + "false otherwise", extended = "Example:\n"
+ + " > SELECT 'fb' _FUNC_ '.*' FROM src LIMIT 1;\n" + " true")
+@VectorizedExpressions({FilterStringColRegExpStringScalar.class})
+public class GenericUDFRegExp extends GenericUDF {
+ static final Log LOG = LogFactory.getLog(GenericUDFRegExp.class.getName());
+ private transient PrimitiveCategory[] inputTypes = new PrimitiveCategory[2];
+ private transient Converter[] converters = new Converter[2];
+ private final BooleanWritable output = new BooleanWritable();
+ private transient boolean isRegexConst;
+ private transient String regexConst;
+ private transient Pattern patternConst;
+ private transient boolean warned;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
+ checkArgsSize(arguments, 2, 2);
+
+ checkArgPrimitive(arguments, 0);
+ checkArgPrimitive(arguments, 1);
+
+ checkArgGroups(arguments, 0, inputTypes, STRING_GROUP);
+ checkArgGroups(arguments, 1, inputTypes, STRING_GROUP);
+
+ obtainStringConverter(arguments, 0, inputTypes, converters);
+ obtainStringConverter(arguments, 1, inputTypes, converters);
+
+ if (arguments[1] instanceof ConstantObjectInspector) {
+ regexConst = getConstantStringValue(arguments, 1);
+ if (regexConst != null) {
+ patternConst = Pattern.compile(regexConst);
+ }
+ isRegexConst = true;
+ }
+
+ ObjectInspector outputOI = PrimitiveObjectInspectorFactory.writableBooleanObjectInspector;
+ return outputOI;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments) throws HiveException {
+ String s = getStringValue(arguments, 0, converters);
+ if (s == null) {
+ return null;
+ }
+
+ String regex;
+ if (isRegexConst) {
+ regex = regexConst;
+ } else {
+ regex = getStringValue(arguments, 1, converters);
+ }
+ if (regex == null) {
+ return null;
+ }
+
+ if (regex.length() == 0) {
+ if (!warned) {
+ warned = true;
+ LOG.warn(getClass().getSimpleName() + " regex is empty. Additional "
+ + "warnings for an empty regex will be suppressed.");
+ }
+ output.set(false);
+ return output;
+ }
+
+ Pattern p;
+ if (isRegexConst) {
+ p = patternConst;
+ } else {
+ p = Pattern.compile(regex);
+ }
+
+ Matcher m = p.matcher(s);
+ output.set(m.find(0));
+ return output;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return children[0] + " regexp " + children[1];
+ }
+
+ @Override
+ protected String getFuncName() {
+ return "regexp";
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/312711b7/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFRegexp.java
----------------------------------------------------------------------
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFRegexp.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFRegexp.java
new file mode 100644
index 0000000..4e3be90
--- /dev/null
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFRegexp.java
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.ql.udf.generic;
+
+import junit.framework.TestCase;
+
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.BooleanWritable;
+import org.apache.hadoop.io.Text;
+
+public class TestGenericUDFRegexp extends TestCase {
+
+ public void testConstant() throws HiveException {
+ GenericUDFRegExp udf = new GenericUDFRegExp();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ Text regexText = new Text("^fo");
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.stringTypeInfo, regexText);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ runAndVerifyConst("fofo", regexText, true, udf);
+ runAndVerifyConst("fofofo", regexText, true, udf);
+ runAndVerifyConst("fobar", regexText, true, udf);
+ runAndVerifyConst("barfobar", regexText, false, udf);
+ // null
+ runAndVerifyConst(null, regexText, null, udf);
+ }
+
+ public void testEmptyConstant() throws HiveException {
+ GenericUDFRegExp udf = new GenericUDFRegExp();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ Text regexText = new Text("");
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.stringTypeInfo, regexText);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ // empty regex (should be one WARN message)
+ runAndVerifyConst("foo", regexText, false, udf);
+ runAndVerifyConst("bar", regexText, false, udf);
+ // null
+ runAndVerifyConst(null, regexText, null, udf);
+ }
+
+ public void testNullConstant() throws HiveException {
+ GenericUDFRegExp udf = new GenericUDFRegExp();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ Text regexText = null;
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory
+ .getPrimitiveWritableConstantObjectInspector(TypeInfoFactory.stringTypeInfo, regexText);
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+ // null
+ runAndVerifyConst("fofo", regexText, null, udf);
+ runAndVerifyConst("fofofo", regexText, null, udf);
+ runAndVerifyConst("fobar", regexText, null, udf);
+ runAndVerifyConst(null, regexText, null, udf);
+ }
+
+ public void testNonConstant() throws HiveException {
+ GenericUDFRegExp udf = new GenericUDFRegExp();
+ ObjectInspector valueOI0 = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ ObjectInspector valueOI1 = PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ ObjectInspector[] arguments = { valueOI0, valueOI1 };
+
+ udf.initialize(arguments);
+
+ runAndVerify("fofo", "^fo", true, udf);
+ runAndVerify("fo\no", "^fo\no$", true, udf);
+ runAndVerify("Bn", "^Ba*n", true, udf);
+ runAndVerify("afofo", "fo", true, udf);
+ runAndVerify("afofo", "^fo", false, udf);
+ runAndVerify("Baan", "^Ba?n", false, udf);
+ runAndVerify("axe", "pi|apa", false, udf);
+ runAndVerify("pip", "^(pi)*$", false, udf);
+ // empty regex (should be one WARN message)
+ runAndVerify("bar", "", false, udf);
+ runAndVerify("foo", "", false, udf);
+ // null
+ runAndVerify(null, "^fo", null, udf);
+ runAndVerify("fofo", null, null, udf);
+ }
+
+ private void runAndVerifyConst(String str, Text regexText, Boolean expResult, GenericUDF udf)
+ throws HiveException {
+ DeferredObject valueObj0 = new DeferredJavaObject(str != null ? new Text(str) : null);
+ DeferredObject valueObj1 = new DeferredJavaObject(regexText);
+ DeferredObject[] args = { valueObj0, valueObj1 };
+ BooleanWritable output = (BooleanWritable) udf.evaluate(args);
+ if (expResult == null) {
+ assertNull(output);
+ } else {
+ assertNotNull(output);
+ assertEquals("regexp() const test ", expResult.booleanValue(), output.get());
+ }
+ }
+
+ private void runAndVerify(String str, String regex, Boolean expResult, GenericUDF udf)
+ throws HiveException {
+ DeferredObject valueObj0 = new DeferredJavaObject(str != null ? new Text(str) : null);
+ DeferredObject valueObj1 = new DeferredJavaObject(regex != null ? new Text(regex) : null);
+ DeferredObject[] args = { valueObj0, valueObj1 };
+ BooleanWritable output = (BooleanWritable) udf.evaluate(args);
+ if (expResult == null) {
+ assertNull(output);
+ } else {
+ assertNotNull(output);
+ assertEquals("regexp() test ", expResult.booleanValue(), output.get());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hive/blob/312711b7/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
index a4b8e05..25eb161 100644
--- a/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/spark/vectorization_short_regress.q.out
@@ -360,7 +360,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((((cbigint <= 197) and (cint < cbigint)) or ((cdouble >= -26.28) and (csmallint > cdouble))) or ((ctinyint > cfloat) and (cstring1 rlike '.*ss.*'))) or ((cfloat > 79.553) and (cstring2 like '10%'))) (type: boolean)
+ predicate: (((((cbigint <= 197) and (cint < cbigint)) or ((cdouble >= -26.28) and (csmallint > cdouble))) or ((ctinyint > cfloat) and cstring1 regexp '.*ss.*')) or ((cfloat > 79.553) and (cstring2 like '10%'))) (type: boolean)
Statistics: Num rows: 6826 Data size: 209555 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cbigint (type: bigint), csmallint (type: smallint), cdouble (type: double), ctinyint (type: tinyint)
@@ -935,7 +935,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((((cstring1 rlike 'a.*') and (cstring2 like '%ss%')) or ((1 <> cboolean2) and ((csmallint < 79.553) and (-257 <> ctinyint)))) or ((cdouble > ctinyint) and (cfloat >= cint))) or ((cint < cbigint) and (ctinyint > cbigint))) (type: boolean)
+ predicate: ((((cstring1 regexp 'a.*' and (cstring2 like '%ss%')) or ((1 <> cboolean2) and ((csmallint < 79.553) and (-257 <> ctinyint)))) or ((cdouble > ctinyint) and (cfloat >= cint))) or ((cint < cbigint) and (ctinyint > cbigint))) (type: boolean)
Statistics: Num rows: 9898 Data size: 303864 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cdouble (type: double), ctimestamp2 (type: timestamp), cstring1 (type: string), cboolean2 (type: boolean), ctinyint (type: tinyint), cfloat (type: float), ctimestamp1 (type: timestamp), csmallint (type: smallint), cbigint (type: bigint), (-3728 * cbigint) (type: bigint), (- cint) (type: int), (-863.257 - cint) (type: double), (- csmallint) (type: smallint), (csmallint - (- csmallint)) (type: smallint), ((csmallint - (- csmallint)) + (- csmallint)) (type: smallint), (cint / cint) (type: double), ((-863.257 - cint) - -26.28) (type: double), (- cfloat) (type: float), (cdouble * -89010) (type: double), (ctinyint / 988888) (type: double), (- ctinyint) (type: tinyint), (79.553 / ctinyint) (type: double)
@@ -2339,7 +2339,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((ctimestamp1 <> 0) and (((((((-257 <> ctinyint) and cboolean2 is not null) and ((cstring1 rlike '.*ss') and (-3 < ctimestamp1))) or (ctimestamp2 = -5)) or ((ctimestamp1 < 0) and (cstring2 like '%b%'))) or (cdouble = cint)) or (cboolean1 is null and (cfloat < cint)))) (type: boolean)
+ predicate: ((ctimestamp1 <> 0) and (((((((-257 <> ctinyint) and cboolean2 is not null) and (cstring1 regexp '.*ss' and (-3 < ctimestamp1))) or (ctimestamp2 = -5)) or ((ctimestamp1 < 0) and (cstring2 like '%b%'))) or (cdouble = cint)) or (cboolean1 is null and (cfloat < cint)))) (type: boolean)
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctimestamp1 (type: timestamp), cstring1 (type: string), cint (type: int), csmallint (type: smallint), ctinyint (type: tinyint), cfloat (type: float), cdouble (type: double)
@@ -2672,7 +2672,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 377237 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cboolean1 is not null and (((((cdouble < csmallint) and ((cboolean2 = cboolean1) and (cbigint <= -863.257))) or ((cint >= -257) and (cstring1 is not null and (cboolean1 >= 1)))) or (cstring2 rlike 'b')) or ((csmallint >= ctinyint) and ctimestamp2 is null))) (type: boolean)
+ predicate: (cboolean1 is not null and (((((cdouble < csmallint) and ((cboolean2 = cboolean1) and (cbigint <= -863.257))) or ((cint >= -257) and (cstring1 is not null and (cboolean1 >= 1)))) or cstring2 regexp 'b') or ((csmallint >= ctinyint) and ctimestamp2 is null))) (type: boolean)
Statistics: Num rows: 4778 Data size: 146682 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cboolean1 (type: boolean), cfloat (type: float), cbigint (type: bigint), cint (type: int), cdouble (type: double), ctinyint (type: tinyint), csmallint (type: smallint)
http://git-wip-us.apache.org/repos/asf/hive/blob/312711b7/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
index a3c723d..bf01f78 100644
--- a/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/tez/vectorization_short_regress.q.out
@@ -360,7 +360,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((cbigint <= 197) and (UDFToLong(cint) < cbigint)) or (((cdouble >= -26.28) and (UDFToDouble(csmallint) > cdouble)) or (((UDFToFloat(ctinyint) > cfloat) and (cstring1 rlike '.*ss.*')) or ((cfloat > 79.553) and (cstring2 like '10%'))))) (type: boolean)
+ predicate: (((cbigint <= 197) and (UDFToLong(cint) < cbigint)) or (((cdouble >= -26.28) and (UDFToDouble(csmallint) > cdouble)) or (((UDFToFloat(ctinyint) > cfloat) and cstring1 regexp '.*ss.*') or ((cfloat > 79.553) and (cstring2 like '10%'))))) (type: boolean)
Statistics: Num rows: 6826 Data size: 1467614 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cbigint (type: bigint), csmallint (type: smallint), cdouble (type: double), ctinyint (type: tinyint)
@@ -935,7 +935,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((cstring1 rlike 'a.*') and (cstring2 like '%ss%')) or (((1 <> cboolean2) and ((UDFToDouble(csmallint) < 79.553) and (-257 <> UDFToInteger(ctinyint)))) or (((cdouble > UDFToDouble(ctinyint)) and (cfloat >= UDFToFloat(cint))) or ((UDFToLong(cint) < cbigint) and (UDFToLong(ctinyint) > cbigint))))) (type: boolean)
+ predicate: ((cstring1 regexp 'a.*' and (cstring2 like '%ss%')) or (((1 <> cboolean2) and ((UDFToDouble(csmallint) < 79.553) and (-257 <> UDFToInteger(ctinyint)))) or (((cdouble > UDFToDouble(ctinyint)) and (cfloat >= UDFToFloat(cint))) or ((UDFToLong(cint) < cbigint) and (UDFToLong(ctinyint) > cbigint))))) (type: boolean)
Statistics: Num rows: 9898 Data size: 2128105 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cdouble (type: double), ctimestamp2 (type: timestamp), cstring1 (type: string), cboolean2 (type: boolean), ctinyint (type: tinyint), cfloat (type: float), ctimestamp1 (type: timestamp), csmallint (type: smallint), cbigint (type: bigint), (-3728 * cbigint) (type: bigint), (- cint) (type: int), (-863.257 - UDFToDouble(cint)) (type: double), (- csmallint) (type: smallint), (csmallint - (- csmallint)) (type: smallint), ((csmallint - (- csmallint)) + (- csmallint)) (type: smallint), (UDFToDouble(cint) / UDFToDouble(cint)) (type: double), ((-863.257 - UDFToDouble(cint)) - -26.28) (type: double), (- cfloat) (type: float), (cdouble * -89010.0) (type: double), (UDFToDouble(ctinyint) / 988888.0) (type: double), (- ctinyint) (type: tinyint), (79.553 / UDFToDouble(ctinyint)) (type: double)
@@ -2339,7 +2339,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(ctimestamp1) <> 0.0) and (((-257 <> UDFToInteger(ctinyint)) and (cboolean2 is not null and ((cstring1 rlike '.*ss') and (-3.0 < UDFToDouble(ctimestamp1))))) or ((UDFToDouble(ctimestamp2) = -5.0) or (((UDFToDouble(ctimestamp1) < 0.0) and (cstring2 like '%b%')) or ((cdouble = UDFToDouble(cint)) or (cboolean1 is null and (cfloat < UDFToFloat(cint)))))))) (type: boolean)
+ predicate: ((UDFToDouble(ctimestamp1) <> 0.0) and (((-257 <> UDFToInteger(ctinyint)) and (cboolean2 is not null and (cstring1 regexp '.*ss' and (-3.0 < UDFToDouble(ctimestamp1))))) or ((UDFToDouble(ctimestamp2) = -5.0) or (((UDFToDouble(ctimestamp1) < 0.0) and (cstring2 like '%b%')) or ((cdouble = UDFToDouble(cint)) or (cboolean1 is null and (cfloat < UDFToFloat(cint)))))))) (type: boolean)
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctimestamp1 (type: timestamp), cstring1 (type: string), cint (type: int), csmallint (type: smallint), ctinyint (type: tinyint), cfloat (type: float), cdouble (type: double)
@@ -2672,7 +2672,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cboolean1 is not null and (((cdouble < UDFToDouble(csmallint)) and ((cboolean2 = cboolean1) and (UDFToDouble(cbigint) <= -863.257))) or (((cint >= -257) and (cstring1 is not null and (cboolean1 >= 1))) or ((cstring2 rlike 'b') or ((csmallint >= UDFToShort(ctinyint)) and ctimestamp2 is null))))) (type: boolean)
+ predicate: (cboolean1 is not null and (((cdouble < UDFToDouble(csmallint)) and ((cboolean2 = cboolean1) and (UDFToDouble(cbigint) <= -863.257))) or (((cint >= -257) and (cstring1 is not null and (cboolean1 >= 1))) or (cstring2 regexp 'b' or ((csmallint >= UDFToShort(ctinyint)) and ctimestamp2 is null))))) (type: boolean)
Statistics: Num rows: 4778 Data size: 1027287 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cboolean1 (type: boolean), cfloat (type: float), cbigint (type: bigint), cint (type: int), cdouble (type: double), ctinyint (type: tinyint), csmallint (type: smallint)
http://git-wip-us.apache.org/repos/asf/hive/blob/312711b7/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/vectorization_short_regress.q.out b/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
index b9ab174..b823d4b 100644
--- a/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
+++ b/ql/src/test/results/clientpositive/vectorization_short_regress.q.out
@@ -349,7 +349,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((cbigint <= 197) and (UDFToLong(cint) < cbigint)) or (((cdouble >= -26.28) and (UDFToDouble(csmallint) > cdouble)) or (((UDFToFloat(ctinyint) > cfloat) and (cstring1 rlike '.*ss.*')) or ((cfloat > 79.553) and (cstring2 like '10%'))))) (type: boolean)
+ predicate: (((cbigint <= 197) and (UDFToLong(cint) < cbigint)) or (((cdouble >= -26.28) and (UDFToDouble(csmallint) > cdouble)) or (((UDFToFloat(ctinyint) > cfloat) and cstring1 regexp '.*ss.*') or ((cfloat > 79.553) and (cstring2 like '10%'))))) (type: boolean)
Statistics: Num rows: 6826 Data size: 1467614 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cbigint (type: bigint), csmallint (type: smallint), cdouble (type: double), ctinyint (type: tinyint)
@@ -906,7 +906,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (((cstring1 rlike 'a.*') and (cstring2 like '%ss%')) or (((1 <> cboolean2) and ((UDFToDouble(csmallint) < 79.553) and (-257 <> UDFToInteger(ctinyint)))) or (((cdouble > UDFToDouble(ctinyint)) and (cfloat >= UDFToFloat(cint))) or ((UDFToLong(cint) < cbigint) and (UDFToLong(ctinyint) > cbigint))))) (type: boolean)
+ predicate: ((cstring1 regexp 'a.*' and (cstring2 like '%ss%')) or (((1 <> cboolean2) and ((UDFToDouble(csmallint) < 79.553) and (-257 <> UDFToInteger(ctinyint)))) or (((cdouble > UDFToDouble(ctinyint)) and (cfloat >= UDFToFloat(cint))) or ((UDFToLong(cint) < cbigint) and (UDFToLong(ctinyint) > cbigint))))) (type: boolean)
Statistics: Num rows: 9898 Data size: 2128105 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cint (type: int), cdouble (type: double), ctimestamp2 (type: timestamp), cstring1 (type: string), cboolean2 (type: boolean), ctinyint (type: tinyint), cfloat (type: float), ctimestamp1 (type: timestamp), csmallint (type: smallint), cbigint (type: bigint), (-3728 * cbigint) (type: bigint), (- cint) (type: int), (-863.257 - UDFToDouble(cint)) (type: double), (- csmallint) (type: smallint), (csmallint - (- csmallint)) (type: smallint), ((csmallint - (- csmallint)) + (- csmallint)) (type: smallint), (UDFToDouble(cint) / UDFToDouble(cint)) (type: double), ((-863.257 - UDFToDouble(cint)) - -26.28) (type: double), (- cfloat) (type: float), (cdouble * -89010.0) (type: double), (UDFToDouble(ctinyint) / 988888.0) (type: double), (- ctinyint) (type: tinyint), (79.553 / UDFToDouble(ctinyint)) (type: double)
@@ -2288,7 +2288,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: ((UDFToDouble(ctimestamp1) <> 0.0) and (((-257 <> UDFToInteger(ctinyint)) and (cboolean2 is not null and ((cstring1 rlike '.*ss') and (-3.0 < UDFToDouble(ctimestamp1))))) or ((UDFToDouble(ctimestamp2) = -5.0) or (((UDFToDouble(ctimestamp1) < 0.0) and (cstring2 like '%b%')) or ((cdouble = UDFToDouble(cint)) or (cboolean1 is null and (cfloat < UDFToFloat(cint)))))))) (type: boolean)
+ predicate: ((UDFToDouble(ctimestamp1) <> 0.0) and (((-257 <> UDFToInteger(ctinyint)) and (cboolean2 is not null and (cstring1 regexp '.*ss' and (-3.0 < UDFToDouble(ctimestamp1))))) or ((UDFToDouble(ctimestamp2) = -5.0) or (((UDFToDouble(ctimestamp1) < 0.0) and (cstring2 like '%b%')) or ((cdouble = UDFToDouble(cint)) or (cboolean1 is null and (cfloat < UDFToFloat(cint)))))))) (type: boolean)
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: ctimestamp1 (type: timestamp), cstring1 (type: string), cint (type: int), csmallint (type: smallint), ctinyint (type: tinyint), cfloat (type: float), cdouble (type: double)
@@ -2624,7 +2624,7 @@ STAGE PLANS:
alias: alltypesorc
Statistics: Num rows: 12288 Data size: 2641964 Basic stats: COMPLETE Column stats: NONE
Filter Operator
- predicate: (cboolean1 is not null and (((cdouble < UDFToDouble(csmallint)) and ((cboolean2 = cboolean1) and (UDFToDouble(cbigint) <= -863.257))) or (((cint >= -257) and (cstring1 is not null and (cboolean1 >= 1))) or ((cstring2 rlike 'b') or ((csmallint >= UDFToShort(ctinyint)) and ctimestamp2 is null))))) (type: boolean)
+ predicate: (cboolean1 is not null and (((cdouble < UDFToDouble(csmallint)) and ((cboolean2 = cboolean1) and (UDFToDouble(cbigint) <= -863.257))) or (((cint >= -257) and (cstring1 is not null and (cboolean1 >= 1))) or (cstring2 regexp 'b' or ((csmallint >= UDFToShort(ctinyint)) and ctimestamp2 is null))))) (type: boolean)
Statistics: Num rows: 4778 Data size: 1027287 Basic stats: COMPLETE Column stats: NONE
Select Operator
expressions: cboolean1 (type: boolean), cfloat (type: float), cbigint (type: bigint), cint (type: int), cdouble (type: double), ctinyint (type: tinyint), csmallint (type: smallint)
[46/50] [abbrv] hive git commit: HIVE-10665 Continue to make
udaf_percentile_approx_23.q test more stable (Alexander Pivovarov,
reviewed by Jason Dere and Swarnim Kulkarni)
Posted by sp...@apache.org.
HIVE-10665 Continue to make udaf_percentile_approx_23.q test more stable (Alexander Pivovarov, reviewed by Jason Dere and Swarnim Kulkarni)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/3d95e106
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/3d95e106
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/3d95e106
Branch: refs/heads/parquet
Commit: 3d95e1061883ab9d7265ac7d678f103c4565fc9c
Parents: d117612
Author: Alexander Pivovarov <ap...@gmail.com>
Authored: Wed May 13 17:42:12 2015 -0700
Committer: Alexander Pivovarov <ap...@gmail.com>
Committed: Tue May 19 23:15:39 2015 -0700
----------------------------------------------------------------------
ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q | 2 +-
.../results/clientpositive/udaf_percentile_approx_23.q.out | 6 +++---
2 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/3d95e106/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q b/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q
index b345b37..028acdb 100644
--- a/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q
+++ b/ql/src/test/queries/clientpositive/udaf_percentile_approx_23.q
@@ -90,7 +90,7 @@ set hive.cbo.enable=false;
-- NaN
explain
select percentile_approx(case when key < 100 then cast('NaN' as double) else key end, 0.5) from bucket;
-select percentile_approx(case when key < 100 then cast('NaN' as double) else key end, 0.5) from bucket;
+select percentile_approx(case when key < 100 then cast('NaN' as double) else key end, 0.5) between 340.5 and 343.0 from bucket;
-- with CBO
explain
http://git-wip-us.apache.org/repos/asf/hive/blob/3d95e106/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out b/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out
index 022a884..14a6874 100644
--- a/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out
+++ b/ql/src/test/results/clientpositive/udaf_percentile_approx_23.q.out
@@ -556,15 +556,15 @@ STAGE PLANS:
Processor Tree:
ListSink
-PREHOOK: query: select percentile_approx(case when key < 100 then cast('NaN' as double) else key end, 0.5) from bucket
+PREHOOK: query: select percentile_approx(case when key < 100 then cast('NaN' as double) else key end, 0.5) between 340.5 and 343.0 from bucket
PREHOOK: type: QUERY
PREHOOK: Input: default@bucket
#### A masked pattern was here ####
-POSTHOOK: query: select percentile_approx(case when key < 100 then cast('NaN' as double) else key end, 0.5) from bucket
+POSTHOOK: query: select percentile_approx(case when key < 100 then cast('NaN' as double) else key end, 0.5) between 340.5 and 343.0 from bucket
POSTHOOK: type: QUERY
POSTHOOK: Input: default@bucket
#### A masked pattern was here ####
-341.5
+true
PREHOOK: query: -- with CBO
explain
select percentile_approx(key, 0.5) from bucket