You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by yc...@apache.org on 2016/09/12 13:08:14 UTC
hive git commit: HIVE-14715: Hive throws NumberFormatException with
query with Null value (Yongzhi Chen, reviewed by Aihua Xu)
Repository: hive
Updated Branches:
refs/heads/master bc75e46ad -> d4ffa5d3d
HIVE-14715: Hive throws NumberFormatException with query with Null value (Yongzhi Chen, reviewed by Aihua Xu)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/d4ffa5d3
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/d4ffa5d3
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/d4ffa5d3
Branch: refs/heads/master
Commit: d4ffa5d3d8c272e33754335aa5174ed5fe092f87
Parents: bc75e46
Author: Yongzhi Chen <yc...@apache.org>
Authored: Wed Sep 7 21:30:01 2016 -0400
Committer: Yongzhi Chen <yc...@apache.org>
Committed: Mon Sep 12 09:06:57 2016 -0400
----------------------------------------------------------------------
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 3 +-
.../queries/clientpositive/groupby_nullvalues.q | 29 +++++++
.../clientpositive/groupby_nullvalues.q.out | 83 ++++++++++++++++++++
3 files changed, 114 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/d4ffa5d3/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index e4a37f4..b071ddf 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -5128,6 +5128,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
colExprMap);
int keyLength = reduceKeys.size();
+ int numOfColsRmedFromkey = grpByExprs.size() - keyLength;
// add a key for reduce sink
if (groupingSetsPresent) {
@@ -5157,7 +5158,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
reduceSinkOutputRowResolver, outputValueColumnNames, reduceValues, colExprMap);
} else {
// Put partial aggregation results in reduceValues
- int inputField = reduceKeys.size();
+ int inputField = reduceKeys.size() + numOfColsRmedFromkey;
for (Map.Entry<String, ASTNode> entry : aggregationTrees.entrySet()) {
http://git-wip-us.apache.org/repos/asf/hive/blob/d4ffa5d3/ql/src/test/queries/clientpositive/groupby_nullvalues.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/groupby_nullvalues.q b/ql/src/test/queries/clientpositive/groupby_nullvalues.q
new file mode 100644
index 0000000..85ccb38
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/groupby_nullvalues.q
@@ -0,0 +1,29 @@
+set hive.cbo.enable=false;
+drop table if exists paqtest;
+CREATE TABLE paqtest(
+c1 int,
+s1 string,
+s2 string,
+bn1 bigint)
+ROW FORMAT SERDE
+'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat';
+
+insert into paqtest values (58, '', 'ABC', 0);
+
+SELECT
+'PM' AS cy,
+c1,
+NULL AS iused,
+NULL AS itp,
+s2,
+NULL AS cvg,
+NULL AS acavg,
+sum(bn1) AS cca
+FROM paqtest
+WHERE (s1 IS NULL OR length(s1) = 0)
+GROUP BY 'Pricing mismatch', c1, NULL, NULL, s2, NULL, NULL;
+drop table paqtest;
http://git-wip-us.apache.org/repos/asf/hive/blob/d4ffa5d3/ql/src/test/results/clientpositive/groupby_nullvalues.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/groupby_nullvalues.q.out b/ql/src/test/results/clientpositive/groupby_nullvalues.q.out
new file mode 100644
index 0000000..aae2a12
--- /dev/null
+++ b/ql/src/test/results/clientpositive/groupby_nullvalues.q.out
@@ -0,0 +1,83 @@
+PREHOOK: query: drop table if exists paqtest
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists paqtest
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE paqtest(
+c1 int,
+s1 string,
+s2 string,
+bn1 bigint)
+ROW FORMAT SERDE
+'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@paqtest
+POSTHOOK: query: CREATE TABLE paqtest(
+c1 int,
+s1 string,
+s2 string,
+bn1 bigint)
+ROW FORMAT SERDE
+'org.apache.hadoop.hive.ql.io.parquet.serde.ParquetHiveSerDe'
+STORED AS INPUTFORMAT
+'org.apache.hadoop.hive.ql.io.parquet.MapredParquetInputFormat'
+OUTPUTFORMAT
+'org.apache.hadoop.hive.ql.io.parquet.MapredParquetOutputFormat'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@paqtest
+PREHOOK: query: insert into paqtest values (58, '', 'ABC', 0)
+PREHOOK: type: QUERY
+PREHOOK: Input: default@values__tmp__table__1
+PREHOOK: Output: default@paqtest
+POSTHOOK: query: insert into paqtest values (58, '', 'ABC', 0)
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@values__tmp__table__1
+POSTHOOK: Output: default@paqtest
+POSTHOOK: Lineage: paqtest.bn1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col4, type:string, comment:), ]
+POSTHOOK: Lineage: paqtest.c1 EXPRESSION [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col1, type:string, comment:), ]
+POSTHOOK: Lineage: paqtest.s1 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col2, type:string, comment:), ]
+POSTHOOK: Lineage: paqtest.s2 SIMPLE [(values__tmp__table__1)values__tmp__table__1.FieldSchema(name:tmp_values_col3, type:string, comment:), ]
+PREHOOK: query: SELECT
+'PM' AS cy,
+c1,
+NULL AS iused,
+NULL AS itp,
+s2,
+NULL AS cvg,
+NULL AS acavg,
+sum(bn1) AS cca
+FROM paqtest
+WHERE (s1 IS NULL OR length(s1) = 0)
+GROUP BY 'Pricing mismatch', c1, NULL, NULL, s2, NULL, NULL
+PREHOOK: type: QUERY
+PREHOOK: Input: default@paqtest
+#### A masked pattern was here ####
+POSTHOOK: query: SELECT
+'PM' AS cy,
+c1,
+NULL AS iused,
+NULL AS itp,
+s2,
+NULL AS cvg,
+NULL AS acavg,
+sum(bn1) AS cca
+FROM paqtest
+WHERE (s1 IS NULL OR length(s1) = 0)
+GROUP BY 'Pricing mismatch', c1, NULL, NULL, s2, NULL, NULL
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@paqtest
+#### A masked pattern was here ####
+PM 58 NULL NULL ABC NULL NULL 0
+PREHOOK: query: drop table paqtest
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@paqtest
+PREHOOK: Output: default@paqtest
+POSTHOOK: query: drop table paqtest
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@paqtest
+POSTHOOK: Output: default@paqtest