You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by yc...@apache.org on 2018/03/20 17:39:07 UTC
hive git commit: HIVE-18975: NPE when inserting NULL value in
structure and array with HBase table (Oleksiy Sayankin,
reviewed by Ashutosh Chauhan and Yongzhi Chen)
Repository: hive
Updated Branches:
refs/heads/master dec3625b9 -> 8f804af05
HIVE-18975: NPE when inserting NULL value in structure and array with HBase table (Oleksiy Sayankin, reviewed by Ashutosh Chauhan and Yongzhi Chen)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/8f804af0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/8f804af0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/8f804af0
Branch: refs/heads/master
Commit: 8f804af05389edd7d5fa91e21ab9bdd71cc2c7a5
Parents: dec3625
Author: Yongzhi Chen <yc...@apache.org>
Authored: Tue Mar 20 13:12:05 2018 -0400
Committer: Yongzhi Chen <yc...@apache.org>
Committed: Tue Mar 20 13:12:05 2018 -0400
----------------------------------------------------------------------
.../hadoop/hive/hbase/HBaseRowSerializer.java | 136 +++++++++----------
.../src/test/queries/positive/hbase_queries.q | 16 +++
.../test/results/positive/hbase_queries.q.out | 82 +++++++++++
3 files changed, 166 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/8f804af0/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java
----------------------------------------------------------------------
diff --git a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java
index be6aa1e..bc4e146 100644
--- a/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java
+++ b/hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseRowSerializer.java
@@ -226,83 +226,83 @@ public class HBaseRowSerializer {
return output.toByteArray();
}
- private boolean serialize(
- Object obj,
- ObjectInspector objInspector,
- int level, ByteStream.Output ss) throws IOException {
+ private boolean serialize(Object obj, ObjectInspector objInspector, int level, ByteStream.Output ss)
+ throws IOException {
switch (objInspector.getCategory()) {
- case PRIMITIVE:
- LazyUtils.writePrimitiveUTF8(ss, obj,
- (PrimitiveObjectInspector) objInspector, escaped, escapeChar, needsEscape);
- return true;
- case LIST:
- char separator = (char) separators[level];
- ListObjectInspector loi = (ListObjectInspector)objInspector;
- List<?> list = loi.getList(obj);
- ObjectInspector eoi = loi.getListElementObjectInspector();
- if (list == null) {
- return false;
- } else {
- for (int i = 0; i < list.size(); i++) {
- if (i > 0) {
- ss.write(separator);
- }
- serialize(list.get(i), eoi, level + 1, ss);
+ case PRIMITIVE:
+ LazyUtils.writePrimitiveUTF8(ss, obj, (PrimitiveObjectInspector) objInspector, escaped, escapeChar, needsEscape);
+ return true;
+ case LIST:
+ char separator = (char) separators[level];
+ ListObjectInspector loi = (ListObjectInspector) objInspector;
+ List<?> list = loi.getList(obj);
+ ObjectInspector eoi = loi.getListElementObjectInspector();
+ if (list == null) {
+ return false;
+ } else {
+ for (int i = 0; i < list.size(); i++) {
+ if (i > 0) {
+ ss.write(separator);
}
- }
- return true;
- case MAP:
- char sep = (char) separators[level];
- char keyValueSeparator = (char) separators[level+1];
- MapObjectInspector moi = (MapObjectInspector) objInspector;
- ObjectInspector koi = moi.getMapKeyObjectInspector();
- ObjectInspector voi = moi.getMapValueObjectInspector();
-
- Map<?, ?> map = moi.getMap(obj);
- if (map == null) {
- return false;
- } else {
- boolean first = true;
- for (Map.Entry<?, ?> entry: map.entrySet()) {
- if (first) {
- first = false;
- } else {
- ss.write(sep);
- }
- serialize(entry.getKey(), koi, level+2, ss);
-
- if ( entry.getValue() != null) {
- ss.write(keyValueSeparator);
- serialize(entry.getValue(), voi, level+2, ss);
- }
+ Object currentItem = list.get(i);
+ if (currentItem != null) {
+ serialize(currentItem, eoi, level + 1, ss);
}
}
- return true;
- case STRUCT:
- sep = (char)separators[level];
- StructObjectInspector soi = (StructObjectInspector)objInspector;
- List<? extends StructField> fields = soi.getAllStructFieldRefs();
- list = soi.getStructFieldsDataAsList(obj);
- if (list == null) {
- return false;
- } else {
- for (int i = 0; i < list.size(); i++) {
- if (i > 0) {
- ss.write(sep);
- }
+ }
+ return true;
+ case MAP:
+ char sep = (char) separators[level];
+ char keyValueSeparator = (char) separators[level + 1];
+ MapObjectInspector moi = (MapObjectInspector) objInspector;
+ ObjectInspector koi = moi.getMapKeyObjectInspector();
+ ObjectInspector voi = moi.getMapValueObjectInspector();
- serialize(list.get(i), fields.get(i).getFieldObjectInspector(),
- level + 1, ss);
+ Map<?, ?> map = moi.getMap(obj);
+ if (map == null) {
+ return false;
+ } else {
+ boolean first = true;
+ for (Map.Entry<?, ?> entry : map.entrySet()) {
+ if (first) {
+ first = false;
+ } else {
+ ss.write(sep);
+ }
+ serialize(entry.getKey(), koi, level + 2, ss);
+ Object currentValue = entry.getValue();
+ if (currentValue != null) {
+ ss.write(keyValueSeparator);
+ serialize(currentValue, voi, level + 2, ss);
}
}
- return true;
- case UNION: {
- // union type currently not totally supported. See HIVE-2390
+ }
+ return true;
+ case STRUCT:
+ sep = (char) separators[level];
+ StructObjectInspector soi = (StructObjectInspector) objInspector;
+ List<? extends StructField> fields = soi.getAllStructFieldRefs();
+ list = soi.getStructFieldsDataAsList(obj);
+ if (list == null) {
return false;
- }
- default:
- throw new RuntimeException("Unknown category type: " + objInspector.getCategory());
+ } else {
+ for (int i = 0; i < list.size(); i++) {
+ if (i > 0) {
+ ss.write(sep);
+ }
+ Object currentItem = list.get(i);
+ if (currentItem != null) {
+ serialize(currentItem, fields.get(i).getFieldObjectInspector(), level + 1, ss);
+ }
+ }
+ }
+ return true;
+ case UNION:
+ // union type currently not totally supported. See HIVE-2390
+ return false;
+ default:
+ throw new RuntimeException("Unknown category type: " + objInspector.getCategory());
}
}
}
http://git-wip-us.apache.org/repos/asf/hive/blob/8f804af0/hbase-handler/src/test/queries/positive/hbase_queries.q
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/queries/positive/hbase_queries.q b/hbase-handler/src/test/queries/positive/hbase_queries.q
index 43efd6c..4604f3e 100644
--- a/hbase-handler/src/test/queries/positive/hbase_queries.q
+++ b/hbase-handler/src/test/queries/positive/hbase_queries.q
@@ -189,6 +189,20 @@ insert into table hbase_table_10 select 5 as id, map(50,cast(null as int), 55, 5
select * from hbase_table_10;
+DROP TABLE IF EXISTS hbase_table_11;
+CREATE TABLE hbase_table_11(id INT, map_column STRUCT<s_int:INT,s_string:STRING,s_date:DATE>)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,id:id');
+INSERT INTO hbase_table_11 SELECT 2,NAMED_STRUCT("s_int",CAST(NULL AS INT),"s_string","s1","s_date",CAST('2018-03-12' AS DATE)) FROM src LIMIT 1;
+select * from hbase_table_11;
+
+DROP TABLE IF EXISTS hbase_table_12;
+CREATE TABLE hbase_table_12(id INT, list_column ARRAY <STRING>)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,id:id');
+INSERT INTO hbase_table_12 SELECT 2, ARRAY("a", CAST (NULL AS STRING), "b") FROM src LIMIT 1;
+select * from hbase_table_12;
+
DROP TABLE hbase_table_1;
DROP TABLE hbase_table_1_like;
DROP TABLE hbase_table_2;
@@ -203,3 +217,5 @@ DROP TABLE empty_hbase_table;
DROP TABLE empty_normal_table;
DROP TABLE hbase_table_9;
DROP TABLE hbase_table_10;
+DROP TABLE hbase_table_11;
+DROP TABLE hbase_table_12;
http://git-wip-us.apache.org/repos/asf/hive/blob/8f804af0/hbase-handler/src/test/results/positive/hbase_queries.q.out
----------------------------------------------------------------------
diff --git a/hbase-handler/src/test/results/positive/hbase_queries.q.out b/hbase-handler/src/test/results/positive/hbase_queries.q.out
index 8fcd350..41e29f9 100644
--- a/hbase-handler/src/test/results/positive/hbase_queries.q.out
+++ b/hbase-handler/src/test/results/positive/hbase_queries.q.out
@@ -1095,6 +1095,72 @@ POSTHOOK: Input: default@hbase_table_10
3 {30:31} 1234
4 {40:null,45:null} 1234
5 {50:null,55:58} 1234
+PREHOOK: query: DROP TABLE IF EXISTS hbase_table_11
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS hbase_table_11
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hbase_table_11(id INT, map_column STRUCT<s_int:INT,s_string:STRING,s_date:DATE>)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,id:id')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hbase_table_11
+POSTHOOK: query: CREATE TABLE hbase_table_11(id INT, map_column STRUCT<s_int:INT,s_string:STRING,s_date:DATE>)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,id:id')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_table_11
+PREHOOK: query: INSERT INTO hbase_table_11 SELECT 2,NAMED_STRUCT("s_int",CAST(NULL AS INT),"s_string","s1","s_date",CAST('2018-03-12' AS DATE)) FROM src LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_11
+POSTHOOK: query: INSERT INTO hbase_table_11 SELECT 2,NAMED_STRUCT("s_int",CAST(NULL AS INT),"s_string","s1","s_date",CAST('2018-03-12' AS DATE)) FROM src LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_11
+PREHOOK: query: select * from hbase_table_11
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_11
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_table_11
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_11
+#### A masked pattern was here ####
+2 {"s_int":null,"s_string":"s1","s_date":"2018-03-12"}
+PREHOOK: query: DROP TABLE IF EXISTS hbase_table_12
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: DROP TABLE IF EXISTS hbase_table_12
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: CREATE TABLE hbase_table_12(id INT, list_column ARRAY <STRING>)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,id:id')
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@hbase_table_12
+POSTHOOK: query: CREATE TABLE hbase_table_12(id INT, list_column ARRAY <STRING>)
+STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
+WITH SERDEPROPERTIES ('hbase.columns.mapping'=':key,id:id')
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@hbase_table_12
+PREHOOK: query: INSERT INTO hbase_table_12 SELECT 2, ARRAY("a", CAST (NULL AS STRING), "b") FROM src LIMIT 1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@hbase_table_12
+POSTHOOK: query: INSERT INTO hbase_table_12 SELECT 2, ARRAY("a", CAST (NULL AS STRING), "b") FROM src LIMIT 1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@hbase_table_12
+PREHOOK: query: select * from hbase_table_12
+PREHOOK: type: QUERY
+PREHOOK: Input: default@hbase_table_12
+#### A masked pattern was here ####
+POSTHOOK: query: select * from hbase_table_12
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@hbase_table_12
+#### A masked pattern was here ####
+2 ["a","","b"]
PREHOOK: query: DROP TABLE hbase_table_1
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@hbase_table_1
@@ -1207,3 +1273,19 @@ POSTHOOK: query: DROP TABLE hbase_table_10
POSTHOOK: type: DROPTABLE
POSTHOOK: Input: default@hbase_table_10
POSTHOOK: Output: default@hbase_table_10
+PREHOOK: query: DROP TABLE hbase_table_11
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hbase_table_11
+PREHOOK: Output: default@hbase_table_11
+POSTHOOK: query: DROP TABLE hbase_table_11
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hbase_table_11
+POSTHOOK: Output: default@hbase_table_11
+PREHOOK: query: DROP TABLE hbase_table_12
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@hbase_table_12
+PREHOOK: Output: default@hbase_table_12
+POSTHOOK: query: DROP TABLE hbase_table_12
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@hbase_table_12
+POSTHOOK: Output: default@hbase_table_12