You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by jd...@apache.org on 2015/10/09 19:00:18 UTC
hive git commit: HIVE-12012: select query on json table with map
containing numeric values fails (Jason Dere, reviewed by Sushanth Sowmyan)
Repository: hive
Updated Branches:
refs/heads/master be05e32e5 -> cc2adc732
HIVE-12012: select query on json table with map containing numeric values fails (Jason Dere, reviewed by Sushanth Sowmyan)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/cc2adc73
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/cc2adc73
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/cc2adc73
Branch: refs/heads/master
Commit: cc2adc732790338cdc198e7c45d463a3ee6b0e4f
Parents: be05e32
Author: Jason Dere <jd...@hortonworks.com>
Authored: Fri Oct 9 09:59:35 2015 -0700
Committer: Jason Dere <jd...@hortonworks.com>
Committed: Fri Oct 9 09:59:35 2015 -0700
----------------------------------------------------------------------
.../apache/hive/hcatalog/data/JsonSerDe.java | 8 +-
.../hive/hcatalog/data/TestJsonSerDe.java | 36 ++++++
.../test/queries/clientpositive/json_serde1.q | 36 ++++++
.../results/clientpositive/json_serde1.q.out | 113 +++++++++++++++++++
4 files changed, 186 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/cc2adc73/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
index 9b325b6..1b47b28 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
@@ -349,13 +349,7 @@ public class JsonSerDe implements SerDe {
HCatFieldSchema valueSchema = hcatFieldSchema.getMapValueSchema().get(0);
while ((valueToken = p.nextToken()) != JsonToken.END_OBJECT) {
Object k = getObjectOfCorrespondingPrimitiveType(p.getCurrentName(), hcatFieldSchema.getMapKeyTypeInfo());
- Object v;
- if (valueSchema.getType() == HCatFieldSchema.Type.STRUCT) {
- v = extractCurrentField(p, valueSchema, false);
- } else {
- v = extractCurrentField(p, valueSchema, true);
- }
-
+ Object v = extractCurrentField(p, valueSchema, false);
map.put(k, v);
}
val = map;
http://git-wip-us.apache.org/repos/asf/hive/blob/cc2adc73/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
----------------------------------------------------------------------
diff --git a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
index 618f39b..5ececb5 100644
--- a/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
+++ b/hcatalog/core/src/test/java/org/apache/hive/hcatalog/data/TestJsonSerDe.java
@@ -307,4 +307,40 @@ public class TestJsonSerDe extends TestCase {
assertTrue(HCatDataCheckUtil.recordsEqual((HCatRecord)rjsd.deserialize(text2), expected2));
}
+
+ private static HashMap<String, Integer> createHashMapStringInteger(Object...vals) {
+ assertTrue(vals.length % 2 == 0);
+ HashMap<String, Integer> retval = new HashMap<String, Integer>();
+ for (int idx = 0; idx < vals.length; idx += 2) {
+ retval.put((String) vals[idx], (Integer) vals[idx+1]);
+ }
+ return retval;
+ }
+
+ public void testMapValues() throws Exception {
+ Configuration conf = new Configuration();
+ Properties props = new Properties();
+
+ props.put(serdeConstants.LIST_COLUMNS, "a,b");
+ props.put(serdeConstants.LIST_COLUMN_TYPES, "array<string>,map<string,int>");
+ JsonSerDe rjsd = new JsonSerDe();
+ SerDeUtils.initializeSerDe(rjsd, conf, props, null);
+
+ Text text1 = new Text("{ \"a\":[\"aaa\"],\"b\":{\"bbb\":1}} ");
+ Text text2 = new Text("{\"a\":[\"yyy\"],\"b\":{\"zzz\":123}}");
+ Text text3 = new Text("{\"a\":[\"a\"],\"b\":{\"x\":11, \"y\": 22, \"z\": null}}");
+
+ HCatRecord expected1 = new DefaultHCatRecord(Arrays.<Object>asList(
+ Arrays.<String>asList("aaa"),
+ createHashMapStringInteger("bbb", 1)));
+ HCatRecord expected2 = new DefaultHCatRecord(Arrays.<Object>asList(
+ Arrays.<String>asList("yyy"),
+ createHashMapStringInteger("zzz", 123)));
+ HCatRecord expected3 = new DefaultHCatRecord(Arrays.<Object>asList(
+ Arrays.<String>asList("a"),
+ createHashMapStringInteger("x", 11, "y", 22, "z", null)));
+
+ assertTrue(HCatDataCheckUtil.recordsEqual((HCatRecord)rjsd.deserialize(text1), expected1));
+ assertTrue(HCatDataCheckUtil.recordsEqual((HCatRecord)rjsd.deserialize(text2), expected2));
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/cc2adc73/ql/src/test/queries/clientpositive/json_serde1.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/json_serde1.q b/ql/src/test/queries/clientpositive/json_serde1.q
new file mode 100644
index 0000000..85f5af2
--- /dev/null
+++ b/ql/src/test/queries/clientpositive/json_serde1.q
@@ -0,0 +1,36 @@
+
+add jar ${system:maven.local.repository}/org/apache/hive/hcatalog/hive-hcatalog-core/${system:hive.version}/hive-hcatalog-core-${system:hive.version}.jar;
+
+drop table if exists json_serde1_1;
+drop table if exists json_serde1_2;
+
+create table json_serde1_1 (a array<string>,b map<string,int>)
+ row format serde 'org.apache.hive.hcatalog.data.JsonSerDe';
+
+insert into table json_serde1_1
+ select array('aaa'),map('aaa',1) from src limit 2;
+
+select * from json_serde1_1;
+
+create table json_serde1_2 (
+ a array<int>,
+ b map<int,date>,
+ c struct<c1:int, c2:string, c3:array<string>, c4:map<string, int>, c5:struct<c5_1:string, c5_2:int>>
+) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe';
+
+insert into table json_serde1_2
+ select
+ array(3, 2, 1),
+ map(1, date '2001-01-01', 2, null),
+ named_struct(
+ 'c1', 123456,
+ 'c2', 'hello',
+ 'c3', array('aa', 'bb', 'cc'),
+ 'c4', map('abc', 123, 'xyz', 456),
+ 'c5', named_struct('c5_1', 'bye', 'c5_2', 88))
+ from src limit 2;
+
+select * from json_serde1_2;
+
+drop table json_serde1_1;
+drop table json_serde1_2;
http://git-wip-us.apache.org/repos/asf/hive/blob/cc2adc73/ql/src/test/results/clientpositive/json_serde1.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/json_serde1.q.out b/ql/src/test/results/clientpositive/json_serde1.q.out
new file mode 100644
index 0000000..6235aff
--- /dev/null
+++ b/ql/src/test/results/clientpositive/json_serde1.q.out
@@ -0,0 +1,113 @@
+PREHOOK: query: drop table if exists json_serde1_1
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists json_serde1_1
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: drop table if exists json_serde1_2
+PREHOOK: type: DROPTABLE
+POSTHOOK: query: drop table if exists json_serde1_2
+POSTHOOK: type: DROPTABLE
+PREHOOK: query: create table json_serde1_1 (a array<string>,b map<string,int>)
+ row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@json_serde1_1
+POSTHOOK: query: create table json_serde1_1 (a array<string>,b map<string,int>)
+ row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@json_serde1_1
+PREHOOK: query: insert into table json_serde1_1
+ select array('aaa'),map('aaa',1) from src limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@json_serde1_1
+POSTHOOK: query: insert into table json_serde1_1
+ select array('aaa'),map('aaa',1) from src limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@json_serde1_1
+POSTHOOK: Lineage: json_serde1_1.a EXPRESSION []
+POSTHOOK: Lineage: json_serde1_1.b EXPRESSION []
+PREHOOK: query: select * from json_serde1_1
+PREHOOK: type: QUERY
+PREHOOK: Input: default@json_serde1_1
+#### A masked pattern was here ####
+POSTHOOK: query: select * from json_serde1_1
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@json_serde1_1
+#### A masked pattern was here ####
+["aaa"] {"aaa":1}
+["aaa"] {"aaa":1}
+PREHOOK: query: create table json_serde1_2 (
+ a array<int>,
+ b map<int,date>,
+ c struct<c1:int, c2:string, c3:array<string>, c4:map<string, int>, c5:struct<c5_1:string, c5_2:int>>
+) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@json_serde1_2
+POSTHOOK: query: create table json_serde1_2 (
+ a array<int>,
+ b map<int,date>,
+ c struct<c1:int, c2:string, c3:array<string>, c4:map<string, int>, c5:struct<c5_1:string, c5_2:int>>
+) row format serde 'org.apache.hive.hcatalog.data.JsonSerDe'
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@json_serde1_2
+PREHOOK: query: insert into table json_serde1_2
+ select
+ array(3, 2, 1),
+ map(1, date '2001-01-01', 2, null),
+ named_struct(
+ 'c1', 123456,
+ 'c2', 'hello',
+ 'c3', array('aa', 'bb', 'cc'),
+ 'c4', map('abc', 123, 'xyz', 456),
+ 'c5', named_struct('c5_1', 'bye', 'c5_2', 88))
+ from src limit 2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@json_serde1_2
+POSTHOOK: query: insert into table json_serde1_2
+ select
+ array(3, 2, 1),
+ map(1, date '2001-01-01', 2, null),
+ named_struct(
+ 'c1', 123456,
+ 'c2', 'hello',
+ 'c3', array('aa', 'bb', 'cc'),
+ 'c4', map('abc', 123, 'xyz', 456),
+ 'c5', named_struct('c5_1', 'bye', 'c5_2', 88))
+ from src limit 2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@json_serde1_2
+POSTHOOK: Lineage: json_serde1_2.a EXPRESSION []
+POSTHOOK: Lineage: json_serde1_2.b EXPRESSION []
+POSTHOOK: Lineage: json_serde1_2.c EXPRESSION []
+PREHOOK: query: select * from json_serde1_2
+PREHOOK: type: QUERY
+PREHOOK: Input: default@json_serde1_2
+#### A masked pattern was here ####
+POSTHOOK: query: select * from json_serde1_2
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@json_serde1_2
+#### A masked pattern was here ####
+[3,2,1] {1:"2001-01-01",2:null} {"c1":123456,"c2":"hello","c3":["aa","bb","cc"],"c4":{"xyz":456,"abc":123},"c5":{"c5_1":"bye","c5_2":88}}
+[3,2,1] {1:"2001-01-01",2:null} {"c1":123456,"c2":"hello","c3":["aa","bb","cc"],"c4":{"xyz":456,"abc":123},"c5":{"c5_1":"bye","c5_2":88}}
+PREHOOK: query: drop table json_serde1_1
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@json_serde1_1
+PREHOOK: Output: default@json_serde1_1
+POSTHOOK: query: drop table json_serde1_1
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@json_serde1_1
+POSTHOOK: Output: default@json_serde1_1
+PREHOOK: query: drop table json_serde1_2
+PREHOOK: type: DROPTABLE
+PREHOOK: Input: default@json_serde1_2
+PREHOOK: Output: default@json_serde1_2
+POSTHOOK: query: drop table json_serde1_2
+POSTHOOK: type: DROPTABLE
+POSTHOOK: Input: default@json_serde1_2
+POSTHOOK: Output: default@json_serde1_2