You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by bs...@apache.org on 2019/05/08 16:23:08 UTC
[hive] branch master updated: HIVE-21240: JSON SerDe Re-Write and
Fixup timestamp parsing issue (David Mollitor via Slim Bouguerra)
This is an automated email from the ASF dual-hosted git repository.
bslim pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 2b7e997 HIVE-21240: JSON SerDe Re-Write and Fixup timestamp parsing issue (David Mollitor via Slim Bouguerra)
2b7e997 is described below
commit 2b7e99708f546fa7dbc4b6dfd3a9fcfe69b186dd
Author: David Mollitor <da...@gmail.com>
AuthorDate: Wed May 8 09:21:44 2019 -0700
HIVE-21240: JSON SerDe Re-Write and Fixup timestamp parsing issue (David Mollitor via Slim Bouguerra)
---
.../org/apache/hive/hcatalog/data/JsonSerDe.java | 36 +-
.../hive/ql/udf/generic/GenericUDFJsonRead.java | 13 +-
.../ql/udf/generic/TestGenericUDFJsonRead.java | 34 +-
.../queries/clientpositive/kafka_storage_handler.q | 1 +
.../kafka/kafka_storage_handler.q.out | 154 +++---
.../org/apache/hadoop/hive/serde2/JsonSerDe.java | 412 +++++----------
.../hadoop/hive/serde2/json/BinaryEncoding.java | 33 ++
.../hadoop/hive/serde2/json/HiveJsonReader.java | 557 +++++++++++++++++++++
.../hive/serde2/json/HiveJsonStructReader.java | 402 ---------------
.../hadoop/hive/serde2/json/HiveJsonWriter.java | 386 ++++++++++++++
.../apache/hadoop/hive/serde2/TestJsonSerDe.java | 315 ++++++++++++
serde/src/test/resources/json/array.json | 8 +
serde/src/test/resources/json/complex_write.json | 1 +
serde/src/test/resources/json/map_int_key.json | 6 +
serde/src/test/resources/json/simple.json | 7 +
serde/src/test/resources/json/single_pixel.json | 1 +
16 files changed, 1568 insertions(+), 798 deletions(-)
diff --git a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
index 87611ad..dd7690e 100644
--- a/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
+++ b/hcatalog/core/src/main/java/org/apache/hive/hcatalog/data/JsonSerDe.java
@@ -60,8 +60,7 @@ public class JsonSerDe extends AbstractSerDe {
public void initialize(Configuration conf, Properties tbl)
throws SerDeException {
- jsonSerde.initialize(conf, tbl);
- jsonSerde.setWriteablesUsage(false);
+ jsonSerde.initialize(conf, tbl, false);
StructTypeInfo rowTypeInfo = jsonSerde.getTypeInfo();
cachedObjectInspector = HCatRecordObjectInspectorFactory.getHCatRecordObjectInspector(rowTypeInfo);
@@ -84,28 +83,32 @@ public class JsonSerDe extends AbstractSerDe {
@Override
public Object deserialize(Writable blob) throws SerDeException {
try {
- Object row = jsonSerde.deserialize(blob);
- List fatRow = fatLand((Object[]) row);
+ List<?> row = (List<?>) jsonSerde.deserialize(blob);
+ List<Object> fatRow = fatLand(row);
return new DefaultHCatRecord(fatRow);
} catch (Exception e) {
throw new SerDeException(e);
}
}
+
@SuppressWarnings({"rawtypes", "unchecked" })
- private static List fatLand(Object[] arr) {
- List ret = new ArrayList<>();
- for (Object o : arr) {
- if (o != null && o instanceof Map<?, ?>) {
+ private static List<Object> fatLand(final List<?> arr) {
+ final List ret = new ArrayList<>();
+ for (final Object o : arr) {
+ if (o == null) {
+ ret.add(null);
+ } else if (o instanceof Map<?, ?>) {
ret.add(fatMap(((Map) o)));
- } else if (o != null && o instanceof List<?>) {
- ret.add(fatLand(((List) o).toArray()));
- } else if (o != null && o.getClass().isArray() && o.getClass().getComponentType() != byte.class) {
+ } else if (o instanceof List<?>) {
+ ret.add(fatLand((List) o));
+ } else if (o.getClass().isArray()
+ && o.getClass().getComponentType() != byte.class) {
Class<?> ct = o.getClass().getComponentType();
if (ct.isPrimitive()) {
ret.add(primitiveArrayToList(o));
} else {
- ret.add(fatLand((Object[]) o));
+ ret.add(fatLand(Arrays.asList((Object[]) o)));
}
} else {
ret.add(o);
@@ -114,15 +117,14 @@ public class JsonSerDe extends AbstractSerDe {
return ret;
}
- @SuppressWarnings("rawtypes")
private static Object fatMap(Map<Object, Object> map) {
- Map ret = new LinkedHashMap<>();
+ Map<Object, Object> ret = new LinkedHashMap<>();
Set<Entry<Object, Object>> es = map.entrySet();
for (Entry<Object, Object> e : es) {
- Object oldV = e.getValue();
- Object newV;
+ final Object oldV = e.getValue();
+ final Object newV;
if (oldV != null && oldV.getClass().isArray()) {
- newV = fatLand((Object[]) oldV);
+ newV = fatLand(Arrays.asList((Object[]) oldV));
} else {
newV = oldV;
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFJsonRead.java b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFJsonRead.java
index f5814ed..7064c80 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFJsonRead.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/udf/generic/GenericUDFJsonRead.java
@@ -21,7 +21,8 @@ import org.apache.hadoop.hive.ql.exec.Description;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.exec.UDFArgumentTypeException;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde2.json.HiveJsonStructReader;
+import org.apache.hadoop.hive.serde2.json.HiveJsonReader;
+import org.apache.hadoop.hive.serde2.json.HiveJsonReader.Feature;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
@@ -39,7 +40,7 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
public class GenericUDFJsonRead extends GenericUDF {
private TextConverter inputConverter;
- private HiveJsonStructReader jsonReader;
+ private HiveJsonReader jsonReader;
@Override
public ObjectInspector initialize(ObjectInspector[] arguments) throws UDFArgumentException {
@@ -55,9 +56,11 @@ public class GenericUDFJsonRead extends GenericUDF {
String typeStr = getConstantStringValue(arguments, 1);
try {
- TypeInfo t = TypeInfoUtils.getTypeInfoFromTypeString(typeStr);
- jsonReader = new HiveJsonStructReader(t);
- jsonReader.setWritablesUsage(true);
+ final TypeInfo t = TypeInfoUtils.getTypeInfoFromTypeString(typeStr);
+ final ObjectInspector oi =
+ TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(t);
+ jsonReader = new HiveJsonReader(oi);
+ jsonReader.enable(Feature.PRIMITIVE_TO_WRITABLE);
} catch (Exception e) {
throw new UDFArgumentException(getFuncName() + ": Error parsing typestring: " + e.getMessage());
}
diff --git a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFJsonRead.java b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFJsonRead.java
index 3016eaf..7f0b1a7 100644
--- a/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFJsonRead.java
+++ b/ql/src/test/org/apache/hadoop/hive/ql/udf/generic/TestGenericUDFJsonRead.java
@@ -96,9 +96,9 @@ public class TestGenericUDFJsonRead {
udf.initialize(arguments);
Object res = udf.evaluate(evalArgs("{\"a\":\"b\"}"));
- assertTrue(res instanceof Object[]);
- Object o[] = (Object[]) res;
- assertEquals(new Text("b"), o[0]);
+ assertTrue(res instanceof List<?>);
+ List<?> o = (List<?>) res;
+ assertEquals(new Text("b"), o.get(0));
}
}
@@ -109,9 +109,10 @@ public class TestGenericUDFJsonRead {
udf.initialize(arguments);
Object res = udf.evaluate(evalArgs("{\"a\":null}"));
- assertTrue(res instanceof Object[]);
- Object o[] = (Object[]) res;
- assertEquals(null, o[0]);
+ assertTrue(res instanceof List<?>);
+
+ List<?> o = (List<?>) res;
+ assertEquals(null, o.get(0));
}
}
@@ -144,9 +145,10 @@ public class TestGenericUDFJsonRead {
udf.initialize(arguments);
Object res = udf.evaluate(evalArgs("{\"a\":null}"));
- assertTrue(res instanceof Object[]);
- Object o[] = (Object[]) res;
- assertEquals(null, o[0]);
+ assertTrue(res instanceof List<?>);
+
+ List<?> o = (List<?>) res;
+ assertEquals(null, o.get(0));
}
}
@@ -156,10 +158,8 @@ public class TestGenericUDFJsonRead {
ObjectInspector[] arguments = buildArguments("struct<a:int>");
udf.initialize(arguments);
- Object res = udf.evaluate(evalArgs("{\"b\":null}"));
- assertTrue(res instanceof Object[]);
- Object o[] = (Object[]) res;
- assertEquals(null, o[0]);
+ // Invalid - should throw Exception
+ udf.evaluate(evalArgs("{\"b\":null}"));
}
}
@@ -169,10 +169,8 @@ public class TestGenericUDFJsonRead {
ObjectInspector[] arguments = buildArguments("array<int>");
udf.initialize(arguments);
- Object res = udf.evaluate(evalArgs("[1,22,2,{\"b\":null}]"));
- assertTrue(res instanceof Object[]);
- Object o[] = (Object[]) res;
- assertEquals(null, o[0]);
+ // Invalid - should throw Exception
+ udf.evaluate(evalArgs("[1,22,2,{\"b\":null}]"));
}
}
@@ -184,7 +182,7 @@ public class TestGenericUDFJsonRead {
Object res = udf.evaluate(evalArgs("{\"a\":\"v\"}"));
assertTrue(res instanceof Map);
- Map o = (Map) res;
+ Map<?, ?> o = (Map<?, ?>) res;
assertEquals(1, o.size());
assertEquals(new Text("v"), o.get(new Text("a")));
}
diff --git a/ql/src/test/queries/clientpositive/kafka_storage_handler.q b/ql/src/test/queries/clientpositive/kafka_storage_handler.q
index c1d5f9d..f4670f5 100644
--- a/ql/src/test/queries/clientpositive/kafka_storage_handler.q
+++ b/ql/src/test/queries/clientpositive/kafka_storage_handler.q
@@ -140,6 +140,7 @@ CREATE EXTERNAL TABLE kafka_table_2
`country` string,`continent` string, `namespace` string, `newPage` boolean, `unpatrolled` boolean,
`anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint)
STORED BY 'org.apache.hadoop.hive.kafka.KafkaStorageHandler'
+WITH SERDEPROPERTIES ("timestamp.formats"="yyyy-MM-dd\'T\'HH:mm:ss\'Z\'")
TBLPROPERTIES
("kafka.topic" = "test-topic",
"kafka.bootstrap.servers"="localhost:9093");
diff --git a/ql/src/test/results/clientpositive/kafka/kafka_storage_handler.q.out b/ql/src/test/results/clientpositive/kafka/kafka_storage_handler.q.out
index 7b71851..44023de 100644
--- a/ql/src/test/results/clientpositive/kafka/kafka_storage_handler.q.out
+++ b/ql/src/test/results/clientpositive/kafka/kafka_storage_handler.q.out
@@ -62,16 +62,16 @@ POSTHOOK: query: Select `__partition` , `__offset`,`__key`, `__time`, `page`, `u
POSTHOOK: type: QUERY
POSTHOOK: Input: default@kafka_table
POSTHOOK: Output: hdfs://### HDFS PATH ###
-0 0 key NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
-0 1 key NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
-0 2 key NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
-0 3 key NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
-0 4 key NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
-0 5 key NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
-0 6 key NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
-0 7 key NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
-0 8 key NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
-0 9 key NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
+0 0 key 2013-08-31 01:02:33 Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
+0 1 key 2013-08-31 03:32:45 Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
+0 2 key 2013-08-31 07:11:21 Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
+0 3 key 2013-08-31 11:58:39 Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
+0 4 key 2013-08-31 12:41:27 Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
+0 5 key 2013-09-01 01:02:33 Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
+0 6 key 2013-09-01 03:32:45 Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
+0 7 key 2013-09-01 07:11:21 Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
+0 8 key 2013-09-01 11:58:39 Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
+0 9 key 2013-09-01 12:41:27 Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
PREHOOK: query: Select count(*) FROM kafka_table
PREHOOK: type: QUERY
PREHOOK: Input: default@kafka_table
@@ -93,16 +93,16 @@ from kafka_table where `__timestamp` > 1533960760123
POSTHOOK: type: QUERY
POSTHOOK: Input: default@kafka_table
POSTHOOK: Output: hdfs://### HDFS PATH ###
-0 0 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
-0 1 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
-0 2 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
-0 3 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
-0 4 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
-0 5 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
-0 6 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
-0 7 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
-0 8 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
-0 9 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
+0 0 2013-08-31 01:02:33 Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
+0 1 2013-08-31 03:32:45 Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
+0 2 2013-08-31 07:11:21 Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
+0 3 2013-08-31 11:58:39 Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
+0 4 2013-08-31 12:41:27 Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
+0 5 2013-09-01 01:02:33 Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
+0 6 2013-09-01 03:32:45 Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
+0 7 2013-09-01 07:11:21 Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
+0 8 2013-09-01 11:58:39 Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
+0 9 2013-09-01 12:41:27 Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
PREHOOK: query: Select `__partition`, `__offset` ,`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` ,
`unpatrolled` , `anonymous` , `robot` , added , deleted , delta
from kafka_table where `__timestamp` > 533960760123
@@ -115,16 +115,16 @@ from kafka_table where `__timestamp` > 533960760123
POSTHOOK: type: QUERY
POSTHOOK: Input: default@kafka_table
POSTHOOK: Output: hdfs://### HDFS PATH ###
-0 0 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
-0 1 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
-0 2 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
-0 3 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
-0 4 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
-0 5 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
-0 6 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
-0 7 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
-0 8 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
-0 9 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
+0 0 2013-08-31 01:02:33 Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
+0 1 2013-08-31 03:32:45 Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
+0 2 2013-08-31 07:11:21 Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
+0 3 2013-08-31 11:58:39 Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
+0 4 2013-08-31 12:41:27 Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
+0 5 2013-09-01 01:02:33 Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
+0 6 2013-09-01 03:32:45 Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
+0 7 2013-09-01 07:11:21 Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
+0 8 2013-09-01 11:58:39 Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
+0 9 2013-09-01 12:41:27 Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
PREHOOK: query: Select `__partition`, `__offset`,`__time`, `page`, `user`, `language`, `country`,`continent`, `namespace`, `newPage` ,
`unpatrolled` , `anonymous` , `robot` , added , deleted , delta
from kafka_table where (`__offset` > 7 and `__partition` = 0 and `__offset` <9 ) OR
@@ -139,9 +139,9 @@ from kafka_table where (`__offset` > 7 and `__partition` = 0 and `__offset` <9 )
POSTHOOK: type: QUERY
POSTHOOK: Input: default@kafka_table
POSTHOOK: Output: hdfs://### HDFS PATH ###
-0 1 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
-0 4 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
-0 8 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
+0 1 2013-08-31 03:32:45 Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
+0 4 2013-08-31 12:41:27 Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
+0 8 2013-09-01 11:58:39 Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
PREHOOK: query: Select `__key`,`__partition`, `__offset`,`__time`, `page`, `user` from kafka_table where `__offset` = 5
PREHOOK: type: QUERY
PREHOOK: Input: default@kafka_table
@@ -150,7 +150,7 @@ POSTHOOK: query: Select `__key`,`__partition`, `__offset`,`__time`, `page`, `use
POSTHOOK: type: QUERY
POSTHOOK: Input: default@kafka_table
POSTHOOK: Output: hdfs://### HDFS PATH ###
-key 0 5 NULL Gypsy Danger nuclear
+key 0 5 2013-09-01 01:02:33 Gypsy Danger nuclear
PREHOOK: query: Select `__key`,`__partition`, `__offset`,`__time`, `page`, `user` from kafka_table where `__offset` < 5
PREHOOK: type: QUERY
PREHOOK: Input: default@kafka_table
@@ -159,11 +159,11 @@ POSTHOOK: query: Select `__key`,`__partition`, `__offset`,`__time`, `page`, `use
POSTHOOK: type: QUERY
POSTHOOK: Input: default@kafka_table
POSTHOOK: Output: hdfs://### HDFS PATH ###
-key 0 0 NULL Gypsy Danger nuclear
-key 0 1 NULL Striker Eureka speed
-key 0 2 NULL Cherno Alpha masterYi
-key 0 3 NULL Crimson Typhoon triplets
-key 0 4 NULL Coyote Tango stringer
+key 0 0 2013-08-31 01:02:33 Gypsy Danger nuclear
+key 0 1 2013-08-31 03:32:45 Striker Eureka speed
+key 0 2 2013-08-31 07:11:21 Cherno Alpha masterYi
+key 0 3 2013-08-31 11:58:39 Crimson Typhoon triplets
+key 0 4 2013-08-31 12:41:27 Coyote Tango stringer
PREHOOK: query: Select `__key`,`__partition`, `__offset`,`__time`, `page`, `user` from kafka_table where `__offset` > 5
PREHOOK: type: QUERY
PREHOOK: Input: default@kafka_table
@@ -172,10 +172,10 @@ POSTHOOK: query: Select `__key`,`__partition`, `__offset`,`__time`, `page`, `use
POSTHOOK: type: QUERY
POSTHOOK: Input: default@kafka_table
POSTHOOK: Output: hdfs://### HDFS PATH ###
-key 0 6 NULL Striker Eureka speed
-key 0 7 NULL Cherno Alpha masterYi
-key 0 8 NULL Crimson Typhoon triplets
-key 0 9 NULL Coyote Tango stringer
+key 0 6 2013-09-01 03:32:45 Striker Eureka speed
+key 0 7 2013-09-01 07:11:21 Cherno Alpha masterYi
+key 0 8 2013-09-01 11:58:39 Crimson Typhoon triplets
+key 0 9 2013-09-01 12:41:27 Coyote Tango stringer
PREHOOK: query: Select `__partition`, `__offset`, `user` from kafka_table where
`__timestamp` > to_epoch_milli(CURRENT_TIMESTAMP - interval '1' HOURS)
PREHOOK: type: QUERY
@@ -347,9 +347,9 @@ POSTHOOK: query: select `partition_id`, `row_offset`,`__time`, `page`, `user`, `
POSTHOOK: type: QUERY
POSTHOOK: Input: default@orc_kafka_table
POSTHOOK: Output: hdfs://### HDFS PATH ###
-0 0 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
-0 1 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
-0 2 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
+0 0 2013-08-31 01:02:33 Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
+0 1 2013-08-31 03:32:45 Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
+0 2 2013-08-31 07:11:21 Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
PREHOOK: query: From kafka_table ktable JOIN kafka_table_offsets offset_table
on (ktable.`__partition` = offset_table.partition_id and ktable.`__offset` > offset_table.max_offset)
insert into table orc_kafka_table select `__partition`, `__offset`, `__timestamp`,
@@ -422,16 +422,16 @@ POSTHOOK: query: select `partition_id`, `row_offset`,`__time`, `page`, `user`, `
POSTHOOK: type: QUERY
POSTHOOK: Input: default@orc_kafka_table
POSTHOOK: Output: hdfs://### HDFS PATH ###
-0 0 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
-0 1 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
-0 2 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
-0 3 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
-0 4 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
-0 5 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
-0 6 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
-0 7 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
-0 8 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
-0 9 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
+0 0 2013-08-31 01:02:33 Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
+0 1 2013-08-31 03:32:45 Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
+0 2 2013-08-31 07:11:21 Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
+0 3 2013-08-31 11:58:39 Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
+0 4 2013-08-31 12:41:27 Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
+0 5 2013-09-01 01:02:33 Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
+0 6 2013-09-01 03:32:45 Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
+0 7 2013-09-01 07:11:21 Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
+0 8 2013-09-01 11:58:39 Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
+0 9 2013-09-01 12:41:27 Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
PREHOOK: query: Drop table kafka_table_offsets
PREHOOK: type: DROPTABLE
PREHOOK: Input: default@kafka_table_offsets
@@ -544,21 +544,22 @@ POSTHOOK: query: select `partition_id`, `row_offset`,`__time`, `page`, `user`, `
POSTHOOK: type: QUERY
POSTHOOK: Input: default@orc_kafka_table
POSTHOOK: Output: hdfs://### HDFS PATH ###
-0 0 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
-0 1 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
-0 2 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
-0 3 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
-0 4 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
-0 5 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
-0 6 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
-0 7 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
-0 8 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
-0 9 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
+0 0 2013-08-31 01:02:33 Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
+0 1 2013-08-31 03:32:45 Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
+0 2 2013-08-31 07:11:21 Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
+0 3 2013-08-31 11:58:39 Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
+0 4 2013-08-31 12:41:27 Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
+0 5 2013-09-01 01:02:33 Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
+0 6 2013-09-01 03:32:45 Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
+0 7 2013-09-01 07:11:21 Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
+0 8 2013-09-01 11:58:39 Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
+0 9 2013-09-01 12:41:27 Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
PREHOOK: query: CREATE EXTERNAL TABLE kafka_table_2
(`__time` timestamp with local time zone , `page` string, `user` string, `language` string,
`country` string,`continent` string, `namespace` string, `newPage` boolean, `unpatrolled` boolean,
`anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint)
STORED BY 'org.apache.hadoop.hive.kafka.KafkaStorageHandler'
+WITH SERDEPROPERTIES ("timestamp.formats"="yyyy-MM-dd\'T\'HH:mm:ss\'Z\'")
TBLPROPERTIES
("kafka.topic" = "test-topic",
"kafka.bootstrap.servers"="localhost:9093")
@@ -570,6 +571,7 @@ POSTHOOK: query: CREATE EXTERNAL TABLE kafka_table_2
`country` string,`continent` string, `namespace` string, `newPage` boolean, `unpatrolled` boolean,
`anonymous` boolean, `robot` boolean, added int, deleted int, delta bigint)
STORED BY 'org.apache.hadoop.hive.kafka.KafkaStorageHandler'
+WITH SERDEPROPERTIES ("timestamp.formats"="yyyy-MM-dd\'T\'HH:mm:ss\'Z\'")
TBLPROPERTIES
("kafka.topic" = "test-topic",
"kafka.bootstrap.servers"="localhost:9093")
@@ -588,16 +590,16 @@ FROM kafka_table_2
POSTHOOK: type: QUERY
POSTHOOK: Input: default@kafka_table_2
POSTHOOK: Output: hdfs://### HDFS PATH ###
-0 0 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
-0 1 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
-0 2 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
-0 3 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
-0 4 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
-0 5 NULL Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
-0 6 NULL Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
-0 7 NULL Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
-0 8 NULL Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
-0 9 NULL Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
+0 0 2013-08-30 18:02:33.0 US/Pacific Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
+0 1 2013-08-30 20:32:45.0 US/Pacific Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
+0 2 2013-08-31 00:11:21.0 US/Pacific Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
+0 3 2013-08-31 04:58:39.0 US/Pacific Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
+0 4 2013-08-31 05:41:27.0 US/Pacific Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
+0 5 2013-08-31 18:02:33.0 US/Pacific Gypsy Danger nuclear en United States North America article true true false false 57 200 -143
+0 6 2013-08-31 20:32:45.0 US/Pacific Striker Eureka speed en Australia Australia wikipedia true false false true 459 129 330
+0 7 2013-09-01 00:11:21.0 US/Pacific Cherno Alpha masterYi ru Russia Asia article true false false true 123 12 111
+0 8 2013-09-01 04:58:39.0 US/Pacific Crimson Typhoon triplets zh China Asia wikipedia false true false true 905 5 900
+0 9 2013-09-01 05:41:27.0 US/Pacific Coyote Tango stringer ja Japan Asia wikipedia false true false true 1 10 -9
PREHOOK: query: Select count(*) FROM kafka_table_2
PREHOOK: type: QUERY
PREHOOK: Input: default@kafka_table_2
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java b/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java
index f1c8477..9cbd344 100644
--- a/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/JsonSerDe.java
@@ -16,42 +16,22 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.hadoop.hive.serde2;
import java.io.ByteArrayInputStream;
-import java.io.IOException;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import java.util.Map;
import java.util.Properties;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.common.type.Date;
-import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.serde.serdeConstants;
-import org.apache.hadoop.hive.serde2.json.HiveJsonStructReader;
-import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.json.BinaryEncoding;
+import org.apache.hadoop.hive.serde2.json.HiveJsonReader;
+import org.apache.hadoop.hive.serde2.json.HiveJsonWriter;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.StructTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
@@ -63,76 +43,151 @@ import org.apache.hive.common.util.TimestampParser;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-@SerDeSpec(schemaProps = {serdeConstants.LIST_COLUMNS,
- serdeConstants.LIST_COLUMN_TYPES,
- serdeConstants.TIMESTAMP_FORMATS })
-
+/**
+ * Hive SerDe for processing JSON formatted data. This is typically paired with
+ * the TextInputFormat and therefore each line provided to this SerDe must be a
+ * single, and complete JSON object.<br/>
+ * <h2>Example</h2>
+ * <p>
+ * {"name="john","age"=30}<br/>
+ * {"name="sue","age"=32}
+ * </p>
+ */
+@SerDeSpec(schemaProps = { serdeConstants.LIST_COLUMNS,
+ serdeConstants.LIST_COLUMN_TYPES, serdeConstants.TIMESTAMP_FORMATS,
+ JsonSerDe.BINARY_FORMAT, JsonSerDe.IGNORE_EXTRA })
public class JsonSerDe extends AbstractSerDe {
private static final Logger LOG = LoggerFactory.getLogger(JsonSerDe.class);
+
+ public static final String BINARY_FORMAT = "json.binary.format";
+ public static final String IGNORE_EXTRA = "text.ignore.extra.fields";
+ public static final String NULL_EMPTY_LINES = "text.null.empty.line";
+
private List<String> columnNames;
- private HiveJsonStructReader structReader;
+ private BinaryEncoding binaryEncoding;
+ private boolean nullEmptyLines;
+
+ private HiveJsonReader jsonReader;
+ private HiveJsonWriter jsonWriter;
private StructTypeInfo rowTypeInfo;
+ private StructObjectInspector soi;
+ /**
+ * Initialize the SerDe. By default, items being deserialized are expected to
+ * be wrapped in Hadoop Writable objects and objects being serialized are
+ * expected to be Java primitive objects.
+ */
@Override
- public void initialize(Configuration conf, Properties tbl)
- throws SerDeException {
- List<TypeInfo> columnTypes;
+ public void initialize(final Configuration conf, final Properties tbl)
+ throws SerDeException {
+ initialize(conf, tbl, true);
+ }
+
+ /**
+ * Initialize the SerDe.
+ *
+ * @param conf System properties; can be null in compile time
+ * @param tbl table properties
+ * @param writeablePrimitivesDeserialize true if outputs are Hadoop Writable
+ */
+ public void initialize(final Configuration conf, final Properties tbl,
+ final boolean writeablePrimitivesDeserialize) {
+
LOG.debug("Initializing JsonSerDe: {}", tbl.entrySet());
// Get column names
- String columnNameProperty = tbl.getProperty(serdeConstants.LIST_COLUMNS);
- final String columnNameDelimiter = tbl.containsKey(serdeConstants.COLUMN_NAME_DELIMITER) ? tbl
- .getProperty(serdeConstants.COLUMN_NAME_DELIMITER)
- : String.valueOf(SerDeUtils.COMMA);
- // all table column names
- if (columnNameProperty.isEmpty()) {
- columnNames = Collections.emptyList();
- } else {
- columnNames = Arrays.asList(columnNameProperty.split(columnNameDelimiter));
- }
+ final String columnNameProperty =
+ tbl.getProperty(serdeConstants.LIST_COLUMNS);
+ final String columnNameDelimiter = tbl.getProperty(
+ serdeConstants.COLUMN_NAME_DELIMITER, String.valueOf(SerDeUtils.COMMA));
+
+ this.columnNames = columnNameProperty.isEmpty() ? Collections.emptyList()
+ : Arrays.asList(columnNameProperty.split(columnNameDelimiter));
// all column types
- String columnTypeProperty = tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);
- if (columnTypeProperty.isEmpty()) {
- columnTypes = Collections.emptyList();
- } else {
- columnTypes = TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
- }
+ final String columnTypeProperty =
+ tbl.getProperty(serdeConstants.LIST_COLUMN_TYPES);
+
+ final List<TypeInfo> columnTypes =
+ columnTypeProperty.isEmpty() ? Collections.emptyList()
+ : TypeInfoUtils.getTypeInfosFromTypeString(columnTypeProperty);
LOG.debug("columns: {}, {}", columnNameProperty, columnNames);
LOG.debug("types: {}, {} ", columnTypeProperty, columnTypes);
assert (columnNames.size() == columnTypes.size());
- rowTypeInfo = (StructTypeInfo) TypeInfoFactory.getStructTypeInfo(columnNames, columnTypes);
+ final String nullEmpty = tbl.getProperty(NULL_EMPTY_LINES, "false");
+ this.nullEmptyLines = Boolean.parseBoolean(nullEmpty);
+
+ this.rowTypeInfo = (StructTypeInfo) TypeInfoFactory
+ .getStructTypeInfo(columnNames, columnTypes);
+
+ this.soi = (StructObjectInspector) TypeInfoUtils
+ .getStandardWritableObjectInspectorFromTypeInfo(this.rowTypeInfo);
+
+ final TimestampParser tsParser;
+ final String parserFormats =
+ tbl.getProperty(serdeConstants.TIMESTAMP_FORMATS);
+ if (parserFormats != null) {
+ tsParser =
+ new TimestampParser(HiveStringUtils.splitAndUnEscape(parserFormats));
+ } else {
+ tsParser = new TimestampParser();
+ }
+
+ final String binaryEncodingStr = tbl.getProperty(BINARY_FORMAT, "base64");
+ this.binaryEncoding =
+ BinaryEncoding.valueOf(binaryEncodingStr.toUpperCase());
+
+ this.jsonReader = new HiveJsonReader(this.soi, tsParser);
+ this.jsonWriter = new HiveJsonWriter(this.binaryEncoding, columnNames);
+
+ this.jsonReader.setBinaryEncoding(binaryEncoding);
+ this.jsonReader.enable(HiveJsonReader.Feature.COL_INDEX_PARSING);
- TimestampParser tsParser = new TimestampParser(
- HiveStringUtils.splitAndUnEscape(tbl.getProperty(serdeConstants.TIMESTAMP_FORMATS)));
- structReader = new HiveJsonStructReader(rowTypeInfo, tsParser);
- structReader.setIgnoreUnknownFields(true);
- structReader.enableHiveColIndexParsing(true);
- structReader.setWritablesUsage(true);
+ if (writeablePrimitivesDeserialize) {
+ this.jsonReader.enable(HiveJsonReader.Feature.PRIMITIVE_TO_WRITABLE);
+ }
+
+ final String ignoreExtras = tbl.getProperty(IGNORE_EXTRA, "true");
+ if (Boolean.parseBoolean(ignoreExtras)) {
+ this.jsonReader.enable(HiveJsonReader.Feature.IGNORE_UKNOWN_FIELDS);
+ }
+
+ LOG.debug("JSON Struct Reader: {}", jsonReader);
+ LOG.debug("JSON Struct Writer: {}", jsonWriter);
}
/**
- * Takes JSON string in Text form, and has to return an object representation above
- * it that's readable by the corresponding object inspector.
+ * Deserialize an object out of a Writable blob containing JSON text. The
+ * return value of this function will be constant since the function will
+ * reuse the returned object. If the client wants to keep a copy of the
+ * object, the client needs to clone the returned value by calling
+ * ObjectInspectorUtils.getStandardObject().
*
- * For this implementation, since we're using the jackson parser, we can construct
- * our own object implementation, and we use HCatRecord for it
+ * @param blob The Writable (Text) object containing a serialized object
+ * @return A List containing all the values of the row
*/
@Override
- public Object deserialize(Writable blob) throws SerDeException {
+ public Object deserialize(final Writable blob) throws SerDeException {
+ final Text t = (Text) blob;
+
+ if (t.getLength() == 0) {
+ if (!this.nullEmptyLines) {
+ throw new SerDeException("Encountered an empty row in the text file");
+ }
+ final int fieldCount = soi.getAllStructFieldRefs().size();
+ return Collections.nCopies(fieldCount, null);
+ }
- Object row;
- Text t = (Text) blob;
try {
- row = structReader.parseStruct(new ByteArrayInputStream((t.getBytes()), 0, t.getLength()));
- return row;
+ return jsonReader.parseStruct(
+ new ByteArrayInputStream((t.getBytes()), 0, t.getLength()));
} catch (Exception e) {
- LOG.warn("Error [{}] parsing json text [{}].", e, t);
+ LOG.debug("Problem parsing JSON text [{}].", t, e);
throw new SerDeException(e);
}
}
@@ -142,227 +197,21 @@ public class JsonSerDe extends AbstractSerDe {
* and generate a Text representation of the object.
*/
@Override
- public Writable serialize(Object obj, ObjectInspector objInspector)
- throws SerDeException {
- StringBuilder sb = new StringBuilder();
- try {
+ public Writable serialize(final Object obj,
+ final ObjectInspector objInspector) throws SerDeException {
- StructObjectInspector soi = (StructObjectInspector) objInspector;
- List<? extends StructField> structFields = soi.getAllStructFieldRefs();
- assert (columnNames.size() == structFields.size());
- if (obj == null) {
- sb.append("null");
- } else {
- sb.append(SerDeUtils.LBRACE);
- for (int i = 0; i < structFields.size(); i++) {
- if (i > 0) {
- sb.append(SerDeUtils.COMMA);
- }
- appendWithQuotes(sb, columnNames.get(i));
- sb.append(SerDeUtils.COLON);
- buildJSONString(sb, soi.getStructFieldData(obj, structFields.get(i)),
- structFields.get(i).getFieldObjectInspector());
- }
- sb.append(SerDeUtils.RBRACE);
- }
+ final String jsonText = this.jsonWriter.write(obj, objInspector);
- } catch (IOException e) {
- LOG.warn("Error generating json text from object.", e);
- throw new SerDeException(e);
- }
- return new Text(sb.toString());
+ return new Text(jsonText);
}
- private static StringBuilder appendWithQuotes(StringBuilder sb, String value) {
- return sb == null ? null : sb.append(SerDeUtils.QUOTE).append(value).append(SerDeUtils.QUOTE);
- }
- // TODO : code section copied over from SerDeUtils because of non-standard json production there
- // should use quotes for all field names. We should fix this there, and then remove this copy.
- // See http://jackson.codehaus.org/1.7.3/javadoc/org/codehaus/jackson/JsonParser.Feature.html#ALLOW_UNQUOTED_FIELD_NAMES
- // for details - trying to enable Jackson to ignore that doesn't seem to work(compilation failure
- // when attempting to use that feature, so having to change the production itself.
- // Also, throws IOException when Binary is detected.
- private static void buildJSONString(StringBuilder sb, Object o, ObjectInspector oi) throws IOException {
-
- switch (oi.getCategory()) {
- case PRIMITIVE: {
- PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
- if (o == null) {
- sb.append("null");
- } else {
- switch (poi.getPrimitiveCategory()) {
- case BOOLEAN: {
- boolean b = ((BooleanObjectInspector) poi).get(o);
- sb.append(b ? "true" : "false");
- break;
- }
- case BYTE: {
- sb.append(((ByteObjectInspector) poi).get(o));
- break;
- }
- case SHORT: {
- sb.append(((ShortObjectInspector) poi).get(o));
- break;
- }
- case INT: {
- sb.append(((IntObjectInspector) poi).get(o));
- break;
- }
- case LONG: {
- sb.append(((LongObjectInspector) poi).get(o));
- break;
- }
- case FLOAT: {
- sb.append(((FloatObjectInspector) poi).get(o));
- break;
- }
- case DOUBLE: {
- sb.append(((DoubleObjectInspector) poi).get(o));
- break;
- }
- case STRING: {
- String s =
- SerDeUtils.escapeString(((StringObjectInspector) poi).getPrimitiveJavaObject(o));
- appendWithQuotes(sb, s);
- break;
- }
- case BINARY:
- byte[] b = ((BinaryObjectInspector) oi).getPrimitiveJavaObject(o);
- Text txt = new Text();
- txt.set(b, 0, b.length);
- appendWithQuotes(sb, SerDeUtils.escapeString(txt.toString()));
- break;
- case DATE:
- Date d = ((DateObjectInspector) poi).getPrimitiveJavaObject(o);
- appendWithQuotes(sb, d.toString());
- break;
- case TIMESTAMP: {
- Timestamp t = ((TimestampObjectInspector) poi).getPrimitiveJavaObject(o);
- appendWithQuotes(sb, t.toString());
- break;
- }
- case DECIMAL:
- sb.append(((HiveDecimalObjectInspector) poi).getPrimitiveJavaObject(o));
- break;
- case VARCHAR: {
- String s = SerDeUtils.escapeString(
- ((HiveVarcharObjectInspector) poi).getPrimitiveJavaObject(o).toString());
- appendWithQuotes(sb, s);
- break;
- }
- case CHAR: {
- //this should use HiveChar.getPaddedValue() but it's protected; currently (v0.13)
- // HiveChar.toString() returns getPaddedValue()
- String s = SerDeUtils.escapeString(
- ((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString());
- appendWithQuotes(sb, s);
- break;
- }
- default:
- throw new RuntimeException("Unknown primitive type: " + poi.getPrimitiveCategory());
- }
- }
- break;
- }
- case LIST: {
- ListObjectInspector loi = (ListObjectInspector) oi;
- ObjectInspector listElementObjectInspector = loi
- .getListElementObjectInspector();
- List<?> olist = loi.getList(o);
- if (olist == null) {
- sb.append("null");
- } else {
- sb.append(SerDeUtils.LBRACKET);
- for (int i = 0; i < olist.size(); i++) {
- if (i > 0) {
- sb.append(SerDeUtils.COMMA);
- }
- buildJSONString(sb, olist.get(i), listElementObjectInspector);
- }
- sb.append(SerDeUtils.RBRACKET);
- }
- break;
- }
- case MAP: {
- MapObjectInspector moi = (MapObjectInspector) oi;
- ObjectInspector mapKeyObjectInspector = moi.getMapKeyObjectInspector();
- ObjectInspector mapValueObjectInspector = moi
- .getMapValueObjectInspector();
- Map<?, ?> omap = moi.getMap(o);
- if (omap == null) {
- sb.append("null");
- } else {
- sb.append(SerDeUtils.LBRACE);
- boolean first = true;
- for (Object entry : omap.entrySet()) {
- if (first) {
- first = false;
- } else {
- sb.append(SerDeUtils.COMMA);
- }
- Map.Entry<?, ?> e = (Map.Entry<?, ?>) entry;
- StringBuilder keyBuilder = new StringBuilder();
- buildJSONString(keyBuilder, e.getKey(), mapKeyObjectInspector);
- String keyString = keyBuilder.toString().trim();
- if ((!keyString.isEmpty()) && (keyString.charAt(0) != SerDeUtils.QUOTE)) {
- appendWithQuotes(sb, keyString);
- } else {
- sb.append(keyString);
- }
- sb.append(SerDeUtils.COLON);
- buildJSONString(sb, e.getValue(), mapValueObjectInspector);
- }
- sb.append(SerDeUtils.RBRACE);
- }
- break;
- }
- case STRUCT: {
- StructObjectInspector soi = (StructObjectInspector) oi;
- List<? extends StructField> structFields = soi.getAllStructFieldRefs();
- if (o == null) {
- sb.append("null");
- } else {
- sb.append(SerDeUtils.LBRACE);
- for (int i = 0; i < structFields.size(); i++) {
- if (i > 0) {
- sb.append(SerDeUtils.COMMA);
- }
- appendWithQuotes(sb, structFields.get(i).getFieldName());
- sb.append(SerDeUtils.COLON);
- buildJSONString(sb, soi.getStructFieldData(o, structFields.get(i)),
- structFields.get(i).getFieldObjectInspector());
- }
- sb.append(SerDeUtils.RBRACE);
- }
- break;
- }
- case UNION: {
- UnionObjectInspector uoi = (UnionObjectInspector) oi;
- if (o == null) {
- sb.append("null");
- } else {
- sb.append(SerDeUtils.LBRACE);
- sb.append(uoi.getTag(o));
- sb.append(SerDeUtils.COLON);
- buildJSONString(sb, uoi.getField(o),
- uoi.getObjectInspectors().get(uoi.getTag(o)));
- sb.append(SerDeUtils.RBRACE);
- }
- break;
- }
- default:
- throw new RuntimeException("Unknown type in ObjectInspector!");
- }
- }
-
-
/**
- * Returns an object inspector for the specified schema that
- * is capable of reading in the object representation of the JSON string
+ * Returns an object inspector for the specified schema that is capable of
+ * reading in the object representation of the JSON string.
*/
@Override
public ObjectInspector getObjectInspector() throws SerDeException {
- return structReader.getObjectInspector();
+ return jsonReader.getObjectInspector();
}
@Override
@@ -380,8 +229,11 @@ public class JsonSerDe extends AbstractSerDe {
return rowTypeInfo;
}
- public void setWriteablesUsage(boolean b) {
- structReader.setWritablesUsage(b);
+ public BinaryEncoding getBinaryEncoding() {
+ return binaryEncoding;
}
+ public boolean isNullEmptyLines() {
+ return nullEmptyLines;
+ }
}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/json/BinaryEncoding.java b/serde/src/java/org/apache/hadoop/hive/serde2/json/BinaryEncoding.java
new file mode 100644
index 0000000..02f26b0
--- /dev/null
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/json/BinaryEncoding.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.json;
+
+/**
+ * Enums describing the available String->Bytes encoding available for JSON
+ * parsing. This base-64 variant is what most people would think of "the
+ * standard" Base64 encoding for JSON: the specific MIME content transfer
+ * encoding. The Raw String encoding produces an array of bytes by reading the
+ * JSON value as a String and transforming it using Java's String getBytes()
+ * method.
+ *
+ * @see String#getBytes(java.nio.charset.Charset)
+ */
+public enum BinaryEncoding {
+ RAWSTRING, BASE64
+}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java b/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java
new file mode 100644
index 0000000..29be63e
--- /dev/null
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonReader.java
@@ -0,0 +1,557 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.json;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.time.ZoneId;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.hadoop.hive.common.type.Date;
+import org.apache.hadoop.hive.common.type.HiveChar;
+import org.apache.hadoop.hive.common.type.HiveDecimal;
+import org.apache.hadoop.hive.common.type.HiveVarchar;
+import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TimestampLocalTZTypeInfo;
+import org.apache.hive.common.util.TimestampParser;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.JsonNodeType;
+import com.fasterxml.jackson.databind.node.TextNode;
+import com.google.common.base.Preconditions;
+
+/**
+ * This class converts JSON strings into Java or Hive Primitive objects.
+ *
+ * Support types are:<br/>
+ * <br/>
+ * <table border="1">
+ * <tr>
+ * <th>JSON Type</th>
+ * <th>Java Type</th>
+ * <th>Notes</th>
+ * </tr>
+ * <tr>
+ * <td>Object</td>
+ * <td>java.util.List</td>
+ * <td>Each element may be different type
+ * </tr>
+ * <tr>
+ * <td>Array</td>
+ * <td>java.util.List</td>
+ * <td>Each element is same type</td>
+ * </tr>
+ * <tr>
+ * <td>Map</td>
+ * <td>java.util.Map</td>
+ * <td>Keys must be same primitive type; every value is the same type</td>
+ * </tr>
+ * </table>
+ */
+public class HiveJsonReader {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HiveJsonReader.class);
+
+ private final Map<Pair<StructObjectInspector, String>, StructField> discoveredFields =
+ new HashMap<>();
+
+ private final Set<Pair<StructObjectInspector, String>> discoveredUnknownFields =
+ new HashSet<>();
+
+ private final EnumSet<Feature> features = EnumSet.noneOf(Feature.class);
+
+ private final ObjectMapper objectMapper;
+
+ private final TimestampParser tsParser;
+ private BinaryEncoding binaryEncoding;
+ private final ObjectInspector oi;
+
+ /**
+ * Enumeration that defines all on/off features for this reader.
+ * <ul>
+ * <li>{@link #COL_INDEX_PARSING}</li>
+ * <li>{@link #PRIMITIVE_TO_WRITABLE}</li>
+ * <li>{@link #IGNORE_UKNOWN_FIELDS}</li>
+ * </ul>
+ */
+ public enum Feature {
+ /**
+ * Enables an optimization to look up each JSON field based on its index in
+ * the Hive schema.
+ */
+ COL_INDEX_PARSING,
+
+ /**
+ * If this feature is enabled, when a JSON node is parsed, its value will be
+ * returned as a Hadoop Writable object. Otherwise, the Java native value is
+ * returned.
+ */
+ PRIMITIVE_TO_WRITABLE,
+
+ /**
+ * If the JSON object being parsed includes a field that is not included in
+ * the Hive schema, enabling this feature will cause the JSON reader to
+ * produce a log warnings. If this feature is disabled, an Exception will be
+ * thrown and parsing will stop.
+ */
+ IGNORE_UKNOWN_FIELDS
+ }
+
+ /**
+ * Constructor with default the Hive default timestamp parser.
+ *
+ * @param oi ObjectInspector for all the fields in the JSON object
+ */
+ public HiveJsonReader(ObjectInspector oi) {
+ this(oi, new TimestampParser());
+ }
+
+ /**
+ * Constructor with default the Hive default timestamp parser.
+ *
+ * @param oi ObjectInspector info for all the fields in the JSON object
+ * @param tsParser Custom timestamp parser
+ */
+ public HiveJsonReader(ObjectInspector oi, TimestampParser tsParser) {
+ this.binaryEncoding = BinaryEncoding.BASE64;
+ this.tsParser = tsParser;
+ this.oi = oi;
+ this.objectMapper = new ObjectMapper();
+ }
+
+ /**
+ * Parse text containing a complete JSON object.
+ *
+ * @param text The text to parse
+ * @return A List of Objects, one for each field in the JSON object
+ * @throws IOException Unable to parse the JSON text
+ * @throws SerDeException The SerDe is not configured correctly
+ */
+ public Object parseStruct(final String text)
+ throws IOException, SerDeException {
+ Preconditions.checkNotNull(text);
+ Preconditions.checkState(this.oi != null);
+ final JsonNode rootNode = this.objectMapper.reader().readTree(text);
+ return visitNode(rootNode, this.oi);
+ }
+
+ /**
+ * Parse text containing a complete JSON object.
+ *
+ * @param in The InputStream to read the text from
+ * @return A List of Objects, one for each field in the JSON object
+ * @throws IOException Unable to parse the JSON text
+ * @throws SerDeException The SerDe is not configured correctly
+ */
+ public Object parseStruct(final InputStream in)
+ throws IOException, SerDeException {
+ Preconditions.checkNotNull(in);
+ Preconditions.checkState(this.oi != null);
+ final JsonNode rootNode = this.objectMapper.reader().readTree(in);
+ return visitNode(rootNode, this.oi);
+ }
+
+ /**
+ * Visit a node and parse it based on the provided ObjectInspector.
+ *
+ * @param rootNode The root node to process
+ * @param oi The ObjectInspector to use
+ * @return The value in this node. Return value may be null, primitive, and
+ * may be a complex type if nested.
+ * @throws SerDeException The SerDe is not configured correctly
+ */
+ private Object visitNode(final JsonNode rootNode, final ObjectInspector oi)
+ throws SerDeException {
+
+ if (!rootNode.isNull()) {
+ switch (oi.getCategory()) {
+ case PRIMITIVE:
+ final Object value = visitLeafNode(rootNode, oi);
+ return optionallyWrapWritable(value, oi);
+ case LIST:
+ return visitArrayNode(rootNode, oi);
+ case STRUCT:
+ return visitStructNode(rootNode, oi);
+ case MAP:
+ return visitMapNode(rootNode, oi);
+ default:
+ throw new SerDeException(
+ "Parsing of: " + oi.getCategory() + " is not supported");
+ }
+ }
+
+ return null;
+ }
+
+ /**
+ * The typical usage of this SerDe requires that it return Hadoop Writable
+ * objects. However, some uses of this SerDe want the return values to be Java
+ * primitive objects. This SerDe works explicitly in Java primitive objects
+ * and will wrap the objects in Writable containers if required.
+ *
+ * @param value The Java primitive object to wrap
+ * @param oi The ObjectInspector provides the type to wrap into
+ * @return A Hadoop Writable if required; otherwise the object itself
+ */
+ private Object optionallyWrapWritable(final Object value,
+ final ObjectInspector oi) {
+ if (!isEnabled(Feature.PRIMITIVE_TO_WRITABLE)) {
+ return value;
+ }
+
+ final PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
+ final PrimitiveTypeInfo typeInfo = poi.getTypeInfo();
+
+ return PrimitiveObjectInspectorFactory
+ .getPrimitiveJavaObjectInspector(typeInfo)
+ .getPrimitiveWritableObject(value);
+ }
+
+ /**
+ * Visit a node if it is expected to be a Map (a.k.a. JSON Object)
+ *
+ * @param rootNode The node pointing at the JSON object
+ * @param oi The ObjectInspector to parse the Map (must be a
+ * MapObjectInspector)
+ * @return A Java Map containing the contents of the JSON map
+ * @throws SerDeException The SerDe is not configured correctly
+ */
+ private Map<Object, Object> visitMapNode(final JsonNode rootNode,
+ final ObjectInspector oi) throws SerDeException {
+ Preconditions.checkArgument(JsonNodeType.OBJECT == rootNode.getNodeType());
+
+ final Map<Object, Object> ret = new LinkedHashMap<>();
+
+ final ObjectInspector mapKeyInspector =
+ ((MapObjectInspector) oi).getMapKeyObjectInspector();
+
+ final ObjectInspector mapValueInspector =
+ ((MapObjectInspector) oi).getMapValueObjectInspector();
+
+ if (!(mapKeyInspector instanceof PrimitiveObjectInspector)) {
+ throw new SerDeException("Map key must be a primitive type");
+ }
+
+ final Iterator<Entry<String, JsonNode>> it = rootNode.fields();
+ while (it.hasNext()) {
+ final Entry<String, JsonNode> field = it.next();
+ final Object key =
+ visitNode(new TextNode(field.getKey()), mapKeyInspector);
+ final Object val = visitNode(field.getValue(), mapValueInspector);
+ ret.put(key, val);
+ }
+
+ return ret;
+ }
+
+ /**
+ * Visit a node if it is expected to be a Struct data type (a.k.a. JSON
+ * Object)
+ *
+ * @param rootNode The node pointing at the JSON object
+ * @param oi The ObjectInspector to parse the Map (must be a
+ * StructObjectInspector)
+ * @return A primitive array of Objects, each element is an element of the
+ * struct
+ * @throws SerDeException The SerDe is not configured correctly
+ */
+ private List<Object> visitStructNode(final JsonNode rootNode,
+ final ObjectInspector oi) throws SerDeException {
+
+ Preconditions.checkArgument(JsonNodeType.OBJECT == rootNode.getNodeType());
+
+ final StructObjectInspector structInspector = (StructObjectInspector) oi;
+
+ final int fieldCount = structInspector.getAllStructFieldRefs().size();
+ final List<Object> ret = Arrays.asList(new Object[fieldCount]);
+
+ final Iterator<Entry<String, JsonNode>> it = rootNode.fields();
+ while (it.hasNext()) {
+ final Entry<String, JsonNode> field = it.next();
+ final String fieldName = field.getKey();
+ final JsonNode childNode = field.getValue();
+ final StructField structField =
+ getStructField(structInspector, fieldName);
+
+ // If the struct field is null it is because there is a field defined in
+ // the JSON object that was not defined in the table definition. Ignore.
+ if (structField != null) {
+ final Object childValue =
+ visitNode(childNode, structField.getFieldObjectInspector());
+ ret.set(structField.getFieldID(), childValue);
+ }
+ }
+
+ return ret;
+ }
+
+ /**
+ * Visit a node if it is expected to be a JSON Array data type (a.k.a. Hive
+ * Array type)
+ *
+ * @param rootNode The node pointing at the JSON object
+ * @param oi The ObjectInspector to parse the List (must be a
+ * ListObjectInspector)
+ * @return A Java List of Objects, each element is an element of the array
+ * @throws SerDeException The SerDe is not configured correctly
+ */
+ private List<Object> visitArrayNode(final JsonNode rootNode,
+ final ObjectInspector oi) throws SerDeException {
+ Preconditions.checkArgument(JsonNodeType.ARRAY == rootNode.getNodeType());
+
+ final ObjectInspector loi =
+ ((ListObjectInspector) oi).getListElementObjectInspector();
+
+ final List<Object> ret = new ArrayList<>(rootNode.size());
+ final Iterator<JsonNode> it = rootNode.elements();
+
+ while (it.hasNext()) {
+ final JsonNode element = it.next();
+ ret.add(visitNode(element, loi));
+ }
+
+ return ret;
+ }
+
+ /**
+ * Visit a node if it is expected to be a primitive value (JSON leaf node).
+ *
+ * @param leafNode The node pointing at the JSON object
+ * @param oi The ObjectInspector to parse the value (must be a
+ * PrimitiveObjectInspector)
+ * @return A Java primitive Object
+ * @throws SerDeException The SerDe is not configured correctly
+ */
+ private Object visitLeafNode(final JsonNode leafNode,
+ final ObjectInspector oi) throws SerDeException {
+ Preconditions.checkArgument(leafNode.getNodeType() != JsonNodeType.OBJECT);
+ Preconditions.checkArgument(leafNode.getNodeType() != JsonNodeType.ARRAY);
+
+ final PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
+ final PrimitiveTypeInfo typeInfo = poi.getTypeInfo();
+
+ switch (typeInfo.getPrimitiveCategory()) {
+ case INT:
+ return Integer.valueOf(leafNode.asInt());
+ case BYTE:
+ return Byte.valueOf((byte) leafNode.asInt());
+ case SHORT:
+ return Short.valueOf((short) leafNode.asInt());
+ case LONG:
+ return Long.valueOf(leafNode.asLong());
+ case BOOLEAN:
+ return Boolean.valueOf(leafNode.asBoolean());
+ case FLOAT:
+ return Float.valueOf((float) leafNode.asDouble());
+ case DOUBLE:
+ return Double.valueOf(leafNode.asDouble());
+ case STRING:
+ return leafNode.asText();
+ case BINARY:
+ return getByteValue(leafNode);
+ case DATE:
+ return Date.valueOf(leafNode.asText());
+ case TIMESTAMP:
+ return tsParser.parseTimestamp(leafNode.asText());
+ case DECIMAL:
+ return HiveDecimal.create(leafNode.asText());
+ case TIMESTAMPLOCALTZ:
+ final Timestamp ts = tsParser.parseTimestamp(leafNode.asText());
+ final ZoneId zid = ((TimestampLocalTZTypeInfo) typeInfo).timeZone();
+ final TimestampTZ tstz = new TimestampTZ();
+ tstz.set(ts.toEpochSecond(), ts.getNanos(), zid);
+ return tstz;
+ case VARCHAR:
+ return new HiveVarchar(leafNode.asText(),
+ ((BaseCharTypeInfo) typeInfo).getLength());
+ case CHAR:
+ return new HiveChar(leafNode.asText(),
+ ((BaseCharTypeInfo) typeInfo).getLength());
+ default:
+ throw new SerDeException(
+ "Could not convert from string to type: " + typeInfo.getTypeName());
+ }
+ }
+
+ /**
+ * A user may configure the encoding for binary data represented as text
+ * within a JSON object. This method applies that encoding to the text.
+ *
+ * @param binaryNode JSON Node containing the binary data
+ * @return A byte array with the binary data
+ * @throws SerDeException The SerDe is not configured correctly
+ */
+ private byte[] getByteValue(final JsonNode binaryNode) throws SerDeException {
+ try {
+ switch (this.binaryEncoding) {
+ case RAWSTRING:
+ final String byteText = binaryNode.textValue();
+ return byteText.getBytes(StandardCharsets.UTF_8);
+ case BASE64:
+ return binaryNode.binaryValue();
+ default:
+ throw new SerDeException(
+ "No such binary encoding: " + this.binaryEncoding);
+ }
+ } catch (IOException e) {
+ throw new SerDeException("Error generating JSON binary type from record.",
+ e);
+ }
+ }
+
+ /**
+ * Matches the JSON object's field name with the Hive data type.
+ *
+ * @param oi The ObjectInsepctor to lookup the matching in
+ * @param fieldName The name of the field parsed from the JSON text
+ * @return The meta data of regarding this field
+ * @throws SerDeException The SerDe is not configured correctly
+ */
+ private StructField getStructField(final StructObjectInspector oi,
+ final String fieldName) throws SerDeException {
+
+ final Pair<StructObjectInspector, String> pair =
+ ImmutablePair.of(oi, fieldName);
+
+ // Ignore the field if it has been ignored before
+ if (this.discoveredUnknownFields.contains(pair)) {
+ return null;
+ }
+
+ // Return from cache if the field has already been discovered
+ StructField structField = this.discoveredFields.get(pair);
+ if (structField != null) {
+ return structField;
+ }
+
+ // Otherwise attempt to discover the field
+ if (isEnabled(Feature.COL_INDEX_PARSING)) {
+ int colIndex = getColIndex(fieldName);
+ if (colIndex >= 0) {
+ structField = oi.getAllStructFieldRefs().get(colIndex);
+ }
+ }
+ if (structField == null) {
+ try {
+ structField = oi.getStructFieldRef(fieldName);
+ } catch (Exception e) {
+ // No such field
+ }
+ }
+ if (structField != null) {
+ // cache it for next time
+ this.discoveredFields.put(pair, structField);
+ } else {
+ // Tried everything and did not discover this field
+ if (isEnabled(Feature.IGNORE_UKNOWN_FIELDS)
+ && this.discoveredUnknownFields.add(pair)) {
+ LOG.warn("Discovered unknown field: {}. Ignoring.", fieldName);
+ } else {
+ throw new SerDeException(
+ "Field found in JSON does not match table definition: "
+ + fieldName);
+ }
+ }
+
+ return structField;
+ }
+
+ private final Pattern internalPattern = Pattern.compile("^_col([0-9]+)$");
+
+ /**
+ * Look up a column based on its index.
+ *
+ * @param internalName The name of the column
+ * @return The index of the field or -1 if the field name does not contain its
+ * index number too
+ */
+ private int getColIndex(final String internalName) {
+ // The above line should have been all the implementation that
+ // we need, but due to a bug in that impl which recognizes
+ // only single-digit columns, we need another impl here.
+ final Matcher m = internalPattern.matcher(internalName);
+ return m.matches() ? Integer.parseInt(m.group(1)) : -1;
+ }
+
+ public void enable(Feature feature) {
+ this.features.add(feature);
+ }
+
+ public void disable(Feature feature) {
+ this.features.remove(feature);
+ }
+
+ public Set<Feature> getFeatures() {
+ return Collections.unmodifiableSet(this.features);
+ }
+
+ public boolean isEnabled(Feature feature) {
+ return this.features.contains(feature);
+ }
+
+ public ObjectInspector getObjectInspector() {
+ return oi;
+ }
+
+ public BinaryEncoding getBinaryEncodingType() {
+ return binaryEncoding;
+ }
+
+ public void setBinaryEncoding(BinaryEncoding encoding) {
+ this.binaryEncoding = encoding;
+ }
+
+ @Override
+ public String toString() {
+ return "HiveJsonReader [features=" + features + ", tsParser=" + tsParser
+ + ", binaryEncoding=" + binaryEncoding + "]";
+ }
+
+}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonStructReader.java b/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonStructReader.java
deleted file mode 100644
index ec4efad..0000000
--- a/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonStructReader.java
+++ /dev/null
@@ -1,402 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hive.serde2.json;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.CharacterCodingException;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.apache.hadoop.hive.common.type.Date;
-import org.apache.hadoop.hive.common.type.HiveChar;
-import org.apache.hadoop.hive.common.type.HiveDecimal;
-import org.apache.hadoop.hive.common.type.HiveVarchar;
-import org.apache.hadoop.hive.serde2.SerDeException;
-import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorConverters.Converter;
-import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.StructField;
-import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.hive.serde2.typeinfo.BaseCharTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
-import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
-import org.apache.hadoop.io.Text;
-import org.apache.hive.common.util.TimestampParser;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParseException;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonToken;
-
-public class HiveJsonStructReader {
-
- private static final Logger LOG = LoggerFactory.getLogger(HiveJsonStructReader.class);
-
- private ObjectInspector oi;
- private JsonFactory factory;
-
-
- Set<String> reportedUnknownFieldNames = new HashSet<>();
-
- private static boolean ignoreUnknownFields;
- private static boolean hiveColIndexParsing;
- private boolean writeablePrimitives;
-
- private TimestampParser tsParser;
-
- public HiveJsonStructReader(TypeInfo t) {
- this(t, new TimestampParser());
- }
-
- public HiveJsonStructReader(TypeInfo t, TimestampParser tsParser) {
- this.tsParser = tsParser;
- oi = TypeInfoUtils.getStandardWritableObjectInspectorFromTypeInfo(t);
- factory = new JsonFactory();
- }
-
- public Object parseStruct(String text) throws JsonParseException, IOException, SerDeException {
- JsonParser parser = factory.createParser(text);
- return parseInternal(parser);
- }
-
- public Object parseStruct(InputStream is) throws JsonParseException, IOException, SerDeException {
- JsonParser parser = factory.createParser(is);
- return parseInternal(parser);
- }
-
- private Object parseInternal(JsonParser parser) throws SerDeException {
- try {
- parser.nextToken();
- Object res = parseDispatcher(parser, oi);
- return res;
- } catch (Exception e) {
- String locationStr = parser.getCurrentLocation().getLineNr() + "," + parser.getCurrentLocation().getColumnNr();
- throw new SerDeException("at[" + locationStr + "]: " + e.getMessage(), e);
- }
- }
-
- private Object parseDispatcher(JsonParser parser, ObjectInspector oi)
- throws JsonParseException, IOException, SerDeException {
-
- switch (oi.getCategory()) {
- case PRIMITIVE:
- return parsePrimitive(parser, (PrimitiveObjectInspector) oi);
- case LIST:
- return parseList(parser, (ListObjectInspector) oi);
- case STRUCT:
- return parseStruct(parser, (StructObjectInspector) oi);
- case MAP:
- return parseMap(parser, (MapObjectInspector) oi);
- default:
- throw new SerDeException("parsing of: " + oi.getCategory() + " is not handled");
- }
- }
-
- private Object parseMap(JsonParser parser, MapObjectInspector oi) throws IOException, SerDeException {
-
- if (parser.getCurrentToken() == JsonToken.VALUE_NULL) {
- parser.nextToken();
- return null;
- }
-
- Map<Object, Object> ret = new LinkedHashMap<>();
-
- if (parser.getCurrentToken() != JsonToken.START_OBJECT) {
- throw new SerDeException("struct expected");
- }
-
- if (!(oi.getMapKeyObjectInspector() instanceof PrimitiveObjectInspector)) {
- throw new SerDeException("map key must be a primitive");
- }
- PrimitiveObjectInspector keyOI = (PrimitiveObjectInspector) oi.getMapKeyObjectInspector();
- ObjectInspector valOI = oi.getMapValueObjectInspector();
-
- JsonToken currentToken = parser.nextToken();
- while (currentToken != null && currentToken != JsonToken.END_OBJECT) {
-
- if (currentToken != JsonToken.FIELD_NAME) {
- throw new SerDeException("unexpected token: " + currentToken);
- }
-
- Object key = parseMapKey(parser, keyOI);
- Object val = parseDispatcher(parser, valOI);
- ret.put(key, val);
-
- currentToken = parser.getCurrentToken();
- }
- if (currentToken != null) {
- parser.nextToken();
- }
- return ret;
-
- }
-
- private Object parseStruct(JsonParser parser, StructObjectInspector oi)
- throws JsonParseException, IOException, SerDeException {
-
- Object[] ret = new Object[oi.getAllStructFieldRefs().size()];
-
- if (parser.getCurrentToken() == JsonToken.VALUE_NULL) {
- parser.nextToken();
- return null;
- }
- if (parser.getCurrentToken() != JsonToken.START_OBJECT) {
- throw new SerDeException("struct expected");
- }
- JsonToken currentToken = parser.nextToken();
- while (currentToken != null && currentToken != JsonToken.END_OBJECT) {
-
- switch (currentToken) {
- case FIELD_NAME:
- String name = parser.getCurrentName();
- try {
- StructField field = null;
- try {
- field = getStructField(oi, name);
- } catch (RuntimeException e) {
- if (ignoreUnknownFields) {
- if (!reportedUnknownFieldNames.contains(name)) {
- LOG.warn("ignoring field:" + name);
- reportedUnknownFieldNames.add(name);
- }
- parser.nextToken();
- skipValue(parser);
- break;
- }
- }
- if (field == null) {
- throw new SerDeException("undeclared field");
- }
- parser.nextToken();
- ret[field.getFieldID()] = parseDispatcher(parser, field.getFieldObjectInspector());
- } catch (Exception e) {
- throw new SerDeException("struct field " + name + ": " + e.getMessage(), e);
- }
- break;
- default:
- throw new SerDeException("unexpected token: " + currentToken);
- }
- currentToken = parser.getCurrentToken();
- }
- if (currentToken != null) {
- parser.nextToken();
- }
- return ret;
- }
-
- private StructField getStructField(StructObjectInspector oi, String name) {
- if (hiveColIndexParsing) {
- int colIndex = getColIndex(name);
- if (colIndex >= 0) {
- return oi.getAllStructFieldRefs().get(colIndex);
- }
- }
- // FIXME: linear scan inside the below method...get a map here or something..
- return oi.getStructFieldRef(name);
- }
-
- Pattern internalPattern = Pattern.compile("^_col([0-9]+)$");
-
- private int getColIndex(String internalName) {
- // The above line should have been all the implementation that
- // we need, but due to a bug in that impl which recognizes
- // only single-digit columns, we need another impl here.
- Matcher m = internalPattern.matcher(internalName);
- if (!m.matches()) {
- return -1;
- } else {
- return Integer.parseInt(m.group(1));
- }
- }
-
- private static void skipValue(JsonParser parser) throws JsonParseException, IOException {
-
- int array = 0;
- int object = 0;
- do {
- JsonToken currentToken = parser.getCurrentToken();
- if(currentToken == JsonToken.START_ARRAY) {
- array++;
- }
- if (currentToken == JsonToken.END_ARRAY) {
- array--;
- }
- if (currentToken == JsonToken.START_OBJECT) {
- object++;
- }
- if (currentToken == JsonToken.END_OBJECT) {
- object--;
- }
-
- parser.nextToken();
-
- } while (array > 0 || object > 0);
-
- }
-
- private Object parseList(JsonParser parser, ListObjectInspector oi)
- throws JsonParseException, IOException, SerDeException {
- List<Object> ret = new ArrayList<>();
-
- if (parser.getCurrentToken() == JsonToken.VALUE_NULL) {
- parser.nextToken();
- return null;
- }
-
- if (parser.getCurrentToken() != JsonToken.START_ARRAY) {
- throw new SerDeException("array expected");
- }
- ObjectInspector eOI = oi.getListElementObjectInspector();
- JsonToken currentToken = parser.nextToken();
- try {
- while (currentToken != null && currentToken != JsonToken.END_ARRAY) {
- ret.add(parseDispatcher(parser, eOI));
- currentToken = parser.getCurrentToken();
- }
- } catch (Exception e) {
- throw new SerDeException("array: " + e.getMessage(), e);
- }
-
- currentToken = parser.nextToken();
-
- return ret;
- }
-
- private Object parsePrimitive(JsonParser parser, PrimitiveObjectInspector oi)
- throws SerDeException, IOException {
- JsonToken currentToken = parser.getCurrentToken();
- if (currentToken == null) {
- return null;
- }
- try {
- switch (parser.getCurrentToken()) {
- case VALUE_FALSE:
- case VALUE_TRUE:
- case VALUE_NUMBER_INT:
- case VALUE_NUMBER_FLOAT:
- case VALUE_STRING:
- return getObjectOfCorrespondingPrimitiveType(parser.getValueAsString(), oi);
- case VALUE_NULL:
- return null;
- default:
- throw new SerDeException("unexpected token type: " + currentToken);
- }
- } finally {
- parser.nextToken();
-
- }
- }
-
- private Object getObjectOfCorrespondingPrimitiveType(String s, PrimitiveObjectInspector oi)
- throws IOException {
- PrimitiveTypeInfo typeInfo = oi.getTypeInfo();
- if (writeablePrimitives) {
- Converter c = ObjectInspectorConverters.getConverter(PrimitiveObjectInspectorFactory.javaStringObjectInspector, oi);
- return c.convert(s);
- }
-
- switch (typeInfo.getPrimitiveCategory()) {
- case INT:
- return Integer.valueOf(s);
- case BYTE:
- return Byte.valueOf(s);
- case SHORT:
- return Short.valueOf(s);
- case LONG:
- return Long.valueOf(s);
- case BOOLEAN:
- return (s.equalsIgnoreCase("true"));
- case FLOAT:
- return Float.valueOf(s);
- case DOUBLE:
- return Double.valueOf(s);
- case STRING:
- return s;
- case BINARY:
- try {
- String t = Text.decode(s.getBytes(), 0, s.getBytes().length);
- return t.getBytes();
- } catch (CharacterCodingException e) {
- LOG.warn("Error generating json binary type from object.", e);
- return null;
- }
- case DATE:
- return Date.valueOf(s);
- case TIMESTAMP:
- return tsParser.parseTimestamp(s);
- case DECIMAL:
- return HiveDecimal.create(s);
- case VARCHAR:
- return new HiveVarchar(s, ((BaseCharTypeInfo) typeInfo).getLength());
- case CHAR:
- return new HiveChar(s, ((BaseCharTypeInfo) typeInfo).getLength());
- }
- throw new IOException("Could not convert from string to map type " + typeInfo.getTypeName());
- }
-
- private Object parseMapKey(JsonParser parser, PrimitiveObjectInspector oi) throws SerDeException, IOException {
- JsonToken currentToken = parser.getCurrentToken();
- if (currentToken == null) {
- return null;
- }
- try {
- switch (parser.getCurrentToken()) {
- case FIELD_NAME:
- return getObjectOfCorrespondingPrimitiveType(parser.getValueAsString(), oi);
- case VALUE_NULL:
- return null;
- default:
- throw new SerDeException("unexpected token type: " + currentToken);
- }
- } finally {
- parser.nextToken();
-
- }
- }
-
- public void setIgnoreUnknownFields(boolean b) {
- ignoreUnknownFields = b;
- }
-
- public void enableHiveColIndexParsing(boolean b) {
- hiveColIndexParsing = b;
- }
-
- public void setWritablesUsage(boolean b) {
- writeablePrimitives = b;
- }
-
- public ObjectInspector getObjectInspector() {
- return oi;
- }
-}
diff --git a/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonWriter.java b/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonWriter.java
new file mode 100644
index 0000000..cacc70b
--- /dev/null
+++ b/serde/src/java/org/apache/hadoop/hive/serde2/json/HiveJsonWriter.java
@@ -0,0 +1,386 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2.json;
+
+import java.util.Collections;
+import java.util.EnumSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.objectinspector.ListObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.MapObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.StructField;
+import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.UnionObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.BooleanObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ByteObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DateObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.DoubleObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.FloatObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveCharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveDecimalObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.HiveVarcharObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.IntObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.LongObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.ShortObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
+import org.apache.hadoop.io.Text;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.ObjectWriter;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ContainerNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.ValueNode;
+
+/**
+ * A class which takes Java Objects and a ObjectInspector to produce a JSON
+ * string representation of the structure. The Java Object can be a Collection
+ * containing other Collections and therefore can be thought of as a tree or as
+ * a directed acyclic graph that must be walked and each node recorded as JSON.
+ */
+public class HiveJsonWriter {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(HiveJsonWriter.class);
+
+ private final EnumSet<Feature> features = EnumSet.noneOf(Feature.class);
+
+ private final List<String> rootStructFieldNames;
+
+ private final JsonNodeFactory nodeFactory;
+ private final ObjectMapper mapper;
+ private BinaryEncoding binaryEncoding;
+
+ /**
+ * Enumeration that defines all on/off features for this writer.
+ */
+ public enum Feature {
+ PRETTY_PRINT
+ }
+
+ /**
+ * Default constructor. Uses base-64 mime encoding for any binary data.
+ */
+ public HiveJsonWriter() {
+ this(BinaryEncoding.BASE64, Collections.emptyList());
+ }
+
+ /**
+ * Constructs a JSON writer.
+ *
+ * @param encoding The encoding to use for binary data
+ * @param rootStructFieldNames The names of the fields at the root level
+ */
+ public HiveJsonWriter(final BinaryEncoding encoding,
+ final List<String> rootStructFieldNames) {
+ this.binaryEncoding = encoding;
+ this.rootStructFieldNames = rootStructFieldNames;
+ this.nodeFactory = JsonNodeFactory.instance;
+ this.mapper = new ObjectMapper();
+ }
+
+ /**
+ * Given an Object and an ObjectInspector, convert the structure into a JSON
+ * text string.
+ *
+ * @param o The object to convert
+ * @param objInspector The ObjectInspector describing the object
+ * @return A String containing the JSON text
+ * @throws SerDeException The object cannot be transformed
+ */
+ public String write(final Object o, final ObjectInspector objInspector)
+ throws SerDeException {
+
+ final JsonNode rootNode =
+ walkObjectGraph(objInspector, o, rootStructFieldNames);
+
+ LOG.debug("Create JSON tree from Object tree: {}", rootNode);
+
+ try {
+ final ObjectWriter ow = isEnabled(Feature.PRETTY_PRINT)
+ ? this.mapper.writerWithDefaultPrettyPrinter()
+ : this.mapper.writer();
+
+ return ow.writeValueAsString(rootNode);
+ } catch (JsonProcessingException e) {
+ throw new SerDeException(e);
+ }
+ }
+
+ /**
+ * Walk the object graph.
+ *
+ * @param oi The ObjectInspector describing the Object
+ * @param o The object to convert
+ * @param fieldNames List of field names to use, default names used otherwise
+ * @return A JsonNode representation of the Object
+ * @throws SerDeException The Object cannot be parsed
+ */
+ private JsonNode walkObjectGraph(final ObjectInspector oi, final Object o,
+ final List<String> fieldNames)
+ throws SerDeException {
+
+ if (o != null) {
+ switch (oi.getCategory()) {
+ case LIST:
+ return visitList(oi, o);
+ case STRUCT:
+ return visitStruct(oi, o, fieldNames);
+ case MAP:
+ return visitMap(oi, o);
+ case PRIMITIVE:
+ return primitiveToNode(oi, o);
+ case UNION:
+ return visitUnion(oi, o);
+ default:
+ throw new SerDeException(
+ "Parsing of: " + oi.getCategory() + " is not supported");
+ }
+ }
+ return this.nodeFactory.nullNode();
+ }
+
+ /**
+ * Visit a vertex in the graph that is a Java Map.
+ *
+ * @param oi The map's OjectInspector
+ * @param o The Map object
+ * @return A JsonNode representation of the Map
+ * @throws SerDeException The Map cannot be parsed
+ */
+ private ObjectNode visitMap(final ObjectInspector oi, final Object o)
+ throws SerDeException {
+ final MapObjectInspector moi = (MapObjectInspector) oi;
+
+ final ObjectInspector mapKeyInspector = moi.getMapKeyObjectInspector();
+ final ObjectInspector mapValueInspector = moi.getMapValueObjectInspector();
+
+ final Map<?, ?> map = moi.getMap(o);
+ final ObjectNode objectNode = this.nodeFactory.objectNode();
+
+ for (Map.Entry<?, ?> entry : map.entrySet()) {
+ final JsonNode keyNode = primitiveToNode(mapKeyInspector, entry.getKey());
+ final JsonNode valueNode = walkObjectGraph(mapValueInspector,
+ entry.getValue(), Collections.emptyList());
+ objectNode.set(keyNode.asText(), valueNode);
+ }
+
+ return objectNode;
+ }
+
+ /**
+ * Visit a vertex in the graph that is a Java List.
+ *
+ * @param oi The list's OjectInspector
+ * @param o The List object
+ * @return A JsonNode representation of the List
+ * @throws SerDeException The List cannot be parsed
+ */
+ private ContainerNode<?> visitList(final ObjectInspector oi, final Object o)
+ throws SerDeException {
+ final ListObjectInspector loi = (ListObjectInspector) oi;
+
+ final List<?> list = loi.getList(o);
+ final ArrayNode arrayNode = this.nodeFactory.arrayNode(list.size());
+
+ for (final Object item : list) {
+ final JsonNode valueNode = walkObjectGraph(
+ loi.getListElementObjectInspector(), item, Collections.emptyList());
+ arrayNode.add(valueNode);
+ }
+
+ return arrayNode;
+ }
+
+ /**
+ * Visit a vertex in the graph that is a struct data type. A struct is
+ * represented as a Java List where the name associated with each element in
+ * the list is stored in the ObjectInspector.
+ *
+ * @param oi The struct's OjectInspector
+ * @param o The List object
+ * @param fieldNames List of names to override the default field names
+ * @return A JsonNode representation of the List
+ * @throws SerDeException The struct cannot be parsed
+ */
+ private ObjectNode visitStruct(final ObjectInspector oi, final Object o,
+ final List<String> fieldNames) throws SerDeException {
+ final StructObjectInspector structInspector = (StructObjectInspector) oi;
+
+ final ObjectNode structNode = this.nodeFactory.objectNode();
+
+ for (final StructField field : structInspector.getAllStructFieldRefs()) {
+ final Object fieldValue = structInspector.getStructFieldData(o, field);
+ final ObjectInspector coi = field.getFieldObjectInspector();
+ final JsonNode fieldNode =
+ walkObjectGraph(coi, fieldValue, Collections.emptyList());
+
+ // Map field names to something else if require
+ final String fieldName = (fieldNames.isEmpty()) ? field.getFieldName()
+ : fieldNames.get(field.getFieldID());
+
+ structNode.set(fieldName, fieldNode);
+ }
+
+ return structNode;
+ }
+
+ /**
+ * Visit a vertex in the graph that is a union data type.
+ *
+ * @param oi The union's OjectInspector
+ * @param o The Union object
+ * @return A JsonNode representation of the union
+ * @throws SerDeException The union cannot be parsed
+ */
+ private ObjectNode visitUnion(final ObjectInspector oi, final Object o)
+ throws SerDeException {
+ final UnionObjectInspector unionInspector = (UnionObjectInspector) oi;
+
+ final ObjectNode unionNode = this.nodeFactory.objectNode();
+
+ final byte tag = unionInspector.getTag(o);
+ final String tagText = Byte.toString(tag);
+ final JsonNode valueNode =
+ walkObjectGraph(unionInspector.getObjectInspectors().get(tag),
+ unionInspector.getField(o), Collections.emptyList());
+
+ unionNode.set(tagText, valueNode);
+
+ return unionNode;
+ }
+
+ /**
+ * Convert a primitive Java object to a JsonNode.
+ *
+ * @param oi The primitive ObjectInspector
+ * @param o The primitive Object
+ * @return A JSON node containing the value of the primitive Object
+ * @throws SerDeException The primitive value cannot be parsed
+ */
+ private ValueNode primitiveToNode(final ObjectInspector oi, final Object o)
+ throws SerDeException {
+ final PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
+
+ switch (poi.getPrimitiveCategory()) {
+ case BINARY:
+ return getByteValue(
+ ((BinaryObjectInspector) poi).getPrimitiveJavaObject(o));
+ case BOOLEAN:
+ return this.nodeFactory.booleanNode(((BooleanObjectInspector) poi).get(o));
+ case BYTE:
+ return this.nodeFactory.numberNode(((ByteObjectInspector) poi).get(o));
+ case DATE:
+ return this.nodeFactory.textNode(
+ ((DateObjectInspector) poi).getPrimitiveJavaObject(o).toString());
+ case DECIMAL:
+ return this.nodeFactory.numberNode(((HiveDecimalObjectInspector) poi)
+ .getPrimitiveJavaObject(o).bigDecimalValue());
+ case DOUBLE:
+ return this.nodeFactory.numberNode(((DoubleObjectInspector) poi).get(o));
+ case FLOAT:
+ return this.nodeFactory.numberNode(((FloatObjectInspector) poi).get(o));
+ case INT:
+ return this.nodeFactory.numberNode(((IntObjectInspector) poi).get(o));
+ case LONG:
+ return this.nodeFactory.numberNode(((LongObjectInspector) poi).get(o));
+ case SHORT:
+ return this.nodeFactory.numberNode(((ShortObjectInspector) poi).get(o));
+ case STRING:
+ return this.nodeFactory
+ .textNode(((StringObjectInspector) poi).getPrimitiveJavaObject(o));
+ case CHAR:
+ return this.nodeFactory.textNode(
+ ((HiveCharObjectInspector) poi).getPrimitiveJavaObject(o).toString());
+ case VARCHAR:
+ return this.nodeFactory.textNode(((HiveVarcharObjectInspector) poi)
+ .getPrimitiveJavaObject(o).toString());
+ case TIMESTAMP:
+ return this.nodeFactory.textNode(((TimestampObjectInspector) poi)
+ .getPrimitiveJavaObject(o).toString());
+ default:
+ throw new SerDeException(
+ "Unsupported type: " + poi.getPrimitiveCategory());
+ }
+ }
+
+ /**
+ * Convert a byte array into a text representation.
+ *
+ * @param o The byte array to convert to text
+ * @return A JsonNode with the binary data encoded
+ * @throws SerDeException The struct cannot be parsed
+ */
+ private ValueNode getByteValue(final byte[] buf) throws SerDeException {
+ switch (this.binaryEncoding) {
+ case RAWSTRING:
+ final Text txt = new Text();
+ txt.set(buf, 0, buf.length);
+ return this.nodeFactory.textNode(txt.toString());
+ case BASE64:
+ return this.nodeFactory.binaryNode(buf);
+ default:
+ throw new SerDeException(
+ "Error generating JSON binary type from record.");
+ }
+ }
+
+ public void enable(Feature feature) {
+ this.features.add(feature);
+ }
+
+ public void disable(Feature feature) {
+ this.features.remove(feature);
+ }
+
+ public Set<Feature> getFeatures() {
+ return Collections.unmodifiableSet(this.features);
+ }
+
+ public boolean isEnabled(Feature feature) {
+ return this.features.contains(feature);
+ }
+
+ public BinaryEncoding getBinaryEncodingType() {
+ return binaryEncoding;
+ }
+
+ public void setBinaryEncoding(BinaryEncoding encoding) {
+ this.binaryEncoding = encoding;
+ }
+
+ @Override
+ public String toString() {
+ return "HiveJsonWriter [features=" + features + ", rootStructFieldNames="
+ + rootStructFieldNames + ", binaryEncoding=" + binaryEncoding + "]";
+ }
+
+}
diff --git a/serde/src/test/org/apache/hadoop/hive/serde2/TestJsonSerDe.java b/serde/src/test/org/apache/hadoop/hive/serde2/TestJsonSerDe.java
new file mode 100644
index 0000000..709871f
--- /dev/null
+++ b/serde/src/test/org/apache/hadoop/hive/serde2/TestJsonSerDe.java
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.serde2;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+import java.util.TimeZone;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.hive.common.type.Timestamp;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.FloatWritable;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test suite for the JSON SerDe class.
+ */
+public class TestJsonSerDe {
+
+ @Test
+ public void testPrimativeDataTypes() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.LIST_COLUMNS,
+ "name,height,weight,endangered,born");
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES,
+ "string,float,int,boolean,timestamp");
+ props.setProperty(serdeConstants.TIMESTAMP_FORMATS, "millis");
+
+ JsonSerDe serde = new JsonSerDe();
+ serde.initialize(null, props, false);
+
+ final String jsonText = loadJson("simple.json");
+
+ final Text text = new Text(jsonText);
+ final List<?> results = (List<?>) serde.deserialize(text);
+
+ Assert.assertNotNull(results);
+ Assert.assertEquals(5, results.size());
+ Assert.assertEquals("giraffe", results.get(0));
+ Assert.assertEquals(5.5f, results.get(1));
+ Assert.assertEquals(1360, results.get(2));
+ Assert.assertEquals(true, results.get(3));
+ Assert.assertEquals(Timestamp.ofEpochMilli(1549751270013L), results.get(4));
+ }
+
+ @Test
+ public void testArray() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.LIST_COLUMNS, "list,items");
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,array<string>");
+ props.setProperty(serdeConstants.TIMESTAMP_FORMATS, "millis");
+
+ JsonSerDe serde = new JsonSerDe();
+ serde.initialize(null, props, false);
+
+ final String jsonText = loadJson("array.json");
+
+ final Text text = new Text(jsonText);
+ final List<?> results = (List<?>) serde.deserialize(text);
+
+ Assert.assertNotNull(results);
+ Assert.assertEquals(2, results.size());
+ Assert.assertEquals("grocery", results.get(0));
+ Assert.assertEquals(Arrays.asList("milk", "eggs", "bread"), results.get(1));
+ }
+
+ /**
+ * Test when a map has a key defined as a numeric value. Technically, JSON
+ * does not support this because each key in a map must be a quoted string.
+ * Unquoted strings (hence an int value) is allowed by Javascript, but not by
+ * JSON specification. For Hive, the int map key type is stored as a string
+ * and must be converted back into an int type.
+ */
+ @Test
+ public void testMapNumericKey() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.LIST_COLUMNS, "map");
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES, "map<int,string>");
+ props.setProperty(serdeConstants.TIMESTAMP_FORMATS, "millis");
+
+ JsonSerDe serde = new JsonSerDe();
+ serde.initialize(null, props, false);
+
+ final String jsonText = loadJson("map_int_key.json");
+
+ final Text text = new Text(jsonText);
+ final List<?> results = (List<?>) serde.deserialize(text);
+
+ Assert.assertNotNull(results);
+ Assert.assertEquals(1, results.size());
+
+ Map<?, ?> resultMap = (Map<?, ?>) results.get(0);
+ Object value1 = resultMap.get(1);
+ Object value2 = resultMap.get(2);
+
+ Assert.assertEquals(2, resultMap.size());
+ Assert.assertEquals("2001-01-01", value1);
+ Assert.assertEquals(null, value2);
+ }
+
+ @Test
+ public void testBlankLineAllowed() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.LIST_COLUMNS, "a,b,c");
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES, "int,int,int");
+ props.setProperty(serdeConstants.TIMESTAMP_FORMATS, "millis");
+
+ // This is the test parameter
+ props.setProperty(JsonSerDe.NULL_EMPTY_LINES, "true");
+
+ JsonSerDe serde = new JsonSerDe();
+ serde.initialize(null, props, false);
+
+ final Text text = new Text("");
+ final List<?> results = (List<?>) serde.deserialize(text);
+ Assert.assertEquals(Arrays.asList(null, null, null), results);
+ }
+
+ @Test(expected = SerDeException.class)
+ public void testBlankLineException() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.LIST_COLUMNS, "a,b,c");
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES, "int,int,int");
+ props.setProperty(serdeConstants.TIMESTAMP_FORMATS, "millis");
+
+ // This is the test parameter
+ props.setProperty(JsonSerDe.NULL_EMPTY_LINES, "false");
+
+ JsonSerDe serde = new JsonSerDe();
+ serde.initialize(null, props, false);
+
+ serde.deserialize(new Text(""));
+ }
+
+ @Test
+ public void testChar() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.LIST_COLUMNS, "a");
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES, "char(5)");
+ props.setProperty(serdeConstants.TIMESTAMP_FORMATS, "millis");
+
+ JsonSerDe serde = new JsonSerDe();
+ serde.initialize(null, props, true);
+
+ List<?> results = (List<?>) serde.deserialize(new Text("{\"a\":\"xxx\"}"));
+ Assert.assertNotNull(results);
+ Assert.assertEquals(1, results.size());
+ Assert.assertEquals("xxx ", results.get(0).toString());
+ }
+
+ /**
+ * When parsing the JSON object, a cache is kept for the definition of each
+ * field and it index in its most immediate struct. Check that if two names
+ * have the same name, in the same index of their respective structs, that
+ * they are not confused with one-another.
+ */
+ @Test
+ public void testCacheIndexSameFieldName() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.LIST_COLUMNS, "a,b");
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES,
+ "int,struct<a:boolean>");
+ props.setProperty(serdeConstants.TIMESTAMP_FORMATS, "millis");
+
+ JsonSerDe serde = new JsonSerDe();
+ serde.initialize(null, props, false);
+
+ List<?> results =
+ (List<?>) serde.deserialize(new Text("{\"a\":5,\"b\":{\"a\":true}}"));
+ Assert.assertNotNull(results);
+ Assert.assertEquals(2, results.size());
+ Assert.assertEquals(5, results.get(0));
+ Assert.assertEquals(Arrays.asList(true), results.get(1));
+ }
+
+ @Test
+ public void testSerialize() throws SerDeException, IOException {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.LIST_COLUMNS, "a,b,c,d,e,f");
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES,
+ "int,int,struct<c:int,d:int>,string,array<int>,map<int,float>");
+ props.setProperty(serdeConstants.TIMESTAMP_FORMATS, "millis");
+
+ final String jsonText = loadJson("complex_write.json");
+
+ JsonSerDe serde = new JsonSerDe();
+ serde.initialize(null, props, false);
+
+ ObjectInspector oi = serde.getObjectInspector();
+
+ Map<IntWritable, FloatWritable> map = new LinkedHashMap<>();
+ map.put(new IntWritable(666), new FloatWritable(0.12345f));
+ map.put(new IntWritable(999), new FloatWritable(0.6789f));
+
+ Object testStructure = Arrays.asList(new IntWritable(2), new IntWritable(3),
+ Arrays.asList(new IntWritable(2), new IntWritable(3)),
+ new Text("SomeData"),
+ Arrays.asList(new IntWritable(101), new IntWritable(102)), map);
+
+ Object result = serde.serialize(testStructure, oi);
+ Text text = (Text) result;
+ Assert.assertEquals(jsonText, text.toString());
+ }
+
+ /**
+ * Test serializing a information about a file to JSON. The name of the file
+ * is a "single_pixel.png" and the binary data of the PNG file are stored in
+ * JSON. By default, the writer uses base-64 mime encoding.
+ */
+ @Test
+ public void testSerializeBinaryData() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.LIST_COLUMNS, "name,content");
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES, "string,binary");
+ props.setProperty(serdeConstants.TIMESTAMP_FORMATS, "millis");
+
+ final String jsonText = loadJson("single_pixel.json");
+
+ JsonSerDe serde = new JsonSerDe();
+ serde.initialize(null, props, false);
+
+ ObjectInspector oi = serde.getObjectInspector();
+
+ final byte[] buf = Base64.getMimeDecoder()
+ .decode("iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAf"
+ + "FcSJAAAADUlEQVR42mNUuXDhPwAFqQLFTWHXeAAAAABJRU5ErkJggg==");
+
+ Object testObject =
+ Arrays.asList(new Text("single_pixel.png"), new BytesWritable(buf));
+
+ Object result = serde.serialize(testObject, oi);
+ Text text = (Text) result;
+ Assert.assertEquals(jsonText, text.toString());
+ }
+
+ /**
+ * Test serializing "timestamp with local time zone". Take a time in GMT and
+ * have it convert it to the local time.
+ */
+ @Test
+ public void testTimestampWithLocalTimeZone() throws Exception {
+ Properties props = new Properties();
+ props.setProperty(serdeConstants.LIST_COLUMNS, "__time");
+ props.setProperty(serdeConstants.LIST_COLUMN_TYPES,
+ "timestamp with local time zone");
+ props.setProperty(serdeConstants.TIMESTAMP_FORMATS,
+ "yyyy-MM-dd'T'HH:mm:ss'Z");
+
+ final TimeZone localTz = TimeZone.getDefault();
+ TimeZone.setDefault(TimeZone.getTimeZone("US/Pacific"));
+
+ try {
+ JsonSerDe serde = new JsonSerDe();
+ serde.initialize(null, props, false);
+
+ List<?> results = (List<?>) serde
+ .deserialize(new Text("{\"__time\":\"2013-08-31T01:02:33Z\"}"));
+
+ Assert.assertNotNull(results);
+ Assert.assertEquals(1, results.size());
+ Assert.assertTrue(results.get(0) instanceof TimestampTZ);
+ Assert.assertEquals("2013-08-30 18:02:33.0 US/Pacific",
+ results.get(0).toString());
+
+ } finally {
+ TimeZone.setDefault(localTz);
+ }
+ }
+
+ /**
+ * Accepts a file name and loads it from /src/test/resource/json
+ *
+ * @param resourceName The name of the file to load
+ * @return A JSON string, all whitespace removed
+ * @throws IOException Failed to load the file
+ */
+ private String loadJson(final String resourceName) throws IOException {
+ final String path = "/json/" + resourceName;
+ final URL url = this.getClass().getResource(path);
+ final File testJson = new File(url.getFile());
+ final String jsonText =
+ FileUtils.readFileToString(testJson, StandardCharsets.UTF_8);
+ return jsonText.replaceAll("\\s+", "");
+ }
+}
diff --git a/serde/src/test/resources/json/array.json b/serde/src/test/resources/json/array.json
new file mode 100644
index 0000000..f0b2ac7
--- /dev/null
+++ b/serde/src/test/resources/json/array.json
@@ -0,0 +1,8 @@
+{
+ "list": "grocery",
+ "items": [
+ "milk",
+ "eggs",
+ "bread"
+ ]
+}
\ No newline at end of file
diff --git a/serde/src/test/resources/json/complex_write.json b/serde/src/test/resources/json/complex_write.json
new file mode 100644
index 0000000..aa02840
--- /dev/null
+++ b/serde/src/test/resources/json/complex_write.json
@@ -0,0 +1 @@
+{"a":2,"b":3,"c":{"c":2,"d":3},"d":"SomeData","e":[101,102],"f":{"666":0.12345,"999":0.6789}}
\ No newline at end of file
diff --git a/serde/src/test/resources/json/map_int_key.json b/serde/src/test/resources/json/map_int_key.json
new file mode 100644
index 0000000..786b6e4
--- /dev/null
+++ b/serde/src/test/resources/json/map_int_key.json
@@ -0,0 +1,6 @@
+{
+ "map": {
+ "1": "2001-01-01",
+ "2": null
+ }
+}
\ No newline at end of file
diff --git a/serde/src/test/resources/json/simple.json b/serde/src/test/resources/json/simple.json
new file mode 100644
index 0000000..c35d847
--- /dev/null
+++ b/serde/src/test/resources/json/simple.json
@@ -0,0 +1,7 @@
+{
+ "name": "giraffe",
+ "height": 5.5,
+ "weight": 1360,
+ "endangered": true,
+ "born": 1549751270013
+}
\ No newline at end of file
diff --git a/serde/src/test/resources/json/single_pixel.json b/serde/src/test/resources/json/single_pixel.json
new file mode 100644
index 0000000..244ba69
--- /dev/null
+++ b/serde/src/test/resources/json/single_pixel.json
@@ -0,0 +1 @@
+{"name":"single_pixel.png","content":"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNUuXDhPwAFqQLFTWHXeAAAAABJRU5ErkJggg=="}
\ No newline at end of file