You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/05/14 02:26:52 UTC
[4/4] git commit: DRILL-718: Return complex types as JSON string.
DRILL-718: Return complex types as JSON string.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ea72a380
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ea72a380
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ea72a380
Branch: refs/heads/master
Commit: ea72a380e875aaaa126510116742be659016d98f
Parents: 4b57652
Author: Mehant Baid <me...@gmail.com>
Authored: Tue May 13 15:07:54 2014 -0700
Committer: Aditya Kishore <ad...@maprtech.com>
Committed: Tue May 13 16:06:03 2014 -0700
----------------------------------------------------------------------
.../src/main/codegen/includes/vv_imports.ftl | 1 +
.../codegen/templates/RepeatedValueVectors.java | 2 +-
.../src/main/codegen/templates/TypeHelper.java | 5 ++
.../drill/exec/util/JsonStringArrayList.java | 46 ++++++++++++++++++
.../drill/exec/util/JsonStringHashMap.java | 50 ++++++++++++++++++++
.../apache/drill/exec/util/TextSerializer.java | 38 +++++++++++++++
.../drill/exec/vector/complex/MapVector.java | 3 +-
.../exec/vector/complex/RepeatedListVector.java | 3 +-
.../exec/vector/complex/RepeatedMapVector.java | 3 +-
.../exec/vector/complex/fn/JsonReader.java | 2 +-
.../exec/store/json/JsonRecordReader2Test.java | 2 +
11 files changed, 150 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
index 71404d3..114ecbe 100644
--- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
+++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
@@ -38,6 +38,7 @@ import org.apache.drill.exec.vector.complex.impl.*;
import org.apache.drill.exec.vector.complex.writer.*;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
+import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index 537c7fd..cac87c5 100644
--- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -296,7 +296,7 @@ package org.apache.drill.exec.vector;
}
public List<${friendlyType}> getObject(int index) {
- List<${friendlyType}> vals = Lists.newArrayList();
+ List<${friendlyType}> vals = new JsonStringArrayList();
int start = offsets.getAccessor().get(index) - sliceOffset;
int end = offsets.getAccessor().get(index+1) - sliceOffset;
for(int i = start; i < end; i++){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/templates/TypeHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
index 4755e92..f2d66d2 100644
--- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java
+++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
@@ -66,7 +66,12 @@ public class TypeHelper {
}
</#list>
</#list>
+ case MAP:
+ case REPEATMAP:
+ case LIST:
+ return new GenericAccessor(vector);
}
+
throw new UnsupportedOperationException();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
new file mode 100644
index 0000000..bc378a1
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import org.apache.hadoop.io.Text;
+
+import java.util.ArrayList;
+
+public class JsonStringArrayList extends ArrayList {
+
+ private static ObjectMapper mapper;
+
+ static {
+ mapper = new ObjectMapper();
+ SimpleModule serializer = new SimpleModule("TextSerializer")
+ .addSerializer(Text.class, new TextSerializer());
+ mapper.registerModule(serializer);
+ }
+
+ @Override
+ public final String toString() {
+ try {
+ return mapper.writeValueAsString(this);
+ } catch(JsonProcessingException e) {
+ throw new IllegalStateException("Cannot serialize array list to JSON string", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
new file mode 100644
index 0000000..124252a
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.module.SimpleModule;
+import org.apache.hadoop.io.Text;
+
+import java.util.HashMap;
+
+/*
+ * Simple class that extends the regular java.util.HashMap but overrides the
+ * toString() method of the HashMap class to produce a JSON string instead
+ */
+public class JsonStringHashMap extends HashMap {
+
+ private static ObjectMapper mapper;
+
+ static {
+ mapper = new ObjectMapper();
+ SimpleModule serializer = new SimpleModule("TextSerializer")
+ .addSerializer(Text.class, new TextSerializer());
+ mapper.registerModule(serializer);
+ }
+
+ @Override
+ public final String toString() {
+ try {
+ return mapper.writeValueAsString(this);
+ } catch(JsonProcessingException e) {
+ throw new IllegalStateException("Cannot serialize hash map to JSON string", e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
new file mode 100644
index 0000000..3070592
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.util;
+
+import com.fasterxml.jackson.core.JsonGenerationException;
+import com.fasterxml.jackson.core.JsonGenerator;
+import com.fasterxml.jackson.databind.SerializerProvider;
+import com.fasterxml.jackson.databind.ser.std.StdSerializer;
+import org.apache.hadoop.io.Text;
+
+import java.io.IOException;
+
+public class TextSerializer extends StdSerializer<Text> {
+
+ public TextSerializer() {
+ super(Text.class);
+ }
+
+ @Override
+ public void serialize(Text text, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonGenerationException {
+ jsonGenerator.writeString(text.toString());
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index 91c0be5..9b2e119 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -36,6 +36,7 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.util.JsonStringHashMap;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.RepeatedMapVector.MapSingleCopier;
import org.apache.drill.exec.vector.complex.impl.SingleMapReaderImpl;
@@ -316,7 +317,7 @@ public class MapVector extends AbstractContainerVector {
@Override
public Object getObject(int index) {
- Map<String, Object> vv = Maps.newHashMap();
+ Map<String, Object> vv = new JsonStringHashMap();
for(Map.Entry<String, ValueVector> e : vectors.entrySet()){
ValueVector v = e.getValue();
String k = e.getKey();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
index 93930b5..88d858a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
@@ -37,6 +37,7 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.exec.vector.RepeatedFixedWidthVector;
import org.apache.drill.exec.vector.UInt4Vector;
import org.apache.drill.exec.vector.ValueVector;
@@ -155,7 +156,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea
@Override
public Object getObject(int index) {
- List<Object> l = Lists.newArrayList();
+ List<Object> l = new JsonStringArrayList();
int end = offsets.getAccessor().get(index+1);
for(int i = offsets.getAccessor().get(index); i < end; i++){
l.add(vector.getAccessor().getObject(i));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index 2492cc8..22471f0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -36,6 +36,7 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.exec.vector.RepeatedFixedWidthVector;
import org.apache.drill.exec.vector.UInt4Vector;
import org.apache.drill.exec.vector.ValueVector;
@@ -348,7 +349,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
@Override
public Object getObject(int index) {
- List<Object> l = Lists.newArrayList();
+ List<Object> l = new JsonStringArrayList();
int end = offsets.getAccessor().get(index+1);
for(int i = offsets.getAccessor().get(index); i < end; i++){
Map<String, Object> vv = Maps.newHashMap();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
index 761bc79..e5d4b5b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
@@ -221,7 +221,7 @@ public class JsonReader {
vh.start = 0;
vh.end = b.length;
list.varChar().write(vh);
-
+ break;
default:
throw new IllegalStateException("Unexpected token " + parser.getCurrentToken());
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
index 1372219..ad46326 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
@@ -27,6 +27,8 @@ public class JsonRecordReader2Test extends BaseTestQuery{
public void testComplexJsonInput() throws Exception{
// test("select z[0]['orange'] from cp.`jsoninput/input2.json` limit 10");
test("select `integer`, x['y'] as x1, x['y'] as x2, z[0], z[0]['orange'], z[1]['pink'] from cp.`jsoninput/input2.json` limit 10");
+ test("select x from cp.`jsoninput/input2.json`");
+
// test("select z[0] from cp.`jsoninput/input2.json` limit 10");
}
Re: [4/4] git commit: DRILL-718: Return complex types as JSON string.
Posted by Mehant Baid <ba...@gmail.com>.
Yes!
We sub-classed HashMap and ArrayList used internally by Map, RepeatedMap
and RepeatedList vectors to override the toString() method to return
JSON string.
Thanks
Mehant
On 5/13/14, 6:45 PM, Timothy Chen wrote:
> Does this means any complex type will be displayed JSON strings to the
> user from the interactive shell as well?
>
> Tim
>
> On Tue, May 13, 2014 at 5:26 PM, <ja...@apache.org> wrote:
>> DRILL-718: Return complex types as JSON string.
>>
>>
>> Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
>> Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ea72a380
>> Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ea72a380
>> Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ea72a380
>>
>> Branch: refs/heads/master
>> Commit: ea72a380e875aaaa126510116742be659016d98f
>> Parents: 4b57652
>> Author: Mehant Baid <me...@gmail.com>
>> Authored: Tue May 13 15:07:54 2014 -0700
>> Committer: Aditya Kishore <ad...@maprtech.com>
>> Committed: Tue May 13 16:06:03 2014 -0700
>>
>> ----------------------------------------------------------------------
>> .../src/main/codegen/includes/vv_imports.ftl | 1 +
>> .../codegen/templates/RepeatedValueVectors.java | 2 +-
>> .../src/main/codegen/templates/TypeHelper.java | 5 ++
>> .../drill/exec/util/JsonStringArrayList.java | 46 ++++++++++++++++++
>> .../drill/exec/util/JsonStringHashMap.java | 50 ++++++++++++++++++++
>> .../apache/drill/exec/util/TextSerializer.java | 38 +++++++++++++++
>> .../drill/exec/vector/complex/MapVector.java | 3 +-
>> .../exec/vector/complex/RepeatedListVector.java | 3 +-
>> .../exec/vector/complex/RepeatedMapVector.java | 3 +-
>> .../exec/vector/complex/fn/JsonReader.java | 2 +-
>> .../exec/store/json/JsonRecordReader2Test.java | 2 +
>> 11 files changed, 150 insertions(+), 5 deletions(-)
>> ----------------------------------------------------------------------
>>
>>
>> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
>> ----------------------------------------------------------------------
>> diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
>> index 71404d3..114ecbe 100644
>> --- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
>> +++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
>> @@ -38,6 +38,7 @@ import org.apache.drill.exec.vector.complex.impl.*;
>> import org.apache.drill.exec.vector.complex.writer.*;
>> import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
>> import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
>> +import org.apache.drill.exec.util.JsonStringArrayList;
>>
>> import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
>>
>>
>> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
>> ----------------------------------------------------------------------
>> diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
>> index 537c7fd..cac87c5 100644
>> --- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
>> +++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
>> @@ -296,7 +296,7 @@ package org.apache.drill.exec.vector;
>> }
>>
>> public List<${friendlyType}> getObject(int index) {
>> - List<${friendlyType}> vals = Lists.newArrayList();
>> + List<${friendlyType}> vals = new JsonStringArrayList();
>> int start = offsets.getAccessor().get(index) - sliceOffset;
>> int end = offsets.getAccessor().get(index+1) - sliceOffset;
>> for(int i = start; i < end; i++){
>>
>> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/templates/TypeHelper.java
>> ----------------------------------------------------------------------
>> diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
>> index 4755e92..f2d66d2 100644
>> --- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java
>> +++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
>> @@ -66,7 +66,12 @@ public class TypeHelper {
>> }
>> </#list>
>> </#list>
>> + case MAP:
>> + case REPEATMAP:
>> + case LIST:
>> + return new GenericAccessor(vector);
>> }
>> +
>> throw new UnsupportedOperationException();
>> }
>>
>>
>> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
>> ----------------------------------------------------------------------
>> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
>> new file mode 100644
>> index 0000000..bc378a1
>> --- /dev/null
>> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
>> @@ -0,0 +1,46 @@
>> +/**
>> + * Licensed to the Apache Software Foundation (ASF) under one
>> + * or more contributor license agreements. See the NOTICE file
>> + * distributed with this work for additional information
>> + * regarding copyright ownership. The ASF licenses this file
>> + * to you under the Apache License, Version 2.0 (the
>> + * "License"); you may not use this file except in compliance
>> + * with the License. You may obtain a copy of the License at
>> + *
>> + * http://www.apache.org/licenses/LICENSE-2.0
>> + *
>> + * Unless required by applicable law or agreed to in writing, software
>> + * distributed under the License is distributed on an "AS IS" BASIS,
>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>> + * See the License for the specific language governing permissions and
>> + * limitations under the License.
>> + */
>> +package org.apache.drill.exec.util;
>> +
>> +import com.fasterxml.jackson.core.JsonProcessingException;
>> +import com.fasterxml.jackson.databind.ObjectMapper;
>> +import com.fasterxml.jackson.databind.module.SimpleModule;
>> +import org.apache.hadoop.io.Text;
>> +
>> +import java.util.ArrayList;
>> +
>> +public class JsonStringArrayList extends ArrayList {
>> +
>> + private static ObjectMapper mapper;
>> +
>> + static {
>> + mapper = new ObjectMapper();
>> + SimpleModule serializer = new SimpleModule("TextSerializer")
>> + .addSerializer(Text.class, new TextSerializer());
>> + mapper.registerModule(serializer);
>> + }
>> +
>> + @Override
>> + public final String toString() {
>> + try {
>> + return mapper.writeValueAsString(this);
>> + } catch(JsonProcessingException e) {
>> + throw new IllegalStateException("Cannot serialize array list to JSON string", e);
>> + }
>> + }
>> +}
>>
>> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
>> ----------------------------------------------------------------------
>> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
>> new file mode 100644
>> index 0000000..124252a
>> --- /dev/null
>> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
>> @@ -0,0 +1,50 @@
>> +/**
>> + * Licensed to the Apache Software Foundation (ASF) under one
>> + * or more contributor license agreements. See the NOTICE file
>> + * distributed with this work for additional information
>> + * regarding copyright ownership. The ASF licenses this file
>> + * to you under the Apache License, Version 2.0 (the
>> + * "License"); you may not use this file except in compliance
>> + * with the License. You may obtain a copy of the License at
>> + *
>> + * http://www.apache.org/licenses/LICENSE-2.0
>> + *
>> + * Unless required by applicable law or agreed to in writing, software
>> + * distributed under the License is distributed on an "AS IS" BASIS,
>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>> + * See the License for the specific language governing permissions and
>> + * limitations under the License.
>> + */
>> +package org.apache.drill.exec.util;
>> +
>> +import com.fasterxml.jackson.core.JsonProcessingException;
>> +import com.fasterxml.jackson.databind.ObjectMapper;
>> +import com.fasterxml.jackson.databind.module.SimpleModule;
>> +import org.apache.hadoop.io.Text;
>> +
>> +import java.util.HashMap;
>> +
>> +/*
>> + * Simple class that extends the regular java.util.HashMap but overrides the
>> + * toString() method of the HashMap class to produce a JSON string instead
>> + */
>> +public class JsonStringHashMap extends HashMap {
>> +
>> + private static ObjectMapper mapper;
>> +
>> + static {
>> + mapper = new ObjectMapper();
>> + SimpleModule serializer = new SimpleModule("TextSerializer")
>> + .addSerializer(Text.class, new TextSerializer());
>> + mapper.registerModule(serializer);
>> + }
>> +
>> + @Override
>> + public final String toString() {
>> + try {
>> + return mapper.writeValueAsString(this);
>> + } catch(JsonProcessingException e) {
>> + throw new IllegalStateException("Cannot serialize hash map to JSON string", e);
>> + }
>> + }
>> +}
>>
>> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
>> ----------------------------------------------------------------------
>> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
>> new file mode 100644
>> index 0000000..3070592
>> --- /dev/null
>> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
>> @@ -0,0 +1,38 @@
>> +/**
>> + * Licensed to the Apache Software Foundation (ASF) under one
>> + * or more contributor license agreements. See the NOTICE file
>> + * distributed with this work for additional information
>> + * regarding copyright ownership. The ASF licenses this file
>> + * to you under the Apache License, Version 2.0 (the
>> + * "License"); you may not use this file except in compliance
>> + * with the License. You may obtain a copy of the License at
>> + *
>> + * http://www.apache.org/licenses/LICENSE-2.0
>> + *
>> + * Unless required by applicable law or agreed to in writing, software
>> + * distributed under the License is distributed on an "AS IS" BASIS,
>> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
>> + * See the License for the specific language governing permissions and
>> + * limitations under the License.
>> + */
>> +package org.apache.drill.exec.util;
>> +
>> +import com.fasterxml.jackson.core.JsonGenerationException;
>> +import com.fasterxml.jackson.core.JsonGenerator;
>> +import com.fasterxml.jackson.databind.SerializerProvider;
>> +import com.fasterxml.jackson.databind.ser.std.StdSerializer;
>> +import org.apache.hadoop.io.Text;
>> +
>> +import java.io.IOException;
>> +
>> +public class TextSerializer extends StdSerializer<Text> {
>> +
>> + public TextSerializer() {
>> + super(Text.class);
>> + }
>> +
>> + @Override
>> + public void serialize(Text text, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonGenerationException {
>> + jsonGenerator.writeString(text.toString());
>> + }
>> +}
>>
>> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
>> ----------------------------------------------------------------------
>> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
>> index 91c0be5..9b2e119 100644
>> --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
>> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
>> @@ -36,6 +36,7 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
>> import org.apache.drill.exec.proto.UserBitShared.SerializedField;
>> import org.apache.drill.exec.record.MaterializedField;
>> import org.apache.drill.exec.record.TransferPair;
>> +import org.apache.drill.exec.util.JsonStringHashMap;
>> import org.apache.drill.exec.vector.ValueVector;
>> import org.apache.drill.exec.vector.complex.RepeatedMapVector.MapSingleCopier;
>> import org.apache.drill.exec.vector.complex.impl.SingleMapReaderImpl;
>> @@ -316,7 +317,7 @@ public class MapVector extends AbstractContainerVector {
>>
>> @Override
>> public Object getObject(int index) {
>> - Map<String, Object> vv = Maps.newHashMap();
>> + Map<String, Object> vv = new JsonStringHashMap();
>> for(Map.Entry<String, ValueVector> e : vectors.entrySet()){
>> ValueVector v = e.getValue();
>> String k = e.getKey();
>>
>> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
>> ----------------------------------------------------------------------
>> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
>> index 93930b5..88d858a 100644
>> --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
>> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
>> @@ -37,6 +37,7 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
>> import org.apache.drill.exec.proto.UserBitShared.SerializedField;
>> import org.apache.drill.exec.record.MaterializedField;
>> import org.apache.drill.exec.record.TransferPair;
>> +import org.apache.drill.exec.util.JsonStringArrayList;
>> import org.apache.drill.exec.vector.RepeatedFixedWidthVector;
>> import org.apache.drill.exec.vector.UInt4Vector;
>> import org.apache.drill.exec.vector.ValueVector;
>> @@ -155,7 +156,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea
>>
>> @Override
>> public Object getObject(int index) {
>> - List<Object> l = Lists.newArrayList();
>> + List<Object> l = new JsonStringArrayList();
>> int end = offsets.getAccessor().get(index+1);
>> for(int i = offsets.getAccessor().get(index); i < end; i++){
>> l.add(vector.getAccessor().getObject(i));
>>
>> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
>> ----------------------------------------------------------------------
>> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
>> index 2492cc8..22471f0 100644
>> --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
>> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
>> @@ -36,6 +36,7 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
>> import org.apache.drill.exec.proto.UserBitShared.SerializedField;
>> import org.apache.drill.exec.record.MaterializedField;
>> import org.apache.drill.exec.record.TransferPair;
>> +import org.apache.drill.exec.util.JsonStringArrayList;
>> import org.apache.drill.exec.vector.RepeatedFixedWidthVector;
>> import org.apache.drill.exec.vector.UInt4Vector;
>> import org.apache.drill.exec.vector.ValueVector;
>> @@ -348,7 +349,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
>>
>> @Override
>> public Object getObject(int index) {
>> - List<Object> l = Lists.newArrayList();
>> + List<Object> l = new JsonStringArrayList();
>> int end = offsets.getAccessor().get(index+1);
>> for(int i = offsets.getAccessor().get(index); i < end; i++){
>> Map<String, Object> vv = Maps.newHashMap();
>>
>> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
>> ----------------------------------------------------------------------
>> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
>> index 761bc79..e5d4b5b 100644
>> --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
>> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
>> @@ -221,7 +221,7 @@ public class JsonReader {
>> vh.start = 0;
>> vh.end = b.length;
>> list.varChar().write(vh);
>> -
>> + break;
>> default:
>> throw new IllegalStateException("Unexpected token " + parser.getCurrentToken());
>> }
>>
>> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
>> ----------------------------------------------------------------------
>> diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
>> index 1372219..ad46326 100644
>> --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
>> +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
>> @@ -27,6 +27,8 @@ public class JsonRecordReader2Test extends BaseTestQuery{
>> public void testComplexJsonInput() throws Exception{
>> // test("select z[0]['orange'] from cp.`jsoninput/input2.json` limit 10");
>> test("select `integer`, x['y'] as x1, x['y'] as x2, z[0], z[0]['orange'], z[1]['pink'] from cp.`jsoninput/input2.json` limit 10");
>> + test("select x from cp.`jsoninput/input2.json`");
>> +
>> // test("select z[0] from cp.`jsoninput/input2.json` limit 10");
>> }
>>
>>
Re: [4/4] git commit: DRILL-718: Return complex types as JSON string.
Posted by Timothy Chen <tn...@gmail.com>.
Does this means any complex type will be displayed JSON strings to the
user from the interactive shell as well?
Tim
On Tue, May 13, 2014 at 5:26 PM, <ja...@apache.org> wrote:
> DRILL-718: Return complex types as JSON string.
>
>
> Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
> Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ea72a380
> Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ea72a380
> Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ea72a380
>
> Branch: refs/heads/master
> Commit: ea72a380e875aaaa126510116742be659016d98f
> Parents: 4b57652
> Author: Mehant Baid <me...@gmail.com>
> Authored: Tue May 13 15:07:54 2014 -0700
> Committer: Aditya Kishore <ad...@maprtech.com>
> Committed: Tue May 13 16:06:03 2014 -0700
>
> ----------------------------------------------------------------------
> .../src/main/codegen/includes/vv_imports.ftl | 1 +
> .../codegen/templates/RepeatedValueVectors.java | 2 +-
> .../src/main/codegen/templates/TypeHelper.java | 5 ++
> .../drill/exec/util/JsonStringArrayList.java | 46 ++++++++++++++++++
> .../drill/exec/util/JsonStringHashMap.java | 50 ++++++++++++++++++++
> .../apache/drill/exec/util/TextSerializer.java | 38 +++++++++++++++
> .../drill/exec/vector/complex/MapVector.java | 3 +-
> .../exec/vector/complex/RepeatedListVector.java | 3 +-
> .../exec/vector/complex/RepeatedMapVector.java | 3 +-
> .../exec/vector/complex/fn/JsonReader.java | 2 +-
> .../exec/store/json/JsonRecordReader2Test.java | 2 +
> 11 files changed, 150 insertions(+), 5 deletions(-)
> ----------------------------------------------------------------------
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
> ----------------------------------------------------------------------
> diff --git a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
> index 71404d3..114ecbe 100644
> --- a/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
> +++ b/exec/java-exec/src/main/codegen/includes/vv_imports.ftl
> @@ -38,6 +38,7 @@ import org.apache.drill.exec.vector.complex.impl.*;
> import org.apache.drill.exec.vector.complex.writer.*;
> import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter;
> import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter;
> +import org.apache.drill.exec.util.JsonStringArrayList;
>
> import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
> ----------------------------------------------------------------------
> diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
> index 537c7fd..cac87c5 100644
> --- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
> +++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
> @@ -296,7 +296,7 @@ package org.apache.drill.exec.vector;
> }
>
> public List<${friendlyType}> getObject(int index) {
> - List<${friendlyType}> vals = Lists.newArrayList();
> + List<${friendlyType}> vals = new JsonStringArrayList();
> int start = offsets.getAccessor().get(index) - sliceOffset;
> int end = offsets.getAccessor().get(index+1) - sliceOffset;
> for(int i = start; i < end; i++){
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/codegen/templates/TypeHelper.java
> ----------------------------------------------------------------------
> diff --git a/exec/java-exec/src/main/codegen/templates/TypeHelper.java b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
> index 4755e92..f2d66d2 100644
> --- a/exec/java-exec/src/main/codegen/templates/TypeHelper.java
> +++ b/exec/java-exec/src/main/codegen/templates/TypeHelper.java
> @@ -66,7 +66,12 @@ public class TypeHelper {
> }
> </#list>
> </#list>
> + case MAP:
> + case REPEATMAP:
> + case LIST:
> + return new GenericAccessor(vector);
> }
> +
> throw new UnsupportedOperationException();
> }
>
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
> ----------------------------------------------------------------------
> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
> new file mode 100644
> index 0000000..bc378a1
> --- /dev/null
> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringArrayList.java
> @@ -0,0 +1,46 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.drill.exec.util;
> +
> +import com.fasterxml.jackson.core.JsonProcessingException;
> +import com.fasterxml.jackson.databind.ObjectMapper;
> +import com.fasterxml.jackson.databind.module.SimpleModule;
> +import org.apache.hadoop.io.Text;
> +
> +import java.util.ArrayList;
> +
> +public class JsonStringArrayList extends ArrayList {
> +
> + private static ObjectMapper mapper;
> +
> + static {
> + mapper = new ObjectMapper();
> + SimpleModule serializer = new SimpleModule("TextSerializer")
> + .addSerializer(Text.class, new TextSerializer());
> + mapper.registerModule(serializer);
> + }
> +
> + @Override
> + public final String toString() {
> + try {
> + return mapper.writeValueAsString(this);
> + } catch(JsonProcessingException e) {
> + throw new IllegalStateException("Cannot serialize array list to JSON string", e);
> + }
> + }
> +}
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
> ----------------------------------------------------------------------
> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
> new file mode 100644
> index 0000000..124252a
> --- /dev/null
> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/JsonStringHashMap.java
> @@ -0,0 +1,50 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.drill.exec.util;
> +
> +import com.fasterxml.jackson.core.JsonProcessingException;
> +import com.fasterxml.jackson.databind.ObjectMapper;
> +import com.fasterxml.jackson.databind.module.SimpleModule;
> +import org.apache.hadoop.io.Text;
> +
> +import java.util.HashMap;
> +
> +/*
> + * Simple class that extends the regular java.util.HashMap but overrides the
> + * toString() method of the HashMap class to produce a JSON string instead
> + */
> +public class JsonStringHashMap extends HashMap {
> +
> + private static ObjectMapper mapper;
> +
> + static {
> + mapper = new ObjectMapper();
> + SimpleModule serializer = new SimpleModule("TextSerializer")
> + .addSerializer(Text.class, new TextSerializer());
> + mapper.registerModule(serializer);
> + }
> +
> + @Override
> + public final String toString() {
> + try {
> + return mapper.writeValueAsString(this);
> + } catch(JsonProcessingException e) {
> + throw new IllegalStateException("Cannot serialize hash map to JSON string", e);
> + }
> + }
> +}
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
> ----------------------------------------------------------------------
> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
> new file mode 100644
> index 0000000..3070592
> --- /dev/null
> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/TextSerializer.java
> @@ -0,0 +1,38 @@
> +/**
> + * Licensed to the Apache Software Foundation (ASF) under one
> + * or more contributor license agreements. See the NOTICE file
> + * distributed with this work for additional information
> + * regarding copyright ownership. The ASF licenses this file
> + * to you under the Apache License, Version 2.0 (the
> + * "License"); you may not use this file except in compliance
> + * with the License. You may obtain a copy of the License at
> + *
> + * http://www.apache.org/licenses/LICENSE-2.0
> + *
> + * Unless required by applicable law or agreed to in writing, software
> + * distributed under the License is distributed on an "AS IS" BASIS,
> + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
> + * See the License for the specific language governing permissions and
> + * limitations under the License.
> + */
> +package org.apache.drill.exec.util;
> +
> +import com.fasterxml.jackson.core.JsonGenerationException;
> +import com.fasterxml.jackson.core.JsonGenerator;
> +import com.fasterxml.jackson.databind.SerializerProvider;
> +import com.fasterxml.jackson.databind.ser.std.StdSerializer;
> +import org.apache.hadoop.io.Text;
> +
> +import java.io.IOException;
> +
> +public class TextSerializer extends StdSerializer<Text> {
> +
> + public TextSerializer() {
> + super(Text.class);
> + }
> +
> + @Override
> + public void serialize(Text text, JsonGenerator jsonGenerator, SerializerProvider serializerProvider) throws IOException, JsonGenerationException {
> + jsonGenerator.writeString(text.toString());
> + }
> +}
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
> ----------------------------------------------------------------------
> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
> index 91c0be5..9b2e119 100644
> --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
> @@ -36,6 +36,7 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
> import org.apache.drill.exec.proto.UserBitShared.SerializedField;
> import org.apache.drill.exec.record.MaterializedField;
> import org.apache.drill.exec.record.TransferPair;
> +import org.apache.drill.exec.util.JsonStringHashMap;
> import org.apache.drill.exec.vector.ValueVector;
> import org.apache.drill.exec.vector.complex.RepeatedMapVector.MapSingleCopier;
> import org.apache.drill.exec.vector.complex.impl.SingleMapReaderImpl;
> @@ -316,7 +317,7 @@ public class MapVector extends AbstractContainerVector {
>
> @Override
> public Object getObject(int index) {
> - Map<String, Object> vv = Maps.newHashMap();
> + Map<String, Object> vv = new JsonStringHashMap();
> for(Map.Entry<String, ValueVector> e : vectors.entrySet()){
> ValueVector v = e.getValue();
> String k = e.getKey();
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
> ----------------------------------------------------------------------
> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
> index 93930b5..88d858a 100644
> --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
> @@ -37,6 +37,7 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
> import org.apache.drill.exec.proto.UserBitShared.SerializedField;
> import org.apache.drill.exec.record.MaterializedField;
> import org.apache.drill.exec.record.TransferPair;
> +import org.apache.drill.exec.util.JsonStringArrayList;
> import org.apache.drill.exec.vector.RepeatedFixedWidthVector;
> import org.apache.drill.exec.vector.UInt4Vector;
> import org.apache.drill.exec.vector.ValueVector;
> @@ -155,7 +156,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea
>
> @Override
> public Object getObject(int index) {
> - List<Object> l = Lists.newArrayList();
> + List<Object> l = new JsonStringArrayList();
> int end = offsets.getAccessor().get(index+1);
> for(int i = offsets.getAccessor().get(index); i < end; i++){
> l.add(vector.getAccessor().getObject(i));
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
> ----------------------------------------------------------------------
> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
> index 2492cc8..22471f0 100644
> --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
> @@ -36,6 +36,7 @@ import org.apache.drill.exec.memory.OutOfMemoryRuntimeException;
> import org.apache.drill.exec.proto.UserBitShared.SerializedField;
> import org.apache.drill.exec.record.MaterializedField;
> import org.apache.drill.exec.record.TransferPair;
> +import org.apache.drill.exec.util.JsonStringArrayList;
> import org.apache.drill.exec.vector.RepeatedFixedWidthVector;
> import org.apache.drill.exec.vector.UInt4Vector;
> import org.apache.drill.exec.vector.ValueVector;
> @@ -348,7 +349,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
>
> @Override
> public Object getObject(int index) {
> - List<Object> l = Lists.newArrayList();
> + List<Object> l = new JsonStringArrayList();
> int end = offsets.getAccessor().get(index+1);
> for(int i = offsets.getAccessor().get(index); i < end; i++){
> Map<String, Object> vv = Maps.newHashMap();
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
> ----------------------------------------------------------------------
> diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
> index 761bc79..e5d4b5b 100644
> --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
> +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
> @@ -221,7 +221,7 @@ public class JsonReader {
> vh.start = 0;
> vh.end = b.length;
> list.varChar().write(vh);
> -
> + break;
> default:
> throw new IllegalStateException("Unexpected token " + parser.getCurrentToken());
> }
>
> http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ea72a380/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
> ----------------------------------------------------------------------
> diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
> index 1372219..ad46326 100644
> --- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
> +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
> @@ -27,6 +27,8 @@ public class JsonRecordReader2Test extends BaseTestQuery{
> public void testComplexJsonInput() throws Exception{
> // test("select z[0]['orange'] from cp.`jsoninput/input2.json` limit 10");
> test("select `integer`, x['y'] as x1, x['y'] as x2, z[0], z[0]['orange'], z[1]['pink'] from cp.`jsoninput/input2.json` limit 10");
> + test("select x from cp.`jsoninput/input2.json`");
> +
> // test("select z[0] from cp.`jsoninput/input2.json` limit 10");
> }
>
>