You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by pa...@apache.org on 2015/03/26 01:22:09 UTC
drill git commit: DRILL-2366: accurately materialize schema path for
complex types + test cases
Repository: drill
Updated Branches:
refs/heads/master 8493713ca -> 7bded6d3c
DRILL-2366: accurately materialize schema path for complex types + test cases
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/7bded6d3
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/7bded6d3
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/7bded6d3
Branch: refs/heads/master
Commit: 7bded6d3c0655ffa6e1f9731bd2d26c86b593700
Parents: 8493713
Author: Hanifi Gunes <hg...@maprtech.com>
Authored: Fri Mar 20 14:17:51 2015 -0700
Committer: Parth Chandra <pc...@maprtech.com>
Committed: Wed Mar 25 13:34:51 2015 -0700
----------------------------------------------------------------------
.../drill/exec/record/HyperVectorWrapper.java | 10 ++-
.../drill/exec/record/SimpleVectorWrapper.java | 10 ++-
.../vector/complex/AbstractContainerVector.java | 2 -
.../drill/exec/vector/complex/MapVector.java | 4 --
.../exec/vector/complex/RepeatedMapVector.java | 22 ++++++
.../expr/TestSchemaPathMaterialization.java | 70 ++++++++++++++++++++
.../resources/complex/json/repeated_list.json | 7 ++
.../complex/json/repeated_list_map.json | 7 ++
8 files changed, 121 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/main/java/org/apache/drill/exec/record/HyperVectorWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/HyperVectorWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/HyperVectorWrapper.java
index 9dbb583..f68bed1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/HyperVectorWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/HyperVectorWrapper.java
@@ -17,10 +17,13 @@
*/
package org.apache.drill.exec.record;
+import java.util.AbstractMap;
+
import org.apache.commons.lang3.ArrayUtils;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.AbstractContainerVector;
+import org.apache.drill.exec.vector.complex.AbstractMapVector;
import org.apache.drill.exec.vector.complex.MapVector;
import com.google.common.base.Preconditions;
@@ -92,8 +95,11 @@ public class HyperVectorWrapper<T extends ValueVector> implements VectorWrapper<
for (ValueVector v : this.vectors) {
ValueVector vector = v;
for (int i = 1; i < ids.length; i++) {
- MapVector map = (MapVector) vector;
- vector = map.getVectorById(ids[i]);
+ final AbstractMapVector mapLike = AbstractMapVector.class.cast(vector);
+ if (mapLike == null) {
+ return null;
+ }
+ vector = mapLike.getChildByOrdinal(ids[i]);
}
vectors[index] = vector;
index++;
http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
index 5bd3e41..6b1d178 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/SimpleVectorWrapper.java
@@ -22,6 +22,7 @@ import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.exec.vector.ValueVector;
import org.apache.drill.exec.vector.complex.AbstractContainerVector;
+import org.apache.drill.exec.vector.complex.AbstractMapVector;
import org.apache.drill.exec.vector.complex.MapVector;
public class SimpleVectorWrapper<T extends ValueVector> implements VectorWrapper<T>{
@@ -85,11 +86,14 @@ public class SimpleVectorWrapper<T extends ValueVector> implements VectorWrapper
ValueVector vector = v;
for (int i = 1; i < ids.length; i++) {
- MapVector map = (MapVector) vector;
- vector = map.getVectorById(ids[i]);
+ final AbstractMapVector mapLike = AbstractMapVector.class.cast(vector);
+ if (mapLike == null) {
+ return null;
+ }
+ vector = mapLike.getChildByOrdinal(ids[i]);
}
- return new SimpleVectorWrapper<ValueVector>(vector);
+ return new SimpleVectorWrapper<>(vector);
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
index 3bc7d11..acf9ec7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/AbstractContainerVector.java
@@ -132,8 +132,6 @@ public abstract class AbstractContainerVector implements ValueVector {
addToBreadCrumb = false;
builder.remainder(seg);
}
- // this is a complex array reference, which means it doesn't correspond directly to a vector by itself.
- seg = seg.getChild();
}
} else {
// name segment.
http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
index b995462..54ffeb2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java
@@ -320,10 +320,6 @@ public class MapVector extends AbstractMapVector {
}
}
- public ValueVector getVectorById(int id) {
- return getChildByOrdinal(id);
- }
-
public class Mutator implements ValueVector.Mutator{
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index c7e7cba..5afc889 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -25,6 +25,7 @@ import java.util.Map;
import org.apache.commons.lang3.ArrayUtils;
import org.apache.drill.common.expression.FieldReference;
+import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.DataMode;
import org.apache.drill.common.types.TypeProtos.MajorType;
@@ -37,6 +38,7 @@ import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.UserBitShared.SerializedField;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.record.TransferPair;
+import org.apache.drill.exec.record.TypedFieldId;
import org.apache.drill.exec.util.CallBack;
import org.apache.drill.exec.util.JsonStringArrayList;
import org.apache.drill.exec.vector.AllocationHelper;
@@ -166,6 +168,26 @@ public class RepeatedMapVector extends AbstractMapVector implements RepeatedFixe
}
}
+ @Override
+ public TypedFieldId getFieldIdIfMatches(TypedFieldId.Builder builder, boolean addToBreadCrumb, PathSegment seg) {
+ if (seg != null && seg.isArray() && !seg.isLastPath()) {
+ if (addToBreadCrumb) {
+ addToBreadCrumb = false;
+ builder.remainder(seg);
+ }
+ // skip the first array segment as there is no corresponding child vector.
+ seg = seg.getChild();
+
+ // multi-level numbered access to a repeated map is not possible so return if the next part is also an array
+ // segment.
+ if (seg.isArray()) {
+ return null;
+ }
+ }
+
+ return super.getFieldIdIfMatches(builder, addToBreadCrumb, seg);
+ }
+
public TransferPair getTransferPairToSingleMap(FieldReference reference) {
return new SingleMapTransferPair(this, reference);
}
http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
new file mode 100644
index 0000000..fb4cad9
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/TestSchemaPathMaterialization.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.expr;
+
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.junit.Ignore;
+import org.junit.Test;
+
+public class TestSchemaPathMaterialization extends BaseTestQuery {
+
+ @Test
+ public void testSingleProjectionFromMultiLevelRepeatedList() throws Exception {
+ final String query = "select t.odd[2][0][0] v1 " +
+ " from cp.`complex/json/repeated_list.json` t";
+
+ testBuilder()
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns("v1")
+ .baselineValues(5L)
+ .go();
+ }
+
+ @Test
+ @Ignore("Ignored until DRILL-2375 is fixed")
+ public void testProjectionFromMultiLevelRepeatedList() throws Exception {
+ final String query = "select t.odd[0][0][0] v1, t.odd[0][0][1] v2, t.odd[0][0][2] v3, " +
+ " t.odd[1] v4, t.odd[2][0][0] v5, t.odd[2][1][0] v6" +
+ " from cp.`complex/json/repeated_list.json` t";
+
+ testRunAndPrint(UserBitShared.QueryType.SQL, query);
+ testBuilder()
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns("v1", "v2", "v3", "v4", "v5", "v6")
+ .baselineValues(1L, null, 3L, null, 5L, null)
+ .go();
+ }
+
+ @Test
+ @Ignore("Ignored until DRILL-2375 is fixed")
+ public void testProjectionFromMultiLevelRepeatedListMap() throws Exception {
+ final String query = "select t.odd[0][0].val[0] v1, t.odd[0][0].val[0] v2, " +
+ " from cp.`complex/json/repeated_list_map.json` t";
+
+ testRunAndPrint(UserBitShared.QueryType.SQL, query);
+ testBuilder()
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns("v1", "v2")
+ .baselineValues(1L, 3L)
+ .go();
+ }
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/test/resources/complex/json/repeated_list.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/complex/json/repeated_list.json b/exec/java-exec/src/test/resources/complex/json/repeated_list.json
new file mode 100644
index 0000000..696d069
--- /dev/null
+++ b/exec/java-exec/src/test/resources/complex/json/repeated_list.json
@@ -0,0 +1,7 @@
+{
+ "odd": [
+ [[1],[],[3]],
+ [],
+ [[5]]
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/7bded6d3/exec/java-exec/src/test/resources/complex/json/repeated_list_map.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/complex/json/repeated_list_map.json b/exec/java-exec/src/test/resources/complex/json/repeated_list_map.json
new file mode 100644
index 0000000..ef6c0ee
--- /dev/null
+++ b/exec/java-exec/src/test/resources/complex/json/repeated_list_map.json
@@ -0,0 +1,7 @@
+{
+ "odd": [
+ [{"val": [1, 3]}, {}, {"val": [5]}],
+ [],
+ [{"val": [7]}]
+ ]
+}
\ No newline at end of file