You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by hg...@apache.org on 2015/05/29 04:44:52 UTC
[1/2] drill git commit: DRILL-3143: MaterializedField#clone should
deep copy itself without disregarding its children
Repository: drill
Updated Branches:
refs/heads/master b7c99dde0 -> b56086436
DRILL-3143: MaterializedField#clone should deep copy itself without disregarding its children
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a84eb583
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a84eb583
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a84eb583
Branch: refs/heads/master
Commit: a84eb583ad15eece9dca64b10eabdd32584cc20d
Parents: b7c99dd
Author: Hanifi Gunes <hg...@maprtech.com>
Authored: Mon May 18 17:25:31 2015 -0700
Committer: Hanifi Gunes <hg...@maprtech.com>
Committed: Thu May 28 19:47:23 2015 -0700
----------------------------------------------------------------------
.../codegen/templates/FixedValueVectors.java | 2 +-
.../codegen/templates/NullableValueVectors.java | 2 +-
.../codegen/templates/RepeatedValueVectors.java | 2 +-
.../templates/VariableLengthVectors.java | 2 +-
.../exec/physical/impl/join/HashJoinBatch.java | 2 +-
.../drill/exec/record/MaterializedField.java | 40 ++++++---
.../drill/exec/vector/BaseValueVector.java | 2 +-
.../org/apache/drill/exec/vector/BitVector.java | 2 +-
.../exec/record/TestMaterializedField.java | 87 ++++++++++++++++++++
9 files changed, 122 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/a84eb583/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
index 0dffa0b..7103a17 100644
--- a/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/FixedValueVectors.java
@@ -168,7 +168,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
return new TransferImpl(getField());
}
public TransferPair getTransferPair(FieldReference ref){
- return new TransferImpl(getField().clone(ref));
+ return new TransferImpl(getField().withPath(ref));
}
public TransferPair makeTransferPair(ValueVector to) {
http://git-wip-us.apache.org/repos/asf/drill/blob/a84eb583/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
index ce6a3a7..90ec6be 100644
--- a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
@@ -234,7 +234,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
return new TransferImpl(getField());
}
public TransferPair getTransferPair(FieldReference ref){
- return new TransferImpl(getField().clone(ref));
+ return new TransferImpl(getField().withPath(ref));
}
public TransferPair makeTransferPair(ValueVector to) {
http://git-wip-us.apache.org/repos/asf/drill/blob/a84eb583/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index 37b8fac..7b2b78d 100644
--- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -90,7 +90,7 @@ public final class Repeated${minor.class}Vector extends BaseRepeatedValueVector
@Override
public TransferPair getTransferPair(FieldReference ref){
- return new TransferImpl(getField().clone(ref));
+ return new TransferImpl(getField().withPath(ref));
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/a84eb583/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
index 529f21b..b3389e2 100644
--- a/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/VariableLengthVectors.java
@@ -172,7 +172,7 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements V
return new TransferImpl(getField());
}
public TransferPair getTransferPair(FieldReference ref){
- return new TransferImpl(getField().clone(ref));
+ return new TransferImpl(getField().withPath(ref));
}
public TransferPair makeTransferPair(ValueVector to) {
http://git-wip-us.apache.org/repos/asf/drill/blob/a84eb583/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
index 5fd866f..73f3435 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/join/HashJoinBatch.java
@@ -443,7 +443,7 @@ public class HashJoinBatch extends AbstractRecordBatch<HashJoinPOP> {
}
// make sure to project field with children for children to show up in the schema
- final MaterializedField projected = field.cloneWithType(outputType);
+ final MaterializedField projected = field.withType(outputType);
// Add the vector to our output container
container.addOrGet(projected);
http://git-wip-us.apache.org/repos/asf/drill/blob/a84eb583/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
index 64ba861..39b0af5 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
@@ -19,6 +19,7 @@ package org.apache.drill.exec.record;
import java.util.Collection;
import java.util.Iterator;
+import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Set;
@@ -34,13 +35,17 @@ import org.apache.drill.exec.proto.UserBitShared.SerializedField;
public class MaterializedField {
- private Key key;
+ private final Key key;
// use an ordered set as existing code relies on order (e,g. parquet writer)
- private Set<MaterializedField> children = Sets.newLinkedHashSet();
+ private final LinkedHashSet<MaterializedField> children;
private MaterializedField(SchemaPath path, MajorType type) {
- super();
- key = new Key(path, type);
+ this(path, type, new LinkedHashSet<MaterializedField>());
+ }
+
+ private MaterializedField(SchemaPath path, MajorType type, LinkedHashSet<MaterializedField> children) {
+ this.key = new Key(path, type);
+ this.children = children;
}
public static MaterializedField create(SerializedField serField){
@@ -77,14 +82,25 @@ public class MaterializedField {
children.add(field);
}
- public MaterializedField cloneWithType(MajorType type) {
- final MaterializedField clone = new MaterializedField(key.path, type);
- clone.children = Sets.newLinkedHashSet(children);
- return clone;
+ public MaterializedField clone() {
+ return withPathAndType(getPath(), getType());
+ }
+
+ public MaterializedField withType(MajorType type) {
+ return withPathAndType(getPath(), type);
}
- public MaterializedField clone(FieldReference ref){
- return create(ref, key.type);
+ public MaterializedField withPath(SchemaPath path) {
+ return withPathAndType(path, getType());
+ }
+
+ public MaterializedField withPathAndType(final SchemaPath path, final MajorType type) {
+ final LinkedHashSet<MaterializedField> newChildren = new LinkedHashSet<>(children.size());
+ final MaterializedField clone = new MaterializedField(path, type, newChildren);
+ for (final MaterializedField child:children) {
+ newChildren.add(child.clone());
+ }
+ return clone;
}
public String getLastName(){
@@ -268,8 +284,8 @@ public class MaterializedField {
*/
public class Key {
- private SchemaPath path;
- private MajorType type;
+ private final SchemaPath path;
+ private final MajorType type;
private Key(SchemaPath path, MajorType type) {
this.path = path;
http://git-wip-us.apache.org/repos/asf/drill/blob/a84eb583/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
index c8b6d4d..ec409a3 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -55,7 +55,7 @@ public abstract class BaseValueVector implements ValueVector {
}
public MaterializedField getField(FieldReference ref){
- return getField().clone(ref);
+ return getField().withPath(ref);
}
@Override
http://git-wip-us.apache.org/repos/asf/drill/blob/a84eb583/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
index f88a7bc..10bdf07 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -191,7 +191,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
return new TransferImpl(getField());
}
public TransferPair getTransferPair(FieldReference ref) {
- return new TransferImpl(getField().clone(ref));
+ return new TransferImpl(getField().withPath(ref));
}
public TransferPair makeTransferPair(ValueVector to) {
http://git-wip-us.apache.org/repos/asf/drill/blob/a84eb583/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestMaterializedField.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestMaterializedField.java b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestMaterializedField.java
new file mode 100644
index 0000000..dbe4dc2
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/record/TestMaterializedField.java
@@ -0,0 +1,87 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p/>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p/>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.record;
+
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.common.types.Types;
+import static org.junit.Assert.assertTrue;
+
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestMaterializedField {
+
+ private static final String PARENT_NAME = "parent";
+ private static final String PARENT_SECOND_NAME = "parent2";
+ private static final String CHILD_NAME = "child";
+ private static final String CHILD_SECOND_NAME = "child2";
+ private static final TypeProtos.MajorType PARENT_TYPE = Types.repeated(TypeProtos.MinorType.MAP);
+ private static final TypeProtos.MajorType PARENT_SECOND_TYPE = Types.repeated(TypeProtos.MinorType.LIST);
+ private static final TypeProtos.MajorType CHILD_TYPE = Types.repeated(TypeProtos.MinorType.MAP);
+ private static final TypeProtos.MajorType CHILD_SECOND_TYPE = Types.repeated(TypeProtos.MinorType.MAP);
+
+ // set of (name, type) tuples representing a test case
+ private static final Object[][] matrix = {
+ {PARENT_SECOND_NAME, PARENT_TYPE},
+ {PARENT_NAME, PARENT_SECOND_TYPE},
+ {CHILD_SECOND_NAME, CHILD_TYPE},
+ {CHILD_NAME, CHILD_SECOND_TYPE},
+ };
+
+ private MaterializedField parent;
+ private MaterializedField child;
+
+ @Before
+ public void initialize() {
+ parent = MaterializedField.create(PARENT_NAME, PARENT_TYPE);
+ child = MaterializedField.create(CHILD_NAME, CHILD_TYPE);
+ parent.addChild(child);
+ }
+
+ @Test
+ public void testClone() {
+ final MaterializedField cloneParent = parent.clone();
+ final boolean isParentEqual = parent.equals(cloneParent);
+ assertTrue("Cloned parent does not match the original", isParentEqual);
+
+ final MaterializedField cloneChild = child.clone();
+ final boolean isChildEqual = child.equals(cloneChild);
+ assertTrue("Cloned child does not match the original", isChildEqual);
+
+ for (final MaterializedField field:new MaterializedField[]{parent, child}) {
+ for (Object[] args:matrix) {
+ final SchemaPath path = SchemaPath.getSimplePath(args[0].toString());
+ final TypeProtos.MajorType type = TypeProtos.MajorType.class.cast(args[1]);
+
+ final MaterializedField clone = field.withPathAndType(path, type);
+
+ final boolean isPathEqual = path.equals(clone.getPath());
+ assertTrue("Cloned path does not match the original", isPathEqual);
+
+ final boolean isTypeEqual = type.equals(clone.getType());
+ assertTrue("Cloned type does not match the original", isTypeEqual);
+
+ final boolean isChildrenEqual = field.getChildren().equals(clone.getChildren());
+ assertTrue("Cloned children do not match the original", isChildrenEqual);
+ }
+ }
+
+ }
+
+}
[2/2] drill git commit: DRILL-3032: repeated vectors should handle
late type && instantiate its children upon construction
Posted by hg...@apache.org.
DRILL-3032: repeated vectors should handle late type && instantiate its children upon construction
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/b5608643
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/b5608643
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/b5608643
Branch: refs/heads/master
Commit: b560864369abfd8d2d9234c78f0b97e0f6d4cfda
Parents: a84eb58
Author: Hanifi Gunes <hg...@maprtech.com>
Authored: Fri May 22 16:27:23 2015 -0700
Committer: Hanifi Gunes <hg...@maprtech.com>
Committed: Thu May 28 19:47:43 2015 -0700
----------------------------------------------------------------------
.../drill/exec/record/MaterializedField.java | 4 -
.../drill/exec/record/VectorContainer.java | 22 ++--
.../exec/vector/BaseRepeatedValueVector.java | 9 +-
.../drill/exec/vector/VectorDescriptor.java | 50 ++++++--
.../exec/vector/complex/RepeatedListVector.java | 22 ++--
.../org/apache/drill/TestExampleQueries.java | 121 ++++++++++---------
.../resources/join/join-left-drill-3032.json | 4 +
.../resources/join/join-right-drill-3032.json | 3 +
8 files changed, 140 insertions(+), 95 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/b5608643/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
index 39b0af5..26e1257 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
@@ -21,11 +21,7 @@ import java.util.Collection;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.Map;
-import java.util.Set;
-import com.google.common.collect.Sets;
-
-import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
import org.apache.drill.common.types.TypeProtos.DataMode;
http://git-wip-us.apache.org/repos/asf/drill/blob/b5608643/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
index e5f4be1..c4f9ed9 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
@@ -76,22 +76,22 @@ public class VectorContainer implements Iterable<VectorWrapper<?>>, VectorAccess
return addOrGet(field, null);
}
- public <T extends ValueVector> T addOrGet(MaterializedField field, SchemaChangeCallBack callBack) {
- TypedFieldId id = getValueVectorId(field.getPath());
- ValueVector v = null;
- Class clazz = TypeHelper.getValueVectorClass(field.getType().getMinorType(), field.getType().getMode());
+ public <T extends ValueVector> T addOrGet(final MaterializedField field, final SchemaChangeCallBack callBack) {
+ final TypedFieldId id = getValueVectorId(field.getPath());
+ final ValueVector vector;
+ final Class clazz = TypeHelper.getValueVectorClass(field.getType().getMinorType(), field.getType().getMode());
if (id != null) {
- v = getValueAccessorById(id.getFieldIds()).getValueVector();
- if (id.getFieldIds().length == 1 && clazz != null && !clazz.isAssignableFrom(v.getClass())) {
- ValueVector newVector = TypeHelper.getNewVector(field, this.oContext.getAllocator(), callBack);
- replace(v, newVector);
+ vector = getValueAccessorById(id.getFieldIds()).getValueVector();
+ if (id.getFieldIds().length == 1 && clazz != null && !clazz.isAssignableFrom(vector.getClass())) {
+ final ValueVector newVector = TypeHelper.getNewVector(field, this.oContext.getAllocator(), callBack);
+ replace(vector, newVector);
return (T) newVector;
}
} else {
- v = TypeHelper.getNewVector(field, this.oContext.getAllocator(), callBack);
- add(v);
+ vector = TypeHelper.getNewVector(field, this.oContext.getAllocator(), callBack);
+ add(vector);
}
- return (T) v;
+ return (T) vector;
}
public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
http://git-wip-us.apache.org/repos/asf/drill/blob/b5608643/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseRepeatedValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseRepeatedValueVector.java
index be04680..d5a0d62 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseRepeatedValueVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseRepeatedValueVector.java
@@ -140,9 +140,12 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
@Override
public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
boolean created = false;
- if (vector == DEFAULT_DATA_VECTOR) {
- vector = TypeHelper.getNewVector(MaterializedField.create(DATA_VECTOR_NAME, descriptor.getType()), allocator);
- getField().addChild(vector.getField());
+ if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != TypeProtos.MinorType.LATE) {
+ final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField();
+ vector = TypeHelper.getNewVector(field, allocator);
+ // returned vector must have the same field
+ assert field.equals(vector.getField());
+ getField().addChild(field);
created = true;
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b5608643/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java
index 9a29848..2ecaef6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorDescriptor.java
@@ -17,41 +17,67 @@
*/
package org.apache.drill.exec.vector;
+import java.util.Collection;
+
import com.google.common.base.Preconditions;
+import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.types.TypeProtos;
+import org.apache.drill.exec.record.MaterializedField;
public class VectorDescriptor {
- private static final String DEFAULT_NAME = new String("NONE");
+ private static final String DEFAULT_NAME = "NONE";
- private final TypeProtos.MajorType type;
- private final String name;
+ private final MaterializedField field;
- public VectorDescriptor(TypeProtos.MajorType type) {
+ public VectorDescriptor(final TypeProtos.MajorType type) {
this(DEFAULT_NAME, type);
}
- public VectorDescriptor(String name,TypeProtos.MajorType type) {
- this.name = Preconditions.checkNotNull(name, "name cannot be null");
- this.type = Preconditions.checkNotNull(type, "type cannot be null");
+ public VectorDescriptor(final String name, final TypeProtos.MajorType type) {
+ this(MaterializedField.create(name, type));
+ }
+
+ public VectorDescriptor(final MaterializedField field) {
+ this.field = Preconditions.checkNotNull(field, "field cannot be null");
+ }
+
+ public MaterializedField getField() {
+ return field;
}
public TypeProtos.MajorType getType() {
- return type;
+ return field.getType();
}
public String getName() {
- return name;
+ return field.getLastName();
+ }
+
+ public Collection<MaterializedField> getChildren() {
+ return field.getChildren();
}
public boolean hasName() {
- return name != DEFAULT_NAME;
+ return getName() != DEFAULT_NAME;
}
- public static VectorDescriptor create(String name, TypeProtos.MajorType type) {
+ public VectorDescriptor withName(final String name) {
+ return new VectorDescriptor(field.withPath(new FieldReference(name)));
+ }
+
+ public VectorDescriptor withType(final TypeProtos.MajorType type) {
+ return new VectorDescriptor(field.withType(type));
+ }
+
+ public static VectorDescriptor create(final String name, final TypeProtos.MajorType type) {
return new VectorDescriptor(name, type);
}
- public static VectorDescriptor create(TypeProtos.MajorType type) {
+ public static VectorDescriptor create(final TypeProtos.MajorType type) {
return new VectorDescriptor(type);
}
+
+ public static VectorDescriptor create(final MaterializedField field) {
+ return new VectorDescriptor(field);
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b5608643/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
index 443a761..a5553b2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.vector.complex;
import com.google.common.base.Preconditions;
import io.netty.buffer.DrillBuf;
+import java.util.Collection;
import java.util.Iterator;
import java.util.List;
@@ -69,7 +70,7 @@ public class RepeatedListVector extends AbstractContainerVector
@Override
public Object getObject(int index) {
- List<Object> list = new JsonStringArrayList();
+ final List<Object> list = new JsonStringArrayList();
final int start = offsets.getAccessor().get(index);
final int until = offsets.getAccessor().get(index+1);
for (int i = start; i < until; i++) {
@@ -133,7 +134,7 @@ public class RepeatedListVector extends AbstractContainerVector
public DelegateTransferPair(DelegateRepeatedVector target) {
this.target = Preconditions.checkNotNull(target);
if (target.getDataVector() == DEFAULT_DATA_VECTOR) {
- target.addOrGetVector(VectorDescriptor.create(getDataVector().getField().getType()));
+ target.addOrGetVector(VectorDescriptor.create(getDataVector().getField()));
target.getDataVector().allocateNew();
}
this.children = new TransferPair[] {
@@ -284,15 +285,16 @@ public class RepeatedListVector extends AbstractContainerVector
protected RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack, DelegateRepeatedVector delegate) {
super(field, allocator, callBack);
- int childrenSize = field.getChildren().size();
-
- // repeated list vector should not have more than one child
- assert childrenSize <= 1;
this.delegate = Preconditions.checkNotNull(delegate);
- if (childrenSize > 0) {
- MaterializedField child = field.getChildren().iterator().next();
- addOrGetVector(VectorDescriptor.create(child.getType()));
-// setVector(TypeHelper.getNewVector(child, allocator, callBack));
+
+ final Collection<MaterializedField> children = field.getChildren();
+ final int childSize = children.size();
+ // repeated list vector cannot have more than one child
+ assert childSize < 2;
+ final boolean hasChild = childSize == 1;
+ if (hasChild) {
+ final MaterializedField child = children.iterator().next();
+ addOrGetVector(VectorDescriptor.create(child));
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b5608643/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index d80e752..f0422d3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -27,8 +27,9 @@ import org.junit.Ignore;
import org.junit.Test;
import java.math.BigDecimal;
+import static org.apache.drill.TestBuilder.listOf;
-public class TestExampleQueries extends BaseTestQuery{
+public class TestExampleQueries extends BaseTestQuery {
// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExampleQueries.class);
@Test // see DRILL-2328
@@ -41,7 +42,7 @@ public class TestExampleQueries extends BaseTestQuery{
testBuilder()
.sqlQuery("select (mi || lname) as CONCATOperator, mi, lname, concat(mi, lname) as CONCAT from concatNull")
.ordered()
- .baselineColumns("CONCATOperator", "mi", "lname","CONCAT")
+ .baselineColumns("CONCATOperator", "mi", "lname", "CONCAT")
.baselineValues("A.Nowmer", "A.", "Nowmer", "A.Nowmer")
.baselineValues("I.Whelply", "I.", "Whelply", "I.Whelply")
.baselineValues(null, null, "Derry", "Derry")
@@ -157,7 +158,7 @@ public class TestExampleQueries extends BaseTestQuery{
}
@Test
- public void testJoinMerge() throws Exception{
+ public void testJoinMerge() throws Exception {
test("alter session set `planner.enable_hashjoin` = false");
test("select count(*) \n" +
" from (select l.l_orderkey as x, c.c_custkey as y \n" +
@@ -170,12 +171,12 @@ public class TestExampleQueries extends BaseTestQuery{
}
@Test
- public void testJoinExpOn() throws Exception{
+ public void testJoinExpOn() throws Exception {
test("select a.n_nationkey from cp.`tpch/nation.parquet` a join cp.`tpch/region.parquet` b on a.n_regionkey + 1 = b.r_regionkey and a.n_regionkey + 1 = b.r_regionkey;");
}
@Test
- public void testJoinExpWhere() throws Exception{
+ public void testJoinExpWhere() throws Exception {
test("select a.n_nationkey from cp.`tpch/nation.parquet` a , cp.`tpch/region.parquet` b where a.n_regionkey + 1 = b.r_regionkey and a.n_regionkey + 1 = b.r_regionkey;");
}
@@ -201,8 +202,8 @@ public class TestExampleQueries extends BaseTestQuery{
public void testPushExpInJoinConditionLeftJoin() throws Exception {
test("select a.n_nationkey, b.r_regionkey from cp.`tpch/nation.parquet` a left join cp.`tpch/region.parquet` b " + "" +
" on a.n_regionkey +100 = b.r_regionkey +200 " + // expressions in both sides of equal join filter
- // " and (substr(a.n_name,1,3)= 'L1' or substr(a.n_name,2,2) = 'L2') " + // left filter
- " and (substr(b.r_name,1,3)= 'R1' or substr(b.r_name,2,2) = 'R2') ") ; // right filter
+ // " and (substr(a.n_name,1,3)= 'L1' or substr(a.n_name,2,2) = 'L2') " + // left filter
+ " and (substr(b.r_name,1,3)= 'R1' or substr(b.r_name,2,2) = 'R2') "); // right filter
// " and (substr(a.n_name,2,3)= 'L3' or substr(b.r_name,3,2) = 'R3');"); // non-equal join filter
}
@@ -211,52 +212,52 @@ public class TestExampleQueries extends BaseTestQuery{
test("select a.n_nationkey, b.r_regionkey from cp.`tpch/nation.parquet` a right join cp.`tpch/region.parquet` b " + "" +
" on a.n_regionkey +100 = b.r_regionkey +200 " + // expressions in both sides of equal join filter
" and (substr(a.n_name,1,3)= 'L1' or substr(a.n_name,2,2) = 'L2') "); // left filter
- // " and (substr(b.r_name,1,3)= 'R1' or substr(b.r_name,2,2) = 'R2') " + // right filter
- // " and (substr(a.n_name,2,3)= 'L3' or substr(b.r_name,3,2) = 'R3');"); // non-equal join filter
+ // " and (substr(b.r_name,1,3)= 'R1' or substr(b.r_name,2,2) = 'R2') " + // right filter
+ // " and (substr(a.n_name,2,3)= 'L3' or substr(b.r_name,3,2) = 'R3');"); // non-equal join filter
}
@Test
- public void testCaseReturnValueVarChar() throws Exception{
+ public void testCaseReturnValueVarChar() throws Exception {
test("select case when employee_id < 1000 then 'ABC' else 'DEF' end from cp.`employee.json` limit 5");
}
@Test
- public void testCaseReturnValueBigInt() throws Exception{
- test("select case when employee_id < 1000 then 1000 else 2000 end from cp.`employee.json` limit 5" );
+ public void testCaseReturnValueBigInt() throws Exception {
+ test("select case when employee_id < 1000 then 1000 else 2000 end from cp.`employee.json` limit 5");
}
@Test
- public void testHashPartitionSV2 () throws Exception{
+ public void testHashPartitionSV2() throws Exception {
test("select count(n_nationkey) from cp.`tpch/nation.parquet` where n_nationkey > 8 group by n_regionkey");
}
@Test
- public void testHashPartitionSV4 () throws Exception{
+ public void testHashPartitionSV4() throws Exception {
test("select count(n_nationkey) as cnt from cp.`tpch/nation.parquet` group by n_regionkey order by cnt");
}
@Test
- public void testSelectWithLimit() throws Exception{
+ public void testSelectWithLimit() throws Exception {
test("select employee_id, first_name, last_name from cp.`employee.json` limit 5 ");
}
@Test
- public void testSelectWithLimit2() throws Exception{
+ public void testSelectWithLimit2() throws Exception {
test("select l_comment, l_orderkey from cp.`tpch/lineitem.parquet` limit 10000 ");
}
@Test
- public void testSVRV4() throws Exception{
+ public void testSVRV4() throws Exception {
test("select employee_id, first_name from cp.`employee.json` order by employee_id ");
}
@Test
- public void testSVRV4MultBatch() throws Exception{
+ public void testSVRV4MultBatch() throws Exception {
test("select l_orderkey from cp.`tpch/lineitem.parquet` order by l_orderkey limit 10000 ");
}
@Test
- public void testSVRV4Join() throws Exception{
+ public void testSVRV4Join() throws Exception {
test("select count(*) from cp.`tpch/lineitem.parquet` l, cp.`tpch/partsupp.parquet` ps \n" +
" where l.l_partkey = ps.ps_partkey and l.l_suppkey = ps.ps_suppkey ;");
}
@@ -285,7 +286,7 @@ public class TestExampleQueries extends BaseTestQuery{
@Test
@Ignore("DRILL-3004")
- public void testJoin() throws Exception{
+ public void testJoin() throws Exception {
test("alter session set `planner.enable_hashjoin` = false");
test("SELECT\n" +
" nations.N_NAME,\n" +
@@ -299,22 +300,22 @@ public class TestExampleQueries extends BaseTestQuery{
@Test
- public void testWhere() throws Exception{
+ public void testWhere() throws Exception {
test("select * from cp.`employee.json` ");
}
@Test
- public void testGroupBy() throws Exception{
+ public void testGroupBy() throws Exception {
test("select marital_status, COUNT(1) as cnt from cp.`employee.json` group by marital_status");
}
@Test
- public void testExplainPhysical() throws Exception{
+ public void testExplainPhysical() throws Exception {
test("explain plan for select marital_status, COUNT(1) as cnt from cp.`employee.json` group by marital_status");
}
@Test
- public void testExplainLogical() throws Exception{
+ public void testExplainLogical() throws Exception {
test("explain plan without implementation for select marital_status, COUNT(1) as cnt from cp.`employee.json` group by marital_status");
}
@@ -490,7 +491,7 @@ public class TestExampleQueries extends BaseTestQuery{
}
@Test // DRILL-1488
- public void testIdentifierMaxLength() throws Exception {
+ public void testIdentifierMaxLength() throws Exception {
// use long column alias name (approx 160 chars)
test("select employee_id as aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa from cp.`employee.json` limit 1");
@@ -517,7 +518,7 @@ public class TestExampleQueries extends BaseTestQuery{
@Test // DRILL-1788
public void testCaseInsensitiveJoin() throws Exception {
test("select n3.n_name from (select n2.n_name from cp.`tpch/nation.parquet` n1, cp.`tpch/nation.parquet` n2 where n1.N_name = n2.n_name) n3 " +
- " join cp.`tpch/nation.parquet` n4 on n3.n_name = n4.n_name");
+ " join cp.`tpch/nation.parquet` n4 on n3.n_name = n4.n_name");
}
@Test // DRILL-1561
@@ -558,7 +559,7 @@ public class TestExampleQueries extends BaseTestQuery{
assertEquals(String.format("Received unexpected number of rows in output: expected=%d, received=%s",
expectedRecordCount, actualRecordCount), expectedRecordCount, actualRecordCount);
- // source is CSV
+ // source is CSV
String root = FileUtils.getResourceAsFile("/store/text/data/regions.csv").toURI().toString();
String query = String.format("select rid, x.name from (select columns[0] as RID, columns[1] as NAME from dfs_test.`%s`) X where X.rid = 2", root);
actualRecordCount = testSql(query);
@@ -572,9 +573,9 @@ public class TestExampleQueries extends BaseTestQuery{
public void testMultipleCountDistinctWithGroupBy() throws Exception {
String query = "select n_regionkey, count(distinct n_nationkey), count(distinct n_name) from cp.`tpch/nation.parquet` group by n_regionkey;";
String hashagg_only = "alter session set `planner.enable_hashagg` = true; " +
- "alter session set `planner.enable_streamagg` = false;";
+ "alter session set `planner.enable_streamagg` = false;";
String streamagg_only = "alter session set `planner.enable_hashagg` = false; " +
- "alter session set `planner.enable_streamagg` = true;";
+ "alter session set `planner.enable_streamagg` = true;";
// hash agg and streaming agg with default slice target (single phase aggregate)
test(hashagg_only + query);
@@ -617,16 +618,16 @@ public class TestExampleQueries extends BaseTestQuery{
@Test // DRILL-2063
public void testAggExpressionWithGroupBy() throws Exception {
String query = "select l_suppkey, sum(l_extendedprice)/sum(l_quantity) as avg_price \n" +
- " from cp.`tpch/lineitem.parquet` where l_orderkey in \n" +
- " (select o_orderkey from cp.`tpch/orders.parquet` where o_custkey = 2) \n" +
- " and l_suppkey = 4 group by l_suppkey";
+ " from cp.`tpch/lineitem.parquet` where l_orderkey in \n" +
+ " (select o_orderkey from cp.`tpch/orders.parquet` where o_custkey = 2) \n" +
+ " and l_suppkey = 4 group by l_suppkey";
testBuilder()
- .sqlQuery(query)
- .ordered()
- .baselineColumns("l_suppkey", "avg_price")
- .baselineValues(4, 1374.47)
- .build().run();
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns("l_suppkey", "avg_price")
+ .baselineValues(4, 1374.47)
+ .build().run();
}
@@ -638,11 +639,11 @@ public class TestExampleQueries extends BaseTestQuery{
" group by l_suppkey having sum(l_extendedprice)/sum(l_quantity) > 1850.0";
testBuilder()
- .sqlQuery(query)
- .ordered()
- .baselineColumns("l_suppkey", "avg_price")
- .baselineValues(98, 1854.95)
- .build().run();
+ .sqlQuery(query)
+ .ordered()
+ .baselineColumns("l_suppkey", "avg_price")
+ .baselineValues(98, 1854.95)
+ .build().run();
}
@Test
@@ -695,7 +696,7 @@ public class TestExampleQueries extends BaseTestQuery{
}
@Test // DRILL-2311
- @Ignore ("Move to TestParquetWriter. Have to ensure same file name does not exist on filesystem.")
+ @Ignore("Move to TestParquetWriter. Have to ensure same file name does not exist on filesystem.")
public void testCreateTableSameColumnNames() throws Exception {
String creatTable = "CREATE TABLE CaseInsensitiveColumnNames as " +
"select cast(r_regionkey as BIGINT) BIGINT_col, cast(r_regionkey as DECIMAL) bigint_col \n" +
@@ -765,11 +766,11 @@ public class TestExampleQueries extends BaseTestQuery{
+ " in (select p.p_partkey from cp.`tpch/part.parquet` p where p.p_type like '%NICKEL'))";
testBuilder()
- .sqlQuery(query)
- .unOrdered()
- .baselineColumns("cnt")
- .baselineValues(60175l)
- .go();
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns("cnt")
+ .baselineValues(60175l)
+ .go();
}
@Test // DRILL-2094
@@ -913,10 +914,10 @@ public class TestExampleQueries extends BaseTestQuery{
+ "from cp.`tpch/nation.parquet` \n"
+ "group by n_nationkey \n"
+ "having n_nationkey in \n"
- + "(select r_regionkey \n"
- + "from cp.`tpch/region.parquet` \n"
- + "group by r_regionkey \n"
- + "having sum(r_regionkey) > 0)";
+ + "(select r_regionkey \n"
+ + "from cp.`tpch/region.parquet` \n"
+ + "group by r_regionkey \n"
+ + "having sum(r_regionkey) > 0)";
String query3 = "select n_nationkey as col \n"
+ "from cp.`tpch/nation.parquet` \n"
@@ -966,9 +967,9 @@ public class TestExampleQueries extends BaseTestQuery{
String root = FileUtils.getResourceAsFile("/store/text/data/nations.csv").toURI().toString();
String query = String.format(
"select cast(columns[0] as int) as nation_key " +
- " from dfs_test.`%s` " +
- " group by columns[0] " +
- " order by cast(columns[0] as int)", root);
+ " from dfs_test.`%s` " +
+ " group by columns[0] " +
+ " order by cast(columns[0] as int)", root);
testBuilder()
.sqlQuery(query)
@@ -1020,4 +1021,14 @@ public class TestExampleQueries extends BaseTestQuery{
}
+ @Test
+ public void testRepeatedListProjectionPastJoin() throws Exception {
+ final String query = "select * from cp.`join/join-left-drill-3032.json` f1 inner join cp.`join/join-right-drill-3032.json` f2 on f1.id = f2.id";
+ testBuilder()
+ .sqlQuery(query)
+ .unOrdered()
+ .baselineColumns("id", "id0", "aaa")
+ .baselineValues(1L, 1L, listOf(listOf(listOf("val1"), listOf("val2"))))
+ .go();
+ }
}
http://git-wip-us.apache.org/repos/asf/drill/blob/b5608643/exec/java-exec/src/test/resources/join/join-left-drill-3032.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/join/join-left-drill-3032.json b/exec/java-exec/src/test/resources/join/join-left-drill-3032.json
new file mode 100644
index 0000000..e19acb8
--- /dev/null
+++ b/exec/java-exec/src/test/resources/join/join-left-drill-3032.json
@@ -0,0 +1,4 @@
+{
+ "id":1,
+ "aaa":[[["val1"], ["val2"]]]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/b5608643/exec/java-exec/src/test/resources/join/join-right-drill-3032.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/join/join-right-drill-3032.json b/exec/java-exec/src/test/resources/join/join-right-drill-3032.json
new file mode 100644
index 0000000..439e837
--- /dev/null
+++ b/exec/java-exec/src/test/resources/join/join-right-drill-3032.json
@@ -0,0 +1,3 @@
+{
+ "id":1
+}
\ No newline at end of file