You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/11/12 05:11:18 UTC
[01/16] incubator-drill git commit: DRILL-1638: display list of files
in the logical plan
Repository: incubator-drill
Updated Branches:
refs/heads/master c305c794a -> 1e21045bf
DRILL-1638: display list of files in the logical plan
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/23d5c38f
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/23d5c38f
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/23d5c38f
Branch: refs/heads/master
Commit: 23d5c38f5379048e2ca69aa99d3720c99a1f677a
Parents: c305c79
Author: Hanifi Gunes <hg...@maprtech.com>
Authored: Mon Nov 10 18:00:02 2014 -0800
Committer: Hanifi Gunes <hg...@maprtech.com>
Committed: Tue Nov 11 16:14:33 2014 -0800
----------------------------------------------------------------------
.../java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/23d5c38f/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
index e22e07e..b505535 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/easy/EasyGroupScan.java
@@ -225,7 +225,8 @@ public class EasyGroupScan extends AbstractFileGroupScan{
@Override
public String toString() {
- return "EasyGroupScan [selectionRoot=" + selectionRoot + ", numFiles=" + getFiles().size() + ", columns = " + columns + "]";
+ final String pattern = "EasyGroupScan [selectionRoot=%s, numFiles=%s, columns=%s, files=%s]";
+ return String.format(pattern, selectionRoot, getFiles().size(), columns, getFiles());
}
@Override
[06/16] incubator-drill git commit: DRILL-1639: Close Hive resources
Posted by ja...@apache.org.
DRILL-1639: Close Hive resources
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/25fca623
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/25fca623
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/25fca623
Branch: refs/heads/master
Commit: 25fca623f579570807a6ca8f61b7a81c7b019f4a
Parents: abab3d3
Author: Jacques Nadeau <ja...@apache.org>
Authored: Wed Nov 5 08:55:19 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 16:48:44 2014 -0800
----------------------------------------------------------------------
.../org/apache/drill/exec/store/hive/HiveRecordReader.java | 8 ++++++++
1 file changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/25fca623/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
----------------------------------------------------------------------
diff --git a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
index 0f59095..625a7b2 100644
--- a/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
+++ b/contrib/storage-hive/core/src/main/java/org/apache/drill/exec/store/hive/HiveRecordReader.java
@@ -42,6 +42,7 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.rpc.ProtobufLengthDecoder;
import org.apache.drill.exec.store.AbstractRecordReader;
import org.apache.drill.exec.util.DecimalUtility;
import org.apache.drill.exec.vector.BigIntVector;
@@ -91,6 +92,8 @@ import com.google.common.collect.Lists;
public class HiveRecordReader extends AbstractRecordReader {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(HiveRecordReader.class);
+
protected Table table;
protected Partition partition;
protected InputSplit inputSplit;
@@ -348,6 +351,11 @@ public class HiveRecordReader extends AbstractRecordReader {
@Override
public void cleanup() {
+ try {
+ reader.close();
+ } catch (Exception e) {
+ logger.warn("Failure while closing Hive Record reader.", e);
+ }
}
public static MinorType getMinorTypeFromHivePrimitiveTypeInfo(PrimitiveTypeInfo primitiveTypeInfo) {
[09/16] incubator-drill git commit: DRILL-1671, DRILL-1653,
DRILL-1652: Fixes for flatten bugs
Posted by ja...@apache.org.
DRILL-1671, DRILL-1653, DRILL-1652: Fixes for flatten bugs
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/60736dea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/60736dea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/60736dea
Branch: refs/heads/master
Commit: 60736dea0fc7099f3773616c1995df4789221cdb
Parents: ed96249
Author: Jacques Nadeau <ja...@apache.org>
Authored: Mon Nov 10 22:16:19 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 16:48:45 2014 -0800
----------------------------------------------------------------------
.../drill/common/expression/SchemaPath.java | 3 +-
.../codegen/templates/RepeatedValueVectors.java | 30 +--
.../exec/planner/physical/DrillFlattenPrel.java | 75 -------
.../exec/planner/physical/FlattenPrel.java | 80 ++++++++
.../visitor/RewriteProjectToFlatten.java | 4 +-
.../visitor/SplitUpComplexExpressions.java | 2 +-
.../drill/exec/record/MaterializedField.java | 22 +-
.../exec/vector/complex/RepeatedMapVector.java | 35 +++-
.../exec/vector/complex/fn/JsonReader.java | 3 +-
.../exec/physical/impl/flatten/TestFlatten.java | 51 +++++
.../src/test/resources/flatten/empty-rm.json | 7 +
.../test/resources/flatten/many-arrays-50.json | 203 +++++++++++++++++++
.../src/test/resources/flatten/missing-map.json | 11 +
.../flatten/single-user-transactions.json | 18 ++
14 files changed, 445 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
index 3b31a15..1d7f922 100644
--- a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -233,8 +233,7 @@ public class SchemaPath extends LogicalExpressionBase {
@Override
public String toString() {
- String expr = ExpressionStringBuilder.toString(this);
- return "SchemaPath ["+ expr + "]";
+ return ExpressionStringBuilder.toString(this);
}
public String toExpr() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index 2853e83..0f86db9 100644
--- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -18,6 +18,7 @@
import java.lang.Override;
+import org.apache.drill.exec.record.TransferPair;
import org.apache.drill.exec.vector.RepeatedFixedWidthVector;
import org.apache.drill.exec.vector.UInt4Vector;
import org.mortbay.jetty.servlet.Holder;
@@ -102,20 +103,25 @@ public final class Repeated${minor.class}Vector extends BaseValueVector implemen
clear();
}
- public void splitAndTransferTo(int startIndex, int length, Repeated${minor.class}Vector target) {
- int startPos = offsets.getAccessor().get(startIndex);
- int endPos = offsets.getAccessor().get(startIndex+length);
- values.splitAndTransferTo(startIndex, endPos-startPos, target.values);
- target.offsets.clear();
- target.offsets.allocateNew(endPos - startPos + 1);
+ public void splitAndTransferTo(final int startIndex, final int groups, Repeated${minor.class}Vector to) {
+ final UInt4Vector.Accessor a = offsets.getAccessor();
+ final UInt4Vector.Mutator m = to.offsets.getMutator();
+
+ final int startPos = offsets.getAccessor().get(startIndex);
+ final int endPos = offsets.getAccessor().get(startIndex + groups);
+ final int valuesToCopy = endPos - startPos;
+
+ values.splitAndTransferTo(startPos, valuesToCopy, to.values);
+ to.offsets.clear();
+ to.offsets.allocateNew(valuesToCopy + 1);
int normalizedPos = 0;
- for (int i=0; i<length+1;i++) {
- normalizedPos = offsets.getAccessor().get(startIndex+i) - startPos;
- target.offsets.getMutator().set(i, normalizedPos);
+ for (int i=0; i < groups + 1;i++ ) {
+ normalizedPos = a.get(startIndex+i) - startPos;
+ m.set(i, normalizedPos);
}
- target.parentValueCount = length;
- target.childValueCount = offsets.getAccessor().get(startIndex+length) - startPos;
- target.offsets.getMutator().setValueCount(length);
+ to.parentValueCount = groups;
+ to.childValueCount = valuesToCopy;
+ m.setValueCount(groups);
}
private class TransferImpl implements TransferPair{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java
deleted file mode 100644
index 1409347..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.planner.physical;
-
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.config.FlattenPOP;
-import org.apache.drill.exec.planner.common.DrillFilterRelBase;
-import org.apache.drill.exec.planner.logical.DrillOptiq;
-import org.apache.drill.exec.planner.logical.DrillParseContext;
-import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
-import org.apache.drill.exec.record.BatchSchema;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.rex.RexNode;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-
-public class DrillFlattenPrel extends SinglePrel implements Prel {
-
- RexNode toFlatten;
-
- public DrillFlattenPrel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode toFlatten) {
- super(cluster, traits, child);
- this.toFlatten = toFlatten;
- }
-
- @Override
- public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
- return new DrillFlattenPrel(getCluster(), traitSet, sole(inputs), toFlatten);
- }
-
- @Override
- public Iterator<Prel> iterator() {
- return PrelUtil.iter(getChild());
- }
-
- @Override
- public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
- Prel child = (Prel) this.getChild();
-
- PhysicalOperator childPOP = child.getPhysicalOperator(creator);
- FlattenPOP f = new FlattenPOP(childPOP, (SchemaPath) getFlattenExpression(new DrillParseContext()));
- return creator.addMetadata(this, f);
- }
-
- @Override
- public BatchSchema.SelectionVectorMode getEncoding() {
- return BatchSchema.SelectionVectorMode.NONE;
- }
-
- protected LogicalExpression getFlattenExpression(DrillParseContext context){
- return DrillOptiq.toDrill(context, getChild(), toFlatten);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
new file mode 100644
index 0000000..18a2cc2
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
@@ -0,0 +1,80 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.planner.physical;
+
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.config.FlattenPOP;
+import org.apache.drill.exec.planner.common.DrillFilterRelBase;
+import org.apache.drill.exec.planner.logical.DrillOptiq;
+import org.apache.drill.exec.planner.logical.DrillParseContext;
+import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
+import org.apache.drill.exec.record.BatchSchema;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.RelWriter;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.rex.RexNode;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+public class FlattenPrel extends SinglePrel implements Prel {
+
+ RexNode toFlatten;
+
+ public FlattenPrel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode toFlatten) {
+ super(cluster, traits, child);
+ this.toFlatten = toFlatten;
+ }
+
+ @Override
+ public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+ return new FlattenPrel(getCluster(), traitSet, sole(inputs), toFlatten);
+ }
+
+ @Override
+ public Iterator<Prel> iterator() {
+ return PrelUtil.iter(getChild());
+ }
+
+ @Override
+ public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
+ Prel child = (Prel) this.getChild();
+
+ PhysicalOperator childPOP = child.getPhysicalOperator(creator);
+ FlattenPOP f = new FlattenPOP(childPOP, (SchemaPath) getFlattenExpression(new DrillParseContext()));
+ return creator.addMetadata(this, f);
+ }
+
+ public RelWriter explainTerms(RelWriter pw) {
+ return super.explainTerms(pw).item("flattenField", this.toFlatten);
+ }
+
+ @Override
+ public BatchSchema.SelectionVectorMode getEncoding() {
+ return BatchSchema.SelectionVectorMode.NONE;
+ }
+
+ protected LogicalExpression getFlattenExpression(DrillParseContext context){
+ return DrillOptiq.toDrill(context, getChild(), toFlatten);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
index d4b3573..1055411 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
@@ -21,7 +21,7 @@ import com.google.common.collect.Lists;
import net.hydromatic.optiq.tools.RelConversionException;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.planner.physical.DrillFlattenPrel;
+import org.apache.drill.exec.planner.physical.FlattenPrel;
import org.apache.drill.exec.planner.physical.Prel;
import org.apache.drill.exec.planner.physical.ProjectPrel;
import org.apache.drill.exec.planner.physical.visitor.BasePrelVisitor;
@@ -103,7 +103,7 @@ public class RewriteProjectToFlatten extends BasePrelVisitor<Prel, Object, RelCo
// TODO - figure out what is the right setting for the traits
Prel newChild = ((Prel)project.getInput(0)).accept(this, null);
ProjectPrel newProject = new ProjectPrel(node.getCluster(), project.getTraitSet(), newChild, exprList, new RelRecordType(relDataTypes));
- DrillFlattenPrel flatten = new DrillFlattenPrel(project.getCluster(), project.getTraitSet(), newProject, flatttenExpr);
+ FlattenPrel flatten = new FlattenPrel(project.getCluster(), project.getTraitSet(), newProject, flatttenExpr);
return flatten;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
index 0a49f3a..89c8447 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
@@ -26,7 +26,7 @@ import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.planner.logical.DrillRel;
import org.apache.drill.exec.planner.logical.DrillScanRel;
import org.apache.drill.exec.planner.logical.DrillTable;
-import org.apache.drill.exec.planner.physical.DrillFlattenPrel;
+import org.apache.drill.exec.planner.physical.FlattenPrel;
import org.apache.drill.exec.planner.physical.Prel;
import org.apache.drill.exec.planner.physical.PrelUtil;
import org.apache.drill.exec.planner.physical.ProjectPrel;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
index b4da2b4..7f7dcfa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
@@ -18,10 +18,12 @@
package org.apache.drill.exec.record;
import java.util.Collection;
+import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import com.google.common.collect.Sets;
+
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
@@ -215,9 +217,27 @@ public class MaterializedField {
return true;
}
+
@Override
public String toString() {
- return "MaterializedField [path=" + key.path + ", type=" + Types.toString(key.type) + "]";
+ final int maxLen = 10;
+ String childStr = children != null && !children.isEmpty() ? toString(children, maxLen) : "";
+ return key.path + "(" + key.type.getMinorType().name() + ":" + key.type.getMode().name() + ")" + childStr;
+ }
+
+
+ private String toString(Collection<?> collection, int maxLen) {
+ StringBuilder builder = new StringBuilder();
+ builder.append("[");
+ int i = 0;
+ for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) {
+ if (i > 0){
+ builder.append(", ");
+ }
+ builder.append(iterator.next());
+ }
+ builder.append("]");
+ return builder.toString();
}
public Key key() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index 9b7011c..7ddac23 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -360,6 +360,9 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
public boolean copyValueSafe(int srcIndex, int destIndex) {
RepeatedMapHolder holder = new RepeatedMapHolder();
accessor.get(srcIndex, holder);
+ if(destIndex >= to.getValueCapacity()){
+ return false;
+ }
to.populateEmpties(destIndex+1);
int newIndex = to.offsets.getAccessor().get(destIndex);
//todo: make these bulk copies
@@ -377,8 +380,30 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
}
@Override
- public void splitAndTransfer(int startIndex, int length) {
- throw new UnsupportedOperationException();
+ public void splitAndTransfer(final int groupStart, final int groups) {
+ final UInt4Vector.Accessor a = offsets.getAccessor();
+ final UInt4Vector.Mutator m = to.offsets.getMutator();
+
+ final int startPos = a.get(groupStart);
+ final int endPos = a.get(groupStart+groups);
+ final int valuesToCopy = endPos - startPos;
+
+ to.offsets.clear();
+ to.offsets.allocateNew(groups + 1);
+ int normalizedPos = 0;
+
+ for (int i=0; i < groups+1; i++) {
+ normalizedPos = a.get(groupStart+i) - startPos;
+ m.set(i, normalizedPos);
+ }
+
+ m.setValueCount(groups + 1);
+ to.lastSet = groups;
+
+ for (TransferPair p : pairs) {
+ p.splitAndTransfer(startPos, valuesToCopy);
+ }
+
}
}
@@ -444,7 +469,9 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
SerializedField.Builder b = getField() //
.getAsBuilder() //
.setBufferLength(getBufferSize()) //
- .setGroupCount(accessor.getGroupCount());
+ .setGroupCount(accessor.getGroupCount())
+ // while we don't need to actually read this on load, we need it to make sure we don't skip deserialization of this vector
+ .setValueCount(accessor.getValueCount());
for (ValueVector v : vectors.values()) {
b.addChild(v.getMetadata());
}
@@ -503,7 +530,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
}
public void get(int index, RepeatedMapHolder holder) {
- assert index < getValueCapacity()-1;
+ assert index < getValueCapacity() : String.format("Attempted to access index %d when value capacity is %d", index, getValueCapacity());
holder.start = offsets.getAccessor().get(index);
holder.end = offsets.getAccessor().get(index+1);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
index 17e266f..0ca24e7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
@@ -197,7 +197,6 @@ private boolean writeToVector(ComplexWriter writer, JsonToken t) throws JsonPars
map.start();
outside: while(true) {
if (!map.ok()) {
- logger.warn("Error reported. Quit writing");
break;
}
JsonToken t = parser.nextToken();
@@ -320,7 +319,7 @@ private boolean writeToVector(ComplexWriter writer, JsonToken t) throws JsonPars
list.start();
outside: while (true) {
if (!list.ok()) {
- logger.warn("Error reported. Quit writing");
+// logger.warn("Error reported. Quit writing");
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
index d4c19a3..960da65 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
@@ -17,7 +17,10 @@
******************************************************************************/
package org.apache.drill.exec.physical.impl.flatten;
+import static org.junit.Assert.assertEquals;
+
import org.apache.drill.BaseTestQuery;
+import org.junit.Ignore;
import org.junit.Test;
public class TestFlatten extends BaseTestQuery {
@@ -27,6 +30,7 @@ public class TestFlatten extends BaseTestQuery {
* - /tmp/yelp_academic_dataset_business.json
* - /tmp/mapkv.json
* - /tmp/drill1665.json
+ * - /tmp/bigfile.json
*/
public static boolean RUN_ADVANCED_TESTS = false;
@@ -38,6 +42,53 @@ public class TestFlatten extends BaseTestQuery {
}
@Test
+ public void drill1671() throws Exception{
+ int rowCount = testSql("select * from (select count(*) as cnt from (select id, flatten(evnts1), flatten(evnts2), flatten(evnts3), flatten(evnts4), flatten(evnts5), flatten(evnts6), flatten(evnts7), flatten(evnts8), flatten(evnts9), flatten(evnts10), flatten(evnts11) from cp.`/flatten/many-arrays-50.json`)x )y where cnt = 2048");
+ assertEquals(rowCount, 1);
+ }
+
+ @Test
+ @Ignore("not yet fixed")
+ public void drill1660() throws Exception {
+ test("select * from cp.`/flatten/empty-rm.json`");
+ }
+
+ @Test
+ public void drill1653() throws Exception{
+ int rowCount = testSql("select * from (select sum(t.flat.`value`) as sm from (select id, flatten(kvgen(m)) as flat from cp.`/flatten/missing-map.json`)t) where sm = 10 ");
+ assertEquals(rowCount, 1);
+ }
+
+ @Test
+ public void drill1652() throws Exception {
+ if(RUN_ADVANCED_TESTS){
+ test("select uid, flatten(transactions) from dfs.`/tmp/bigfile.json`");
+ }
+ }
+
+ @Test
+ @Ignore("Still not working.")
+ public void drill1649() throws Exception {
+ test("select event_info.uid, transaction_info.trans_id, event_info.event.evnt_id\n" +
+ "from (\n" +
+ " select userinfo.transaction.trans_id trans_id, max(userinfo.event.event_time) max_event_time\n" +
+ " from (\n" +
+ " select uid, flatten(events) event, flatten(transactions) transaction from cp.`/flatten/single-user-transactions.json`\n" +
+ " ) userinfo\n" +
+ " where userinfo.transaction.trans_time >= userinfo.event.event_time\n" +
+ " group by userinfo.transaction.trans_id\n" +
+ ") transaction_info\n" +
+ "inner join\n" +
+ "(\n" +
+ " select uid, flatten(events) event\n" +
+ " from cp.`/flatten/single-user-transactions.json`\n" +
+ ") event_info\n" +
+ "on transaction_info.max_event_time = event_info.event.event_time;");
+ }
+
+
+
+ @Test
public void testKVGenFlatten1() throws Exception {
// works - TODO and verify results
test("select flatten(kvgen(f1)) as monkey, x " +
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/empty-rm.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/flatten/empty-rm.json b/exec/java-exec/src/test/resources/flatten/empty-rm.json
new file mode 100644
index 0000000..7927a3f
--- /dev/null
+++ b/exec/java-exec/src/test/resources/flatten/empty-rm.json
@@ -0,0 +1,7 @@
+{
+ "id":1,
+ "evnts":[
+ {},
+ {"key":null}
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/many-arrays-50.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/flatten/many-arrays-50.json b/exec/java-exec/src/test/resources/flatten/many-arrays-50.json
new file mode 100644
index 0000000..c25565e
--- /dev/null
+++ b/exec/java-exec/src/test/resources/flatten/many-arrays-50.json
@@ -0,0 +1,203 @@
+{
+ "id":1,
+ "evnts1":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts2":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts3":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts4":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts5":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts6":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts7":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts8":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts9":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts10":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts11":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts12":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts13":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts14":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts15":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts16":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts17":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts18":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts19":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts20":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts21":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts22":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts23":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts24":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts25":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts26":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts27":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts28":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts29":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts30":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts31":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts32":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts33":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts34":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts35":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts36":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts37":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts38":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts39":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts40":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts41":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts42":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts43":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts44":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts45":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts46":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts47":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts48":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts49":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts50":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/missing-map.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/flatten/missing-map.json b/exec/java-exec/src/test/resources/flatten/missing-map.json
new file mode 100644
index 0000000..5d8f815
--- /dev/null
+++ b/exec/java-exec/src/test/resources/flatten/missing-map.json
@@ -0,0 +1,11 @@
+{
+ "id": 1,
+ "m": {"a":1,"b":2}
+}
+{
+ "id": 2
+}
+{
+ "id": 3,
+ "m": {"c":3,"d":4}
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/single-user-transactions.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/flatten/single-user-transactions.json b/exec/java-exec/src/test/resources/flatten/single-user-transactions.json
new file mode 100644
index 0000000..cff5532
--- /dev/null
+++ b/exec/java-exec/src/test/resources/flatten/single-user-transactions.json
@@ -0,0 +1,18 @@
+{
+ "uid" : 1,
+ "events" : [
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e2", "campaign_id":"c1", "event_name":"e2_name", "event_time":2000000},
+ { "evnt_id":"e3", "campaign_id":"c1", "event_name":"e3_name", "event_time":3000000},
+ { "evnt_id":"e4", "campaign_id":"c1", "event_name":"e4_name", "event_time":4000000},
+ { "evnt_id":"e5", "campaign_id":"c2", "event_name":"e5_name", "event_time":5000000},
+ { "evnt_id":"e6", "campaign_id":"c1", "event_name":"e6_name", "event_time":6000000},
+ { "evnt_id":"e7", "campaign_id":"c1", "event_name":"e7_name", "event_time":7000000},
+ { "evnt_id":"e8", "campaign_id":"c2", "event_name":"e8_name", "event_time":8000000},
+ { "evnt_id":"e9", "campaign_id":"c2", "event_name":"e9_name", "event_time":9000000}
+ ],
+ "transactions" : [
+ { "trans_id":"t1", "amount":100, "trans_time":7777777},
+ { "trans_id":"t2", "amount":100, "trans_time":8888888}
+ ]
+}
[13/16] incubator-drill git commit: Add handling when a null
exception is logged.
Posted by ja...@apache.org.
Add handling when a null exception is logged.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ab154b5e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ab154b5e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ab154b5e
Branch: refs/heads/master
Commit: ab154b5e3acc3461e327643f30211fa5904d462b
Parents: 60736de
Author: Jacques Nadeau <ja...@apache.org>
Authored: Tue Nov 11 09:45:03 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 16:48:45 2014 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/drill/exec/work/ErrorHelper.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ab154b5e/exec/java-exec/src/main/java/org/apache/drill/exec/work/ErrorHelper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/ErrorHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/ErrorHelper.java
index 6a9cab5..025a8de 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/ErrorHelper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/ErrorHelper.java
@@ -20,6 +20,7 @@ package org.apache.drill.exec.work;
import java.util.UUID;
import java.util.regex.Pattern;
+import org.apache.drill.common.exceptions.DrillException;
import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.UserBitShared.DrillPBError;
import org.apache.drill.exec.proto.UserBitShared.ExceptionWrapper;
@@ -61,6 +62,9 @@ public class ErrorHelper {
builder.setEndpoint(endpoint);
builder.setErrorId(id);
builder.setMessage(message);
+ if(t == null){
+ t = new DrillException("Undefined failure occurred.");
+ }
builder.setException(getWrapper(t));
// record the error to the log for later reference.
[02/16] incubator-drill git commit: DRILL-1660: adding a test case to
cover an array of empty and null valued objects. i) remove redundant guard
statement from JsonRecordReader2 ii) rename unit test suite
Posted by ja...@apache.org.
DRILL-1660: adding a test case to cover an array of empty and null valued objects.
i) remove redundant guard statement from JsonRecordReader2
ii) rename unit test suite
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/0cddfc22
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/0cddfc22
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/0cddfc22
Branch: refs/heads/master
Commit: 0cddfc2228859bb636dcf6ed03a227626f69e480
Parents: 23d5c38
Author: Hanifi Gunes <hg...@maprtech.com>
Authored: Tue Nov 11 15:46:10 2014 -0800
Committer: Hanifi Gunes <hg...@maprtech.com>
Committed: Tue Nov 11 16:14:43 2014 -0800
----------------------------------------------------------------------
.../exec/store/easy/json/JSONRecordReader2.java | 2 +-
.../exec/vector/complex/fn/JsonReaderTests.java | 127 -------------------
.../fn/TestJsonReaderWithSparseFiles.java | 127 +++++++++++++++++++
.../vector/complex/fn/nested-with-nulls.json | 3 +-
4 files changed, 130 insertions(+), 129 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0cddfc22/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
index ff7d3f2..d1502d4 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/easy/json/JSONRecordReader2.java
@@ -105,7 +105,7 @@ public class JSONRecordReader2 extends AbstractRecordReader {
recordCount = 0;
try{
- outside: while(true && recordCount < BaseValueVector.INITIAL_VALUE_ALLOCATION){
+ outside: while(recordCount < BaseValueVector.INITIAL_VALUE_ALLOCATION){
writer.setPosition(recordCount);
switch(jsonReader.write(writer)){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0cddfc22/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/JsonReaderTests.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/JsonReaderTests.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/JsonReaderTests.java
deleted file mode 100644
index 9a8eedf..0000000
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/JsonReaderTests.java
+++ /dev/null
@@ -1,127 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.drill.exec.vector.complex.fn;
-
-
-import java.util.List;
-import java.util.Objects;
-
-import org.apache.drill.BaseTestQuery;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.proto.beans.QueryResult;
-import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.rpc.user.QueryResultBatch;
-import org.apache.drill.exec.vector.ValueVector;
-import org.junit.Test;
-
-public class JsonReaderTests extends BaseTestQuery {
-
- static interface Function<T> {
- void apply(T param);
- }
-
- protected void query(final String query, final Function<RecordBatchLoader> testBody) throws Exception {
- List<QueryResultBatch> batches = testSqlWithResults(query);
- RecordBatchLoader loader = new RecordBatchLoader(client.getAllocator());
- try {
- QueryResultBatch batch = batches.get(0);
- loader.load(batch.getHeader().getDef(), batch.getData());
-
- testBody.apply(loader);
-
- } finally {
- for (QueryResultBatch batch:batches) {
- batch.release();
- }
- loader.clear();
- }
- }
-
- @Test
- public void testIfDrillCanReadSparseRecords() throws Exception {
- final String sql = "select * from cp.`vector/complex/fn/sparse.json`";
- query(sql, new Function<RecordBatchLoader>() {
- @Override
- public void apply(RecordBatchLoader loader) {
- assert loader.getRecordCount() == 4 : "invalid record count returned";
-
- //XXX: make sure value order matches vector order
- final Object[][] values = new Object[][] {
- {null, null},
- {1L, null},
- {null, 2L},
- {3L, 3L}
- };
-
- Object[] row;
- Object expected;
- Object actual;
- for (int r=0;r<values.length;r++) {
- row = values[r];
- for (int c=0; c<values[r].length; c++) {
- expected = row[c];
- actual = loader.getValueAccessorById(ValueVector.class, c).getValueVector().getAccessor().getObject(r);
- assert Objects.equals(expected, actual) : String.format("row:%d - col:%d - expected:%s[%s] - actual:%s[%s]",
- r, c,
- expected,
- expected==null?"null":expected.getClass().getSimpleName(),
- actual,
- actual==null?"null":actual.getClass().getSimpleName());
- }
- }
- }
- });
- }
-
- @Test
- public void testIfDrillCanReadSparseNestedRecordsWithoutRaisingException() throws Exception {
- final String sql = "select * from cp.`vector/complex/fn/nested-with-nulls.json`";
- query(sql, new Function<RecordBatchLoader>() {
- @Override
- public void apply(RecordBatchLoader loader) {
- assert loader.getRecordCount() == 3 : "invalid record count returned";
-
- //XXX: make sure value order matches vector order
- final Object[][] values = new Object[][] {
- {"[{},{},{},{\"name\":\"doe\"},{}]"},
- {"[]"},
- {"[{\"name\":\"john\",\"id\":10}]"},
- };
-
- Object[] row;
- Object expected;
- Object actual;
- for (int r=0;r<values.length;r++) {
- row = values[r];
- for (int c = 0; c < values[r].length; c++) {
- expected = row[c];
- actual = loader.getValueAccessorById(ValueVector.class, c).getValueVector().getAccessor().getObject(r);
- assert Objects.equals(actual, expected) : String.format("row:%d - col:%d - expected:%s[%s] - actual:%s[%s]",
- r, c,
- expected,
- expected == null ? "null" : expected.getClass().getSimpleName(),
- actual,
- actual == null ? "null" : actual.getClass().getSimpleName());
- }
- }
- }
- });
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0cddfc22/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
new file mode 100644
index 0000000..3cfdc1d
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.drill.exec.vector.complex.fn;
+
+
+import java.util.List;
+import java.util.Objects;
+
+import org.apache.drill.BaseTestQuery;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.proto.UserBitShared;
+import org.apache.drill.exec.proto.beans.QueryResult;
+import org.apache.drill.exec.record.RecordBatchLoader;
+import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.vector.ValueVector;
+import org.junit.Test;
+
+public class TestJsonReaderWithSparseFiles extends BaseTestQuery {
+
+ static interface Function<T> {
+ void apply(T param);
+ }
+
+ protected void query(final String query, final Function<RecordBatchLoader> testBody) throws Exception {
+ List<QueryResultBatch> batches = testSqlWithResults(query);
+ RecordBatchLoader loader = new RecordBatchLoader(client.getAllocator());
+ try {
+ // first batch at index 0 is empty and used for fast schema return. Load the second one for the tests
+ QueryResultBatch batch = batches.get(1);
+ loader.load(batch.getHeader().getDef(), batch.getData());
+ testBody.apply(loader);
+ } finally {
+ for (QueryResultBatch batch:batches) {
+ batch.release();
+ }
+ loader.clear();
+ }
+ }
+
+ @Test
+ public void testIfDrillCanReadSparseRecords() throws Exception {
+ final String sql = "select * from cp.`vector/complex/fn/sparse.json`";
+ query(sql, new Function<RecordBatchLoader>() {
+ @Override
+ public void apply(RecordBatchLoader loader) {
+ assert loader.getRecordCount() == 4 : "invalid record count returned";
+
+ //XXX: make sure value order matches vector order
+ final Object[][] values = new Object[][] {
+ {null, null},
+ {1L, null},
+ {null, 2L},
+ {3L, 3L}
+ };
+
+ Object[] row;
+ Object expected;
+ Object actual;
+ for (int r=0;r<values.length;r++) {
+ row = values[r];
+ for (int c=0; c<values[r].length; c++) {
+ expected = row[c];
+ actual = loader.getValueAccessorById(ValueVector.class, c).getValueVector().getAccessor().getObject(r);
+ assert Objects.equals(expected, actual) : String.format("row:%d - col:%d - expected:%s[%s] - actual:%s[%s]",
+ r, c,
+ expected,
+ expected==null?"null":expected.getClass().getSimpleName(),
+ actual,
+ actual==null?"null":actual.getClass().getSimpleName());
+ }
+ }
+ }
+ });
+ }
+
+ @Test
+ public void testIfDrillCanReadSparseNestedRecordsWithoutRaisingException() throws Exception {
+ final String sql = "select * from cp.`vector/complex/fn/nested-with-nulls.json`";
+ query(sql, new Function<RecordBatchLoader>() {
+ @Override
+ public void apply(RecordBatchLoader loader) {
+ assert loader.getRecordCount() == 4 : "invalid record count returned";
+
+ //XXX: make sure value order matches vector order
+ final Object[][] values = new Object[][] {
+ {"[{},{},{},{\"name\":\"doe\"},{}]"},
+ {"[]"},
+ {"[{\"name\":\"john\",\"id\":10}]"},
+ {"[{},{}]"},
+ };
+
+ Object[] row;
+ Object expected;
+ Object actual;
+ for (int r=0;r<values.length;r++) {
+ row = values[r];
+ for (int c = 0; c < values[r].length; c++) {
+ expected = row[c];
+ actual = loader.getValueAccessorById(ValueVector.class, c).getValueVector().getAccessor().getObject(r);
+ assert Objects.equals(actual, expected) : String.format("row:%d - col:%d - expected:%s[%s] - actual:%s[%s]",
+ r, c,
+ expected,
+ expected == null ? "null" : expected.getClass().getSimpleName(),
+ actual,
+ actual == null ? "null" : actual.getClass().getSimpleName());
+ }
+ }
+ }
+ });
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0cddfc22/exec/java-exec/src/test/resources/vector/complex/fn/nested-with-nulls.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/vector/complex/fn/nested-with-nulls.json b/exec/java-exec/src/test/resources/vector/complex/fn/nested-with-nulls.json
index 1514a47..7ce785c 100644
--- a/exec/java-exec/src/test/resources/vector/complex/fn/nested-with-nulls.json
+++ b/exec/java-exec/src/test/resources/vector/complex/fn/nested-with-nulls.json
@@ -1,3 +1,4 @@
{"users":[{}, {"id": null, "name":null}, {}, {"name": "doe"}, {}]}
{}
-{"users":[{"id": 10, "name":"john"}]}
\ No newline at end of file
+{"users":[{"id": 10, "name":"john"}]}
+{"users":[{}, {"nothing":null}]}
\ No newline at end of file
[16/16] incubator-drill git commit: Drill-1688: Complex parquet
reader fails to read wide records.
Posted by ja...@apache.org.
Drill-1688: Complex parquet reader fails to read wide records.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/1e21045b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/1e21045b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/1e21045b
Branch: refs/heads/master
Commit: 1e21045bfdab8fde8fc7d3d3a4182ddd4d2e41f3
Parents: 52b729e
Author: Jason Altekruse <al...@gmail.com>
Authored: Tue Nov 11 16:53:55 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 19:27:41 2014 -0800
----------------------------------------------------------------------
.../org/apache/drill/exec/ExecConstants.java | 5 +++++
.../exec/server/options/SystemOptionManager.java | 2 ++
.../store/parquet/ParquetScanBatchCreator.java | 2 +-
.../exec/store/parquet2/DrillParquetReader.java | 19 ++++++++++++++++---
4 files changed, 24 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e21045b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
index f01f577..f204506 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/ExecConstants.java
@@ -85,6 +85,11 @@ public interface ExecConstants {
public static final OptionValidator OUTPUT_FORMAT_VALIDATOR = new StringValidator(OUTPUT_FORMAT_OPTION, "parquet");
public static final String PARQUET_BLOCK_SIZE = "store.parquet.block-size";
public static final OptionValidator PARQUET_BLOCK_SIZE_VALIDATOR = new LongValidator(PARQUET_BLOCK_SIZE, 512*1024*1024);
+
+ public static final String PARQUET_VECTOR_FILL_THRESHOLD = "store.parquet.vector_fill_threshold";
+ public static final OptionValidator PARQUET_VECTOR_FILL_THRESHOLD_VALIDATOR = new PositiveLongValidator(PARQUET_VECTOR_FILL_THRESHOLD, 99l, 85l);
+ public static final String PARQUET_VECTOR_FILL_CHECK_THRESHOLD = "store.parquet.vector_fill_check_threshold";
+ public static final OptionValidator PARQUET_VECTOR_FILL_CHECK_THRESHOLD_VALIDATOR = new PositiveLongValidator(PARQUET_VECTOR_FILL_CHECK_THRESHOLD, 100l, 10l);
public static String PARQUET_NEW_RECORD_READER = "store.parquet.use_new_reader";
public static OptionValidator PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR = new BooleanValidator(PARQUET_NEW_RECORD_READER, false);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e21045b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
index e802b44..9f912e0 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/options/SystemOptionManager.java
@@ -54,6 +54,8 @@ public class SystemOptionManager implements OptionManager {
PlannerSettings.HASH_SINGLE_KEY,
ExecConstants.OUTPUT_FORMAT_VALIDATOR,
ExecConstants.PARQUET_BLOCK_SIZE_VALIDATOR,
+ ExecConstants.PARQUET_VECTOR_FILL_THRESHOLD_VALIDATOR,
+ ExecConstants.PARQUET_VECTOR_FILL_CHECK_THRESHOLD_VALIDATOR,
ExecConstants.PARQUET_RECORD_READER_IMPLEMENTATION_VALIDATOR,
ExecConstants.JSON_READER_ALL_TEXT_MODE_VALIDATOR,
ExecConstants.FILESYSTEM_PARTITION_COLUMN_LABEL_VALIDATOR,
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e21045b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
index 8aebab9..53a6ffc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetScanBatchCreator.java
@@ -118,7 +118,7 @@ public class ParquetScanBatchCreator implements BatchCreator<ParquetRowGroupScan
);
} else {
ParquetMetadata footer = footers.get(e.getPath());
- readers.add(new DrillParquetReader(footer, e, columns, conf));
+ readers.add(new DrillParquetReader(context, footer, e, columns, conf));
}
if (rowGroupScan.getSelectionRoot() != null) {
String[] r = rowGroupScan.getSelectionRoot().split("/");
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/1e21045b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
index 8b5d035..8765935 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
@@ -27,7 +27,9 @@ import java.util.Map;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.ExecConstants;
import org.apache.drill.exec.memory.OutOfMemoryException;
+import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.ops.OperatorContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.record.MaterializedField.Key;
@@ -73,13 +75,23 @@ public class DrillParquetReader extends AbstractRecordReader {
private int recordCount;
private List<ValueVector> primitiveVectors;
private OperatorContext operatorContext;
+ // The interface for the parquet-mr library does not allow re-winding, to enable us to write into our
+ // fixed size value vectors, we must check how full the vectors are after some number of reads, for performance
+ // we avoid doing this every record. These values are populated with system/session settings to allow users to optimize
+ // for performance or allow a wider record size to be suported
+ private final int fillLevelCheckFrequency;
+ private final int fillLevelCheckThreshold;
+ private FragmentContext fragmentContext;
- public DrillParquetReader(ParquetMetadata footer, RowGroupReadEntry entry, List<SchemaPath> columns, Configuration conf) {
+ public DrillParquetReader(FragmentContext fragmentContext, ParquetMetadata footer, RowGroupReadEntry entry, List<SchemaPath> columns, Configuration conf) {
this.footer = footer;
this.conf = conf;
this.entry = entry;
setColumns(columns);
+ this.fragmentContext = fragmentContext;
+ fillLevelCheckFrequency = this.fragmentContext.getOptions().getOption(ExecConstants.PARQUET_VECTOR_FILL_CHECK_THRESHOLD).num_val.intValue();
+ fillLevelCheckThreshold = this.fragmentContext.getOptions().getOption(ExecConstants.PARQUET_VECTOR_FILL_THRESHOLD).num_val.intValue();
}
public static MessageType getProjection(MessageType schema, Collection<SchemaPath> columns) {
@@ -200,8 +212,8 @@ public class DrillParquetReader extends AbstractRecordReader {
recordReader.read();
count++;
totalRead++;
- if (count % 100 == 0) {
- if (getPercentFilled() > 85) {
+ if (count % fillLevelCheckFrequency == 0) {
+ if (getPercentFilled() > fillLevelCheckThreshold) {
break;
}
}
@@ -217,6 +229,7 @@ public class DrillParquetReader extends AbstractRecordReader {
if (v instanceof VariableWidthVector) {
filled = Math.max(filled, ((VariableWidthVector) v).getCurrentSizeInBytes() * 100 / ((VariableWidthVector) v).getByteCapacity());
}
+ // TODO - need to re-enable this
// if (v instanceof RepeatedFixedWidthVector) {
// filled = Math.max(filled, ((RepeatedFixedWidthVector) v).getAccessor().getGroupCount() * 100)
// }
[08/16] incubator-drill git commit: DRILL-1643,
DRILL-1665: Flatten fixes - Fix repeated map vector to correctly
report value count - Update flatten so init variables are reset for each new
batch.
Posted by ja...@apache.org.
DRILL-1643, DRILL-1665: Flatten fixes
- Fix repeated map vector to correctly report value count
- Update flatten so init variables are reset for each new batch.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/ed962497
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/ed962497
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/ed962497
Branch: refs/heads/master
Commit: ed962497f04d432591b33b7532741b07ab46fbfe
Parents: 761156b
Author: Jason Altekruse <al...@gmail.com>
Authored: Wed Nov 5 18:11:22 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 16:48:45 2014 -0800
----------------------------------------------------------------------
.../impl/flatten/FlattenRecordBatch.java | 2 +
.../physical/impl/flatten/FlattenTemplate.java | 40 +++--
.../exec/physical/impl/flatten/Flattener.java | 1 +
.../exec/vector/complex/RepeatedMapVector.java | 9 +-
.../exec/physical/impl/flatten/TestFlatten.java | 99 +++++++++++++
.../store/json/test_flatten_mappify2.json | 148 +++++++++++++++++++
6 files changed, 286 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed962497/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index 129174e..66c6168 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -161,6 +161,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
this.recordCount = remainderIndex;
} else {
setValueCount(outputRecords);
+ flattener.resetGroupIndex();
for(VectorWrapper<?> v: incoming) {
v.clear();
}
@@ -194,6 +195,7 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
for (VectorWrapper<?> v : incoming) {
v.clear();
}
+ flattener.resetGroupIndex();
this.recordCount = remainingRecordCount;
}
// In case of complex writer expression, vectors would be added to batch run-time.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed962497/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java
index af4cead..c5d3d93 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenTemplate.java
@@ -30,6 +30,8 @@ import org.apache.drill.exec.record.selection.SelectionVector2;
import org.apache.drill.exec.record.selection.SelectionVector4;
import com.google.common.collect.ImmutableList;
+
+import org.apache.drill.exec.vector.RepeatedFixedWidthVector.RepeatedAccessor;
import org.apache.drill.exec.vector.RepeatedVector;
public abstract class FlattenTemplate implements Flattener {
@@ -40,7 +42,9 @@ public abstract class FlattenTemplate implements Flattener {
private SelectionVector4 vector4;
private SelectionVectorMode svMode;
RepeatedVector fieldToFlatten;
+ RepeatedAccessor accessor;
private int groupIndex;
+
// this allows for groups to be written between batches if we run out of space, for cases where we have finished
// a batch on the boundary it will be set to 0
private int childIndexWithinCurrGroup;
@@ -56,6 +60,7 @@ public abstract class FlattenTemplate implements Flattener {
@Override
public void setFlattenField(RepeatedVector flattenField) {
this.fieldToFlatten = flattenField;
+ this.accessor = flattenField.getAccessor();
}
public RepeatedVector getFlattenField() {
@@ -76,18 +81,26 @@ public abstract class FlattenTemplate implements Flattener {
if (childIndexWithinCurrGroup == -1) {
childIndexWithinCurrGroup = 0;
}
- outer:
- for ( ; groupIndex < fieldToFlatten.getAccessor().getGroupCount(); groupIndex++) {
- currGroupSize = fieldToFlatten.getAccessor().getGroupSizeAtIndex(groupIndex);
- for ( ; childIndexWithinCurrGroup < currGroupSize; childIndexWithinCurrGroup++) {
- if (!doEval(groupIndex, firstOutputIndex)) {
- break outer;
+ outer: {
+ final int groupCount = accessor.getGroupCount();
+ for ( ; groupIndex < groupCount; groupIndex++) {
+ currGroupSize = accessor.getGroupSizeAtIndex(groupIndex);
+ for ( ; childIndexWithinCurrGroup < currGroupSize; childIndexWithinCurrGroup++) {
+ if (!doEval(groupIndex, firstOutputIndex)) {
+ break outer;
+ }
+ firstOutputIndex++;
+ childIndex++;
}
- firstOutputIndex++;
- childIndex++;
+ childIndexWithinCurrGroup = 0;
}
- childIndexWithinCurrGroup = 0;
}
+// System.out.println(String.format("startIndex %d, recordCount %d, firstOutputIndex: %d, currGroupSize: %d, childIndexWithinCurrGroup: %d, groupIndex: %d", startIndex, recordCount, firstOutputIndex, currGroupSize, childIndexWithinCurrGroup, groupIndex));
+// try{
+//// Thread.sleep(1000);
+// }catch(Exception e){
+//
+// }
for (TransferPair t : transfers) {
t.splitAndTransfer(startIndex, childIndex - startIndex);
@@ -113,6 +126,15 @@ public abstract class FlattenTemplate implements Flattener {
doSetup(context, incoming, outgoing);
}
+
+
+ @Override
+ public void resetGroupIndex() {
+ this.groupIndex = 0;
+ this.currGroupSize = 0;
+ this.childIndex = 0;
+ }
+
public abstract void doSetup(@Named("context") FragmentContext context, @Named("incoming") RecordBatch incoming, @Named("outgoing") RecordBatch outgoing);
public abstract boolean doEval(@Named("inIndex") int inIndex, @Named("outIndex") int outIndex);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed962497/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java
index 49b9c1b..2141ca2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/Flattener.java
@@ -32,6 +32,7 @@ public interface Flattener {
public abstract int flattenRecords(int startIndex, int recordCount, int firstOutputIndex);
public void setFlattenField(RepeatedVector repeatedColumn);
public RepeatedVector getFlattenField();
+ public void resetGroupIndex();
public static TemplateClassDefinition<Flattener> TEMPLATE_DEFINITION = new TemplateClassDefinition<Flattener>(Flattener.class, FlattenTemplate.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed962497/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index 99b9453..9b7011c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -420,7 +420,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
public void load(SerializedField metadata, DrillBuf buf) {
List<SerializedField> fields = metadata.getChildList();
- int bufOffset = offsets.load(metadata.getValueCount()+1, buf);
+ int bufOffset = offsets.load(metadata.getGroupCount()+1, buf);
for (SerializedField fmd : fields) {
MaterializedField fieldDef = MaterializedField.create(fmd);
@@ -444,7 +444,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
SerializedField.Builder b = getField() //
.getAsBuilder() //
.setBufferLength(getBufferSize()) //
- .setValueCount(accessor.getValueCount());
+ .setGroupCount(accessor.getGroupCount());
for (ValueVector v : vectors.values()) {
b.addChild(v.getMetadata());
}
@@ -489,7 +489,8 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
@Override
public int getValueCount() {
- return offsets.getAccessor().getValueCount() - 1;
+ return offsets.getAccessor().get(offsets.getAccessor().getValueCount() - 1);
+// return offsets.getAccessor().getValueCount() - 1;
}
public int getGroupSizeAtIndex(int index) {
@@ -542,7 +543,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
@Override
public int getGroupCount() {
- return size();
+ return offsets.getAccessor().getValueCount() - 1;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed962497/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
index 9514517..d4c19a3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
@@ -22,21 +22,120 @@ import org.junit.Test;
public class TestFlatten extends BaseTestQuery {
+ /**
+ * enable this if you have the following files:
+ * - /tmp/yelp_academic_dataset_business.json
+ * - /tmp/mapkv.json
+ * - /tmp/drill1665.json
+ */
+ public static boolean RUN_ADVANCED_TESTS = false;
+
+
+ @Test
+ public void testFlattenFailure() throws Exception {
+ test("select flatten(complex), rownum from cp.`/store/json/test_flatten_mappify2.json`");
+// test("select complex, rownum from cp.`/store/json/test_flatten_mappify2.json`");
+ }
+
@Test
public void testKVGenFlatten1() throws Exception {
+ // works - TODO and verify results
test("select flatten(kvgen(f1)) as monkey, x " +
"from cp.`/store/json/test_flatten_mapify.json`");
}
@Test
public void testTwoFlattens() throws Exception {
+ // second re-write rule has been added to test the fixes together, this now runs
test("select `integer`, `float`, x, flatten(z), flatten(l) from cp.`/jsoninput/input2_modified.json`");
}
@Test
+ public void testFlattenRepeatedMap() throws Exception {
+ test("select `integer`, `float`, x, flatten(z) from cp.`/jsoninput/input2.json`");
+ }
+
+ @Test
+ public void testFlattenKVGenFlatten() throws Exception {
+ // currently does not fail, but produces incorrect results, requires second re-write rule to split up expressions
+ // with complex outputs
+ test("select `integer`, `float`, x, flatten(kvgen(flatten(z))) from cp.`/jsoninput/input2.json`");
+ }
+
+ @Test
+ public void testKVGenFlatten2() throws Exception {
+ // currently runs
+ // TODO - re-verify results by hand
+ if(RUN_ADVANCED_TESTS){
+ test("select flatten(kvgen(visited_cellid_counts)) as mytb from dfs.`/tmp/mapkv.json`") ;
+ }
+ }
+
+ @Test
public void testFilterFlattenedRecords() throws Exception {
+ // WORKS!!
+ // TODO - hand verify results
test("select t2.key from (select t.monkey.`value` as val, t.monkey.key as key from (select flatten(kvgen(f1)) as monkey, x " +
"from cp.`/store/json/test_flatten_mapify.json`) as t) as t2 where t2.val > 1");
}
+ @Test
+ public void testFilterFlattenedRecords2() throws Exception {
+ // previously failed in generated code
+ // "value" is neither a method, a field, nor a member class of "org.apache.drill.exec.expr.holders.RepeatedVarCharHolder" [ 42eb1fa1-0742-4e4f-8723-609215c18900 on 10.250.0.86:31010 ]
+ // appears to be resolving the data coming out of flatten as repeated, check fast schema stuff
+
+ // FIXED BY RETURNING PROPER SCHEMA DURING FAST SCHEMA STEP
+ // these types of problems are being solved more generally as we develp better support for chaning schema
+ if(RUN_ADVANCED_TESTS){
+ test("select celltbl.catl from (\n" +
+ " select flatten(categories) catl from dfs.`/tmp/yelp_academic_dataset_business.json` b limit 100\n" +
+ " ) celltbl where celltbl.catl = 'Doctors'");
+ }
+ }
+
+ @Test
+ public void countAggFlattened() throws Exception {
+ if(RUN_ADVANCED_TESTS){
+ test("select celltbl.catl, count(celltbl.catl) from ( " +
+ "select business_id, flatten(categories) catl from dfs.`/tmp/yelp_academic_dataset_business.json` b limit 100 " +
+ ") celltbl group by celltbl.catl limit 10 ");
+ }
+ }
+
+
+ @Test
+ public void flattenAndAdditionalColumn() throws Exception {
+ if(RUN_ADVANCED_TESTS){
+ test("select business_id, flatten(categories) from dfs.`/tmp/yelp_academic_dataset_business.json` b");
+ }
+ }
+
+ @Test
+ public void testFailingFlattenAlone() throws Exception {
+ if(RUN_ADVANCED_TESTS){
+ test("select flatten(categories) from dfs.`/tmp/yelp_academic_dataset_business.json` b ");
+ }
+ }
+
+ @Test
+ public void testDistinctAggrFlattened() throws Exception {
+ if(RUN_ADVANCED_TESTS){
+ test(" select distinct(celltbl.catl) from (\n" +
+ " select flatten(categories) catl from dfs.`/tmp/yelp_academic_dataset_business.json` b\n" +
+ " ) celltbl");
+ }
+
+ }
+
+ @Test
+ public void testDrill1665() throws Exception {
+ if(RUN_ADVANCED_TESTS){
+ test("select id, flatten(evnts) as rpt from dfs.`/tmp/drill1665.json`");
+ }
+
+ }
+
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/ed962497/exec/java-exec/src/test/resources/store/json/test_flatten_mappify2.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/json/test_flatten_mappify2.json b/exec/java-exec/src/test/resources/store/json/test_flatten_mappify2.json
new file mode 100644
index 0000000..f53a0c7
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/json/test_flatten_mappify2.json
@@ -0,0 +1,148 @@
+{
+ "rownum": 1,
+ "bigintegercol": {
+ "int_1": 1,
+ "int_2": 2,
+ "int_3": 3
+ },
+ "varcharcol": {
+ "varchar_1": "abc",
+ "varchar_2": "def",
+ "varchar_3": "xyz"
+ },
+ "boolcol": {
+ "boolean_1": true,
+ "boolean_2": false,
+ "boolean_3": true
+ },
+ "float8col": {
+ "f8_1": 1.1,
+ "f8_2": 2.2
+ },
+ "complex": [
+ {
+ "col1": 3
+ },
+ {
+ "col2": 2,
+ "col3": 1
+ },
+ {
+ "col1": 7
+ }
+ ]
+}
+{
+ "rownum": 2,
+ "bigintegercol": {
+ "int_1": 1,
+ "int_2": 2
+ },
+ "varcharcol": {
+ "varchar_1": "abcd"
+ },
+ "boolcol": {
+ "boolean_1": true
+ },
+ "float8col": {
+ "f8_1": 1.1,
+ "f8_2": 2.2,
+ "f8_3": 3.3
+ },
+ "complex": [
+ {
+ "col2": 2,
+ "col3": 1
+ },
+ {
+ "col1": 7
+ }
+ ]
+}
+{
+ "rownum": 3,
+ "bigintegercol": {
+ "int_1": 1,
+ "int_3": 3
+ },
+ "varcharcol": {
+ "varchar_1": "abcde",
+ "varchar_2": null,
+ "varchar_3": "xyz",
+ "varchar_4": "xyz2"
+ },
+ "boolcol": {
+ "boolean_1": true,
+ "boolean_2": false
+ },
+ "float8col": {
+ "f8_1": 1.1,
+ "f8_3": 6.6
+ },
+ "complex": [
+ {
+ "col1": 2,
+ "col3": 1
+ }
+ ]
+}
+{
+ "rownum": 4,
+ "bigintegercol": {
+ "int_2": 2,
+ "int_3": 3
+ },
+ "varcharcol": {
+ "varchar_1": "abc",
+ "varchar_2": "def"
+ },
+ "boolcol": {
+ "boolean_1": true,
+ "boolean_2": false,
+ "boolean_3": null
+ },
+ "float8col": {
+ "f8_1": 1.1,
+ "f8_2": 2.2
+ },
+ "complex": [
+ {
+ "col1": 3,
+ "col2": 2
+ },
+ {
+ "col3": 1,
+ "col1": 7
+ }
+ ]
+}
+{
+ "rownum": 5,
+ "bigintegercol": {
+ "int_2": 2,
+ "int_3": 3
+ },
+ "varcharcol": {
+ "varchar_1": "abc",
+ "varchar_2": "def"
+ },
+ "boolcol": {
+ "boolean_1": true,
+ "boolean_2": false,
+ "boolean_3": null
+ },
+ "float8col": {
+ "f8_1": 1.1,
+ "f8_2": 2.2
+ },
+ "complex": [
+ {
+ "col1": 3,
+ "col2": 2
+ },
+ {
+ "col3": 1,
+ "col1": 7
+ }
+ ]
+}
[11/16] incubator-drill git commit: DRILL-1659: Increase default
maxWidth for sqlline to 10k
Posted by ja...@apache.org.
DRILL-1659: Increase default maxWidth for sqlline to 10k
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/2f6efeaf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/2f6efeaf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/2f6efeaf
Branch: refs/heads/master
Commit: 2f6efeafed36459d81230009b67fde89d6f69728
Parents: 0085ffb
Author: Jacques Nadeau <ja...@apache.org>
Authored: Mon Nov 10 22:46:13 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 16:48:45 2014 -0800
----------------------------------------------------------------------
distribution/src/resources/sqlline | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/2f6efeaf/distribution/src/resources/sqlline
----------------------------------------------------------------------
diff --git a/distribution/src/resources/sqlline b/distribution/src/resources/sqlline
index 0ad3b5c..0852fba 100755
--- a/distribution/src/resources/sqlline
+++ b/distribution/src/resources/sqlline
@@ -37,9 +37,9 @@ bin=`cd "$bin">/dev/null; pwd`
DRILL_SHELL_JAVA_OPTS="$DRILL_SHELL_JAVA_OPTS -Dlog.path=$DRILL_LOG_DIR/sqlline.log"
if [ -n "$QUERY" ] ; then
- echo $QUERY | exec "$JAVA" $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine -d org.apache.drill.jdbc.Driver "${ARGS[@]}"
+ echo $QUERY | exec "$JAVA" $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine -d org.apache.drill.jdbc.Driver --maxWidth=10000 "${ARGS[@]}"
elif [ -n "$FILE" ] ; then
- exec "$JAVA" $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine -d org.apache.drill.jdbc.Driver "${ARGS[@]}" --run=$FILE
+ exec "$JAVA" $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine -d org.apache.drill.jdbc.Driver --maxWidth=10000 "${ARGS[@]}" --run=$FILE
else
- exec "$JAVA" $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine -d org.apache.drill.jdbc.Driver "${ARGS[@]}"
+ exec "$JAVA" $DRILL_SHELL_JAVA_OPTS $DRILL_JAVA_OPTS -cp $CP sqlline.SqlLine -d org.apache.drill.jdbc.Driver --maxWidth=10000 "${ARGS[@]}"
fi
[14/16] incubator-drill git commit: Remove extraneous
System.out.print statements.
Posted by ja...@apache.org.
Remove extraneous System.out.print statements.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/968ba617
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/968ba617
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/968ba617
Branch: refs/heads/master
Commit: 968ba617d0fbb7084cb1f0883bd134c32c62e7e1
Parents: ab154b5
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sun Nov 9 20:37:58 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 16:48:46 2014 -0800
----------------------------------------------------------------------
.../apache/drill/exec/compile/MergeAdapter.java | 2 +-
.../drill/exec/expr/fn/impl/StringFunctions.java | 2 --
.../exec/expr/fn/impl/conv/JsonConvertTo.java | 2 +-
.../physical/impl/filter/FilterTemplate4.java | 1 -
.../drill/exec/record/VectorContainer.java | 3 ---
.../drill/exec/server/rest/QueryWrapper.java | 19 +++++++++++--------
.../drill/exec/vector/VectorValidator.java | 2 +-
7 files changed, 14 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/968ba617/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java
index 6a6be50..62d439c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/compile/MergeAdapter.java
@@ -94,7 +94,7 @@ class MergeAdapter extends ClassVisitor {
@Override
public AnnotationVisitor visitAnnotation(String desc, boolean visible) {
- System.out.println("Annotation");
+// System.out.println("Annotation");
return super.visitAnnotation(desc, visible);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/968ba617/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
index d6afa6a..bad394c 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/StringFunctions.java
@@ -1031,14 +1031,12 @@ public class StringFunctions{
public void eval() {
final int len = in.end - in.start;
int num = nTimes.value;
- System.out.println(len + ":" + num);
out.start = 0;
out.buffer = buffer = buffer.reallocIfNeeded( len * num );
for (int i =0; i < num; i++) {
in.buffer.getBytes(in.start, out.buffer, i * len, len);
}
out.end = len * num;
- System.out.println(out.end);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/968ba617/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertTo.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertTo.java b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertTo.java
index c87bd5c..7f7d6f2 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertTo.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/expr/fn/impl/conv/JsonConvertTo.java
@@ -57,7 +57,7 @@ public class JsonConvertTo {
jsonWriter.write(input);
} catch (Exception e) {
- System.out.println(" msg = " + e.getMessage() + " trace : " + e.getStackTrace());
+ throw new RuntimeException(e);
}
byte [] bytea = stream.toByteArray();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/968ba617/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate4.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate4.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate4.java
index a1769b9..0ea53ac 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate4.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/filter/FilterTemplate4.java
@@ -47,7 +47,6 @@ public abstract class FilterTemplate4 implements Filterer {
for (int i = 0; i < incomingSelectionVector.getCount(); i++) {
int index = incomingSelectionVector.get(i);
if (doEval(index, 0)) {
- System.out.println(" (match): " + index + " (i: " + i + ") ");
outgoingSelectionVector.set(outPos++, index);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/968ba617/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
index fde727f..24b8a4b 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/VectorContainer.java
@@ -308,9 +308,6 @@ public class VectorContainer extends AbstractMapVector implements Iterable<Vecto
@Override
public int getRecordCount() {
- if (recordCount < 0) {
- System.out.println();
- }
Preconditions.checkState(recordCount != -1, "Record count not set for this vector container");
return recordCount;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/968ba617/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
index 7bf3909..da134af 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/server/rest/QueryWrapper.java
@@ -33,6 +33,7 @@ import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.coord.ClusterCoordinator;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.physical.impl.flatten.FlattenRecordBatch;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.VectorWrapper;
@@ -48,6 +49,8 @@ import com.fasterxml.jackson.annotation.JsonProperty;
@XmlRootElement
public class QueryWrapper {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FlattenRecordBatch.class);
+
private String query;
private String queryType;
@@ -77,15 +80,15 @@ public class QueryWrapper {
public List<Map<String, Object>> run(DrillConfig config, ClusterCoordinator coordinator, BufferAllocator allocator)
throws Exception {
- DrillClient client = new DrillClient(config, coordinator, allocator);
- Listener listener = new Listener(new RecordBatchLoader(allocator));
+ try(DrillClient client = new DrillClient(config, coordinator, allocator)){
+ Listener listener = new Listener(new RecordBatchLoader(allocator));
- client.connect();
- client.runQuery(getType(), query, listener);
+ client.connect();
+ client.runQuery(getType(), query, listener);
- List<Map<String, Object>> result = listener.waitForCompletion();
- client.close();
- return result;
+ List<Map<String, Object>> result = listener.waitForCompletion();
+ return result;
+ }
}
@Override
@@ -110,7 +113,7 @@ public class QueryWrapper {
@Override
public void submissionFailed(RpcException ex) {
exception = ex;
- System.out.println("Query failed: " + ex.getMessage());
+ logger.error("Query Failed", ex);
latch.countDown();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/968ba617/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorValidator.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorValidator.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorValidator.java
index 6283f35..7856fad 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorValidator.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorValidator.java
@@ -67,7 +67,7 @@ public class VectorValidator {
}
}
if (hash == 0) {
- System.out.println(hash);
+// System.out.println(hash);
}
}
}
[15/16] incubator-drill git commit: DRILL-1559: increment counts even
if we break out of the read loop
Posted by ja...@apache.org.
DRILL-1559: increment counts even if we break out of the read loop
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/52b729ea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/52b729ea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/52b729ea
Branch: refs/heads/master
Commit: 52b729eacde87fdd6930b99c30582accb2c99475
Parents: 968ba61
Author: Chris Westin <cw...@yahoo.com>
Authored: Tue Nov 11 11:23:13 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 19:27:32 2014 -0800
----------------------------------------------------------------------
.../org/apache/drill/exec/store/parquet2/DrillParquetReader.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/52b729ea/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
index 6ce4e20..8b5d035 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet2/DrillParquetReader.java
@@ -195,9 +195,11 @@ public class DrillParquetReader extends AbstractRecordReader {
@Override
public int next() {
int count = 0;
- for (; count < 4000 && totalRead < recordCount; count++, totalRead++) {
+ while (count < 4000 && totalRead < recordCount) {
recordMaterializer.setPosition(count);
recordReader.read();
+ count++;
+ totalRead++;
if (count % 100 == 0) {
if (getPercentFilled() > 85) {
break;
[07/16] incubator-drill git commit: DRILL-1647: Enable rewrite rule
to allow multiple flattens in s a select clause. Fixes a few outstanding
issues that were the reason for disabling it.
Posted by ja...@apache.org.
DRILL-1647: Enable rewrite rule to allow multiple flattens in s a select clause. Fixes a few outstanding issues that were the reason for disabling it.
Fix for the failed Regression test run. Was an issue with removing cast information, simply needed to return the original expression rather than try to clone it if rewriting was not needed.
Was not visiting the child of a project appropriately, this was causing complex expressions nested inside of a sub-query to fail.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/761156bf
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/761156bf
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/761156bf
Branch: refs/heads/master
Commit: 761156bf8b09a9b9cbe801bae730da340ea36d03
Parents: e515e62
Author: Jason Altekruse <al...@gmail.com>
Authored: Thu Oct 30 14:05:32 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 16:48:45 2014 -0800
----------------------------------------------------------------------
.../planner/physical/visitor/RexVisitorComplexExprSplitter.java | 2 +-
.../exec/planner/physical/visitor/SplitUpComplexExpressions.java | 2 +-
.../apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java | 2 +-
3 files changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/761156bf/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RexVisitorComplexExprSplitter.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RexVisitorComplexExprSplitter.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RexVisitorComplexExprSplitter.java
index 73242d5..9742a90 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RexVisitorComplexExprSplitter.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RexVisitorComplexExprSplitter.java
@@ -98,7 +98,7 @@ public class RexVisitorComplexExprSplitter extends RexVisitorImpl<RexNode> {
complexExprs.add(call.clone(new RelDataTypeDrillImpl(new RelDataTypeHolder(),factory), newOps));
return ret;
}
- return call.clone(new RelDataTypeDrillImpl(new RelDataTypeHolder(),factory), newOps);
+ return call.clone(call.getType(), newOps);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/761156bf/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
index f53b228..0a49f3a 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
@@ -108,7 +108,7 @@ public class SplitUpComplexExpressions extends BasePrelVisitor<Prel, Object, Rel
}
List<RexNode> complexExprs = exprSplitter.getComplexExprs();
- RelNode originalInput = project.getInput(0);
+ RelNode originalInput = ((Prel)project.getInput(0)).accept(this, null);
ProjectPrel childProject;
List<RexNode> allExprs = new ArrayList();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/761156bf/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
index 5f91e6e..58c80a6 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/sql/handlers/DefaultSqlHandler.java
@@ -200,7 +200,7 @@ public class DefaultSqlHandler extends AbstractSqlHandler {
// within a single query, it also breaks up all expressions with complex outputs into their own project operations.
// It currently appears to be producing good plans, but for the flatten case it is revealing execution errors in the
// project operator.
-// phyRelNode = ((Prel) phyRelNode).accept(new SplitUpComplexExpressions(planner.getTypeFactory(), context.getDrillOperatorTable(), context.getPlannerSettings().functionImplementationRegistry), null);
+ phyRelNode = ((Prel) phyRelNode).accept(new SplitUpComplexExpressions(planner.getTypeFactory(), context.getDrillOperatorTable(), context.getPlannerSettings().functionImplementationRegistry), null);
phyRelNode = ((Prel) phyRelNode).accept(new RewriteProjectToFlatten(planner.getTypeFactory(), context.getDrillOperatorTable()), null);
// Definitely before this one
/*
[03/16] incubator-drill git commit: DRILL-1675: Add method to get
QueryId from DrillResultSet
Posted by ja...@apache.org.
DRILL-1675: Add method to get QueryId from DrillResultSet
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/836d38a3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/836d38a3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/836d38a3
Branch: refs/heads/master
Commit: 836d38a357baac4da93f5f58f91060d58c761781
Parents: b5d5148
Author: Steven Phillips <sp...@maprtech.com>
Authored: Wed Oct 29 13:42:46 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 16:48:44 2014 -0800
----------------------------------------------------------------------
.../src/main/java/org/apache/drill/jdbc/DrillResultSet.java | 9 +++++++++
.../java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java | 2 ++
2 files changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/836d38a3/exec/jdbc/src/main/java/org/apache/drill/jdbc/DrillResultSet.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/main/java/org/apache/drill/jdbc/DrillResultSet.java b/exec/jdbc/src/main/java/org/apache/drill/jdbc/DrillResultSet.java
index 90d183f..88a6c6d 100644
--- a/exec/jdbc/src/main/java/org/apache/drill/jdbc/DrillResultSet.java
+++ b/exec/jdbc/src/main/java/org/apache/drill/jdbc/DrillResultSet.java
@@ -33,6 +33,7 @@ import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.proto.UserBitShared.QueryId;
import org.apache.drill.exec.proto.UserBitShared.QueryResult.QueryState;
import org.apache.drill.exec.proto.UserBitShared.QueryType;
+import org.apache.drill.exec.proto.helper.QueryIdHelper;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.rpc.RpcException;
import org.apache.drill.exec.rpc.user.ConnectionThrottle;
@@ -94,6 +95,14 @@ public class DrillResultSet extends AvaticaResultSet {
return this;
}
+ public String getQueryId() {
+ if (queryId != null) {
+ return QueryIdHelper.getQueryId(queryId);
+ } else {
+ return null;
+ }
+ }
+
class Listener implements UserResultsListener {
private static final int MAX = 100;
private volatile RpcException ex;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/836d38a3/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
----------------------------------------------------------------------
diff --git a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
index 4528cee..64e1f02 100644
--- a/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
+++ b/exec/jdbc/src/test/java/org/apache/drill/jdbc/test/JdbcTestQueryBase.java
@@ -26,6 +26,7 @@ import java.sql.Statement;
import java.util.concurrent.TimeUnit;
import org.apache.drill.common.util.TestTools;
+import org.apache.drill.jdbc.DrillResultSet;
import org.apache.drill.jdbc.Driver;
import org.apache.drill.jdbc.JdbcTest;
import org.junit.Rule;
@@ -52,6 +53,7 @@ public class JdbcTestQueryBase extends JdbcTest {
Stopwatch watch = new Stopwatch().start();
Statement s = c.createStatement();
ResultSet r = s.executeQuery(sql);
+ System.out.println(String.format("QueryId: %s", ((DrillResultSet) r).getQueryId()));
boolean first = true;
while (r.next()) {
ResultSetMetaData md = r.getMetaData();
[04/16] incubator-drill git commit: DRILL-1678: Add fragment and
operator id to label in plan visualizer
Posted by ja...@apache.org.
DRILL-1678: Add fragment and operator id to label in plan visualizer
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/b5d5148b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/b5d5148b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/b5d5148b
Branch: refs/heads/master
Commit: b5d5148b5e49b7b52927f84edf6901be0892bea1
Parents: 25fca62
Author: Steven Phillips <sp...@maprtech.com>
Authored: Wed Oct 29 01:53:41 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 16:48:44 2014 -0800
----------------------------------------------------------------------
exec/java-exec/src/main/resources/rest/www/graph.js | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/b5d5148b/exec/java-exec/src/main/resources/rest/www/graph.js
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/resources/rest/www/graph.js b/exec/java-exec/src/main/resources/rest/www/graph.js
index 55ca7f4..2d38e63 100644
--- a/exec/java-exec/src/main/resources/rest/www/graph.js
+++ b/exec/java-exec/src/main/resources/rest/www/graph.js
@@ -102,7 +102,7 @@ $(window).load(function () {
// nodes
for (var i = 0; i < ps.length; i++) {
g.addNode(ps[i][0], {
- label: ps[i][2],
+ label: ps[i][2] + " " + ps[i][0],
fragment: parseInt(ps[i][0].split("-")[0])
});
}
@@ -138,7 +138,7 @@ $(window).load(function () {
svgNodes.each(function(u) {
var fc = d3.rgb(globalconfig.majorcolorscale(graph.node(u).fragment));
d3.select(this).select("rect")
- .style("fill", graph.node(u).label.endsWith("Exchange") ? "white" : fc)
+ .style("fill", graph.node(u).label.split(" ")[0].endsWith("Exchange") ? "white" : fc)
.style("stroke", "#000")
.style("stroke-width", "1px")
});
[12/16] incubator-drill git commit: DRILL-1648: Fix for fast schema
issue that was causing compilation issues in downstream operators.
Posted by ja...@apache.org.
DRILL-1648: Fix for fast schema issue that was causing compilation issues in downstream operators.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/e515e621
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/e515e621
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/e515e621
Branch: refs/heads/master
Commit: e515e6211201ec764d405c9a04b3ce43e4b3259d
Parents: 2f6efea
Author: Jason Altekruse <al...@gmail.com>
Authored: Tue Nov 4 15:08:27 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 16:48:45 2014 -0800
----------------------------------------------------------------------
.../impl/flatten/FlattenRecordBatch.java | 60 +++++++++++++++-----
.../org/apache/drill/TestExampleQueries.java | 18 ------
.../exec/physical/impl/flatten/TestFlatten.java | 42 ++++++++++++++
3 files changed, 87 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e515e621/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
index 5171a25..129174e 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/flatten/FlattenRecordBatch.java
@@ -53,12 +53,15 @@ import org.apache.drill.exec.record.VectorContainer;
import org.apache.drill.exec.record.VectorWrapper;
import org.apache.drill.exec.vector.RepeatedVector;
import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.complex.MapVector;
import org.apache.drill.exec.vector.complex.RepeatedMapVector;
import org.apache.drill.exec.vector.complex.writer.BaseWriter.ComplexWriter;
import com.google.common.collect.Lists;
import com.sun.codemodel.JExpr;
+// TODO - handle the case where a user tries to flatten a scalar, should just act as a project all of the columns exactly
+// as they come in
public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FlattenRecordBatch.class);
@@ -251,8 +254,19 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
incoming.buildSchema();
if ( ! fastSchemaCalled ) {
for (VectorWrapper vw : incoming) {
- ValueVector vector = container.addOrGet(vw.getField());
- container.add(vector);
+ if (vw.getField().getPath().equals(popConfig.getColumn())) {
+ if (vw.getValueVector() instanceof MapVector) {
+ // fast schema upstream did not report a repeated type
+ // assume it will be repeated in the actual results and it will fail in execution if it is not
+ ValueVector vector = container.addOrGet(vw.getField());
+ container.add(vector);
+ } else {
+ container.add(getFlattenFieldTransferPair().getTo());
+ }
+ } else {
+ ValueVector vector = container.addOrGet(vw.getField());
+ container.add(vector);
+ }
}
fastSchemaCalled = true;
container.buildSchema(SelectionVectorMode.NONE);
@@ -264,6 +278,33 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
}
}
+ /**
+ * The data layout is the same for the actual data within a repeated field, as it is in a scalar vector for
+ * the same sql type. For example, a repeated int vector has a vector of offsets into a regular int vector to
+ * represent the lists. As the data layout for the actual values in the same in the repeated vector as in the
+ * scalar vector of the same type, we can avoid making individual copies for the column being flattened, and just
+ * use vector copies between the inner vector of the repeated field to the resulting scalar vector from the flatten
+ * operation. This is completed after we determine how many records will fit (as we will hit either a batch end, or
+ * the end of one of the other vectors while we are copying the data of the other vectors alongside each new flattened
+ * value coming out of the repeated field.)
+ */
+ private TransferPair getFlattenFieldTransferPair() {
+ ValueVector flattenField = incoming.getValueAccessorById(
+ incoming.getSchema().getColumn(
+ incoming.getValueVectorId(
+ popConfig.getColumn()).getFieldIds()[0]).getValueClass(),
+ incoming.getValueVectorId(popConfig.getColumn()).getFieldIds()).getValueVector();
+
+ TransferPair tp;
+ if (flattenField instanceof RepeatedMapVector) {
+ tp = ((RepeatedMapVector)flattenField).getTransferPairToSingleMap();
+ } else {
+ ValueVector vvIn = ((RepeatedVector)flattenField).getAccessor().getAllChildValues();
+ tp = vvIn.getTransferPair();
+ }
+ return tp;
+ }
+
@Override
protected boolean setupNewSchema() throws SchemaChangeException {
this.allocationVectors = Lists.newArrayList();
@@ -275,25 +316,14 @@ public class FlattenRecordBatch extends AbstractSingleRecordBatch<FlattenPOP> {
final ClassGenerator<Flattener> cg = CodeGenerator.getRoot(Flattener.TEMPLATE_DEFINITION, context.getFunctionRegistry());
IntOpenHashSet transferFieldIds = new IntOpenHashSet();
- RepeatedVector flattenField = ((RepeatedVector) incoming.getValueAccessorById(
- incoming.getSchema().getColumn(
- incoming.getValueVectorId(
- popConfig.getColumn()).getFieldIds()[0]).getValueClass(),
- incoming.getValueVectorId(popConfig.getColumn()).getFieldIds()).getValueVector());
-
NamedExpression namedExpression = new NamedExpression(popConfig.getColumn(), new FieldReference(popConfig.getColumn()));
LogicalExpression expr = ExpressionTreeMaterializer.materialize(namedExpression.getExpr(), incoming, collector, context.getFunctionRegistry(), true);
ValueVectorReadExpression vectorRead = (ValueVectorReadExpression) expr;
TypedFieldId id = vectorRead.getFieldId();
Preconditions.checkNotNull(incoming);
- TransferPair tp = null;
- if (flattenField instanceof RepeatedMapVector) {
- tp = ((RepeatedMapVector)flattenField).getTransferPairToSingleMap();
- } else {
- ValueVector vvIn = flattenField.getAccessor().getAllChildValues();
- tp = vvIn.getTransferPair();
- }
+ TransferPair tp = getFlattenFieldTransferPair();
+
transfers.add(tp);
container.add(tp.getTo());
transferFieldIds.add(vectorRead.getFieldId().getFieldIds()[0]);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e515e621/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index 5b64d15..bb411e7 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -34,24 +34,6 @@ public class TestExampleQueries extends BaseTestQuery{
test("select recipe, c.inventor.name as name, c.inventor.age as age from cp.`parquet/complex.parquet` c");
}
- @Test
- public void testFlatten() throws Exception {
- test("select flatten(kvgen(f1)) as monkey, x " +
- "from cp.`/store/json/test_flatten_mapify.json`");
-
- test("select t2.key from (select t.monkey.`value` as val, t.monkey.key as key from (select flatten(kvgen(f1)) as monkey, x " +
- "from cp.`/store/json/test_flatten_mapify.json`) as t) as t2 where t2.val > 1");
-
- test("select `integer`, `float`, x, flatten(z), flatten(l) from cp.`/jsoninput/input2_modified.json`");
-
- }
-
- @Test
- @Ignore("Can't be run on classpath since that fs doesn't support glob queries.")
- public void testWildcard() throws Exception {
- test("select * from dfs.`/tmp/xx/ab*/*.json`");
- }
-
@Test // see DRILL-553
public void testQueryWithNullValues() throws Exception {
test("select count(*) from cp.`customer.json` limit 1");
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/e515e621/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
new file mode 100644
index 0000000..9514517
--- /dev/null
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
@@ -0,0 +1,42 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.physical.impl.flatten;
+
+import org.apache.drill.BaseTestQuery;
+import org.junit.Test;
+
+public class TestFlatten extends BaseTestQuery {
+
+ @Test
+ public void testKVGenFlatten1() throws Exception {
+ test("select flatten(kvgen(f1)) as monkey, x " +
+ "from cp.`/store/json/test_flatten_mapify.json`");
+ }
+
+ @Test
+ public void testTwoFlattens() throws Exception {
+ test("select `integer`, `float`, x, flatten(z), flatten(l) from cp.`/jsoninput/input2_modified.json`");
+ }
+
+ @Test
+ public void testFilterFlattenedRecords() throws Exception {
+ test("select t2.key from (select t.monkey.`value` as val, t.monkey.key as key from (select flatten(kvgen(f1)) as monkey, x " +
+ "from cp.`/store/json/test_flatten_mapify.json`) as t) as t2 where t2.val > 1");
+ }
+
+}
[10/16] incubator-drill git commit: DRILL-1674: Set schema in
WriterRecordBatch
Posted by ja...@apache.org.
DRILL-1674: Set schema in WriterRecordBatch
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/0085ffb9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/0085ffb9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/0085ffb9
Branch: refs/heads/master
Commit: 0085ffb95d0d4cc7e603c93fc7773de73aa4016f
Parents: 836d38a
Author: Steven Phillips <sp...@maprtech.com>
Authored: Tue Oct 28 18:47:16 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 16:48:45 2014 -0800
----------------------------------------------------------------------
.../apache/drill/exec/physical/impl/WriterRecordBatch.java | 7 +++++++
1 file changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/0085ffb9/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
index 07302d1..cb0de02 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WriterRecordBatch.java
@@ -51,6 +51,7 @@ public class WriterRecordBatch extends AbstractRecordBatch<Writer> {
private final RecordBatch incoming;
private boolean processed = false;
private String fragmentUniqueId;
+ private BatchSchema schema;
public WriterRecordBatch(Writer writer, RecordBatch incoming, FragmentContext context, RecordWriter recordWriter) throws OutOfMemoryException {
super(writer, context);
@@ -72,6 +73,11 @@ public class WriterRecordBatch extends AbstractRecordBatch<Writer> {
}
@Override
+ public BatchSchema getSchema() {
+ return schema;
+ }
+
+ @Override
public IterOutcome buildSchema() throws SchemaChangeException {
incoming.buildSchema();
try {
@@ -174,6 +180,7 @@ public class WriterRecordBatch extends AbstractRecordBatch<Writer> {
eventBasedRecordWriter = new EventBasedRecordWriter(incoming, recordWriter);
container.buildSchema(SelectionVectorMode.NONE);
+ schema = container.getSchema();
}
@Override
[05/16] incubator-drill git commit: DRILL-1656: Fix wildcard queries
to correctly determine selection root.
Posted by ja...@apache.org.
DRILL-1656: Fix wildcard queries to correctly determine selection root.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/abab3d31
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/abab3d31
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/abab3d31
Branch: refs/heads/master
Commit: abab3d31d703602219842580d5bd406944adf4df
Parents: 0cddfc2
Author: Jacques Nadeau <ja...@apache.org>
Authored: Mon Nov 10 12:01:17 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 16:48:44 2014 -0800
----------------------------------------------------------------------
.../apache/drill/exec/store/dfs/FileSelection.java | 17 +++++++++++++++--
.../java/org/apache/drill/TestExampleQueries.java | 9 ++++++++-
2 files changed, 23 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/abab3d31/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
index 36e7efe..edf5dbc 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/store/dfs/FileSelection.java
@@ -141,8 +141,21 @@ public class FileSelection {
return null;
}
String[] s = p.toUri().getPath().split("/");
- String newPath = StringUtils.join(ArrayUtils.subarray(s, 0, s.length - 1), "/");
- Preconditions.checkState(!newPath.contains("*") && !newPath.contains("?"), String.format("Unsupported selection path: %s", p));
+ int i = 0;
+
+ // get a selection root based on the portions of the selection path that don't contain a wildcard.
+ for(; i < s.length; i++){
+ if(s[i].contains("*") || s[i].contains("?")){
+ break;
+ }
+ }
+ String newPath;
+ if(i > 0){
+ newPath = StringUtils.join(ArrayUtils.subarray(s, 0, i), "/");
+ }else{
+ newPath = "/";
+ }
+
return new FileSelection(Lists.newArrayList(status), newPath);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/abab3d31/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
index e134a73..5b64d15 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestExampleQueries.java
@@ -17,11 +17,12 @@
*/
package org.apache.drill;
+import static org.junit.Assert.assertEquals;
+
import org.apache.drill.common.util.FileUtils;
import org.apache.drill.exec.rpc.RpcException;
import org.junit.Ignore;
import org.junit.Test;
-import static org.junit.Assert.assertEquals;
public class TestExampleQueries extends BaseTestQuery{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestExampleQueries.class);
@@ -45,6 +46,12 @@ public class TestExampleQueries extends BaseTestQuery{
}
+ @Test
+ @Ignore("Can't be run on classpath since that fs doesn't support glob queries.")
+ public void testWildcard() throws Exception {
+ test("select * from dfs.`/tmp/xx/ab*/*.json`");
+ }
+
@Test // see DRILL-553
public void testQueryWithNullValues() throws Exception {
test("select count(*) from cp.`customer.json` limit 1");