You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by ja...@apache.org on 2014/11/12 05:11:26 UTC
[09/16] incubator-drill git commit: DRILL-1671, DRILL-1653,
DRILL-1652: Fixes for flatten bugs
DRILL-1671, DRILL-1653, DRILL-1652: Fixes for flatten bugs
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/60736dea
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/60736dea
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/60736dea
Branch: refs/heads/master
Commit: 60736dea0fc7099f3773616c1995df4789221cdb
Parents: ed96249
Author: Jacques Nadeau <ja...@apache.org>
Authored: Mon Nov 10 22:16:19 2014 -0800
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Tue Nov 11 16:48:45 2014 -0800
----------------------------------------------------------------------
.../drill/common/expression/SchemaPath.java | 3 +-
.../codegen/templates/RepeatedValueVectors.java | 30 +--
.../exec/planner/physical/DrillFlattenPrel.java | 75 -------
.../exec/planner/physical/FlattenPrel.java | 80 ++++++++
.../visitor/RewriteProjectToFlatten.java | 4 +-
.../visitor/SplitUpComplexExpressions.java | 2 +-
.../drill/exec/record/MaterializedField.java | 22 +-
.../exec/vector/complex/RepeatedMapVector.java | 35 +++-
.../exec/vector/complex/fn/JsonReader.java | 3 +-
.../exec/physical/impl/flatten/TestFlatten.java | 51 +++++
.../src/test/resources/flatten/empty-rm.json | 7 +
.../test/resources/flatten/many-arrays-50.json | 203 +++++++++++++++++++
.../src/test/resources/flatten/missing-map.json | 11 +
.../flatten/single-user-transactions.json | 18 ++
14 files changed, 445 insertions(+), 99 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
----------------------------------------------------------------------
diff --git a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
index 3b31a15..1d7f922 100644
--- a/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
+++ b/common/src/main/java/org/apache/drill/common/expression/SchemaPath.java
@@ -233,8 +233,7 @@ public class SchemaPath extends LogicalExpressionBase {
@Override
public String toString() {
- String expr = ExpressionStringBuilder.toString(this);
- return "SchemaPath ["+ expr + "]";
+ return ExpressionStringBuilder.toString(this);
}
public String toExpr() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index 2853e83..0f86db9 100644
--- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -18,6 +18,7 @@
import java.lang.Override;
+import org.apache.drill.exec.record.TransferPair;
import org.apache.drill.exec.vector.RepeatedFixedWidthVector;
import org.apache.drill.exec.vector.UInt4Vector;
import org.mortbay.jetty.servlet.Holder;
@@ -102,20 +103,25 @@ public final class Repeated${minor.class}Vector extends BaseValueVector implemen
clear();
}
- public void splitAndTransferTo(int startIndex, int length, Repeated${minor.class}Vector target) {
- int startPos = offsets.getAccessor().get(startIndex);
- int endPos = offsets.getAccessor().get(startIndex+length);
- values.splitAndTransferTo(startIndex, endPos-startPos, target.values);
- target.offsets.clear();
- target.offsets.allocateNew(endPos - startPos + 1);
+ public void splitAndTransferTo(final int startIndex, final int groups, Repeated${minor.class}Vector to) {
+ final UInt4Vector.Accessor a = offsets.getAccessor();
+ final UInt4Vector.Mutator m = to.offsets.getMutator();
+
+ final int startPos = offsets.getAccessor().get(startIndex);
+ final int endPos = offsets.getAccessor().get(startIndex + groups);
+ final int valuesToCopy = endPos - startPos;
+
+ values.splitAndTransferTo(startPos, valuesToCopy, to.values);
+ to.offsets.clear();
+ to.offsets.allocateNew(valuesToCopy + 1);
int normalizedPos = 0;
- for (int i=0; i<length+1;i++) {
- normalizedPos = offsets.getAccessor().get(startIndex+i) - startPos;
- target.offsets.getMutator().set(i, normalizedPos);
+ for (int i=0; i < groups + 1;i++ ) {
+ normalizedPos = a.get(startIndex+i) - startPos;
+ m.set(i, normalizedPos);
}
- target.parentValueCount = length;
- target.childValueCount = offsets.getAccessor().get(startIndex+length) - startPos;
- target.offsets.getMutator().setValueCount(length);
+ to.parentValueCount = groups;
+ to.childValueCount = valuesToCopy;
+ m.setValueCount(groups);
}
private class TransferImpl implements TransferPair{
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java
deleted file mode 100644
index 1409347..0000000
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/DrillFlattenPrel.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.planner.physical;
-
-import org.apache.drill.common.expression.LogicalExpression;
-import org.apache.drill.common.expression.PathSegment;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.physical.base.PhysicalOperator;
-import org.apache.drill.exec.physical.config.FlattenPOP;
-import org.apache.drill.exec.planner.common.DrillFilterRelBase;
-import org.apache.drill.exec.planner.logical.DrillOptiq;
-import org.apache.drill.exec.planner.logical.DrillParseContext;
-import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
-import org.apache.drill.exec.record.BatchSchema;
-import org.eigenbase.rel.RelNode;
-import org.eigenbase.relopt.RelOptCluster;
-import org.eigenbase.relopt.RelTraitSet;
-import org.eigenbase.rex.RexNode;
-
-import java.io.IOException;
-import java.util.Iterator;
-import java.util.List;
-
-public class DrillFlattenPrel extends SinglePrel implements Prel {
-
- RexNode toFlatten;
-
- public DrillFlattenPrel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode toFlatten) {
- super(cluster, traits, child);
- this.toFlatten = toFlatten;
- }
-
- @Override
- public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
- return new DrillFlattenPrel(getCluster(), traitSet, sole(inputs), toFlatten);
- }
-
- @Override
- public Iterator<Prel> iterator() {
- return PrelUtil.iter(getChild());
- }
-
- @Override
- public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
- Prel child = (Prel) this.getChild();
-
- PhysicalOperator childPOP = child.getPhysicalOperator(creator);
- FlattenPOP f = new FlattenPOP(childPOP, (SchemaPath) getFlattenExpression(new DrillParseContext()));
- return creator.addMetadata(this, f);
- }
-
- @Override
- public BatchSchema.SelectionVectorMode getEncoding() {
- return BatchSchema.SelectionVectorMode.NONE;
- }
-
- protected LogicalExpression getFlattenExpression(DrillParseContext context){
- return DrillOptiq.toDrill(context, getChild(), toFlatten);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
new file mode 100644
index 0000000..18a2cc2
--- /dev/null
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/FlattenPrel.java
@@ -0,0 +1,80 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.planner.physical;
+
+import org.apache.drill.common.expression.LogicalExpression;
+import org.apache.drill.common.expression.PathSegment;
+import org.apache.drill.common.expression.SchemaPath;
+import org.apache.drill.exec.physical.base.PhysicalOperator;
+import org.apache.drill.exec.physical.config.FlattenPOP;
+import org.apache.drill.exec.planner.common.DrillFilterRelBase;
+import org.apache.drill.exec.planner.logical.DrillOptiq;
+import org.apache.drill.exec.planner.logical.DrillParseContext;
+import org.apache.drill.exec.planner.physical.visitor.PrelVisitor;
+import org.apache.drill.exec.record.BatchSchema;
+import org.eigenbase.rel.RelNode;
+import org.eigenbase.rel.RelWriter;
+import org.eigenbase.relopt.RelOptCluster;
+import org.eigenbase.relopt.RelTraitSet;
+import org.eigenbase.rex.RexNode;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.List;
+
+public class FlattenPrel extends SinglePrel implements Prel {
+
+ RexNode toFlatten;
+
+ public FlattenPrel(RelOptCluster cluster, RelTraitSet traits, RelNode child, RexNode toFlatten) {
+ super(cluster, traits, child);
+ this.toFlatten = toFlatten;
+ }
+
+ @Override
+ public RelNode copy(RelTraitSet traitSet, List<RelNode> inputs) {
+ return new FlattenPrel(getCluster(), traitSet, sole(inputs), toFlatten);
+ }
+
+ @Override
+ public Iterator<Prel> iterator() {
+ return PrelUtil.iter(getChild());
+ }
+
+ @Override
+ public PhysicalOperator getPhysicalOperator(PhysicalPlanCreator creator) throws IOException {
+ Prel child = (Prel) this.getChild();
+
+ PhysicalOperator childPOP = child.getPhysicalOperator(creator);
+ FlattenPOP f = new FlattenPOP(childPOP, (SchemaPath) getFlattenExpression(new DrillParseContext()));
+ return creator.addMetadata(this, f);
+ }
+
+ public RelWriter explainTerms(RelWriter pw) {
+ return super.explainTerms(pw).item("flattenField", this.toFlatten);
+ }
+
+ @Override
+ public BatchSchema.SelectionVectorMode getEncoding() {
+ return BatchSchema.SelectionVectorMode.NONE;
+ }
+
+ protected LogicalExpression getFlattenExpression(DrillParseContext context){
+ return DrillOptiq.toDrill(context, getChild(), toFlatten);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
index d4b3573..1055411 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/RewriteProjectToFlatten.java
@@ -21,7 +21,7 @@ import com.google.common.collect.Lists;
import net.hydromatic.optiq.tools.RelConversionException;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
-import org.apache.drill.exec.planner.physical.DrillFlattenPrel;
+import org.apache.drill.exec.planner.physical.FlattenPrel;
import org.apache.drill.exec.planner.physical.Prel;
import org.apache.drill.exec.planner.physical.ProjectPrel;
import org.apache.drill.exec.planner.physical.visitor.BasePrelVisitor;
@@ -103,7 +103,7 @@ public class RewriteProjectToFlatten extends BasePrelVisitor<Prel, Object, RelCo
// TODO - figure out what is the right setting for the traits
Prel newChild = ((Prel)project.getInput(0)).accept(this, null);
ProjectPrel newProject = new ProjectPrel(node.getCluster(), project.getTraitSet(), newChild, exprList, new RelRecordType(relDataTypes));
- DrillFlattenPrel flatten = new DrillFlattenPrel(project.getCluster(), project.getTraitSet(), newProject, flatttenExpr);
+ FlattenPrel flatten = new FlattenPrel(project.getCluster(), project.getTraitSet(), newProject, flatttenExpr);
return flatten;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
index 0a49f3a..89c8447 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/planner/physical/visitor/SplitUpComplexExpressions.java
@@ -26,7 +26,7 @@ import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry;
import org.apache.drill.exec.planner.logical.DrillRel;
import org.apache.drill.exec.planner.logical.DrillScanRel;
import org.apache.drill.exec.planner.logical.DrillTable;
-import org.apache.drill.exec.planner.physical.DrillFlattenPrel;
+import org.apache.drill.exec.planner.physical.FlattenPrel;
import org.apache.drill.exec.planner.physical.Prel;
import org.apache.drill.exec.planner.physical.PrelUtil;
import org.apache.drill.exec.planner.physical.ProjectPrel;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
index b4da2b4..7f7dcfa 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
@@ -18,10 +18,12 @@
package org.apache.drill.exec.record;
import java.util.Collection;
+import java.util.Iterator;
import java.util.Map;
import java.util.Set;
import com.google.common.collect.Sets;
+
import org.apache.drill.common.expression.FieldReference;
import org.apache.drill.common.expression.PathSegment;
import org.apache.drill.common.expression.SchemaPath;
@@ -215,9 +217,27 @@ public class MaterializedField {
return true;
}
+
@Override
public String toString() {
- return "MaterializedField [path=" + key.path + ", type=" + Types.toString(key.type) + "]";
+ final int maxLen = 10;
+ String childStr = children != null && !children.isEmpty() ? toString(children, maxLen) : "";
+ return key.path + "(" + key.type.getMinorType().name() + ":" + key.type.getMode().name() + ")" + childStr;
+ }
+
+
+ private String toString(Collection<?> collection, int maxLen) {
+ StringBuilder builder = new StringBuilder();
+ builder.append("[");
+ int i = 0;
+ for (Iterator<?> iterator = collection.iterator(); iterator.hasNext() && i < maxLen; i++) {
+ if (i > 0){
+ builder.append(", ");
+ }
+ builder.append(iterator.next());
+ }
+ builder.append("]");
+ return builder.toString();
}
public Key key() {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index 9b7011c..7ddac23 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -360,6 +360,9 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
public boolean copyValueSafe(int srcIndex, int destIndex) {
RepeatedMapHolder holder = new RepeatedMapHolder();
accessor.get(srcIndex, holder);
+ if(destIndex >= to.getValueCapacity()){
+ return false;
+ }
to.populateEmpties(destIndex+1);
int newIndex = to.offsets.getAccessor().get(destIndex);
//todo: make these bulk copies
@@ -377,8 +380,30 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
}
@Override
- public void splitAndTransfer(int startIndex, int length) {
- throw new UnsupportedOperationException();
+ public void splitAndTransfer(final int groupStart, final int groups) {
+ final UInt4Vector.Accessor a = offsets.getAccessor();
+ final UInt4Vector.Mutator m = to.offsets.getMutator();
+
+ final int startPos = a.get(groupStart);
+ final int endPos = a.get(groupStart+groups);
+ final int valuesToCopy = endPos - startPos;
+
+ to.offsets.clear();
+ to.offsets.allocateNew(groups + 1);
+ int normalizedPos = 0;
+
+ for (int i=0; i < groups+1; i++) {
+ normalizedPos = a.get(groupStart+i) - startPos;
+ m.set(i, normalizedPos);
+ }
+
+ m.setValueCount(groups + 1);
+ to.lastSet = groups;
+
+ for (TransferPair p : pairs) {
+ p.splitAndTransfer(startPos, valuesToCopy);
+ }
+
}
}
@@ -444,7 +469,9 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
SerializedField.Builder b = getField() //
.getAsBuilder() //
.setBufferLength(getBufferSize()) //
- .setGroupCount(accessor.getGroupCount());
+ .setGroupCount(accessor.getGroupCount())
+ // while we don't need to actually read this on load, we need it to make sure we don't skip deserialization of this vector
+ .setValueCount(accessor.getValueCount());
for (ValueVector v : vectors.values()) {
b.addChild(v.getMetadata());
}
@@ -503,7 +530,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
}
public void get(int index, RepeatedMapHolder holder) {
- assert index < getValueCapacity()-1;
+ assert index < getValueCapacity() : String.format("Attempted to access index %d when value capacity is %d", index, getValueCapacity());
holder.start = offsets.getAccessor().get(index);
holder.end = offsets.getAccessor().get(index+1);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
index 17e266f..0ca24e7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java
@@ -197,7 +197,6 @@ private boolean writeToVector(ComplexWriter writer, JsonToken t) throws JsonPars
map.start();
outside: while(true) {
if (!map.ok()) {
- logger.warn("Error reported. Quit writing");
break;
}
JsonToken t = parser.nextToken();
@@ -320,7 +319,7 @@ private boolean writeToVector(ComplexWriter writer, JsonToken t) throws JsonPars
list.start();
outside: while (true) {
if (!list.ok()) {
- logger.warn("Error reported. Quit writing");
+// logger.warn("Error reported. Quit writing");
break;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
index d4c19a3..960da65 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/flatten/TestFlatten.java
@@ -17,7 +17,10 @@
******************************************************************************/
package org.apache.drill.exec.physical.impl.flatten;
+import static org.junit.Assert.assertEquals;
+
import org.apache.drill.BaseTestQuery;
+import org.junit.Ignore;
import org.junit.Test;
public class TestFlatten extends BaseTestQuery {
@@ -27,6 +30,7 @@ public class TestFlatten extends BaseTestQuery {
* - /tmp/yelp_academic_dataset_business.json
* - /tmp/mapkv.json
* - /tmp/drill1665.json
+ * - /tmp/bigfile.json
*/
public static boolean RUN_ADVANCED_TESTS = false;
@@ -38,6 +42,53 @@ public class TestFlatten extends BaseTestQuery {
}
@Test
+ public void drill1671() throws Exception{
+ int rowCount = testSql("select * from (select count(*) as cnt from (select id, flatten(evnts1), flatten(evnts2), flatten(evnts3), flatten(evnts4), flatten(evnts5), flatten(evnts6), flatten(evnts7), flatten(evnts8), flatten(evnts9), flatten(evnts10), flatten(evnts11) from cp.`/flatten/many-arrays-50.json`)x )y where cnt = 2048");
+ assertEquals(rowCount, 1);
+ }
+
+ @Test
+ @Ignore("not yet fixed")
+ public void drill1660() throws Exception {
+ test("select * from cp.`/flatten/empty-rm.json`");
+ }
+
+ @Test
+ public void drill1653() throws Exception{
+ int rowCount = testSql("select * from (select sum(t.flat.`value`) as sm from (select id, flatten(kvgen(m)) as flat from cp.`/flatten/missing-map.json`)t) where sm = 10 ");
+ assertEquals(rowCount, 1);
+ }
+
+ @Test
+ public void drill1652() throws Exception {
+ if(RUN_ADVANCED_TESTS){
+ test("select uid, flatten(transactions) from dfs.`/tmp/bigfile.json`");
+ }
+ }
+
+ @Test
+ @Ignore("Still not working.")
+ public void drill1649() throws Exception {
+ test("select event_info.uid, transaction_info.trans_id, event_info.event.evnt_id\n" +
+ "from (\n" +
+ " select userinfo.transaction.trans_id trans_id, max(userinfo.event.event_time) max_event_time\n" +
+ " from (\n" +
+ " select uid, flatten(events) event, flatten(transactions) transaction from cp.`/flatten/single-user-transactions.json`\n" +
+ " ) userinfo\n" +
+ " where userinfo.transaction.trans_time >= userinfo.event.event_time\n" +
+ " group by userinfo.transaction.trans_id\n" +
+ ") transaction_info\n" +
+ "inner join\n" +
+ "(\n" +
+ " select uid, flatten(events) event\n" +
+ " from cp.`/flatten/single-user-transactions.json`\n" +
+ ") event_info\n" +
+ "on transaction_info.max_event_time = event_info.event.event_time;");
+ }
+
+
+
+ @Test
public void testKVGenFlatten1() throws Exception {
// works - TODO and verify results
test("select flatten(kvgen(f1)) as monkey, x " +
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/empty-rm.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/flatten/empty-rm.json b/exec/java-exec/src/test/resources/flatten/empty-rm.json
new file mode 100644
index 0000000..7927a3f
--- /dev/null
+++ b/exec/java-exec/src/test/resources/flatten/empty-rm.json
@@ -0,0 +1,7 @@
+{
+ "id":1,
+ "evnts":[
+ {},
+ {"key":null}
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/many-arrays-50.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/flatten/many-arrays-50.json b/exec/java-exec/src/test/resources/flatten/many-arrays-50.json
new file mode 100644
index 0000000..c25565e
--- /dev/null
+++ b/exec/java-exec/src/test/resources/flatten/many-arrays-50.json
@@ -0,0 +1,203 @@
+{
+ "id":1,
+ "evnts1":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts2":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts3":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts4":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts5":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts6":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts7":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts8":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts9":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts10":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts11":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts12":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts13":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts14":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts15":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts16":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts17":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts18":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts19":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts20":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts21":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts22":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts23":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts24":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts25":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts26":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts27":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts28":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts29":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts30":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts31":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts32":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts33":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts34":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts35":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts36":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts37":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts38":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts39":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts40":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts41":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts42":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts43":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts44":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts45":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts46":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts47":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts48":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts49":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ],
+ "evnts50":[
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000}
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/missing-map.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/flatten/missing-map.json b/exec/java-exec/src/test/resources/flatten/missing-map.json
new file mode 100644
index 0000000..5d8f815
--- /dev/null
+++ b/exec/java-exec/src/test/resources/flatten/missing-map.json
@@ -0,0 +1,11 @@
+{
+ "id": 1,
+ "m": {"a":1,"b":2}
+}
+{
+ "id": 2
+}
+{
+ "id": 3,
+ "m": {"c":3,"d":4}
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/60736dea/exec/java-exec/src/test/resources/flatten/single-user-transactions.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/flatten/single-user-transactions.json b/exec/java-exec/src/test/resources/flatten/single-user-transactions.json
new file mode 100644
index 0000000..cff5532
--- /dev/null
+++ b/exec/java-exec/src/test/resources/flatten/single-user-transactions.json
@@ -0,0 +1,18 @@
+{
+ "uid" : 1,
+ "events" : [
+ { "evnt_id":"e1", "campaign_id":"c1", "event_name":"e1_name", "event_time":1000000},
+ { "evnt_id":"e2", "campaign_id":"c1", "event_name":"e2_name", "event_time":2000000},
+ { "evnt_id":"e3", "campaign_id":"c1", "event_name":"e3_name", "event_time":3000000},
+ { "evnt_id":"e4", "campaign_id":"c1", "event_name":"e4_name", "event_time":4000000},
+ { "evnt_id":"e5", "campaign_id":"c2", "event_name":"e5_name", "event_time":5000000},
+ { "evnt_id":"e6", "campaign_id":"c1", "event_name":"e6_name", "event_time":6000000},
+ { "evnt_id":"e7", "campaign_id":"c1", "event_name":"e7_name", "event_time":7000000},
+ { "evnt_id":"e8", "campaign_id":"c2", "event_name":"e8_name", "event_time":8000000},
+ { "evnt_id":"e9", "campaign_id":"c2", "event_name":"e9_name", "event_time":9000000}
+ ],
+ "transactions" : [
+ { "trans_id":"t1", "amount":100, "trans_time":7777777},
+ { "trans_id":"t2", "amount":100, "trans_time":8888888}
+ ]
+}