You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by js...@apache.org on 2015/04/20 20:14:11 UTC
drill git commit: DRILL-2754: Correctly allocate offset vector size
for repeated type based on the number of groups being transferred,
not the number of child values.
Repository: drill
Updated Branches:
refs/heads/master 9ec257efb -> 21dfe7ac8
DRILL-2754: Correctly allocate offset vector size for repeated type based on the number of groups being transferred, not the number of child values.
Test various levels of nesting, the core change fixes all three of these previously failing cases.
Test improvements, helper methods for easy construction of maps and lists in baselines.
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/21dfe7ac
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/21dfe7ac
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/21dfe7ac
Branch: refs/heads/master
Commit: 21dfe7ac81f3d6c6b2b27812a8b21023ebd7860e
Parents: 9ec257e
Author: Jason Altekruse <al...@gmail.com>
Authored: Wed Apr 8 17:31:00 2015 -0700
Committer: Jason Altekruse <al...@gmail.com>
Committed: Mon Apr 20 08:58:26 2015 -0700
----------------------------------------------------------------------
.../codegen/templates/RepeatedValueVectors.java | 2 +-
.../java/org/apache/drill/BaseTestQuery.java | 38 +++++++++++++++++++
.../org/apache/drill/TestFrameworkTest.java | 40 +++++---------------
.../vector/complex/writer/TestJsonReader.java | 30 ++++++++++++++-
.../test/resources/store/json/null_list.json | 6 +++
.../test/resources/store/json/null_list_v2.json | 10 +++++
.../test/resources/store/json/null_list_v3.json | 19 ++++++++++
7 files changed, 113 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/21dfe7ac/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
index bf84cf9..c06e29c 100644
--- a/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/RepeatedValueVectors.java
@@ -118,7 +118,7 @@ public final class Repeated${minor.class}Vector extends BaseValueVector implemen
values.splitAndTransferTo(startPos, valuesToCopy, to.values);
to.offsets.clear();
- to.offsets.allocateNew(valuesToCopy + 1);
+ to.offsets.allocateNew(groups + 1);
int normalizedPos = 0;
for (int i=0; i < groups + 1;i++ ) {
normalizedPos = a.get(startIndex+i) - startPos;
http://git-wip-us.apache.org/repos/asf/drill/blob/21dfe7ac/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
index 3931039..d8ed0b3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/BaseTestQuery.java
@@ -19,6 +19,7 @@ package org.apache.drill;
import java.io.IOException;
import java.net.URL;
+import java.util.Arrays;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.CountDownLatch;
@@ -44,7 +45,10 @@ import org.apache.drill.exec.server.DrillbitContext;
import org.apache.drill.exec.server.RemoteServiceSet;
import org.apache.drill.exec.store.StoragePluginRegistry;
import org.apache.drill.exec.util.TestUtilities;
+import org.apache.drill.exec.util.JsonStringArrayList;
+import org.apache.drill.exec.util.JsonStringHashMap;
import org.apache.drill.exec.util.VectorUtil;
+import org.apache.hadoop.io.Text;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.rules.TestRule;
@@ -223,6 +227,40 @@ public class BaseTestQuery extends ExecTest {
}
}
+ private Object wrapStringInHadoopTextObject(Object o) {
+ if (o instanceof String) {
+ o = new Text((String)o);
+ }
+ return o;
+ }
+
+ // convenience method for making a list
+ protected JsonStringArrayList list(Object... values) {
+ JsonStringArrayList ret = new JsonStringArrayList<>();
+ for (int i = 0; i < values.length; i++) {
+ values[i] = wrapStringInHadoopTextObject(values[i]);
+ }
+ ret.addAll(Arrays.asList(values));
+ return ret;
+ }
+
+ protected JsonStringHashMap map(Object... keysAndValues) {
+ JsonStringHashMap ret = new JsonStringHashMap();
+ final String errorMsg = "Must provide a list of keys and values to construct a map, expects" +
+ "an even number or arguments, alternating strings for key names and objects for values.";
+ if (keysAndValues.length % 2 != 0) {
+ throw new RuntimeException(errorMsg);
+ }
+ for (int i = 0; i < keysAndValues.length - 1; i += 2) {
+ if ( ! (keysAndValues[i] instanceof String) ) {
+ throw new RuntimeException(errorMsg);
+ }
+ keysAndValues[i + 1] = wrapStringInHadoopTextObject(keysAndValues[i + 1]);
+ ret.put(keysAndValues[i], keysAndValues[i + 1]);
+ }
+ return ret;
+ }
+
@AfterClass
public static void resetDrillbitCount() {
// some test classes assume this value to be 1 and will fail if run along other tests that increase it
http://git-wip-us.apache.org/repos/asf/drill/blob/21dfe7ac/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java b/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
index f8db533..3abd193 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
@@ -134,40 +134,20 @@ public class TestFrameworkTest extends BaseTestQuery{
@Test
public void testBaselineValsVerificationWithComplexAndNulls() throws Exception {
- JsonStringArrayList list = new JsonStringArrayList();
- JsonStringArrayList innerList1 = new JsonStringArrayList();
- innerList1.add(2l);
- innerList1.add(1l);
- JsonStringArrayList innerList2 = new JsonStringArrayList();
- innerList2.add(4l);
- innerList2.add(6l);
- list.add(innerList1);
- list.add(innerList2);
-
- JsonStringArrayList l_list = new JsonStringArrayList();
- l_list.add(4l);
- l_list.add(2l);
-
- JsonStringHashMap x = new JsonStringHashMap();
- x.put("y", new Text("kevin"));
- x.put("z", new Text("paul"));
-
- // [{"orange":"yellow","pink":"red"},{"pink":"purple"}]
- JsonStringArrayList z = new JsonStringArrayList();
- JsonStringHashMap z_1 = new JsonStringHashMap();
- z_1.put("orange", new Text("yellow"));
- z_1.put("pink", new Text("red"));
-
- JsonStringHashMap z_2 = new JsonStringHashMap();
- z_2.put("pink", new Text("purple"));
- z.add(z_1);
- z.add(z_2);
-
testBuilder()
.sqlQuery("select * from cp.`/jsoninput/input2.json` limit 1")
.ordered()
.baselineColumns("integer", "float", "x", "z", "l", "rl")
- .baselineValues(2010l, 17.4, x, z, l_list, list)
+ .baselineValues(2010l,
+ 17.4,
+ map("y", "kevin",
+ "z", "paul"),
+ list(map("orange", "yellow",
+ "pink", "red"),
+ map("pink", "purple")),
+ list(4l, 2l),
+ list(list(2l, 1l),
+ list(4l, 6l)))
.build().run();
}
http://git-wip-us.apache.org/repos/asf/drill/blob/21dfe7ac/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
index 3f69fd0..dfa89ca 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/writer/TestJsonReader.java
@@ -61,6 +61,34 @@ public class TestJsonReader extends BaseTestQuery {
}
@Test
+ public void testSplitAndTransferFailure() throws Exception {
+ final String testVal = "a string";
+ testBuilder()
+ .sqlQuery("select flatten(config) as flat from cp.`/store/json/null_list.json`")
+ .ordered()
+ .baselineColumns("flat")
+ .baselineValues(list())
+ .baselineValues(list(testVal))
+ .go();
+
+ testBuilder()
+ .sqlQuery("select flatten(config) as flat from cp.`/store/json/null_list_v2.json`")
+ .ordered()
+ .baselineColumns("flat")
+ .baselineValues(map("repeated_varchar", list()))
+ .baselineValues(map("repeated_varchar", list(testVal)))
+ .go();
+
+ testBuilder()
+ .sqlQuery("select flatten(config) as flat from cp.`/store/json/null_list_v3.json`")
+ .ordered()
+ .baselineColumns("flat")
+ .baselineValues(map("repeated_map", list(map("repeated_varchar", list()))))
+ .baselineValues(map("repeated_map", list(map("repeated_varchar", list(testVal)))))
+ .go();
+ }
+
+ @Test
@Ignore("DRILL-1824")
public void schemaChangeValidate() throws Exception {
testBuilder() //
@@ -122,7 +150,7 @@ public class TestJsonReader extends BaseTestQuery {
.build().run();
}
- public void gzipIt(File sourceFile) throws IOException {
+ public static void gzipIt(File sourceFile) throws IOException {
// modified from: http://www.mkyong.com/java/how-to-compress-a-file-in-gzip-format/
byte[] buffer = new byte[1024];
http://git-wip-us.apache.org/repos/asf/drill/blob/21dfe7ac/exec/java-exec/src/test/resources/store/json/null_list.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/json/null_list.json b/exec/java-exec/src/test/resources/store/json/null_list.json
new file mode 100644
index 0000000..54a76fe
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/json/null_list.json
@@ -0,0 +1,6 @@
+{
+ "config": [
+ [],
+ [ "a string" ]
+ ]
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/drill/blob/21dfe7ac/exec/java-exec/src/test/resources/store/json/null_list_v2.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/json/null_list_v2.json b/exec/java-exec/src/test/resources/store/json/null_list_v2.json
new file mode 100644
index 0000000..282fc88
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/json/null_list_v2.json
@@ -0,0 +1,10 @@
+{
+ "config": [
+ {
+ "repeated_varchar": null
+ },
+ {
+ "repeated_varchar": [ "a string" ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/drill/blob/21dfe7ac/exec/java-exec/src/test/resources/store/json/null_list_v3.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/store/json/null_list_v3.json b/exec/java-exec/src/test/resources/store/json/null_list_v3.json
new file mode 100644
index 0000000..0b6e93a
--- /dev/null
+++ b/exec/java-exec/src/test/resources/store/json/null_list_v3.json
@@ -0,0 +1,19 @@
+{
+ "config": [
+ {
+ "repeated_map": [
+ {
+ "repeated_varchar": null
+ }
+ ]
+ },
+ {
+ "repeated_map": [
+ {
+ "repeated_varchar": [ "a string" ]
+ }
+ ]
+ }
+ ]
+}
+