You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by pa...@apache.org on 2014/11/27 18:32:04 UTC
[3/4] drill git commit: DRILL-1660: return accurate buffers size when
repeated map vector has no underlying vectors
DRILL-1660: return accurate buffers size when repeated map vector has no underlying vectors
Project: http://git-wip-us.apache.org/repos/asf/drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/a94a028c
Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/a94a028c
Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/a94a028c
Branch: refs/heads/master
Commit: a94a028cfc37fc5b5e8eed319a389cc74682ba68
Parents: cf9e71d
Author: Hanifi Gunes <hg...@maprtech.com>
Authored: Wed Nov 12 13:03:57 2014 -0800
Committer: Parth Chandra <pc...@maprtech.com>
Committed: Wed Nov 26 14:54:25 2014 -0800
----------------------------------------------------------------------
.../fn/TestJsonReaderWithSparseFiles.java | 142 ++++++++++---------
.../complex/fn/single-record-with-empties.json | 1 +
2 files changed, 78 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/drill/blob/a94a028c/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
index 3cfdc1d..3765a29 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/vector/complex/fn/TestJsonReaderWithSparseFiles.java
@@ -22,11 +22,10 @@ import java.util.List;
import java.util.Objects;
import org.apache.drill.BaseTestQuery;
-import org.apache.drill.common.expression.SchemaPath;
-import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.proto.beans.QueryResult;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.rpc.user.QueryResultBatch;
+import org.apache.drill.exec.util.JsonStringArrayList;
+import org.apache.drill.exec.util.JsonStringHashMap;
import org.apache.drill.exec.vector.ValueVector;
import org.junit.Test;
@@ -36,6 +35,55 @@ public class TestJsonReaderWithSparseFiles extends BaseTestQuery {
void apply(T param);
}
+ static class TypeConverter {
+
+ public Object convert(Object obj) {
+ if (obj == null) {
+ return null;
+ }
+ if (obj instanceof JsonStringArrayList || obj instanceof JsonStringHashMap) {
+ return obj.toString();
+ }
+ return obj;
+ }
+
+ }
+
+ static class Verifier implements Function<RecordBatchLoader> {
+
+ private final int count;
+ private final Object[][] values;
+ private final TypeConverter converter = new TypeConverter();
+
+ protected Verifier(int count, Object[][] values) {
+ this.count = count;
+ this.values = values;
+ }
+
+ @Override
+ public void apply(RecordBatchLoader loader) {
+ assert loader.getRecordCount() == count : "invalid record count returned";
+
+ Object[] row;
+ Object expected;
+ Object actual;
+ for (int r=0;r<values.length;r++) {
+ row = values[r];
+ for (int c=0; c<values[r].length; c++) {
+ expected = row[c];
+ actual = loader.getValueAccessorById(ValueVector.class, c).getValueVector().getAccessor().getObject(r);
+ actual = converter.convert(actual);
+ assert Objects.equals(actual, expected) : String.format("row:%d - col:%d - expected:%s[%s] - actual:%s[%s]",
+ r, c,
+ expected,
+ expected==null?"null":expected.getClass().getSimpleName(),
+ actual,
+ actual==null?"null":actual.getClass().getSimpleName());
+ }
+ }
+ }
+ }
+
protected void query(final String query, final Function<RecordBatchLoader> testBody) throws Exception {
List<QueryResultBatch> batches = testSqlWithResults(query);
RecordBatchLoader loader = new RecordBatchLoader(client.getAllocator());
@@ -55,73 +103,37 @@ public class TestJsonReaderWithSparseFiles extends BaseTestQuery {
@Test
public void testIfDrillCanReadSparseRecords() throws Exception {
final String sql = "select * from cp.`vector/complex/fn/sparse.json`";
- query(sql, new Function<RecordBatchLoader>() {
- @Override
- public void apply(RecordBatchLoader loader) {
- assert loader.getRecordCount() == 4 : "invalid record count returned";
-
- //XXX: make sure value order matches vector order
- final Object[][] values = new Object[][] {
- {null, null},
- {1L, null},
- {null, 2L},
- {3L, 3L}
- };
-
- Object[] row;
- Object expected;
- Object actual;
- for (int r=0;r<values.length;r++) {
- row = values[r];
- for (int c=0; c<values[r].length; c++) {
- expected = row[c];
- actual = loader.getValueAccessorById(ValueVector.class, c).getValueVector().getAccessor().getObject(r);
- assert Objects.equals(expected, actual) : String.format("row:%d - col:%d - expected:%s[%s] - actual:%s[%s]",
- r, c,
- expected,
- expected==null?"null":expected.getClass().getSimpleName(),
- actual,
- actual==null?"null":actual.getClass().getSimpleName());
- }
- }
- }
- });
+ //XXX: make sure value order matches vector order
+ final Object[][] values = new Object[][] {
+ {null, null},
+ {1L, null},
+ {null, 2L},
+ {3L, 3L}
+ };
+ query(sql, new Verifier(4, values));
}
@Test
public void testIfDrillCanReadSparseNestedRecordsWithoutRaisingException() throws Exception {
final String sql = "select * from cp.`vector/complex/fn/nested-with-nulls.json`";
- query(sql, new Function<RecordBatchLoader>() {
- @Override
- public void apply(RecordBatchLoader loader) {
- assert loader.getRecordCount() == 4 : "invalid record count returned";
-
- //XXX: make sure value order matches vector order
- final Object[][] values = new Object[][] {
- {"[{},{},{},{\"name\":\"doe\"},{}]"},
- {"[]"},
- {"[{\"name\":\"john\",\"id\":10}]"},
- {"[{},{}]"},
- };
-
- Object[] row;
- Object expected;
- Object actual;
- for (int r=0;r<values.length;r++) {
- row = values[r];
- for (int c = 0; c < values[r].length; c++) {
- expected = row[c];
- actual = loader.getValueAccessorById(ValueVector.class, c).getValueVector().getAccessor().getObject(r);
- assert Objects.equals(actual, expected) : String.format("row:%d - col:%d - expected:%s[%s] - actual:%s[%s]",
- r, c,
- expected,
- expected == null ? "null" : expected.getClass().getSimpleName(),
- actual,
- actual == null ? "null" : actual.getClass().getSimpleName());
- }
- }
- }
- });
+ //XXX: make sure value order matches vector order
+ final Object[][] values = new Object[][] {
+ {"[{},{},{},{\"name\":\"doe\"},{}]"},
+ {"[]"},
+ {"[{\"name\":\"john\",\"id\":10}]"},
+ {"[{},{}]"},
+ };
+ query(sql, new Verifier(4, values));
+ }
+
+ @Test
+ public void testIfDrillCanQuerySingleRecordWithEmpties() throws Exception {
+ final String sql = "select * from cp.`vector/complex/fn/single-record-with-empties.json`";
+ //XXX: make sure value order matches vector order
+ final Object[][] values = new Object[][] {
+ {"[{},{}]"},
+ };
+ query(sql, new Verifier(1, values));
}
}
http://git-wip-us.apache.org/repos/asf/drill/blob/a94a028c/exec/java-exec/src/test/resources/vector/complex/fn/single-record-with-empties.json
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/resources/vector/complex/fn/single-record-with-empties.json b/exec/java-exec/src/test/resources/vector/complex/fn/single-record-with-empties.json
new file mode 100644
index 0000000..a3c2fbf
--- /dev/null
+++ b/exec/java-exec/src/test/resources/vector/complex/fn/single-record-with-empties.json
@@ -0,0 +1 @@
+{"users":[{}, {"name":null}]}
\ No newline at end of file