You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/08/16 03:44:41 UTC
[03/27] git commit: Fix value vectors for e2e scan json pop
Fix value vectors for e2e scan json pop
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/a15f5b19
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/a15f5b19
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/a15f5b19
Branch: refs/heads/master
Commit: a15f5b19c7362793590d670ba1b932f6f8dfda69
Parents: 38ab96f
Author: Timothy Chen <tn...@gmail.com>
Authored: Sat Aug 10 01:32:13 2013 -0700
Committer: Timothy Chen <tn...@gmail.com>
Committed: Sat Aug 10 01:32:13 2013 -0700
----------------------------------------------------------------------
.../templates/NullableValueVectors.java | 5 +++++
.../templates/RepeatedValueVectors.java | 12 +++++-----
.../apache/drill/exec/store/VectorHolder.java | 8 ++++++-
.../drill/exec/vector/RepeatedMutator.java | 23 ++++++++++++++++++++
.../physical/impl/TestSimpleFragmentRun.java | 7 ++++--
.../resources/physical_json_scan_test1.json | 2 +-
6 files changed, 46 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a15f5b19/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
index ca222df..976c984 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
@@ -58,7 +58,12 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
@Override
public ByteBuf[] getBuffers() {
+ <#if type.major == "VarLen">
+ ByteBuf[] valueBuffers = values.getBuffers();
+ return new ByteBuf[]{bits.data, valueBuffers[0], valueBuffers[1]};
+ <#else>
return new ByteBuf[]{bits.data, values.data};
+ </#if>
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a15f5b19/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
index f4a7049..5def096 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
@@ -249,12 +249,16 @@ import com.google.common.collect.Lists;
}
}
- public final class Mutator implements ValueVector.Mutator {
+ public final class Mutator implements RepeatedMutator {
private Mutator(){
}
+ public void startNewGroup(int index) {
+ offsets.getMutator().set(index+1, offsets.getAccessor().get(index));
+ }
+
/**
* Add an element to the given record index. This is similar to the set() method in other
* value vectors, except that it permits setting multiple values for a single record.
@@ -264,18 +268,12 @@ import com.google.common.collect.Lists;
*/
public void add(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
int nextOffset = offsets.getAccessor().get(index+1);
- if (index > 0 && nextOffset == 0) {
- nextOffset = offsets.getAccessor().get(index);
- }
values.getMutator().set(nextOffset, value);
offsets.getMutator().set(index+1, nextOffset+1);
}
public void add(int index, ${minor.class}Holder holder){
int nextOffset = offsets.getAccessor().get(index+1);
- if (index > 0 && nextOffset == 0) {
- nextOffset = offsets.getAccessor().get(index);
- }
values.getMutator().set(nextOffset, holder);
offsets.getMutator().set(index+1, nextOffset+1);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a15f5b19/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
index 2c28082..be0bea8 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
@@ -45,7 +45,13 @@ public class VectorHolder {
}
public void setGroupCount(int groupCount) {
- this.groupCount = groupCount;
+ if(this.groupCount < groupCount) {
+ RepeatedMutator mutator = (RepeatedMutator) vector.getMutator();
+ while(this.groupCount < groupCount) {
+ mutator.startNewGroup(this.groupCount + 1);
+ this.groupCount++;
+ }
+ }
}
public boolean hasEnoughSpace(int newLength) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a15f5b19/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java
new file mode 100644
index 0000000..3e81d75
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedMutator.java
@@ -0,0 +1,23 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+
+package org.apache.drill.exec.vector;
+
+public interface RepeatedMutator extends ValueVector.Mutator {
+ public void startNewGroup(int index);
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a15f5b19/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
index cabe9b3..db6c437 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
@@ -116,7 +116,10 @@ public class TestSimpleFragmentRun extends PopUnitTestBase {
// run query.
bit.run();
client.connect();
- List<QueryResultBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/physical_json_scan_test1.json"), Charsets.UTF_8));
+ List<QueryResultBatch> results = client.runQuery(QueryType.PHYSICAL,
+ Files.toString(FileUtils.getResourceAsFile("/physical_json_scan_test1.json"), Charsets.UTF_8)
+ .replace("#{TEST_FILE}", FileUtils.getResourceAsFile("/scan_json_test_1.json").toURI().toString())
+ );
// look at records
RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
@@ -156,7 +159,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase {
System.out.println();
- for (int r = 0; i < batchLoader.getRecordCount(); r++) {
+ for (int r = 0; r < batchLoader.getRecordCount(); r++) {
boolean first = true;
recordCount++;
for (VectorWrapper<?> v : batchLoader) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/a15f5b19/sandbox/prototype/exec/java-exec/src/test/resources/physical_json_scan_test1.json
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/resources/physical_json_scan_test1.json b/sandbox/prototype/exec/java-exec/src/test/resources/physical_json_scan_test1.json
index 91eb80c..6f08937 100644
--- a/sandbox/prototype/exec/java-exec/src/test/resources/physical_json_scan_test1.json
+++ b/sandbox/prototype/exec/java-exec/src/test/resources/physical_json_scan_test1.json
@@ -11,7 +11,7 @@
@id:1,
pop:"json-scan",
entries:[
- {url: "file:////home/tnachen/src/incubator-drill/sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_1.json"}
+ {url: "#{TEST_FILE}"}
]
},
{