You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2014/05/15 22:05:41 UTC
[09/14] git commit: Zero out RepeatedMap and RepeatedList offset
vectors. Advance var* lastSet when using copy safe.
Zero out RepeatedMap and RepeatedList offset vectors. Advance var* lastSet when using copy safe.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/85a2e042
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/85a2e042
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/85a2e042
Branch: refs/heads/master
Commit: 85a2e0427a91f39d94a77d6aa4af25f895ec93d7
Parents: d468f6d
Author: Jacques Nadeau <ja...@apache.org>
Authored: Thu May 15 09:18:39 2014 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Thu May 15 09:54:54 2014 -0700
----------------------------------------------------------------------
.../codegen/templates/NullableValueVectors.java | 55 +++++++++-----------
.../exec/vector/complex/RepeatedListVector.java | 1 +
.../exec/vector/complex/RepeatedMapVector.java | 4 ++
.../exec/store/json/JsonRecordReader2Test.java | 8 +--
pom.xml | 4 +-
5 files changed, 37 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/85a2e042/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
index 6086c60..61c488a 100644
--- a/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
+++ b/exec/java-exec/src/main/codegen/templates/NullableValueVectors.java
@@ -284,30 +284,25 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
protected void copyFrom(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
if (!from.getAccessor().isNull(fromIndex)) {
- mutator.set(thisIndex, from.getAccessor().get(fromIndex));
+ mutator.set(thisIndex, from.getAccessor().get(fromIndex));
}
}
+
public boolean copyFromSafe(int fromIndex, int thisIndex, ${minor.class}Vector from){
- boolean success = values.copyFromSafe(fromIndex, thisIndex, from);
- bits.getMutator().set(thisIndex, 1);
-
<#if type.major == "VarLen">
- if (success) {
- mutator.lastSet = thisIndex;
- }
+ if(!mutator.fillEmpties(thisIndex)) return false;
</#if>
+ boolean success = values.copyFromSafe(fromIndex, thisIndex, from);
+ success = success && bits.getMutator().setSafe(thisIndex, 1);
return success;
}
public boolean copyFromSafe(int fromIndex, int thisIndex, Nullable${minor.class}Vector from){
- boolean success = bits.copyFromSafe(fromIndex, thisIndex, from.bits) && values.copyFromSafe(fromIndex, thisIndex, from.values);
-<#if type.major == "VarLen">
- if (success) {
- mutator.lastSet = thisIndex;
- }
-</#if>
- return success;
+ <#if type.major == "VarLen">
+ if(!mutator.fillEmpties(thisIndex)) return false;
+ </#if>
+ return bits.copyFromSafe(fromIndex, thisIndex, from.bits) && values.copyFromSafe(fromIndex, thisIndex, from.values);
}
@@ -408,13 +403,22 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
<#if type.major == "VarLen">lastSet = index;</#if>
}
+ <#if type.major == "VarLen">
+ private boolean fillEmpties(int index){
+ for (int i = lastSet + 1; i < index; i++) {
+ if(!values.getMutator().setSafe(i, new byte[]{})) return false;
+ }
+ lastSet = index;
+
+ return true;
+ }
+ </#if>
+
public boolean setSafe(int index, byte[] value, int start, int length) {
<#if type.major != "VarLen">
throw new UnsupportedOperationException();
<#else>
- for (int i = lastSet + 1; i < index; i++) {
- if(!values.getMutator().setSafe(i, new byte[]{})) return false;
- }
+
boolean b1 = bits.getMutator().setSafe(index, 1);
boolean b2 = values.getMutator().setSafe(index, value, start, length);
if(b1 && b2){
@@ -462,9 +466,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
public boolean setSafe(int index, Nullable${minor.class}Holder value) {
<#if type.major == "VarLen">
- for (int i = lastSet + 1; i < index; i++) {
- if(!values.getMutator().setSafe(i, new byte[]{})) return false;
- }
+ if(!fillEmpties(index)) return false;
</#if>
boolean b1 = bits.getMutator().setSafe(index, 1);
boolean b2 = values.getMutator().setSafe(index, value);
@@ -481,9 +483,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
public boolean setSafe(int index, ${minor.class}Holder value) {
<#if type.major == "VarLen">
- for (int i = lastSet + 1; i < index; i++) {
- if(!values.getMutator().setSafe(i, new byte[]{})) return false;
- }
+ if(!fillEmpties(index)) return false;
</#if>
boolean b1 = bits.getMutator().setSafe(index, 1);
boolean b2 = values.getMutator().setSafe(index, value);
@@ -500,15 +500,12 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
<#if !(type.major == "VarLen" || minor.class == "Decimal28Sparse" || minor.class == "Decimal38Sparse" || minor.class == "Decimal28Dense" || minor.class == "Decimal38Dense" || minor.class == "TimeStampTZ" || minor.class == "Interval" || minor.class == "IntervalDay")>
public boolean setSafe(int index, ${minor.javaType!type.javaType} value) {
<#if type.major == "VarLen">
- for (int i = lastSet + 1; i < index; i++) {
- values.getMutator().set(i, new byte[]{});
- }
+ if(!fillEmpties(index)) return false;
</#if>
boolean b1 = bits.getMutator().setSafe(index, 1);
boolean b2 = values.getMutator().setSafe(index, value);
if(b1 && b2){
setCount++;
- <#if type.major == "VarLen">lastSet = index;</#if>
return true;
}else{
return false;
@@ -520,9 +517,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
public void setValueCount(int valueCount) {
assert valueCount >= 0;
<#if type.major == "VarLen">
- for (int i = lastSet + 1; i < valueCount; i++) {
- values.getMutator().set(i, new byte[]{});
- }
+ fillEmpties(valueCount);
</#if>
Nullable${minor.class}Vector.this.valueCount = valueCount;
values.getMutator().setValueCount(valueCount);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/85a2e042/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
index 88d858a..1f65761 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java
@@ -102,6 +102,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea
@Override
public boolean allocateNewSafe() {
if(!offsets.allocateNewSafe()) return false;
+ offsets.zeroVector();
if(vector != null){
return vector.allocateNewSafe();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/85a2e042/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
index 22471f0..bcf8ad7 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java
@@ -76,6 +76,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
public void allocateNew(int parentValueCount, int childValueCount) {
clear();
offsets.allocateNew(parentValueCount+1);
+ offsets.zeroVector();
mutator.reset();
accessor.reset();
}
@@ -187,6 +188,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
@Override
public boolean allocateNewSafe() {
if(!offsets.allocateNewSafe()) return false;
+ offsets.zeroVector();
for(ValueVector v : vectors.values()){
if(!v.allocateNewSafe()) return false;
}
@@ -216,7 +218,9 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat
pairs = new TransferPair[vectors.size()];
int i =0;
for(Map.Entry<String, ValueVector> e : vectors.entrySet()){
+ int preSize = to.vectors.size();
ValueVector v = to.addOrGet(e.getKey(), e.getValue().getField().getType(), e.getValue().getClass());
+ if(preSize != to.vectors.size()) v.allocateNew();
pairs[i++] = e.getValue().makeTransferPair(v);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/85a2e042/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
----------------------------------------------------------------------
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
index 84195c3..0abdbd3 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/JsonRecordReader2Test.java
@@ -21,22 +21,24 @@ import org.apache.drill.BaseTestQuery;
import org.junit.Ignore;
import org.junit.Test;
-@Ignore("Test case fails intermittently, need to be fixed.")
+
public class JsonRecordReader2Test extends BaseTestQuery{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JsonRecordReader2Test.class);
@Test
public void testComplexJsonInput() throws Exception{
// test("select z[0]['orange'] from cp.`jsoninput/input2.json` limit 10");
- test("select `integer`, x['y'] as x1, x['y'] as x2, z[0], z[0]['orange'], z[1]['pink'] from cp.`jsoninput/input2.json` ");
- test("select x from cp.`jsoninput/input2.json`");
+ test("select `integer`, x['y'] as x1, x['y'] as x2, z[0], z[0]['orange'], z[1]['pink'] from cp.`jsoninput/input2.json` limit 10 ");
+// test("select x from cp.`jsoninput/input2.json`");
// test("select z[0] from cp.`jsoninput/input2.json` limit 10");
}
@Test
public void z() throws Exception{
+ for(int i =0 ; i < 5; i++){
test("select * from cp.`join/merge_join.json`");
+ }
}
@Test
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/85a2e042/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 81054dd..1b42893 100644
--- a/pom.xml
+++ b/pom.xml
@@ -259,8 +259,8 @@
<artifactId>maven-surefire-plugin</artifactId>
<version>2.17</version>
<configuration>
- <argLine>-Xms1g -Xmx2g -XX:MaxDirectMemorySize=13096M -XX:+CMSClassUnloadingEnabled</argLine>
- <forkCount>1</forkCount>
+ <argLine>-Xms1g -Xmx2g -XX:MaxDirectMemorySize=2096M -XX:+CMSClassUnloadingEnabled</argLine>
+ <forkCount>4</forkCount>
<reuseForks>true</reuseForks>
<additionalClasspathElements>
<additionalClasspathElement>./exec/jdbc/src/test/resources/storage-plugins.json</additionalClasspathElement>