You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/08/16 03:44:43 UTC
[05/27] git commit: Fix getBuffers and address comments
Fix getBuffers and address comments
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/6f9dadb9
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/6f9dadb9
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/6f9dadb9
Branch: refs/heads/master
Commit: 6f9dadb952db3740cb187057967785d0ffca8e3e
Parents: b1e48b3
Author: Timothy Chen <tn...@gmail.com>
Authored: Tue Aug 13 20:42:53 2013 -0700
Committer: Timothy Chen <tn...@gmail.com>
Committed: Tue Aug 13 20:42:53 2013 -0700
----------------------------------------------------------------------
.../templates/NullableValueVectors.java | 8 ++------
.../templates/RepeatedValueVectors.java | 3 ++-
.../org/apache/drill/exec/schema/DiffSchema.java | 13 ++++++-------
.../apache/drill/exec/store/JSONRecordReader.java | 16 ++++++++--------
.../org/apache/drill/exec/store/VectorHolder.java | 12 ++++++------
5 files changed, 24 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f9dadb9/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
index 976c984..483166b 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
@@ -17,6 +17,7 @@ import io.netty.buffer.ByteBuf;
import java.io.Closeable;
import java.util.List;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.SchemaDefProtos;
import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
@@ -58,12 +59,7 @@ public final class ${className} extends BaseValueVector implements <#if type.maj
@Override
public ByteBuf[] getBuffers() {
- <#if type.major == "VarLen">
- ByteBuf[] valueBuffers = values.getBuffers();
- return new ByteBuf[]{bits.data, valueBuffers[0], valueBuffers[1]};
- <#else>
- return new ByteBuf[]{bits.data, values.data};
- </#if>
+ return ArrayUtils.addAll(bits.getBuffers(), values.getBuffers());
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f9dadb9/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
index 5def096..c1660e8 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
@@ -17,6 +17,7 @@ import java.io.Closeable;
import java.util.Random;
import java.util.Vector;
+import org.apache.commons.lang3.ArrayUtils;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.SchemaDefProtos;
import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
@@ -176,7 +177,7 @@ import com.google.common.collect.Lists;
</#if>
public ByteBuf[] getBuffers() {
- return new ByteBuf[]{offsets.data, values.data};
+ return ArrayUtils.addAll(offsets.getBuffers(), values.getBuffers());
}
public void clear(){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f9dadb9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/DiffSchema.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/DiffSchema.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/DiffSchema.java
index 68c3e12..81a396e 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/DiffSchema.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/DiffSchema.java
@@ -19,14 +19,13 @@
package org.apache.drill.exec.schema;
import com.google.common.collect.Lists;
-import org.apache.drill.exec.physical.impl.OutputMutator;
import java.util.List;
public class DiffSchema {
List<Field> addedFields;
List<Field> removedFields;
- boolean hasChanged = false;
+ boolean changed = false;
public DiffSchema() {
this.addedFields = Lists.newArrayList();
@@ -35,22 +34,22 @@ public class DiffSchema {
public void recordNewField(Field field) {
addedFields.add(field);
- hasChanged = true;
+ changed = true;
}
public void reset() {
addedFields.clear();
removedFields.clear();
- hasChanged = false;
+ changed = false;
}
public void addRemovedField(Field field) {
removedFields.add(field);
- hasChanged = true;
+ changed = true;
}
- public boolean isHasChanged() {
- return hasChanged;
+ public boolean isChanged() {
+ return changed;
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f9dadb9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
index 21b8c1b..8a2de63 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
@@ -125,7 +125,7 @@ public class JSONRecordReader implements RecordReader {
outputMutator.removeField(field.getAsMaterializedField());
}
- if (diffSchema.isHasChanged()) {
+ if (diffSchema.isChanged()) {
outputMutator.setNewSchema();
}
@@ -358,7 +358,7 @@ public class JSONRecordReader implements RecordReader {
private static <T> boolean addValueToVector(int index, VectorHolder holder, T val, MinorType minorType, int groupCount) {
switch (minorType) {
case INT: {
- holder.incAndCheckLength(32 + 1);
+ holder.incAndCheckLength(NullableIntHolder.WIDTH * 8 + 1);
if (groupCount == 0) {
if (val != null) {
NullableIntVector int4 = (NullableIntVector) holder.getValueVector();
@@ -376,10 +376,10 @@ public class JSONRecordReader implements RecordReader {
m.add(index, (Integer) val);
}
- return holder.hasEnoughSpace(32 + 1);
+ return holder.hasEnoughSpace(NullableIntHolder.WIDTH * 8 + 1);
}
case FLOAT4: {
- holder.incAndCheckLength(32 + 1);
+ holder.incAndCheckLength(NullableFloat4Holder.WIDTH * 8 + 1);
if (groupCount == 0) {
if (val != null) {
NullableFloat4Vector float4 = (NullableFloat4Vector) holder.getValueVector();
@@ -396,7 +396,7 @@ public class JSONRecordReader implements RecordReader {
holder.setGroupCount(index);
m.add(index, (Float) val);
}
- return holder.hasEnoughSpace(32 + 1);
+ return holder.hasEnoughSpace(NullableFloat4Holder.WIDTH * 8 + 1);
}
case VARCHAR: {
if (val == null) {
@@ -419,7 +419,7 @@ public class JSONRecordReader implements RecordReader {
}
}
case BIT: {
- holder.incAndCheckLength(1 + 1);
+ holder.incAndCheckLength(NullableBitHolder.WIDTH + 1);
if (groupCount == 0) {
if (val != null) {
NullableBitVector bit = (NullableBitVector) holder.getValueVector();
@@ -436,7 +436,7 @@ public class JSONRecordReader implements RecordReader {
holder.setGroupCount(index);
m.add(index, (Boolean) val ? 1 : 0);
}
- return holder.hasEnoughSpace(1 + 1);
+ return holder.hasEnoughSpace(NullableBitHolder.WIDTH + 1);
}
default:
throw new DrillRuntimeException("Type not supported to add value. Type: " + minorType);
@@ -476,7 +476,7 @@ public class JSONRecordReader implements RecordReader {
ValueVector v = TypeHelper.getNewVector(f, allocator);
AllocationHelper.allocate(v, batchSize, 50);
- holder = new VectorHolder(batchSize, v);
+ holder = new VectorHolder(v);
valueVectorMap.put(fullFieldName, holder);
outputMutator.addField(v);
return holder;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/6f9dadb9/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
index be0bea8..d2ad72a 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
@@ -27,8 +27,8 @@ public class VectorHolder {
private ValueVector vector;
private int currentLength;
- VectorHolder(int length, ValueVector vector) {
- this.length = length;
+ VectorHolder(ValueVector vector) {
+ this.length = vector.getValueCapacity();
this.vector = vector;
}
@@ -38,18 +38,18 @@ public class VectorHolder {
public void incAndCheckLength(int newLength) {
if (!hasEnoughSpace(newLength)) {
- throw new BatchExceededException(length, currentLength + newLength);
+ throw new BatchExceededException(length, vector.getBufferSize() + newLength);
}
- count += 1;
+
currentLength += newLength;
+ count += 1;
}
public void setGroupCount(int groupCount) {
if(this.groupCount < groupCount) {
RepeatedMutator mutator = (RepeatedMutator) vector.getMutator();
while(this.groupCount < groupCount) {
- mutator.startNewGroup(this.groupCount + 1);
- this.groupCount++;
+ mutator.startNewGroup(++this.groupCount);
}
}
}