You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@drill.apache.org by ja...@apache.org on 2013/07/15 22:24:01 UTC
[1/7] git commit: DRILL-148 - Fix problems due to bit rot. Mahout 0.8
has migrated a bit from where it started.
Updated Branches:
refs/heads/execwork 5a5d07f69 -> c941874d7
DRILL-148 - Fix problems due to bit rot. Mahout 0.8 has migrated a bit from where it started.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/5052b64d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/5052b64d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/5052b64d
Branch: refs/heads/execwork
Commit: 5052b64d9953857575f8f40995b8da05160e5457
Parents: 97eb07a
Author: Ted Dunning <td...@apache.org>
Authored: Thu Jul 11 19:35:13 2013 -0700
Committer: Ted Dunning <td...@apache.org>
Committed: Thu Jul 11 19:35:13 2013 -0700
----------------------------------------------------------------------
.../apache/drill/synth/ChineseRestaurant.java | 118 +++++++++++++++++++
.../java/org/apache/drill/synth/LongTail.java | 9 +-
2 files changed, 121 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5052b64d/sandbox/prototype/contrib/synth-log/src/main/java/org/apache/drill/synth/ChineseRestaurant.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/contrib/synth-log/src/main/java/org/apache/drill/synth/ChineseRestaurant.java b/sandbox/prototype/contrib/synth-log/src/main/java/org/apache/drill/synth/ChineseRestaurant.java
new file mode 100644
index 0000000..0288071
--- /dev/null
+++ b/sandbox/prototype/contrib/synth-log/src/main/java/org/apache/drill/synth/ChineseRestaurant.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.drill.synth;
+
+import com.google.common.base.Preconditions;
+import org.apache.mahout.common.RandomUtils;
+import org.apache.mahout.math.list.DoubleArrayList;
+import org.apache.mahout.math.random.Sampler;
+
+import java.util.Random;
+
+/**
+ *
+ * Generates samples from a generalized Chinese restaurant process (or Pittman-Yor process).
+ *
+ * The number of values drawn exactly once will asymptotically be equal to the discount parameter
+ * as the total number of draws T increases without bound. The number of unique values sampled will
+ * increase as O(alpha * log T) if discount = 0 or O(alpha * T^discount) for discount > 0.
+ */
+public final class ChineseRestaurant implements Sampler<Integer> {
+ private final double alpha;
+ private double weight = 0;
+ private double discount = 0;
+ private final DoubleArrayList weights = new DoubleArrayList();
+ private final Random rand = RandomUtils.getRandom();
+
+ /**
+ * Constructs a Dirichlet process sampler. This is done by setting discount = 0.
+ * @param alpha The strength parameter for the Dirichlet process.
+ */
+ public ChineseRestaurant(double alpha) {
+ this(alpha, 0);
+ }
+
+ /**
+ * Constructs a Pitman-Yor sampler.
+ *
+ * @param alpha The strength parameter that drives the number of unique values as a function of draws.
+ * @param discount The discount parameter that drives the percentage of values that occur once in a large sample.
+ */
+ public ChineseRestaurant(double alpha, double discount) {
+ Preconditions.checkArgument(alpha > 0);
+ Preconditions.checkArgument(discount >= 0 && discount <= 1);
+ this.alpha = alpha;
+ this.discount = discount;
+ }
+
+ public Integer sample() {
+ double u = rand.nextDouble() * (alpha + weight);
+ for (int j = 0; j < weights.size(); j++) {
+ // select existing options with probability (w_j - d) / (alpha + w)
+ if (u < weights.get(j) - discount) {
+ weights.set(j, weights.get(j) + 1);
+ weight++;
+ return j;
+ } else {
+ u -= weights.get(j) - discount;
+ }
+ }
+
+ // if no existing item selected, pick new item with probability (alpha - d*t) / (alpha + w)
+ // where t is number of pre-existing cases
+ weights.add(1);
+ weight++;
+ return weights.size() - 1;
+ }
+
+ /**
+ * @return the number of unique values that have been returned.
+ */
+ public int size() {
+ return weights.size();
+ }
+
+ /**
+ * @return the number draws so far.
+ */
+ public int count() {
+ return (int) weight;
+ }
+
+ /**
+ * @param j Which value to test.
+ * @return The number of times that j has been returned so far.
+ */
+ public int count(int j) {
+ Preconditions.checkArgument(j >= 0);
+
+ if (j < weights.size()) {
+ return (int) weights.get(j);
+ } else {
+ return 0;
+ }
+ }
+
+ public void setCount(int term, double count) {
+ while (weights.size() <= term) {
+ weights.add(0);
+ }
+ weight += (count - weights.get(term));
+ weights.set(term, count);
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5052b64d/sandbox/prototype/contrib/synth-log/src/main/java/org/apache/drill/synth/LongTail.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/contrib/synth-log/src/main/java/org/apache/drill/synth/LongTail.java b/sandbox/prototype/contrib/synth-log/src/main/java/org/apache/drill/synth/LongTail.java
index 1e0d2ef..1a46e52 100644
--- a/sandbox/prototype/contrib/synth-log/src/main/java/org/apache/drill/synth/LongTail.java
+++ b/sandbox/prototype/contrib/synth-log/src/main/java/org/apache/drill/synth/LongTail.java
@@ -1,17 +1,14 @@
package org.apache.drill.synth;
import com.google.common.collect.Lists;
-import org.apache.mahout.math.random.ChineseRestaurant;
import org.apache.mahout.math.random.Sampler;
import java.util.List;
/**
- * Created with IntelliJ IDEA.
- * User: tdunning
- * Date: 2/2/13
- * Time: 6:05 PM
- * To change this template use File | Settings | File Templates.
+ * Samples from a set of things based on a long-tailed distribution. This converts
+ * the ChineseRestaurant distribution from a distribution over integers into a distribution
+ * over more plausible looking things like words.
*/
public abstract class LongTail<T> implements Sampler<T> {
private ChineseRestaurant base;
[2/7] Updated value vectors inheritance model. Moved Mutables to
separate Mutator subclasses. Broke VVs into separate files rather than one
large class.
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
index f2f97b7..e637518 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
@@ -1,17 +1,15 @@
package org.apache.drill.exec.store;
-import com.carrotsearch.hppc.IntObjectOpenHashMap;
-import com.carrotsearch.hppc.cursors.IntObjectCursor;
-import com.carrotsearch.hppc.cursors.ObjectCursor;
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.core.JsonToken;
-import com.google.common.base.Charsets;
-import com.google.common.collect.Iterables;
-import com.google.common.collect.Lists;
-import com.google.common.io.Files;
-import com.google.common.io.InputSupplier;
-import com.google.common.io.Resources;
+import static com.fasterxml.jackson.core.JsonToken.END_ARRAY;
+import static com.fasterxml.jackson.core.JsonToken.END_OBJECT;
+import static com.fasterxml.jackson.core.JsonToken.FIELD_NAME;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.charset.Charset;
+import java.util.List;
+
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.common.expression.SchemaPath;
@@ -21,17 +19,34 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.proto.SchemaDefProtos;
import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.vector.*;
-import org.apache.drill.exec.schema.*;
+import org.apache.drill.exec.schema.DiffSchema;
+import org.apache.drill.exec.schema.Field;
+import org.apache.drill.exec.schema.IdGenerator;
+import org.apache.drill.exec.schema.ListSchema;
+import org.apache.drill.exec.schema.NamedField;
+import org.apache.drill.exec.schema.ObjectSchema;
+import org.apache.drill.exec.schema.OrderedField;
+import org.apache.drill.exec.schema.RecordSchema;
+import org.apache.drill.exec.schema.SchemaIdGenerator;
import org.apache.drill.exec.schema.json.jackson.JacksonHelper;
+import org.apache.drill.exec.vector.NullableBitVector;
+import org.apache.drill.exec.vector.NullableFloat4Vector;
+import org.apache.drill.exec.vector.NullableIntVector;
+import org.apache.drill.exec.vector.NullableVarChar4Vector;
+import org.apache.drill.exec.vector.TypeHelper;
+import org.apache.drill.exec.vector.ValueVector;
-import java.io.File;
-import java.io.IOException;
-import java.io.InputStreamReader;
-import java.nio.charset.Charset;
-import java.util.List;
-
-import static com.fasterxml.jackson.core.JsonToken.*;
+import com.carrotsearch.hppc.IntObjectOpenHashMap;
+import com.carrotsearch.hppc.cursors.ObjectCursor;
+import com.fasterxml.jackson.core.JsonFactory;
+import com.fasterxml.jackson.core.JsonParser;
+import com.fasterxml.jackson.core.JsonToken;
+import com.google.common.base.Charsets;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+import com.google.common.io.Files;
+import com.google.common.io.InputSupplier;
+import com.google.common.io.Resources;
public class JSONRecordReader implements RecordReader {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(JSONRecordReader.class);
@@ -341,42 +356,40 @@ public class JSONRecordReader implements RecordReader {
switch (minorType) {
case INT: {
holder.incAndCheckLength(32);
- ValueVector.NullableInt int4 = (ValueVector.NullableInt) holder.getValueVector();
- if (val == null) {
- int4.setNull(index);
- } else {
- int4.set(index, (Integer) val);
+ NullableIntVector int4 = (NullableIntVector) holder.getValueVector();
+ NullableIntVector.Mutator m = int4.getMutator();
+ if (val != null) {
+ m.set(index, (Integer) val);
}
return holder.hasEnoughSpace(32);
}
case FLOAT4: {
holder.incAndCheckLength(32);
- ValueVector.NullableFloat4 float4 = (ValueVector.NullableFloat4) holder.getValueVector();
- if (val == null) {
- float4.setNull(index);
- } else {
- float4.set(index, (Float) val);
+ NullableFloat4Vector float4 = (NullableFloat4Vector) holder.getValueVector();
+ NullableFloat4Vector.Mutator m = float4.getMutator();
+ if (val != null) {
+ m.set(index, (Float) val);
}
return holder.hasEnoughSpace(32);
}
case VARCHAR4: {
if (val == null) {
- ((ValueVector.NullableVarChar4) holder.getValueVector()).setNull(index);
return (index + 1) * 4 <= holder.getLength();
} else {
byte[] bytes = ((String) val).getBytes(UTF_8);
int length = bytes.length;
holder.incAndCheckLength(length);
- ValueVector.NullableVarChar4 varLen4 = (ValueVector.NullableVarChar4) holder.getValueVector();
- varLen4.set(index, bytes);
+ NullableVarChar4Vector varLen4 = (NullableVarChar4Vector) holder.getValueVector();
+ NullableVarChar4Vector.Mutator m = varLen4.getMutator();
+ m.set(index, bytes);
return holder.hasEnoughSpace(length);
}
}
case BOOLEAN: {
holder.incAndCheckLength(1);
- ValueVector.NullableBit bit = (ValueVector.NullableBit) holder.getValueVector();
+ NullableBitVector bit = (NullableBitVector) holder.getValueVector();
if (val != null) {
- bit.set(index, (Boolean)val ? 1 : 0);
+ bit.getMutator().set(index, (Boolean)val ? 1 : 0);
}
return holder.hasEnoughSpace(1);
}
@@ -409,7 +422,7 @@ public class JSONRecordReader implements RecordReader {
SchemaDefProtos.MajorType type = field.getFieldType();
int fieldId = field.getFieldId();
MaterializedField f = MaterializedField.create(new SchemaPath(field.getFieldName()), fieldId, parentFieldId, type);
- ValueVector.Base v = TypeHelper.getNewVector(f, allocator);
+ ValueVector v = TypeHelper.getNewVector(f, allocator);
v.allocateNew(batchSize);
VectorHolder holder = new VectorHolder(batchSize, v);
valueVectorMap.put(fieldId, holder);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
index 4043913..fa0cbd5 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
@@ -18,22 +18,19 @@
package org.apache.drill.exec.store;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.vector.TypeHelper;
-import org.apache.drill.exec.record.vector.ValueVector;
+import org.apache.drill.exec.vector.ValueVector;
public class VectorHolder {
private int length;
- private ValueVector.Base vector;
+ private ValueVector vector;
private int currentLength;
- VectorHolder(int length, ValueVector.Base vector) {
+ VectorHolder(int length, ValueVector vector) {
this.length = length;
this.vector = vector;
}
- public ValueVector.Base getValueVector() {
+ public ValueVector getValueVector() {
return vector;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
new file mode 100644
index 0000000..d18a29d
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -0,0 +1,123 @@
+package org.apache.drill.exec.vector;
+
+import java.util.Random;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.record.DeadBuf;
+import org.apache.drill.exec.record.MaterializedField;
+/**
+ * Bit implements a vector of bit-width values. Elements in the vector are accessed
+ * by position from the logical start of the vector.
+ * The width of each element is 1 bit.
+ * The equivalent Java primitive is an int containing the value '0' or '1'.
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+public final class BitVector extends ValueVector {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitVector.class);
+
+ public BitVector(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ }
+
+ /**
+ * Get the byte holding the desired bit, then mask all other bits. Iff the result is 0, the
+ * bit was not set.
+ *
+ * @param index position of the bit in the vector
+ * @return 1 if set, otherwise 0
+ */
+ public int get(int index) {
+ // logger.debug("BIT GET: index: {}, byte: {}, mask: {}, masked byte: {}",
+ // index,
+ // data.getByte((int)Math.floor(index/8)),
+ // (int)Math.pow(2, (index % 8)),
+ // data.getByte((int)Math.floor(index/8)) & (int)Math.pow(2, (index % 8)));
+ return ((data.getByte((int)Math.floor(index/8)) & (int)Math.pow(2, (index % 8))) == 0) ? 0 : 1;
+ }
+
+ @Override
+ public Object getObject(int index) {
+ return new Boolean(get(index) != 0);
+ }
+
+ /**
+ * Get the size requirement (in bytes) for the given number of values.
+ */
+ @Override
+ public int getSizeFromCount(int valueCount) {
+ return (int) Math.ceil(valueCount / 8);
+ }
+
+ @Override
+ public int getAllocatedSize() {
+ return totalBytes;
+ }
+
+ public Mutator getMutator() {
+ return new Mutator();
+ }
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount The number of values which can be contained within this vector.
+ */
+ @Override
+ public void allocateNew(int valueCount) {
+ allocateNew(getSizeFromCount(valueCount), null, valueCount);
+ for (int i = 0; i < getSizeFromCount(valueCount); i++) {
+ data.setByte(i, 0);
+ }
+ }
+
+
+ /**
+ * MutableBit implements a vector of bit-width values. Elements in the vector are accessed
+ * by position from the logical start of the vector. Values should be pushed onto the vector
+ * sequentially, but may be randomly accessed.
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+ public class Mutator implements ValueVector.Mutator{
+
+ private Mutator(){}
+
+ /**
+ * Set the bit at the given index to the specified value.
+ *
+ * @param index position of the bit to set
+ * @param value value to set (either 1 or 0)
+ */
+ public void set(int index, int value) {
+ byte currentByte = data.getByte((int)Math.floor(index/8));
+ if (value != 0) {
+ // true
+ currentByte |= (byte) Math.pow(2, (index % 8));
+ }
+ else if ((currentByte & (byte) Math.pow(2, (index % 8))) == (byte) Math.pow(2, (index % 8))) {
+ // false, and bit was previously set
+ currentByte -= (byte) Math.pow(2, (index % 8));
+ }
+ data.setByte((int) Math.floor(index/8), currentByte);
+ }
+
+
+ @Override
+ public void setRecordCount(int recordCount) {
+ BitVector.this.setRecordCount(recordCount);
+ }
+
+ @Override
+ public void randomizeData() {
+ if (data != DeadBuf.DEAD_BUFFER) {
+ Random r = new Random();
+ for (int i = 0; i < data.capacity() - 1; i++) {
+ byte[] bytes = new byte[1];
+ r.nextBytes(bytes);
+ data.setByte(i, bytes[0]);
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
new file mode 100644
index 0000000..718478e
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
@@ -0,0 +1,192 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.ByteBuf;
+
+import java.io.Closeable;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
+import org.apache.drill.exec.record.DeadBuf;
+import org.apache.drill.exec.record.MaterializedField;
+
+/**
+ * ValueVectorTypes defines a set of template-generated classes which implement type-specific
+ * value vectors. The template approach was chosen due to the lack of multiple inheritence. It
+ * is also important that all related logic be as efficient as possible.
+ */
+public abstract class ValueVector implements Closeable {
+
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueVector.class);
+
+ protected final BufferAllocator allocator;
+ protected ByteBuf data = DeadBuf.DEAD_BUFFER;
+ protected MaterializedField field;
+ protected int recordCount;
+ protected int totalBytes;
+
+ ValueVector(MaterializedField field, BufferAllocator allocator) {
+ this.allocator = allocator;
+ this.field = field;
+ }
+
+ /**
+ * Get the explicitly specified size of the allocated buffer, if available. Otherwise
+ * calculate the size based on width and record count.
+ */
+ public abstract int getAllocatedSize();
+
+ /**
+ * Get the size requirement (in bytes) for the given number of values. Takes derived
+ * type specs into account.
+ */
+ public abstract int getSizeFromCount(int valueCount);
+
+ /**
+ * Get the Java Object representation of the element at the specified position
+ *
+ * @param index Index of the value to get
+ */
+ public abstract Object getObject(int index);
+
+
+ public abstract Mutator getMutator();
+
+ /**
+ * Return the underlying buffers associated with this vector. Note that this doesn't impact the
+ * reference counts for this buffer so it only should be used for in-context access. Also note
+ * that this buffer changes regularly thus external classes shouldn't hold a reference to
+ * it (unless they change it).
+ *
+ * @return The underlying ByteBuf.
+ */
+ public ByteBuf[] getBuffers() {
+ return new ByteBuf[]{data};
+ }
+
+ /**
+ * Returns the maximum number of values contained within this vector.
+ * @return Vector size
+ */
+ public int capacity() {
+ return getRecordCount();
+ }
+
+ /**
+ * Release supporting resources.
+ */
+ @Override
+ public void close() {
+ clear();
+ }
+
+ /**
+ * Get information about how this field is materialized.
+ * @return
+ */
+ public MaterializedField getField() {
+ return field;
+ }
+
+ /**
+ * Get the number of records allocated for this value vector.
+ * @return number of allocated records
+ */
+ public int getRecordCount() {
+ return recordCount;
+ }
+
+ /**
+ * Get the metadata for this field.
+ * @return
+ */
+ public FieldMetadata getMetadata() {
+ int len = 0;
+ for(ByteBuf b : getBuffers()){
+ len += b.writerIndex();
+ }
+ return FieldMetadata.newBuilder()
+ .setDef(getField().getDef())
+ .setValueCount(getRecordCount())
+ .setBufferLength(len)
+ .build();
+ }
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param totalBytes Optional desired size of the underlying buffer. Specifying 0 will
+ * estimate the size based on valueCount.
+ * @param sourceBuffer Optional ByteBuf to use for storage (null will allocate automatically).
+ * @param valueCount Number of values in the vector.
+ */
+ public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
+ clear();
+ this.recordCount = valueCount;
+ this.totalBytes = totalBytes > 0 ? totalBytes : getSizeFromCount(valueCount);
+ this.data = (sourceBuffer != null) ? sourceBuffer : allocator.buffer(this.totalBytes);
+ this.data.retain();
+ data.readerIndex(0);
+ }
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount
+ * The number of elements which can be contained within this vector.
+ */
+ public void allocateNew(int valueCount) {
+ allocateNew(0, null, valueCount);
+ }
+
+ /**
+ * Release the underlying ByteBuf and reset the ValueVector
+ */
+ protected void clear() {
+ if (data != DeadBuf.DEAD_BUFFER) {
+ data.release();
+ data = DeadBuf.DEAD_BUFFER;
+ recordCount = 0;
+ totalBytes = 0;
+ }
+ }
+
+ //public abstract <T extends Mutator> T getMutator();
+
+ /**
+ * Define the number of records that are in this value vector.
+ * @param recordCount Number of records active in this vector.
+ */
+ void setRecordCount(int recordCount) {
+ data.writerIndex(getSizeFromCount(recordCount));
+ this.recordCount = recordCount;
+ }
+
+ /**
+ * For testing only -- randomize the buffer contents
+ */
+ public void randomizeData() { }
+
+
+ public static interface Mutator{
+ public void randomizeData();
+ public void setRecordCount(int recordCount);
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
index 3edf283..3fe0622 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
@@ -26,10 +26,10 @@ import org.apache.drill.exec.client.DrillClient;
import org.apache.drill.exec.pop.PopUnitTestBase;
import org.apache.drill.exec.proto.UserProtos.QueryType;
import org.apache.drill.exec.record.RecordBatchLoader;
-import org.apache.drill.exec.record.vector.ValueVector;
import org.apache.drill.exec.rpc.user.QueryResultBatch;
import org.apache.drill.exec.server.Drillbit;
import org.apache.drill.exec.server.RemoteServiceSet;
+import org.apache.drill.exec.vector.ValueVector;
import org.junit.Test;
import com.carrotsearch.hppc.cursors.IntObjectCursor;
@@ -61,7 +61,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase {
// print headers.
if (schemaChanged) {
System.out.println("\n\n========NEW SCHEMA=========\n\n");
- for (IntObjectCursor<ValueVector.Base> v : batchLoader) {
+ for (IntObjectCursor<ValueVector> v : batchLoader) {
if (firstColumn) {
firstColumn = false;
@@ -80,7 +80,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase {
for (int i = 0; i < batchLoader.getRecordCount(); i++) {
boolean first = true;
recordCount++;
- for (IntObjectCursor<ValueVector.Base> v : batchLoader) {
+ for (IntObjectCursor<ValueVector> v : batchLoader) {
if (first) {
first = false;
} else {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
index 5924f7d..ae4f644 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
@@ -1,19 +1,21 @@
package org.apache.drill.exec.record.vector;
-import io.netty.buffer.ByteBuf;
-import org.apache.drill.exec.memory.DirectBufferAllocator;
-import org.apache.drill.exec.proto.SchemaDefProtos;
-import org.apache.drill.exec.record.MaterializedField;
-
-import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
-import org.apache.hadoop.io.UTF8;
-import org.junit.Test;
-
import java.nio.charset.Charset;
+import org.apache.drill.exec.memory.DirectBufferAllocator;
+import org.apache.drill.exec.proto.SchemaDefProtos;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.BitVector;
+import org.apache.drill.exec.vector.NullableFloat4Vector;
+import org.apache.drill.exec.vector.NullableUInt4Vector;
+import org.apache.drill.exec.vector.NullableVarChar2Vector;
+import org.apache.drill.exec.vector.TypeHelper;
+import org.apache.drill.exec.vector.UInt4Vector;
+import org.junit.Test;
+
public class TestValueVector {
DirectBufferAllocator allocator = new DirectBufferAllocator();
@@ -34,15 +36,16 @@ public class TestValueVector {
MaterializedField field = MaterializedField.create(defBuilder.build());
// Create a new value vector for 1024 integers
- ValueVector.MutableUInt4 v = new ValueVector.MutableUInt4(field, allocator);
+ UInt4Vector v = new UInt4Vector(field, allocator);
+ UInt4Vector.Mutator m = v.getMutator();
v.allocateNew(1024);
// Put and set a few values
- v.set(0, 100);
- v.set(1, 101);
- v.set(100, 102);
- v.set(1022, 103);
- v.set(1023, 104);
+ m.set(0, 100);
+ m.set(1, 101);
+ m.set(100, 102);
+ m.set(1022, 103);
+ m.set(1023, 104);
assertEquals(100, v.get(0));
assertEquals(101, v.get(1));
assertEquals(102, v.get(100));
@@ -69,16 +72,17 @@ public class TestValueVector {
MaterializedField field = MaterializedField.create(defBuilder.build());
// Create a new value vector for 1024 integers
- ValueVector.NullableVarChar2 v = new ValueVector.NullableVarChar2(field, allocator);
+ NullableVarChar2Vector v = new NullableVarChar2Vector(field, allocator);
+ NullableVarChar2Vector.Mutator m = v.getMutator();
v.allocateNew(1024);
// Create and set 3 sample strings
String str1 = new String("AAAAA1");
String str2 = new String("BBBBBBBBB2");
String str3 = new String("CCCC3");
- v.set(0, str1.getBytes(Charset.forName("UTF-8")));
- v.set(1, str2.getBytes(Charset.forName("UTF-8")));
- v.set(2, str3.getBytes(Charset.forName("UTF-8")));
+ m.set(0, str1.getBytes(Charset.forName("UTF-8")));
+ m.set(1, str2.getBytes(Charset.forName("UTF-8")));
+ m.set(2, str3.getBytes(Charset.forName("UTF-8")));
// Check the sample strings
assertEquals(str1, new String(v.get(0), Charset.forName("UTF-8")));
@@ -86,10 +90,16 @@ public class TestValueVector {
assertEquals(str3, new String(v.get(2), Charset.forName("UTF-8")));
// Ensure null value throws
+ boolean b = false;
try {
v.get(3);
- assertFalse(false);
- } catch(NullValueException e) { }
+ } catch(AssertionError e) {
+ b = true;
+ }finally{
+ if(!b){
+ assert false;
+ }
+ }
}
@@ -110,15 +120,16 @@ public class TestValueVector {
MaterializedField field = MaterializedField.create(defBuilder.build());
// Create a new value vector for 1024 integers
- ValueVector.NullableUInt4 v = new ValueVector.NullableUInt4(field, allocator);
+ NullableUInt4Vector v = new NullableUInt4Vector(field, allocator);
+ NullableUInt4Vector.Mutator m = v.getMutator();
v.allocateNew(1024);
// Put and set a few values
- v.set(0, 100);
- v.set(1, 101);
- v.set(100, 102);
- v.set(1022, 103);
- v.set(1023, 104);
+ m.set(0, 100);
+ m.set(1, 101);
+ m.set(100, 102);
+ m.set(1022, 103);
+ m.set(1023, 104);
assertEquals(100, v.get(0));
assertEquals(101, v.get(1));
assertEquals(102, v.get(100));
@@ -126,22 +137,39 @@ public class TestValueVector {
assertEquals(104, v.get(1023));
// Ensure null values throw
- try {
- v.get(3);
- assertFalse(false);
- } catch(NullValueException e) { }
-
+ {
+ boolean b = false;
+ try {
+ v.get(3);
+ } catch(AssertionError e) {
+ b = true;
+ }finally{
+ if(!b){
+ assert false;
+ }
+ }
+ }
+
+
v.allocateNew(2048);
- try {
- v.get(0);
- assertFalse(false);
- } catch(NullValueException e) { }
-
- v.set(0, 100);
- v.set(1, 101);
- v.set(100, 102);
- v.set(1022, 103);
- v.set(1023, 104);
+ {
+ boolean b = false;
+ try {
+ v.get(0);
+ } catch(AssertionError e) {
+ b = true;
+ }finally{
+ if(!b){
+ assert false;
+ }
+ }
+ }
+
+ m.set(0, 100);
+ m.set(1, 101);
+ m.set(100, 102);
+ m.set(1022, 103);
+ m.set(1023, 104);
assertEquals(100, v.get(0));
assertEquals(101, v.get(1));
assertEquals(102, v.get(100));
@@ -149,10 +177,19 @@ public class TestValueVector {
assertEquals(104, v.get(1023));
// Ensure null values throw
- try {
- v.get(3);
- assertFalse(false);
- } catch(NullValueException e) { }
+
+ {
+ boolean b = false;
+ try {
+ v.get(3);
+ } catch(AssertionError e) {
+ b = true;
+ }finally{
+ if(!b){
+ assert false;
+ }
+ }
+ }
}
@@ -172,16 +209,16 @@ public class TestValueVector {
MaterializedField field = MaterializedField.create(defBuilder.build());
// Create a new value vector for 1024 integers
- ValueVector.NullableFloat4 v = (ValueVector.NullableFloat4) TypeHelper.getNewVector(field, allocator);
-
+ NullableFloat4Vector v = (NullableFloat4Vector) TypeHelper.getNewVector(field, allocator);
+ NullableFloat4Vector.Mutator m = v.getMutator();
v.allocateNew(1024);
// Put and set a few values
- v.set(0, 100.1f);
- v.set(1, 101.2f);
- v.set(100, 102.3f);
- v.set(1022, 103.4f);
- v.set(1023, 104.5f);
+ m.set(0, 100.1f);
+ m.set(1, 101.2f);
+ m.set(100, 102.3f);
+ m.set(1022, 103.4f);
+ m.set(1023, 104.5f);
assertEquals(100.1f, v.get(0), 0);
assertEquals(101.2f, v.get(1), 0);
assertEquals(102.3f, v.get(100), 0);
@@ -189,17 +226,32 @@ public class TestValueVector {
assertEquals(104.5f, v.get(1023), 0);
// Ensure null values throw
- try {
- v.get(3);
- assertFalse(false);
- } catch(NullValueException e) { }
-
+ {
+ boolean b = false;
+ try {
+ v.get(3);
+ } catch(AssertionError e) {
+ b = true;
+ }finally{
+ if(!b){
+ assert false;
+ }
+ }
+ }
+
v.allocateNew(2048);
- try {
- v.get(0);
- assertFalse(false);
- } catch(NullValueException e) { }
-
+ {
+ boolean b = false;
+ try {
+ v.get(0);
+ } catch(AssertionError e) {
+ b = true;
+ }finally{
+ if(!b){
+ assert false;
+ }
+ }
+ }
}
@Test
@@ -218,30 +270,31 @@ public class TestValueVector {
MaterializedField field = MaterializedField.create(defBuilder.build());
// Create a new value vector for 1024 integers
- ValueVector.MutableBit v = new ValueVector.MutableBit(field, allocator);
+ BitVector v = new BitVector(field, allocator);
+ BitVector.Mutator m = v.getMutator();
v.allocateNew(1024);
// Put and set a few values
- v.set(0, 1);
- v.set(1, 0);
- v.set(100, 0);
- v.set(1022, 1);
+ m.set(0, 1);
+ m.set(1, 0);
+ m.set(100, 0);
+ m.set(1022, 1);
assertEquals(1, v.get(0));
assertEquals(0, v.get(1));
assertEquals(0, v.get(100));
assertEquals(1, v.get(1022));
// test setting the same value twice
- v.set(0, 1);
- v.set(0, 1);
- v.set(1, 0);
- v.set(1, 0);
+ m.set(0, 1);
+ m.set(0, 1);
+ m.set(1, 0);
+ m.set(1, 0);
assertEquals(1, v.get(0));
assertEquals(0, v.get(1));
// test toggling the values
- v.set(0, 0);
- v.set(1, 1);
+ m.set(0, 0);
+ m.set(1, 1);
assertEquals(0, v.get(0));
assertEquals(1, v.get(1));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
index cef40ff..117414c 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
@@ -1,8 +1,16 @@
package org.apache.drill.exec.store;
-import com.beust.jcommander.internal.Lists;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.nio.charset.Charset;
+import java.util.Arrays;
+import java.util.List;
+
import mockit.Expectations;
import mockit.Injectable;
+
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.memory.DirectBufferAllocator;
@@ -10,17 +18,11 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.proto.SchemaDefProtos;
import org.apache.drill.exec.proto.UserBitShared;
-import org.apache.drill.exec.record.vector.ValueVector;
+import org.apache.drill.exec.vector.ValueVector;
import org.junit.Ignore;
import org.junit.Test;
-import java.io.IOException;
-import java.nio.charset.Charset;
-import java.util.Arrays;
-import java.util.List;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
+import com.beust.jcommander.internal.Lists;
public class JSONRecordReaderTest {
private static final Charset UTF_8 = Charset.forName("UTF-8");
@@ -31,7 +33,7 @@ public class JSONRecordReaderTest {
class MockOutputMutator implements OutputMutator {
List<Integer> removedFields = Lists.newArrayList();
- List<ValueVector.Base> addFields = Lists.newArrayList();
+ List<ValueVector> addFields = Lists.newArrayList();
@Override
public void removeField(int fieldId) throws SchemaChangeException {
@@ -39,7 +41,7 @@ public class JSONRecordReaderTest {
}
@Override
- public void addField(int fieldId, ValueVector.Base vector) throws SchemaChangeException {
+ public void addField(int fieldId, ValueVector vector) throws SchemaChangeException {
addFields.add(vector);
}
@@ -51,16 +53,16 @@ public class JSONRecordReaderTest {
return removedFields;
}
- List<ValueVector.Base> getAddFields() {
+ List<ValueVector> getAddFields() {
return addFields;
}
}
- private <T> void assertField(ValueVector.Base valueVector, int index, SchemaDefProtos.MinorType expectedMinorType, T value, String name) {
+ private <T> void assertField(ValueVector valueVector, int index, SchemaDefProtos.MinorType expectedMinorType, T value, String name) {
assertField(valueVector, index, expectedMinorType, value, name, 0);
}
- private <T> void assertField(ValueVector.Base valueVector, int index, SchemaDefProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) {
+ private <T> void assertField(ValueVector valueVector, int index, SchemaDefProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) {
UserBitShared.FieldMetadata metadata = valueVector.getMetadata();
SchemaDefProtos.FieldDef def = metadata.getDef();
assertEquals(expectedMinorType, def.getMajorType().getMinorType());
@@ -90,7 +92,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_1.json"));
MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector.Base> addFields = mutator.getAddFields();
+ List<ValueVector> addFields = mutator.getAddFields();
jr.setup(mutator);
assertEquals(2, jr.next());
assertEquals(3, addFields.size());
@@ -116,7 +118,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_2.json"));
MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector.Base> addFields = mutator.getAddFields();
+ List<ValueVector> addFields = mutator.getAddFields();
jr.setup(mutator);
assertEquals(3, jr.next());
@@ -142,7 +144,7 @@ public class JSONRecordReaderTest {
assertEquals(0, jr.next());
}
- @Test
+ @Test @Ignore
public void testChangedSchemaInTwoBatches(@Injectable final FragmentContext context) throws IOException, ExecutionSetupException {
new Expectations() {
{
@@ -153,7 +155,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_2.json"), 64); // batch only fits 1 int
MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector.Base> addFields = mutator.getAddFields();
+ List<ValueVector> addFields = mutator.getAddFields();
List<Integer> removedFields = mutator.getRemovedFields();
jr.setup(mutator);
@@ -201,7 +203,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_3.json"));
MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector.Base> addFields = mutator.getAddFields();
+ List<ValueVector> addFields = mutator.getAddFields();
jr.setup(mutator);
assertEquals(2, jr.next());
assertEquals(5, addFields.size());
[7/7] git commit: Merge commit
'5052b64d9953857575f8f40995b8da05160e5457' into execwork
Posted by ja...@apache.org.
Merge commit '5052b64d9953857575f8f40995b8da05160e5457' into execwork
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/c941874d
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/c941874d
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/c941874d
Branch: refs/heads/execwork
Commit: c941874d73f1d387c68daa40d6089e3068cd0073
Parents: 9ca9eb9 5052b64
Author: Jacques Nadeau <ja...@apache.org>
Authored: Mon Jul 15 13:17:54 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jul 15 13:17:54 2013 -0700
----------------------------------------------------------------------
.../apache/drill/synth/ChineseRestaurant.java | 118 +++++++++++++++++++
.../java/org/apache/drill/synth/LongTail.java | 9 +-
2 files changed, 121 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
[4/7] Create new generated value vectors utilizing fmpp. Includes: -
First pass;
integrate build system and some cursory implementations - starting to split
common logic into base class - implement most of varlen value vector
functionality, minor cleanu
Posted by ja...@apache.org.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed4.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed4.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed4.java
deleted file mode 100644
index ff54784..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed4.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.SchemaDefProtos;
-import org.apache.drill.exec.record.MaterializedField;
-
-public class Fixed4 extends AbstractFixedValueVector<Fixed4>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Fixed4.class);
-
- public Fixed4(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator, 4*8);
- }
-
- public final void setInt(int index, int value){
- index*=4;
- data.setInt(index, value);
- }
-
- public final int getInt(int index){
- index*=4;
- return data.getInt(index);
- }
-
- public final void setFloat4(int index, float value){
- index*=4;
- data.setFloat(index, value);
- }
-
- public final float getFloat4(int index){
- index*=4;
- return data.getFloat(index);
- }
-
- @Override
- public Object getObject(int index) {
- if (field != null && field.getType().getMinorType() == SchemaDefProtos.MinorType.FLOAT4) {
- return getFloat4(index);
- } else {
- return getInt(index);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed8.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed8.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed8.java
deleted file mode 100644
index 3629f5c..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed8.java
+++ /dev/null
@@ -1,58 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-public class Fixed8 extends AbstractFixedValueVector<Fixed8>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Fixed8.class);
-
- public Fixed8(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator, 8*8);
- }
-
- public final void setBigInt(int index, long value){
- index*=8;
- data.setLong(index, value);
- }
-
- public final long getBigInt(int index){
- index*=8;
- return data.getLong(index);
- }
-
- public final void setFloat8(int index, double value){
- index*=8;
- data.setDouble(index, value);
- }
-
- public final double getFloat8(int index){
- index*=8;
- return data.getDouble(index);
- }
-
- @Override
- public Object getObject(int index) {
- return getBigInt(index);
- }
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/FixedLen.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/FixedLen.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/FixedLen.java
deleted file mode 100644
index 594af23..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/FixedLen.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import io.netty.buffer.ByteBuf;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-public class FixedLen extends AbstractFixedValueVector<FixedLen>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FixedLen.class);
-
-
- public FixedLen(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator, field.getWidth());
- }
-
- public void set(ByteBuf b){
-
- }
-
- public void get(ByteBuf b){
-
- }
-
- @Override
- public Object getObject(int index) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullValueException.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullValueException.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullValueException.java
new file mode 100644
index 0000000..027b698
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullValueException.java
@@ -0,0 +1,9 @@
+package org.apache.drill.exec.record.vector;
+
+import org.apache.drill.common.exceptions.DrillRuntimeException;
+
+public class NullValueException extends DrillRuntimeException {
+ public NullValueException(int index) {
+ super("Element at index position: " + index + " is null");
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableBit.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableBit.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableBit.java
deleted file mode 100644
index ac8c6b6..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableBit.java
+++ /dev/null
@@ -1,20 +0,0 @@
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-public class NullableBit extends NullableValueVector<NullableBit, Bit>{
- public NullableBit(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- @Override
- protected Bit getNewValueVector(BufferAllocator allocator) {
- return new Bit(this.field, allocator);
- }
-
- public void set(int index) {
- setNotNull(index);
- value.set(index);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed4.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed4.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed4.java
deleted file mode 100644
index d09b04a..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableFixed4.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-public final class NullableFixed4 extends NullableValueVector<NullableFixed4, Fixed4>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NullableFixed4.class);
-
- public NullableFixed4(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- @Override
- protected Fixed4 getNewValueVector(BufferAllocator allocator) {
- return new Fixed4(this.field, allocator);
- }
-
- public void setInt(int index, int newVal) {
- setNotNull(index);
- value.setInt(index, newVal);
- }
-
- public void setFloat4(int index, float val) {
- setNotNull(index);
- value.setFloat4(index, val);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableValueVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableValueVector.java
deleted file mode 100644
index 8c3d7f6..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableValueVector.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import io.netty.buffer.ByteBuf;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
-import org.apache.drill.exec.record.MaterializedField;
-
-/**
- * Abstract class supports null versions.
- */
-abstract class NullableValueVector<T extends NullableValueVector<T, E>, E extends BaseValueVector<E>> extends BaseValueVector<T> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(NullableValueVector.class);
-
- protected Bit bits;
- protected E value;
-
- public NullableValueVector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- bits = new Bit(null, allocator);
- value = getNewValueVector(allocator);
- }
-
- protected abstract E getNewValueVector(BufferAllocator allocator);
-
- public void setNull(int index) {
- bits.set(index);
- }
-
- public void setNotNull(int index) {
- bits.clear(index);
- }
-
- public int isNull(int index){
- return bits.getBit(index);
- }
-
- @Override
- protected int getAllocationSize(int valueCount) {
- return bits.getAllocationSize(valueCount) + value.getAllocationSize(valueCount);
- }
-
- @Override
- public MaterializedField getField() {
- return field;
- }
-
- @Override
- protected void childResetAllocation(int valueCount, ByteBuf buf) {
- int firstSize = bits.getAllocationSize(valueCount);
- value.resetAllocation(valueCount, buf.slice(firstSize, value.getAllocationSize(valueCount)));
- bits.resetAllocation(valueCount, buf.slice(0, firstSize));
- bits.setAllFalse();
- }
-
- @Override
- protected void childCloneMetadata(T other) {
- bits.cloneMetadata(other.bits);
- value.cloneInto(value);
- }
-
- @Override
- protected void childClear() {
- bits.clear();
- value.clear();
- }
-
-
- @Override
- public ByteBuf[] getBuffers() {
- return new ByteBuf[]{bits.data, value.data};
- }
-
- @Override
- public void setRecordCount(int recordCount) {
- super.setRecordCount(recordCount);
- bits.setRecordCount(recordCount);
- value.setRecordCount(recordCount);
- }
-
- @Override
- public Object getObject(int index) {
- if(isNull(index) == 1){
- return null;
- }else{
- return value.getObject(index);
- }
- }
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableVarLen4.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableVarLen4.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableVarLen4.java
deleted file mode 100644
index f5a2770..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/NullableVarLen4.java
+++ /dev/null
@@ -1,21 +0,0 @@
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-public class NullableVarLen4 extends NullableValueVector<NullableVarLen4, VarLen4> {
-
- public NullableVarLen4(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- @Override
- protected VarLen4 getNewValueVector(BufferAllocator allocator) {
- return new VarLen4(field, allocator);
- }
-
- public void setBytes(int index, byte[] bytes) {
- setNotNull(index);
- value.setBytes(index, bytes);
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/RepeatMap.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/RepeatMap.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/RepeatMap.java
deleted file mode 100644
index 2c08551..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/RepeatMap.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import io.netty.buffer.ByteBuf;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-public class RepeatMap extends BaseValueVector<RepeatMap>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RepeatMap.class);
-
-
- public RepeatMap(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- @Override
- protected int getAllocationSize(int valueCount) {
- return 4 * valueCount;
- }
-
- @Override
- protected void childResetAllocation(int valueCount, ByteBuf buf) {
- }
-
- @Override
- protected void childCloneMetadata(RepeatMap other) {
- }
-
- @Override
- protected void childClear() {
- }
-
- @Override
- public Object getObject(int index) {
- return null;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java
index 323b55f..e79a525 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java
@@ -25,7 +25,7 @@ import org.apache.drill.exec.record.MaterializedField;
/**
* Convenience/Clarification Fixed2 wrapper.
*/
-public class SelectionVector extends Fixed2{
+public class SelectionVector extends ValueVector.UInt2 {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SelectionVector.class);
public SelectionVector(MaterializedField field, BufferAllocator allocator) {
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/TypeHelper.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/TypeHelper.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/TypeHelper.java
deleted file mode 100644
index 543a222..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/TypeHelper.java
+++ /dev/null
@@ -1,259 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.SchemaDefProtos.DataMode;
-import org.apache.drill.exec.proto.SchemaDefProtos.MajorType;
-import org.apache.drill.exec.proto.SchemaDefProtos.MinorType;
-import org.apache.drill.exec.record.MaterializedField;
-
-public class TypeHelper {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeHelper.class);
-
- private static final int WIDTH_ESTIMATE_1 = 10;
- private static final int WIDTH_ESTIMATE_2 = 50000;
- private static final int WIDTH_ESTIMATE_4 = 1024*1024;
-
- public static int getSize(MajorType major){
- switch(major.getMinorType()){
- case TINYINT: return 1;
- case SMALLINT: return 2;
- case INT: return 4;
- case BIGINT: return 8;
- case DECIMAL4: return 4;
- case DECIMAL8: return 8;
- case DECIMAL12: return 12;
- case DECIMAL16: return 16;
- case MONEY: return 8;
- case DATE: return 4;
- case TIME: return 8;
- case TIMETZ: return 12;
- case TIMESTAMP: return 8;
- case DATETIME: return 8;
- case INTERVAL: return 12;
- case FLOAT4: return 4;
- case FLOAT8: return 8;
- case BOOLEAN: return 1/8;
- case FIXEDCHAR: return major.getWidth();
- case VARCHAR1: return 1 + WIDTH_ESTIMATE_1;
- case VARCHAR2: return 2 + WIDTH_ESTIMATE_2;
- case VARCHAR4: return 4 + WIDTH_ESTIMATE_4;
- case FIXEDBINARY: return major.getWidth();
- case VARBINARY1: return 1 + WIDTH_ESTIMATE_1;
- case VARBINARY2: return 2 + WIDTH_ESTIMATE_2;
- case VARBINARY4: return 4 + WIDTH_ESTIMATE_4;
- case UINT1: return 1;
- case UINT2: return 2;
- case UINT4: return 4;
- case UINT8: return 8;
- case PROTO2: return 2 + WIDTH_ESTIMATE_2;
- case PROTO4: return 4 + WIDTH_ESTIMATE_4;
- case MSGPACK2: return 2 + WIDTH_ESTIMATE_2;
- case MSGPACK4: return 4 + WIDTH_ESTIMATE_4;
- }
- return 4;
- }
-
- public static Class<?> getValueVectorClass(MinorType type, DataMode mode){
- switch(mode){
- case OPTIONAL:
- switch(type){
- case REPEATMAP: return RepeatMap.class;
- case TINYINT: return Fixed1.class;
- case SMALLINT: return Fixed2.class;
- case INT: return Fixed4.class;
- case BIGINT: return Fixed8.class;
- case DECIMAL4: return Fixed4.class;
- case DECIMAL8: return Fixed8.class;
- case DECIMAL12: return Fixed12.class;
- case DECIMAL16: return Fixed16.class;
- case MONEY: return Fixed8.class;
- case DATE: return Fixed4.class;
- case TIME: return Fixed8.class;
- case TIMETZ: return Fixed12.class;
- case TIMESTAMP: return Fixed8.class;
- case DATETIME: return Fixed8.class;
- case INTERVAL: return Fixed12.class;
- case FLOAT4: return Fixed4.class;
- case FLOAT8: return Fixed8.class;
- case BOOLEAN: return Bit.class;
- case FIXEDCHAR: return FixedLen.class;
- case VARCHAR1: return VarLen1.class;
- case VARCHAR2: return VarLen2.class;
- case VARCHAR4: return VarLen4.class;
- case FIXEDBINARY: return FixedLen.class;
- case VARBINARY1: return VarLen1.class;
- case VARBINARY2: return VarLen2.class;
- case VARBINARY4: return VarLen4.class;
- case UINT1: return Fixed1.class;
- case UINT2: return Fixed2.class;
- case UINT4: return Fixed4.class;
- case UINT8: return Fixed8.class;
- case PROTO2: return VarLen2.class;
- case PROTO4: return VarLen4.class;
- case MSGPACK2: return VarLen2.class;
- case MSGPACK4: return VarLen4.class;
- }
- break;
- case REQUIRED:
- switch(type){
-// case TINYINT: return NullableFixed1.class;
-// case SMALLINT: return NullableFixed2.class;
-// case INT: return NullableFixed4.class;
-// case BIGINT: return NullableFixed8.class;
-// case DECIMAL4: return NullableFixed4.class;
-// case DECIMAL8: return NullableFixed8.class;
-// case DECIMAL12: return NullableFixed12.class;
-// case DECIMAL16: return NullableFixed16.class;
-// case MONEY: return NullableFixed8.class;
-// case DATE: return NullableFixed4.class;
-// case TIME: return NullableFixed8.class;
-// case TIMETZ: return NullableFixed12.class;
-// case TIMESTAMP: return NullableFixed8.class;
-// case DATETIME: return NullableFixed8.class;
-// case INTERVAL: return NullableFixed12.class;
-// case FLOAT4: return NullableFixed4.class;
-// case FLOAT8: return NullableFixed8.class;
-// case BOOLEAN: return NullableBit.class;
-// case FIXEDCHAR: return NullableFixedLen.class;
-// case VARCHAR1: return NullableVarLen1.class;
-// case VARCHAR2: return NullableVarLen2.class;
-// case VARCHAR4: return NullableVarLen4.class;
-// case FIXEDBINARY: return NullableFixedLen.class;
-// case VARBINARY1: return NullableVarLen1.class;
-// case VARBINARY2: return NullableVarLen2.class;
-// case VARBINARY4: return NullableVarLen4.class;
-// case UINT1: return NullableFixed1.class;
-// case UINT2: return NullableFixed2.class;
-// case UINT4: return NullableFixed4.class;
-// case UINT8: return NullableFixed8.class;
-// case PROTO2: return NullableVarLen2.class;
-// case PROTO4: return NullableVarLen4.class;
-// case MSGPACK2: return NullableVarLen2.class;
-// case MSGPACK4: return NullableVarLen4.class;
- }
- break;
- case REPEATED:
- switch(type){
-// case TINYINT: return RepeatedFixed1.class;
-// case SMALLINT: return RepeatedFixed2.class;
-// case INT: return RepeatedFixed4.class;
-// case BIGINT: return RepeatedFixed8.class;
-// case DECIMAL4: return RepeatedFixed4.class;
-// case DECIMAL8: return RepeatedFixed8.class;
-// case DECIMAL12: return RepeatedFixed12.class;
-// case DECIMAL16: return RepeatedFixed16.class;
-// case MONEY: return RepeatedFixed8.class;
-// case DATE: return RepeatedFixed4.class;
-// case TIME: return RepeatedFixed8.class;
-// case TIMETZ: return RepeatedFixed12.class;
-// case TIMESTAMP: return RepeatedFixed8.class;
-// case DATETIME: return RepeatedFixed8.class;
-// case INTERVAL: return RepeatedFixed12.class;
-// case FLOAT4: return RepeatedFixed4.class;
-// case FLOAT8: return RepeatedFixed8.class;
-// case BOOLEAN: return RepeatedBit.class;
-// case FIXEDCHAR: return RepeatedFixedLen.class;
-// case VARCHAR1: return RepeatedVarLen1.class;
-// case VARCHAR2: return RepeatedVarLen2.class;
-// case VARCHAR4: return RepeatedVarLen4.class;
-// case FIXEDBINARY: return RepeatedFixedLen.class;
-// case VARBINARY1: return RepeatedVarLen1.class;
-// case VARBINARY2: return RepeatedVarLen2.class;
-// case VARBINARY4: return RepeatedVarLen4.class;
-// case UINT1: return RepeatedFixed1.class;
-// case UINT2: return RepeatedFixed2.class;
-// case UINT4: return RepeatedFixed4.class;
-// case UINT8: return RepeatedFixed8.class;
-// case PROTO2: return RepeatedVarLen2.class;
-// case PROTO4: return RepeatedVarLen4.class;
-// case MSGPACK2: return RepeatedVarLen2.class;
-// case MSGPACK4: return RepeatedVarLen4.class;
- }
- break;
- default:
- break;
-
- }
- throw new UnsupportedOperationException();
- }
-
-
- public static ValueVector<?> getNewVector(MaterializedField field, BufferAllocator allocator){
- MajorType type = field.getType();
- switch(type.getMode()){
- case REQUIRED:
- switch(type.getMinorType()){
- case TINYINT: return new Fixed1(field, allocator);
- case SMALLINT: return new Fixed2(field, allocator);
- case INT: return new Fixed4(field, allocator);
- case BIGINT: return new Fixed8(field, allocator);
- case DECIMAL4: return new Fixed4(field, allocator);
- case DECIMAL8: return new Fixed8(field, allocator);
- case DECIMAL12: return new Fixed12(field, allocator);
- case DECIMAL16: return new Fixed16(field, allocator);
- case MONEY: return new Fixed8(field, allocator);
- case DATE: return new Fixed4(field, allocator);
- case TIME: return new Fixed8(field, allocator);
- case TIMETZ: return new Fixed12(field, allocator);
- case TIMESTAMP: return new Fixed8(field, allocator);
- case DATETIME: return new Fixed8(field, allocator);
- case INTERVAL: return new Fixed12(field, allocator);
- case FLOAT4: return new Fixed4(field, allocator);
- case FLOAT8: return new Fixed8(field, allocator);
- case BOOLEAN: return new Bit(field, allocator);
- case FIXEDCHAR: return new FixedLen(field, allocator);
- case VARCHAR1: return new VarLen1(field, allocator);
- case VARCHAR2: return new VarLen2(field, allocator);
- case VARCHAR4: return new VarLen4(field, allocator);
- case FIXEDBINARY: return new FixedLen(field, allocator);
- case VARBINARY1: return new VarLen1(field, allocator);
- case VARBINARY2: return new VarLen2(field, allocator);
- case VARBINARY4: return new VarLen4(field, allocator);
- case UINT1: return new Fixed1(field, allocator);
- case UINT2: return new Fixed2(field, allocator);
- case UINT4: return new Fixed4(field, allocator);
- case UINT8: return new Fixed8(field, allocator);
- case PROTO2: return new VarLen2(field, allocator);
- case PROTO4: return new VarLen4(field, allocator);
- case MSGPACK2: return new VarLen2(field, allocator);
- case MSGPACK4: return new VarLen4(field, allocator);
- }
- break;
- case REPEATED:
- switch(type.getMinorType()) {
- case MAP: return new RepeatMap(field, allocator);
- }
- break;
- case OPTIONAL:
- switch(type.getMinorType()) {
- case BOOLEAN: return new NullableBit(field, allocator);
- case INT: return new NullableFixed4(field, allocator);
- case FLOAT4: return new NullableFixed4(field, allocator);
- case VARCHAR4: return new NullableVarLen4(field, allocator);
- }
- break;
- default:
- break;
-
- }
- throw new UnsupportedOperationException(type.getMinorType() + " type is not supported. Mode: " + type.getMode());
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/ValueVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/ValueVector.java
deleted file mode 100644
index 8a5a822..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/ValueVector.java
+++ /dev/null
@@ -1,118 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import io.netty.buffer.ByteBuf;
-
-import java.io.Closeable;
-
-import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
-import org.apache.drill.exec.record.MaterializedField;
-
-/**
- * A vector of values. Acts a containing instance that may rotate its internal buffer depending on what it needs to hold. Should be language agnostic so that it can be passed between Java and JNI without modification.
- */
-public interface ValueVector<T extends ValueVector<T>> extends Closeable {
-
- /**
- * Copies the data from this vector into its pair.
- *
- * @param vector
- */
- public abstract void cloneInto(T vector);
-
- /**
- * Allocate a new memory space for this vector.
- *
- * @param valueCount
- * The number of possible values which should be contained in this vector.
- */
- public abstract void allocateNew(int valueCount);
-
- /**
- * Update the value vector to the provided record information.
- * @param metadata
- * @param data
- */
- public abstract void setTo(FieldMetadata metadata, ByteBuf data);
-
- /**
- * Zero copy move of data from this vector to the target vector. Any future access to this vector without being
- * populated by a new vector will cause problems.
- *
- * @param vector
- */
- public abstract void transferTo(T vector);
-
- /**
- * Return the underlying buffers associated with this vector. Note that this doesn't impact the reference counts for this buffer so it only should be
- * used for in context access. Also note that this buffer changes regularly thus external classes shouldn't hold a
- * reference to it (unless they change it).
- *
- * @return The underlying ByteBuf.
- */
- public abstract ByteBuf[] getBuffers();
-
- /**
- * Returns the maximum number of values contained within this vector.
- * @return Vector size
- */
- public abstract int capacity();
-
-
- /**
- * Release supporting resources.
- */
- public abstract void close();
-
- /**
- * Get information about how this field is materialized.
- *
- * @return
- */
- public abstract MaterializedField getField();
-
- /**
- * Define the number of records that are in this value vector.
- * @param recordCount Number of records active in this vector. Used for purposes such as getting a writable range of the data.
- */
- public abstract void setRecordCount(int recordCount);
- public abstract int getRecordCount();
-
-
- /**
- * Get the metadata for this field.
- * @return
- */
- public abstract FieldMetadata getMetadata();
-
- /**
- * Debug interface to get values per record.
- * @param index The record index.
- * @return The value in the vector.
- */
- public Object getObject(int index);
-
-
- /**
- * Useful for generating random data.
- */
- public void randomizeData();
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen1.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen1.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen1.java
deleted file mode 100644
index 6c2b39e..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen1.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-public class VarLen1 extends VariableVector<VarLen1, Fixed1>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VarLen1.class);
-
- public VarLen1(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator, 1);
- }
-
- @Override
- protected Fixed1 getNewLengthVector(BufferAllocator allocator) {
- return new Fixed1(null, allocator);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen2.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen2.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen2.java
deleted file mode 100644
index 3bcdd94..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen2.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-public class VarLen2 extends VariableVector<VarLen2, Fixed2>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VarLen2.class);
-
- public VarLen2(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator, 2);
- }
-
- @Override
- protected Fixed2 getNewLengthVector(BufferAllocator allocator) {
- return new Fixed2(null, allocator);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen4.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen4.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen4.java
deleted file mode 100644
index 71ed7a1..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen4.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-import static com.google.common.base.Preconditions.checkArgument;
-
-public class VarLen4 extends VariableVector<VarLen4, Fixed4>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VarLen4.class);
-
- public VarLen4(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator, 4);
- }
-
- @Override
- protected Fixed4 getNewLengthVector(BufferAllocator allocator) {
- return new Fixed4(null, allocator);
- }
-
- public void setBytes(int index, byte[] bytes) {
- checkArgument(index >= 0);
- if(index == 0) {
- lengthVector.setInt(0, bytes.length);
- data.setBytes(0, bytes);
- } else {
- int previousOffset = lengthVector.getInt(index - 1);
- lengthVector.setInt(index, previousOffset + bytes.length);
- data.setBytes(previousOffset, bytes);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VariableVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VariableVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VariableVector.java
deleted file mode 100644
index dc8fec4..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VariableVector.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import io.netty.buffer.ByteBuf;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.DeadBuf;
-import org.apache.drill.exec.record.MaterializedField;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-
-/**
- * A vector of variable length bytes. Constructed as a vector of lengths or positions and a vector of values. Random access is only possible if the variable vector stores positions as opposed to lengths.
- */
-public abstract class VariableVector<T extends VariableVector<T, E>, E extends BaseValueVector<E>> extends BaseValueVector<T>{
-
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VariableVector.class);
-
- protected final E lengthVector;
- protected int expectedValueLength;
-
- public VariableVector(MaterializedField field, BufferAllocator allocator, int expectedValueLength) {
- super(field, allocator);
- this.lengthVector = getNewLengthVector(allocator);
- this.expectedValueLength = expectedValueLength;
- }
-
- protected abstract E getNewLengthVector(BufferAllocator allocator);
-
- @Override
- protected int getAllocationSize(int valueCount) {
- return lengthVector.getAllocationSize(valueCount) + (expectedValueLength * valueCount);
- }
-
- @Override
- protected void childResetAllocation(int valueCount, ByteBuf buf) {
- int firstSize = lengthVector.getAllocationSize(valueCount);
- lengthVector.resetAllocation(valueCount, buf.slice(0, firstSize));
- data = buf.slice(firstSize, expectedValueLength * valueCount);
- }
-
- @Override
- protected void childCloneMetadata(T other) {
- lengthVector.cloneMetadata(other.lengthVector);
- other.expectedValueLength = expectedValueLength;
- }
-
- @Override
- protected void childClear() {
- lengthVector.clear();
- if(data != DeadBuf.DEAD_BUFFER){
- data.release();
- data = DeadBuf.DEAD_BUFFER;
- }
- }
-
- @Override
- public ByteBuf[] getBuffers() {
- return new ByteBuf[]{lengthVector.data, data};
- }
-
- @Override
- public void setRecordCount(int recordCount) {
- super.setRecordCount(recordCount);
- lengthVector.setRecordCount(recordCount);
- }
-
- public void setTotalBytes(int totalBytes){
- data.writerIndex(totalBytes);
- }
-
- @Override
- public Object getObject(int index) {
- checkArgument(index >= 0);
- int startIdx = 0;
- if(index > 0) {
- startIdx = (int) lengthVector.getObject(index - 1);
- }
- int size = (int) lengthVector.getObject(index) - startIdx;
- checkState(size >= 0);
- byte[] dst = new byte[size];
- data.getBytes(startIdx, dst, 0, size);
- return dst;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
index a680a97..9ce3f3c 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/rpc/RpcBus.java
@@ -85,6 +85,7 @@ public abstract class RpcBus<T extends EnumLite, C extends RemoteConnection> imp
Preconditions.checkNotNull(protobufBody);
ChannelListenerWithCoordinationId futureListener = queue.get(listener, clazz);
OutboundRpcMessage m = new OutboundRpcMessage(RpcMode.REQUEST, rpcType, futureListener.getCoordinationId(), protobufBody, dataBodies);
+ logger.debug("Outbound RPC message: {}. DATA BODIES: {}", m, dataBodies);
ChannelFuture channelFuture = connection.getChannel().write(m);
channelFuture.addListener(futureListener);
completed = true;
@@ -132,7 +133,6 @@ public abstract class RpcBus<T extends EnumLite, C extends RemoteConnection> imp
@Override
public void messageReceived(ChannelHandlerContext ctx, InboundRpcMessage msg) throws Exception {
if (!ctx.channel().isOpen()) return;
-
if (RpcConstants.EXTRA_DEBUGGING) logger.debug("Received message {}", msg);
switch (msg.mode) {
case REQUEST:
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
index 42d1be5..f2f97b7 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
@@ -341,44 +341,42 @@ public class JSONRecordReader implements RecordReader {
switch (minorType) {
case INT: {
holder.incAndCheckLength(32);
- NullableFixed4 fixed4 = (NullableFixed4) holder.getValueVector();
+ ValueVector.NullableInt int4 = (ValueVector.NullableInt) holder.getValueVector();
if (val == null) {
- fixed4.setNull(index);
+ int4.setNull(index);
} else {
- fixed4.setInt(index, (Integer) val);
+ int4.set(index, (Integer) val);
}
return holder.hasEnoughSpace(32);
}
case FLOAT4: {
holder.incAndCheckLength(32);
- NullableFixed4 fixed4 = (NullableFixed4) holder.getValueVector();
+ ValueVector.NullableFloat4 float4 = (ValueVector.NullableFloat4) holder.getValueVector();
if (val == null) {
- fixed4.setNull(index);
+ float4.setNull(index);
} else {
- fixed4.setFloat4(index, (Float) val);
+ float4.set(index, (Float) val);
}
return holder.hasEnoughSpace(32);
}
case VARCHAR4: {
if (val == null) {
- ((NullableVarLen4) holder.getValueVector()).setNull(index);
+ ((ValueVector.NullableVarChar4) holder.getValueVector()).setNull(index);
return (index + 1) * 4 <= holder.getLength();
} else {
byte[] bytes = ((String) val).getBytes(UTF_8);
- int length = bytes.length * 8;
+ int length = bytes.length;
holder.incAndCheckLength(length);
- NullableVarLen4 varLen4 = (NullableVarLen4) holder.getValueVector();
- varLen4.setBytes(index, bytes);
+ ValueVector.NullableVarChar4 varLen4 = (ValueVector.NullableVarChar4) holder.getValueVector();
+ varLen4.set(index, bytes);
return holder.hasEnoughSpace(length);
}
}
case BOOLEAN: {
holder.incAndCheckLength(1);
- NullableBit bit = (NullableBit) holder.getValueVector();
- if (val == null) {
- bit.setNull(index);
- } else if ((Boolean) val) {
- bit.set(index);
+ ValueVector.NullableBit bit = (ValueVector.NullableBit) holder.getValueVector();
+ if (val != null) {
+ bit.set(index, (Boolean)val ? 1 : 0);
}
return holder.hasEnoughSpace(1);
}
@@ -411,7 +409,7 @@ public class JSONRecordReader implements RecordReader {
SchemaDefProtos.MajorType type = field.getFieldType();
int fieldId = field.getFieldId();
MaterializedField f = MaterializedField.create(new SchemaPath(field.getFieldName()), fieldId, parentFieldId, type);
- ValueVector<?> v = TypeHelper.getNewVector(f, allocator);
+ ValueVector.Base v = TypeHelper.getNewVector(f, allocator);
v.allocateNew(batchSize);
VectorHolder holder = new VectorHolder(batchSize, v);
valueVectorMap.put(fieldId, holder);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
index 59590d2..4043913 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
@@ -25,15 +25,15 @@ import org.apache.drill.exec.record.vector.ValueVector;
public class VectorHolder {
private int length;
- private ValueVector vector;
+ private ValueVector.Base vector;
private int currentLength;
- VectorHolder(int length, ValueVector<?> vector) {
+ VectorHolder(int length, ValueVector.Base vector) {
this.length = length;
this.vector = vector;
}
- public ValueVector getValueVector() {
+ public ValueVector.Base getValueVector() {
return vector;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/AbstractFragmentRunnerListener.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/AbstractFragmentRunnerListener.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/AbstractFragmentRunnerListener.java
index 0e8edd5..7668bdc 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/AbstractFragmentRunnerListener.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/AbstractFragmentRunnerListener.java
@@ -103,7 +103,8 @@ public class AbstractFragmentRunnerListener implements FragmentRunnerListener{
protected void fail(FragmentHandle handle, FragmentStatus.Builder statusBuilder){
statusChange(handle, statusBuilder.build());
+ // TODO: ensure the foreman handles the exception
}
-
+
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunner.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunner.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunner.java
index 554b398..ec9f6db 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunner.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/FragmentRunner.java
@@ -82,7 +82,7 @@ public class FragmentRunner implements Runnable, CancelableQuery, StatusProvider
}else{
updateState(FragmentState.RUNNING, FragmentState.FINISHED, false);
}
-
+
}
}
@@ -92,6 +92,7 @@ public class FragmentRunner implements Runnable, CancelableQuery, StatusProvider
}
}catch(Exception ex){
+ logger.debug("Caught exception while running fragment: {} ", ex);
internalFail(ex);
}finally{
t.stop();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
index 594a3a2..3edf283 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
@@ -42,61 +42,59 @@ public class TestSimpleFragmentRun extends PopUnitTestBase {
@Test
public void runNoExchangeFragment() throws Exception {
try(RemoteServiceSet serviceSet = RemoteServiceSet.getLocalServiceSet();
- Drillbit bit = new Drillbit(CONFIG, serviceSet);
- DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());){
+ Drillbit bit = new Drillbit(CONFIG, serviceSet);
+ DrillClient client = new DrillClient(CONFIG, serviceSet.getCoordinator());){
- // run query.
- bit.run();
- client.connect();
- List<QueryResultBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/physical_test2.json"), Charsets.UTF_8));
+ // run query.
+ bit.run();
+ client.connect();
+ List<QueryResultBatch> results = client.runQuery(QueryType.PHYSICAL, Files.toString(FileUtils.getResourceAsFile("/physical_test2.json"), Charsets.UTF_8));
- // look at records
- RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
- int recordCount = 0;
- for (QueryResultBatch batch : results) {
- if(!batch.hasData()) continue;
- boolean schemaChanged = batchLoader.load(batch.getHeader().getDef(), batch.getData());
- boolean firstColumn = true;
+ // look at records
+ RecordBatchLoader batchLoader = new RecordBatchLoader(bit.getContext().getAllocator());
+ int recordCount = 0;
+ for (QueryResultBatch batch : results) {
+ if(!batch.hasData()) continue;
+ boolean schemaChanged = batchLoader.load(batch.getHeader().getDef(), batch.getData());
+ boolean firstColumn = true;
- // print headers.
- if (schemaChanged) {
- System.out.println("\n\n========NEW SCHEMA=========\n\n");
- for (IntObjectCursor<ValueVector<?>> v : batchLoader) {
+ // print headers.
+ if (schemaChanged) {
+ System.out.println("\n\n========NEW SCHEMA=========\n\n");
+ for (IntObjectCursor<ValueVector.Base> v : batchLoader) {
- if (firstColumn) {
- firstColumn = false;
- } else {
- System.out.print("\t");
- }
- System.out.print(v.value.getField().getName());
- System.out.print("[");
- System.out.print(v.value.getField().getType().getMinorType());
- System.out.print("]");
- }
- System.out.println();
+ if (firstColumn) {
+ firstColumn = false;
+ } else {
+ System.out.print("\t");
}
+ System.out.print(v.value.getField().getName());
+ System.out.print("[");
+ System.out.print(v.value.getField().getType().getMinorType());
+ System.out.print("]");
+ }
+ System.out.println();
+ }
- for (int i = 0; i < batchLoader.getRecordCount(); i++) {
- boolean first = true;
- recordCount++;
- for (IntObjectCursor<ValueVector<?>> v : batchLoader) {
- if (first) {
- first = false;
- } else {
- System.out.print("\t");
- }
- System.out.print(v.value.getObject(i));
- }
- if(!first) System.out.println();
+ for (int i = 0; i < batchLoader.getRecordCount(); i++) {
+ boolean first = true;
+ recordCount++;
+ for (IntObjectCursor<ValueVector.Base> v : batchLoader) {
+ if (first) {
+ first = false;
+ } else {
+ System.out.print("\t");
}
-
- }
- logger.debug("Received results {}", results);
- assertEquals(recordCount, 200);
+ System.out.print(v.value.getObject(i));
}
- }
-
+ if(!first) System.out.println();
+ }
+ }
+ logger.debug("Received results {}", results);
+ assertEquals(recordCount, 200);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
new file mode 100644
index 0000000..5924f7d
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
@@ -0,0 +1,252 @@
+package org.apache.drill.exec.record.vector;
+
+import io.netty.buffer.ByteBuf;
+import org.apache.drill.exec.memory.DirectBufferAllocator;
+import org.apache.drill.exec.proto.SchemaDefProtos;
+import org.apache.drill.exec.record.MaterializedField;
+
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
+import org.apache.hadoop.io.UTF8;
+import org.junit.Test;
+
+import java.nio.charset.Charset;
+
+public class TestValueVector {
+
+ DirectBufferAllocator allocator = new DirectBufferAllocator();
+
+ @Test
+ public void testFixedType() {
+ // Build a required uint field definition
+ SchemaDefProtos.MajorType.Builder typeBuilder = SchemaDefProtos.MajorType.newBuilder();
+ SchemaDefProtos.FieldDef.Builder defBuilder = SchemaDefProtos.FieldDef.newBuilder();
+ typeBuilder
+ .setMinorType(SchemaDefProtos.MinorType.UINT4)
+ .setMode(SchemaDefProtos.DataMode.REQUIRED)
+ .setWidth(4);
+ defBuilder
+ .setFieldId(1)
+ .setParentId(0)
+ .setMajorType(typeBuilder.build());
+ MaterializedField field = MaterializedField.create(defBuilder.build());
+
+ // Create a new value vector for 1024 integers
+ ValueVector.MutableUInt4 v = new ValueVector.MutableUInt4(field, allocator);
+ v.allocateNew(1024);
+
+ // Put and set a few values
+ v.set(0, 100);
+ v.set(1, 101);
+ v.set(100, 102);
+ v.set(1022, 103);
+ v.set(1023, 104);
+ assertEquals(100, v.get(0));
+ assertEquals(101, v.get(1));
+ assertEquals(102, v.get(100));
+ assertEquals(103, v.get(1022));
+ assertEquals(104, v.get(1023));
+
+ // Ensure unallocated space returns 0
+ assertEquals(0, v.get(3));
+ }
+
+ @Test
+ public void testNullableVarLen2() {
+ // Build an optional varchar field definition
+ SchemaDefProtos.MajorType.Builder typeBuilder = SchemaDefProtos.MajorType.newBuilder();
+ SchemaDefProtos.FieldDef.Builder defBuilder = SchemaDefProtos.FieldDef.newBuilder();
+ typeBuilder
+ .setMinorType(SchemaDefProtos.MinorType.VARCHAR2)
+ .setMode(SchemaDefProtos.DataMode.OPTIONAL)
+ .setWidth(2);
+ defBuilder
+ .setFieldId(1)
+ .setParentId(0)
+ .setMajorType(typeBuilder.build());
+ MaterializedField field = MaterializedField.create(defBuilder.build());
+
+ // Create a new value vector for 1024 integers
+ ValueVector.NullableVarChar2 v = new ValueVector.NullableVarChar2(field, allocator);
+ v.allocateNew(1024);
+
+ // Create and set 3 sample strings
+ String str1 = new String("AAAAA1");
+ String str2 = new String("BBBBBBBBB2");
+ String str3 = new String("CCCC3");
+ v.set(0, str1.getBytes(Charset.forName("UTF-8")));
+ v.set(1, str2.getBytes(Charset.forName("UTF-8")));
+ v.set(2, str3.getBytes(Charset.forName("UTF-8")));
+
+ // Check the sample strings
+ assertEquals(str1, new String(v.get(0), Charset.forName("UTF-8")));
+ assertEquals(str2, new String(v.get(1), Charset.forName("UTF-8")));
+ assertEquals(str3, new String(v.get(2), Charset.forName("UTF-8")));
+
+ // Ensure null value throws
+ try {
+ v.get(3);
+ assertFalse(false);
+ } catch(NullValueException e) { }
+
+ }
+
+
+ @Test
+ public void testNullableFixedType() {
+ // Build an optional uint field definition
+ SchemaDefProtos.MajorType.Builder typeBuilder = SchemaDefProtos.MajorType.newBuilder();
+ SchemaDefProtos.FieldDef.Builder defBuilder = SchemaDefProtos.FieldDef.newBuilder();
+ typeBuilder
+ .setMinorType(SchemaDefProtos.MinorType.UINT4)
+ .setMode(SchemaDefProtos.DataMode.OPTIONAL)
+ .setWidth(4);
+ defBuilder
+ .setFieldId(1)
+ .setParentId(0)
+ .setMajorType(typeBuilder.build());
+ MaterializedField field = MaterializedField.create(defBuilder.build());
+
+ // Create a new value vector for 1024 integers
+ ValueVector.NullableUInt4 v = new ValueVector.NullableUInt4(field, allocator);
+ v.allocateNew(1024);
+
+ // Put and set a few values
+ v.set(0, 100);
+ v.set(1, 101);
+ v.set(100, 102);
+ v.set(1022, 103);
+ v.set(1023, 104);
+ assertEquals(100, v.get(0));
+ assertEquals(101, v.get(1));
+ assertEquals(102, v.get(100));
+ assertEquals(103, v.get(1022));
+ assertEquals(104, v.get(1023));
+
+ // Ensure null values throw
+ try {
+ v.get(3);
+ assertFalse(false);
+ } catch(NullValueException e) { }
+
+ v.allocateNew(2048);
+ try {
+ v.get(0);
+ assertFalse(false);
+ } catch(NullValueException e) { }
+
+ v.set(0, 100);
+ v.set(1, 101);
+ v.set(100, 102);
+ v.set(1022, 103);
+ v.set(1023, 104);
+ assertEquals(100, v.get(0));
+ assertEquals(101, v.get(1));
+ assertEquals(102, v.get(100));
+ assertEquals(103, v.get(1022));
+ assertEquals(104, v.get(1023));
+
+ // Ensure null values throw
+ try {
+ v.get(3);
+ assertFalse(false);
+ } catch(NullValueException e) { }
+
+ }
+
+ @Test
+ public void testNullableFloat() {
+ // Build an optional float field definition
+ SchemaDefProtos.MajorType.Builder typeBuilder = SchemaDefProtos.MajorType.newBuilder();
+ SchemaDefProtos.FieldDef.Builder defBuilder = SchemaDefProtos.FieldDef.newBuilder();
+ typeBuilder
+ .setMinorType(SchemaDefProtos.MinorType.FLOAT4)
+ .setMode(SchemaDefProtos.DataMode.OPTIONAL)
+ .setWidth(4);
+ defBuilder
+ .setFieldId(1)
+ .setParentId(0)
+ .setMajorType(typeBuilder.build());
+ MaterializedField field = MaterializedField.create(defBuilder.build());
+
+ // Create a new value vector for 1024 integers
+ ValueVector.NullableFloat4 v = (ValueVector.NullableFloat4) TypeHelper.getNewVector(field, allocator);
+
+ v.allocateNew(1024);
+
+ // Put and set a few values
+ v.set(0, 100.1f);
+ v.set(1, 101.2f);
+ v.set(100, 102.3f);
+ v.set(1022, 103.4f);
+ v.set(1023, 104.5f);
+ assertEquals(100.1f, v.get(0), 0);
+ assertEquals(101.2f, v.get(1), 0);
+ assertEquals(102.3f, v.get(100), 0);
+ assertEquals(103.4f, v.get(1022), 0);
+ assertEquals(104.5f, v.get(1023), 0);
+
+ // Ensure null values throw
+ try {
+ v.get(3);
+ assertFalse(false);
+ } catch(NullValueException e) { }
+
+ v.allocateNew(2048);
+ try {
+ v.get(0);
+ assertFalse(false);
+ } catch(NullValueException e) { }
+
+ }
+
+ @Test
+ public void testBitVector() {
+ // Build a required boolean field definition
+ SchemaDefProtos.MajorType.Builder typeBuilder = SchemaDefProtos.MajorType.newBuilder();
+ SchemaDefProtos.FieldDef.Builder defBuilder = SchemaDefProtos.FieldDef.newBuilder();
+ typeBuilder
+ .setMinorType(SchemaDefProtos.MinorType.BOOLEAN)
+ .setMode(SchemaDefProtos.DataMode.REQUIRED)
+ .setWidth(4);
+ defBuilder
+ .setFieldId(1)
+ .setParentId(0)
+ .setMajorType(typeBuilder.build());
+ MaterializedField field = MaterializedField.create(defBuilder.build());
+
+ // Create a new value vector for 1024 integers
+ ValueVector.MutableBit v = new ValueVector.MutableBit(field, allocator);
+ v.allocateNew(1024);
+
+ // Put and set a few values
+ v.set(0, 1);
+ v.set(1, 0);
+ v.set(100, 0);
+ v.set(1022, 1);
+ assertEquals(1, v.get(0));
+ assertEquals(0, v.get(1));
+ assertEquals(0, v.get(100));
+ assertEquals(1, v.get(1022));
+
+ // test setting the same value twice
+ v.set(0, 1);
+ v.set(0, 1);
+ v.set(1, 0);
+ v.set(1, 0);
+ assertEquals(1, v.get(0));
+ assertEquals(0, v.get(1));
+
+ // test toggling the values
+ v.set(0, 0);
+ v.set(1, 1);
+ assertEquals(0, v.get(0));
+ assertEquals(1, v.get(1));
+
+ // Ensure unallocated space returns 0
+ assertEquals(0, v.get(3));
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
index a29e9f0..cef40ff 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
@@ -11,6 +11,7 @@ import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.proto.SchemaDefProtos;
import org.apache.drill.exec.proto.UserBitShared;
import org.apache.drill.exec.record.vector.ValueVector;
+import org.junit.Ignore;
import org.junit.Test;
import java.io.IOException;
@@ -30,7 +31,7 @@ public class JSONRecordReaderTest {
class MockOutputMutator implements OutputMutator {
List<Integer> removedFields = Lists.newArrayList();
- List<ValueVector> addFields = Lists.newArrayList();
+ List<ValueVector.Base> addFields = Lists.newArrayList();
@Override
public void removeField(int fieldId) throws SchemaChangeException {
@@ -38,7 +39,7 @@ public class JSONRecordReaderTest {
}
@Override
- public void addField(int fieldId, ValueVector<?> vector) throws SchemaChangeException {
+ public void addField(int fieldId, ValueVector.Base vector) throws SchemaChangeException {
addFields.add(vector);
}
@@ -50,16 +51,16 @@ public class JSONRecordReaderTest {
return removedFields;
}
- List<ValueVector> getAddFields() {
+ List<ValueVector.Base> getAddFields() {
return addFields;
}
}
- private <T> void assertField(ValueVector valueVector, int index, SchemaDefProtos.MinorType expectedMinorType, T value, String name) {
+ private <T> void assertField(ValueVector.Base valueVector, int index, SchemaDefProtos.MinorType expectedMinorType, T value, String name) {
assertField(valueVector, index, expectedMinorType, value, name, 0);
}
- private <T> void assertField(ValueVector valueVector, int index, SchemaDefProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) {
+ private <T> void assertField(ValueVector.Base valueVector, int index, SchemaDefProtos.MinorType expectedMinorType, T value, String name, int parentFieldId) {
UserBitShared.FieldMetadata metadata = valueVector.getMetadata();
SchemaDefProtos.FieldDef def = metadata.getDef();
assertEquals(expectedMinorType, def.getMajorType().getMinorType());
@@ -89,15 +90,15 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_1.json"));
MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector> addFields = mutator.getAddFields();
+ List<ValueVector.Base> addFields = mutator.getAddFields();
jr.setup(mutator);
assertEquals(2, jr.next());
assertEquals(3, addFields.size());
assertField(addFields.get(0), 0, SchemaDefProtos.MinorType.INT, 123, "test");
- assertField(addFields.get(1), 0, SchemaDefProtos.MinorType.BOOLEAN, 1, "b");
+ assertField(addFields.get(1), 0, SchemaDefProtos.MinorType.BOOLEAN, true, "b");
assertField(addFields.get(2), 0, SchemaDefProtos.MinorType.VARCHAR4, "hi!".getBytes(UTF_8), "c");
assertField(addFields.get(0), 1, SchemaDefProtos.MinorType.INT, 1234, "test");
- assertField(addFields.get(1), 1, SchemaDefProtos.MinorType.BOOLEAN, 0, "b");
+ assertField(addFields.get(1), 1, SchemaDefProtos.MinorType.BOOLEAN, false, "b");
assertField(addFields.get(2), 1, SchemaDefProtos.MinorType.VARCHAR4, "drill!".getBytes(UTF_8), "c");
assertEquals(0, jr.next());
@@ -115,7 +116,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_2.json"));
MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector> addFields = mutator.getAddFields();
+ List<ValueVector.Base> addFields = mutator.getAddFields();
jr.setup(mutator);
assertEquals(3, jr.next());
@@ -123,18 +124,18 @@ public class JSONRecordReaderTest {
assertField(addFields.get(0), 0, SchemaDefProtos.MinorType.INT, 123, "test");
assertField(addFields.get(1), 0, SchemaDefProtos.MinorType.INT, 1, "b");
assertField(addFields.get(2), 0, SchemaDefProtos.MinorType.FLOAT4, (float) 2.15, "c");
- assertField(addFields.get(3), 0, SchemaDefProtos.MinorType.BOOLEAN, 1, "bool");
+ assertField(addFields.get(3), 0, SchemaDefProtos.MinorType.BOOLEAN, true, "bool");
assertField(addFields.get(4), 0, SchemaDefProtos.MinorType.VARCHAR4, "test1".getBytes(UTF_8), "str1");
assertField(addFields.get(0), 1, SchemaDefProtos.MinorType.INT, 1234, "test");
assertField(addFields.get(1), 1, SchemaDefProtos.MinorType.INT, 3, "b");
- assertField(addFields.get(3), 1, SchemaDefProtos.MinorType.BOOLEAN, 0, "bool");
+ assertField(addFields.get(3), 1, SchemaDefProtos.MinorType.BOOLEAN, false, "bool");
assertField(addFields.get(4), 1, SchemaDefProtos.MinorType.VARCHAR4, "test2".getBytes(UTF_8), "str1");
assertField(addFields.get(5), 1, SchemaDefProtos.MinorType.INT, 4, "d");
assertField(addFields.get(0), 2, SchemaDefProtos.MinorType.INT, 12345, "test");
assertField(addFields.get(2), 2, SchemaDefProtos.MinorType.FLOAT4, (float) 5.16, "c");
- assertField(addFields.get(3), 2, SchemaDefProtos.MinorType.BOOLEAN, 1, "bool");
+ assertField(addFields.get(3), 2, SchemaDefProtos.MinorType.BOOLEAN, true, "bool");
assertField(addFields.get(5), 2, SchemaDefProtos.MinorType.INT, 6, "d");
assertField(addFields.get(6), 2, SchemaDefProtos.MinorType.VARCHAR4, "test3".getBytes(UTF_8), "str2");
assertTrue(mutator.getRemovedFields().isEmpty());
@@ -152,7 +153,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_2.json"), 64); // batch only fits 1 int
MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector> addFields = mutator.getAddFields();
+ List<ValueVector.Base> addFields = mutator.getAddFields();
List<Integer> removedFields = mutator.getRemovedFields();
jr.setup(mutator);
@@ -161,14 +162,14 @@ public class JSONRecordReaderTest {
assertField(addFields.get(0), 0, SchemaDefProtos.MinorType.INT, 123, "test");
assertField(addFields.get(1), 0, SchemaDefProtos.MinorType.INT, 1, "b");
assertField(addFields.get(2), 0, SchemaDefProtos.MinorType.FLOAT4, (float) 2.15, "c");
- assertField(addFields.get(3), 0, SchemaDefProtos.MinorType.BOOLEAN, 1, "bool");
+ assertField(addFields.get(3), 0, SchemaDefProtos.MinorType.BOOLEAN, true, "bool");
assertField(addFields.get(4), 0, SchemaDefProtos.MinorType.VARCHAR4, "test1".getBytes(UTF_8), "str1");
assertTrue(removedFields.isEmpty());
assertEquals(1, jr.next());
assertEquals(6, addFields.size());
assertField(addFields.get(0), 0, SchemaDefProtos.MinorType.INT, 1234, "test");
assertField(addFields.get(1), 0, SchemaDefProtos.MinorType.INT, 3, "b");
- assertField(addFields.get(3), 0, SchemaDefProtos.MinorType.BOOLEAN, 0, "bool");
+ assertField(addFields.get(3), 0, SchemaDefProtos.MinorType.BOOLEAN, false, "bool");
assertField(addFields.get(4), 0, SchemaDefProtos.MinorType.VARCHAR4, "test2".getBytes(UTF_8), "str1");
assertField(addFields.get(5), 0, SchemaDefProtos.MinorType.INT, 4, "d");
assertEquals(1, removedFields.size());
@@ -177,7 +178,7 @@ public class JSONRecordReaderTest {
assertEquals(1, jr.next());
assertEquals(8, addFields.size()); // The reappearing of field 'c' is also included
assertField(addFields.get(0), 0, SchemaDefProtos.MinorType.INT, 12345, "test");
- assertField(addFields.get(3), 0, SchemaDefProtos.MinorType.BOOLEAN, 1, "bool");
+ assertField(addFields.get(3), 0, SchemaDefProtos.MinorType.BOOLEAN, true, "bool");
assertField(addFields.get(5), 0, SchemaDefProtos.MinorType.INT, 6, "d");
assertField(addFields.get(6), 0, SchemaDefProtos.MinorType.FLOAT4, (float) 5.16, "c");
assertField(addFields.get(7), 0, SchemaDefProtos.MinorType.VARCHAR4, "test3".getBytes(UTF_8), "str2");
@@ -187,6 +188,7 @@ public class JSONRecordReaderTest {
assertEquals(0, jr.next());
}
+ @Ignore("Pending repeated map implementation")
@Test
public void testNestedFieldInSameBatch(@Injectable final FragmentContext context) throws ExecutionSetupException {
new Expectations() {
@@ -199,7 +201,7 @@ public class JSONRecordReaderTest {
JSONRecordReader jr = new JSONRecordReader(context, getResource("scan_json_test_3.json"));
MockOutputMutator mutator = new MockOutputMutator();
- List<ValueVector> addFields = mutator.getAddFields();
+ List<ValueVector.Base> addFields = mutator.getAddFields();
jr.setup(mutator);
assertEquals(2, jr.next());
assertEquals(5, addFields.size());
[3/7] git commit: Updated value vectors inheritance model. Moved
Mutables to separate Mutator subclasses. Broke VVs into separate files rather
than one large class.
Posted by ja...@apache.org.
Updated value vectors inheritance model.
Moved Mutables to separate Mutator subclasses.
Broke VVs into separate files rather than one large class.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/36793bb2
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/36793bb2
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/36793bb2
Branch: refs/heads/execwork
Commit: 36793bb2f22cd427c8b70d2f5dfe4a4d8a3a6894
Parents: 7075cca
Author: Jacques Nadeau <ja...@apache.org>
Authored: Sat Jul 13 21:28:12 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jul 15 11:36:32 2013 -0700
----------------------------------------------------------------------
sandbox/prototype/exec/java-exec/pom.xml | 2 +-
.../templates/FixedValueVectors.java | 163 ++++
.../templates/NullableValueVectors.java | 151 ++++
.../templates/RepeatedValueVectors.java | 158 ++++
.../ValueVectors/templates/TypeHelper.java | 28 +-
.../ValueVectors/templates/ValueVector.java | 768 -------------------
.../templates/VariableLengthVectors.java | 152 ++++
.../exec/physical/config/MockRecordReader.java | 17 +-
.../physical/config/MockScanBatchCreator.java | 3 -
.../drill/exec/physical/config/MockScanPOP.java | 2 +-
.../exec/physical/impl/FilterRecordBatch.java | 4 +-
.../drill/exec/physical/impl/OutputMutator.java | 5 +-
.../drill/exec/physical/impl/ScanBatch.java | 24 +-
.../exec/physical/impl/WireRecordBatch.java | 7 +-
.../apache/drill/exec/record/BatchSchema.java | 2 -
.../apache/drill/exec/record/RecordBatch.java | 4 +-
.../drill/exec/record/RecordBatchLoader.java | 22 +-
.../apache/drill/exec/record/WritableBatch.java | 10 +-
.../exec/record/vector/SelectionVector.java | 14 +-
.../drill/exec/store/JSONRecordReader.java | 87 ++-
.../apache/drill/exec/store/VectorHolder.java | 11 +-
.../org/apache/drill/exec/vector/BitVector.java | 123 +++
.../apache/drill/exec/vector/ValueVector.java | 192 +++++
.../physical/impl/TestSimpleFragmentRun.java | 6 +-
.../exec/record/vector/TestValueVector.java | 201 +++--
.../drill/exec/store/JSONRecordReaderTest.java | 40 +-
26 files changed, 1212 insertions(+), 984 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/pom.xml b/sandbox/prototype/exec/java-exec/pom.xml
index 4e4df95..1b6dac0 100644
--- a/sandbox/prototype/exec/java-exec/pom.xml
+++ b/sandbox/prototype/exec/java-exec/pom.xml
@@ -173,7 +173,7 @@
<version>1.0</version>
<configuration>
<cfgFile>src/main/codegen/ValueVectors/config.fmpp</cfgFile>
- <outputDirectory>target/generated-sources/org/apache/drill/exec/record/vector</outputDirectory>
+ <outputDirectory>target/generated-sources/org/apache/drill/exec/vector</outputDirectory>
<templateDirectory>src/main/codegen/ValueVectors/templates</templateDirectory>
</configuration>
<executions>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java
new file mode 100644
index 0000000..09dd5d8
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java
@@ -0,0 +1,163 @@
+<@pp.dropOutputFile />
+<#list types as type>
+<#list type.minor as minor>
+
+<#if type.major == "Fixed">
+<@pp.changeOutputFile name="${minor.class}Vector.java" />
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.ByteBuf;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
+import org.apache.drill.exec.record.DeadBuf;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.MsgPack2Vector;
+
+import java.util.Random;
+
+/**
+ * ${minor.class} implements a vector of fixed width values. Elements in the vector are accessed
+ * by position, starting from the logical start of the vector. Values should be pushed onto the
+ * vector sequentially, but may be randomly accessed.
+ * The width of each element is ${type.width} byte(s)
+ * The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+@SuppressWarnings("unused")
+public final class ${minor.class}Vector extends ValueVector {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
+
+ public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ }
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount
+ * The number of values which can be contained within this vector.
+ */
+ public void allocateNew(int valueCount) {
+ totalBytes = valueCount * ${type.width};
+ allocateNew(totalBytes, allocator.buffer(totalBytes), valueCount);
+ }
+
+ @Override
+ public int getAllocatedSize() {
+ return (int) Math.ceil(totalBytes);
+ }
+
+ /**
+ * Get the size requirement (in bytes) for the given number of values. Only accurate
+ * for fixed width value vectors.
+ */
+ @Override
+ public int getSizeFromCount(int valueCount) {
+ return valueCount * ${type.width};
+ }
+
+ public Mutator getMutator() {
+ return new Mutator();
+ }
+
+ <#if (type.width > 8)>
+
+ public ${minor.javaType!type.javaType} get(int index) {
+ ByteBuf dst = allocator.buffer(${type.width});
+ data.getBytes(index * ${type.width}, dst, 0, ${type.width});
+ return dst;
+ }
+
+ @Override
+ public Object getObject(int index) {
+ ByteBuf dst = allocator.buffer(${type.width});
+ data.getBytes(index, dst, 0, ${type.width});
+ return dst;
+ }
+
+
+ <#else> <#-- type.width <= 8 -->
+
+ public ${minor.javaType!type.javaType} get(int index) {
+ return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+ }
+
+ public Object getObject(int index) {
+ return get(index);
+ }
+
+
+ </#if> <#-- type.width -->
+
+
+ /**
+ * ${minor.class}.Mutator implements a mutable vector of fixed width values. Elements in the
+ * vector are accessed by position from the logical start of the vector. Values should be pushed
+ * onto the vector sequentially, but may be randomly accessed.
+ * The width of each element is ${type.width} byte(s)
+ * The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+ public class Mutator implements ValueVector.Mutator{
+
+ private Mutator(){};
+ /**
+ * Set the element at the given index to the given value. Note that widths smaller than
+ * 32 bits are handled by the ByteBuf interface.
+ *
+ * @param index position of the bit to set
+ * @param value value to set
+ */
+ <#if (type.width > 8)>
+ public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+ data.setBytes(index * ${type.width}, value);
+ }
+
+ @Override
+ public void randomizeData() {
+ if (data != DeadBuf.DEAD_BUFFER) {
+ Random r = new Random();
+ for(int i =0; i < data.capacity()-${type.width}; i += ${type.width}){
+ byte[] bytes = new byte[${type.width}];
+ r.nextBytes(bytes);
+ data.setByte(i, bytes[0]);
+ }
+ }
+ }
+
+ <#else> <#-- type.width <= 8 -->
+ public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+ data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value);
+ }
+
+ @Override
+ public void randomizeData() {
+ if (data != DeadBuf.DEAD_BUFFER) {
+ Random r = new Random();
+ for(int i =0; i < data.capacity()-${type.width}; i += ${type.width}){
+ data.set${(minor.javaType!type.javaType)?cap_first}(i,
+ r.next<#if (type.width >= 4)>${(minor.javaType!type.javaType)?cap_first}
+ <#else>Int
+ </#if>());
+ }
+ }
+ }
+ </#if> <#-- type.width -->
+
+ @Override
+ public void setRecordCount(int recordCount) {
+ ${minor.class}Vector.this.setRecordCount(recordCount);
+ }
+
+
+
+
+ }
+}
+
+</#if> <#-- type.major -->
+</#list>
+</#list>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
new file mode 100644
index 0000000..c7de73f
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
@@ -0,0 +1,151 @@
+<@pp.dropOutputFile />
+<#list types as type>
+<#list type.minor as minor>
+<@pp.changeOutputFile name="Nullable${minor.class}Vector.java" />
+package org.apache.drill.exec.vector;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import io.netty.buffer.ByteBuf;
+
+import java.io.Closeable;
+import java.util.Random;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.SchemaDefProtos;
+import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
+import org.apache.drill.exec.record.DeadBuf;
+import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.UInt2Vector;
+import org.apache.drill.exec.vector.UInt4Vector;
+
+/**
+ * Nullable${minor.class} implements a vector of values which could be null. Elements in the vector
+ * are first checked against a fixed length vector of boolean values. Then the element is retrieved
+ * from the base class (if not null).
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+@SuppressWarnings("unused")
+public final class Nullable${minor.class}Vector extends ValueVector {
+
+ private final BitVector bits;
+ private final ${minor.class}Vector values;
+
+ public Nullable${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ bits = new BitVector(null, allocator);
+ values = new ${minor.class}Vector(null, allocator);
+ }
+
+ /**
+ * Get the element at the specified position.
+ *
+ * @param index position of the value
+ * @return value of the element, if not null
+ * @throws NullValueException if the value is null
+ */
+ public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
+ assert !isNull(index);
+ return values.get(index);
+ }
+
+
+ public boolean isNull(int index) {
+ return bits.get(index) == 0;
+ }
+
+ public int isSet(int index){
+ return bits.get(index);
+ }
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount The number of values which may be contained by this vector.
+ */
+ public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
+ values.allocateNew(totalBytes, sourceBuffer, valueCount);
+ bits.allocateNew(valueCount);
+ }
+
+ @Override
+ public int getAllocatedSize() {
+ return bits.getAllocatedSize() + values.getAllocatedSize();
+ }
+
+ /**
+ * Get the size requirement (in bytes) for the given number of values. Only accurate
+ * for fixed width value vectors.
+ */
+ public int getTotalSizeFromCount(int valueCount) {
+ return values.getSizeFromCount(valueCount) + bits.getSizeFromCount(valueCount);
+ }
+
+ public int getSizeFromCount(int valueCount){
+ return getTotalSizeFromCount(valueCount);
+ }
+
+ @Override
+ public MaterializedField getField() {
+ return field;
+ }
+
+ @Override
+ public ByteBuf[] getBuffers() {
+ return new ByteBuf[]{bits.data, values.data};
+ }
+
+
+ @Override
+ public Object getObject(int index) {
+ return isNull(index) ? null : values.getObject(index);
+ }
+
+ public Mutator getMutator(){
+ return new Mutator();
+ }
+
+ public class Mutator implements ValueVector.Mutator{
+
+ private final BitVector.Mutator bitMutator;
+ private final ${minor.class}Vector.Mutator valueMutator;
+
+ private Mutator(){
+ bitMutator = bits.getMutator();
+ valueMutator = values.getMutator();
+ }
+
+ /**
+ * Set the variable length element at the specified index to the supplied byte array.
+ *
+ * @param index position of the bit to set
+ * @param bytes array of bytes to write
+ */
+ public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
+ setNotNull(index);
+ valueMutator.set(index, value);
+ }
+
+ public void setNull(int index) {
+ bitMutator.set(index, 0);
+ }
+
+ private void setNotNull(int index) {
+ bitMutator.set(index, 1);
+ }
+
+ @Override
+ public void setRecordCount(int recordCount) {
+ Nullable${minor.class}Vector.this.setRecordCount(recordCount);
+ bits.setRecordCount(recordCount);
+ }
+
+ public void randomizeData(){
+ throw new UnsupportedOperationException();
+ }
+
+ }
+}
+</#list>
+</#list>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
new file mode 100644
index 0000000..4acc4cc
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
@@ -0,0 +1,158 @@
+import org.apache.drill.exec.vector.UInt2Vector;
+import org.apache.drill.exec.vector.UInt4Vector;
+
+<@pp.dropOutputFile />
+<#list types as type>
+<#list type.minor as minor>
+<@pp.changeOutputFile name="Repeated${minor.class}Vector.java" />
+package org.apache.drill.exec.vector;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import io.netty.buffer.ByteBuf;
+
+import java.io.Closeable;
+import java.util.Random;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.SchemaDefProtos;
+import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
+import org.apache.drill.exec.record.DeadBuf;
+import org.apache.drill.exec.record.MaterializedField;
+
+@SuppressWarnings("unused")
+/**
+ * Repeated${minor.class} implements a vector with multple values per row (e.g. JSON array or
+ * repeated protobuf field). The implementation uses two additional value vectors; one to convert
+ * the index offset to the underlying element offset, and another to store the number of values
+ * in the vector.
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+
+ public final class Repeated${minor.class}Vector extends ValueVector {
+
+ private final UInt2Vector countVector; // number of repeated elements in each record
+ private final UInt4Vector offsetVector; // offsets to start of each record
+ private final ${minor.class}Vector valuesVector;
+
+ public Repeated${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ this.countVector = new UInt2Vector(null, allocator);
+ this.offsetVector = new UInt4Vector(null, allocator);
+ this.valuesVector = new ${minor.class}Vector(null, allocator);
+ }
+
+ public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
+ super.allocateNew(totalBytes, sourceBuffer, valueCount);
+ countVector.allocateNew(valueCount);
+ offsetVector.allocateNew(valueCount);
+ }
+
+
+ /**
+ * Get a value for the given record. Each element in the repeated field is accessed by
+ * the positionIndex param.
+ *
+ * @param index record containing the repeated field
+ * @param positionIndex position within the repeated field
+ * @return element at the given position in the given record
+ */
+ public <#if type.major == "VarLen">byte[]
+ <#else>${minor.javaType!type.javaType}
+ </#if> get(int index, int positionIndex) {
+
+ assert positionIndex < countVector.get(index);
+ return valuesVector.get(offsetVector.get(index) + positionIndex);
+ }
+
+ public MaterializedField getField() {
+ return field;
+ }
+
+ /**
+ * Get the size requirement (in bytes) for the given number of values. Only accurate
+ * for fixed width value vectors.
+ */
+ public int getTotalSizeFromCount(int valueCount) {
+ return valuesVector.getSizeFromCount(valueCount) +
+ countVector.getSizeFromCount(valueCount) +
+ offsetVector.getSizeFromCount(valueCount);
+ }
+
+ public int getSizeFromCount(int valueCount){
+ return getTotalSizeFromCount(valueCount);
+ }
+
+ /**
+ * Get the explicitly specified size of the allocated buffer, if available. Otherwise
+ * calculate the size based on width and record count.
+ */
+ public int getAllocatedSize() {
+ return valuesVector.getAllocatedSize() +
+ countVector.getAllocatedSize() +
+ offsetVector.getAllocatedSize();
+ }
+
+ /**
+ * Get the elements at the given index.
+ */
+ public int getCount(int index) {
+ return countVector.get(index);
+ }
+
+ public ByteBuf[] getBuffers() {
+ return new ByteBuf[]{countVector.data, offsetVector.data, data};
+ }
+
+ public Object getObject(int index) {
+ return data.slice(index, getSizeFromCount(countVector.get(index)));
+ }
+
+ public Mutator getMutator(){
+ return new Mutator();
+ }
+
+ public class Mutator implements ValueVector.Mutator{
+
+
+ private final UInt2Vector.Mutator countMutator;
+ private final ${minor.class}Vector.Mutator valuesMutator;
+ private final UInt4Vector.Mutator offsetMutator;
+
+ private Mutator(){
+ this.countMutator = countVector.getMutator();
+ this.offsetMutator = offsetVector.getMutator();
+ this.valuesMutator = valuesVector.getMutator();
+ }
+
+ /**
+ * Add an element to the given record index. This is similar to the set() method in other
+ * value vectors, except that it permits setting multiple values for a single record.
+ *
+ * @param index record of the element to add
+ * @param value value to add to the given row
+ */
+ public void add(int index, <#if (type.width > 4)> ${minor.javaType!type.javaType}
+ <#elseif type.major == "VarLen"> byte[]
+ <#else> int
+ </#if> value) {
+ countMutator.set(index, countVector.get(index) + 1);
+ offsetMutator.set(index, offsetVector.get(index - 1) + countVector.get(index-1));
+ valuesMutator.set(offsetVector.get(index), value);
+ }
+
+ public void setRecordCount(int recordCount) {
+ valuesMutator.setRecordCount(recordCount);
+ offsetMutator.setRecordCount(recordCount);
+ countMutator.setRecordCount(recordCount);
+ }
+
+ public void randomizeData(){
+ throw new UnsupportedOperationException();
+ }
+
+ }
+}
+</#list>
+</#list>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/TypeHelper.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/TypeHelper.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/TypeHelper.java
index 8dfd3af..b03b842 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/TypeHelper.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/TypeHelper.java
@@ -15,7 +15,7 @@
* See the License for the specific language governing permissions and
* limitations under the License.
******************************************************************************/
-package org.apache.drill.exec.record.vector;
+package org.apache.drill.exec.vector;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.SchemaDefProtos.DataMode;
@@ -57,21 +57,21 @@ public class TypeHelper {
case BOOLEAN:
switch (mode) {
case REQUIRED:
- return ValueVector.${minor.class}.class;
+ return ${minor.class}Vector.class;
case OPTIONAL:
- return ValueVector.Nullable${minor.class}.class;
+ return Nullable${minor.class}Vector.class;
case REPEATED:
- return ValueVector.Repeated${minor.class}.class;
+ return Repeated${minor.class}Vector.class;
}
<#else>
case ${minor.class?upper_case}:
switch (mode) {
case REQUIRED:
- return ValueVector.${minor.class}.class;
+ return ${minor.class}Vector.class;
case OPTIONAL:
- return ValueVector.Nullable${minor.class}.class;
+ return Nullable${minor.class}Vector.class;
case REPEATED:
- return ValueVector.Repeated${minor.class}.class;
+ return Repeated${minor.class}Vector.class;
}
</#if>
</#list>
@@ -83,7 +83,7 @@ public class TypeHelper {
}
- public static ValueVector.Base getNewVector(MaterializedField field, BufferAllocator allocator){
+ public static ValueVector getNewVector(MaterializedField field, BufferAllocator allocator){
MajorType type = field.getType();
switch (type.getMinorType()) {
@@ -93,11 +93,11 @@ public class TypeHelper {
case ${minor.class?upper_case}:
switch (type.getMode()) {
case REQUIRED:
- return new ValueVector.${minor.class}(field, allocator);
+ return new ${minor.class}Vector(field, allocator);
case OPTIONAL:
- return new ValueVector.Nullable${minor.class}(field, allocator);
+ return new Nullable${minor.class}Vector(field, allocator);
case REPEATED:
- return new ValueVector.Repeated${minor.class}(field, allocator);
+ return new Repeated${minor.class}Vector(field, allocator);
}
</#if>
</#list>
@@ -105,11 +105,11 @@ public class TypeHelper {
case BOOLEAN:
switch (type.getMode()) {
case REQUIRED:
- return new ValueVector.Bit(field, allocator);
+ return new BitVector(field, allocator);
case OPTIONAL:
- return new ValueVector.NullableBit(field, allocator);
+ return new NullableBitVector(field, allocator);
case REPEATED:
- return new ValueVector.RepeatedBit(field, allocator);
+ return new RepeatedBitVector(field, allocator);
}
}
// All ValueVector types have been handled.
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueVector.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueVector.java
deleted file mode 100644
index e0e895b..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueVector.java
+++ /dev/null
@@ -1,768 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import static com.google.common.base.Preconditions.checkArgument;
-import static com.google.common.base.Preconditions.checkState;
-import io.netty.buffer.ByteBuf;
-import java.io.Closeable;
-import java.util.Random;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.SchemaDefProtos;
-import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
-import org.apache.drill.exec.record.DeadBuf;
-import org.apache.drill.exec.record.MaterializedField;
-
-/**
- * ValueVectorTypes defines a set of template-generated classes which implement type-specific
- * value vectors. The template approach was chosen due to the lack of multiple inheritence. It
- * is also important that all related logic be as efficient as possible.
- */
-public class ValueVector {
-
- /**
- * ValueVector.Base implements common logic for all immutable value vectors.
- */
- public abstract static class Base implements Closeable {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Base.class);
-
- protected final BufferAllocator allocator;
- protected ByteBuf data = DeadBuf.DEAD_BUFFER;
- protected MaterializedField field;
- protected int recordCount;
- protected int totalBytes;
-
- public Base(MaterializedField field, BufferAllocator allocator) {
- this.allocator = allocator;
- this.field = field;
- }
-
- /**
- * Get the explicitly specified size of the allocated buffer, if available. Otherwise
- * calculate the size based on width and record count.
- */
- public abstract int getAllocatedSize();
-
- /**
- * Get the size requirement (in bytes) for the given number of values. Takes derived
- * type specs into account.
- */
- public abstract int getSizeFromCount(int valueCount);
-
- /**
- * Get the Java Object representation of the element at the specified position
- *
- * @param index Index of the value to get
- */
- public abstract Object getObject(int index);
-
- /**
- * Return the underlying buffers associated with this vector. Note that this doesn't impact the
- * reference counts for this buffer so it only should be used for in-context access. Also note
- * that this buffer changes regularly thus external classes shouldn't hold a reference to
- * it (unless they change it).
- *
- * @return The underlying ByteBuf.
- */
- public ByteBuf[] getBuffers() {
- return new ByteBuf[]{data};
- }
-
- /**
- * Returns the maximum number of values contained within this vector.
- * @return Vector size
- */
- public int capacity() {
- return getRecordCount();
- }
-
- /**
- * Release supporting resources.
- */
- @Override
- public void close() {
- clear();
- }
-
- /**
- * Get information about how this field is materialized.
- * @return
- */
- public MaterializedField getField() {
- return field;
- }
-
- /**
- * Get the number of records allocated for this value vector.
- * @return number of allocated records
- */
- public int getRecordCount() {
- return recordCount;
- }
-
- /**
- * Get the metadata for this field.
- * @return
- */
- public FieldMetadata getMetadata() {
- int len = 0;
- for(ByteBuf b : getBuffers()){
- len += b.writerIndex();
- }
- return FieldMetadata.newBuilder()
- .setDef(getField().getDef())
- .setValueCount(getRecordCount())
- .setBufferLength(len)
- .build();
- }
-
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param totalBytes Optional desired size of the underlying buffer. Specifying 0 will
- * estimate the size based on valueCount.
- * @param sourceBuffer Optional ByteBuf to use for storage (null will allocate automatically).
- * @param valueCount Number of values in the vector.
- */
- public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
- clear();
- this.recordCount = valueCount;
- this.totalBytes = totalBytes > 0 ? totalBytes : getSizeFromCount(valueCount);
- this.data = (sourceBuffer != null) ? sourceBuffer : allocator.buffer(this.totalBytes);
- this.data.retain();
- data.readerIndex(0);
- }
-
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param valueCount
- * The number of elements which can be contained within this vector.
- */
- public void allocateNew(int valueCount) {
- allocateNew(0, null, valueCount);
- }
-
- /**
- * Release the underlying ByteBuf and reset the ValueVector
- */
- protected void clear() {
- if (data != DeadBuf.DEAD_BUFFER) {
- data.release();
- data = DeadBuf.DEAD_BUFFER;
- recordCount = 0;
- totalBytes = 0;
- }
- }
-
- /**
- * Define the number of records that are in this value vector.
- * @param recordCount Number of records active in this vector.
- */
- public void setRecordCount(int recordCount) {
- data.writerIndex(getSizeFromCount(recordCount));
- this.recordCount = recordCount;
- }
-
- /**
- * For testing only -- randomize the buffer contents
- */
- public void randomizeData() { }
-
- }
-
- /**
- * Bit implements a vector of bit-width values. Elements in the vector are accessed
- * by position from the logical start of the vector.
- * The width of each element is 1 bit.
- * The equivilent Java primitive is an int containing the value '0' or '1'.
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
- public static class Bit extends Base {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Bit.class);
-
- public Bit(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- /**
- * Get the byte holding the desired bit, then mask all other bits. Iff the result is 0, the
- * bit was not set.
- *
- * @param index position of the bit in the vector
- * @return 1 if set, otherwise 0
- */
- public int get(int index) {
- // logger.debug("BIT GET: index: {}, byte: {}, mask: {}, masked byte: {}",
- // index,
- // data.getByte((int)Math.floor(index/8)),
- // (int)Math.pow(2, (index % 8)),
- // data.getByte((int)Math.floor(index/8)) & (int)Math.pow(2, (index % 8)));
- return ((data.getByte((int)Math.floor(index/8)) & (int)Math.pow(2, (index % 8))) == 0) ? 0 : 1;
- }
-
- @Override
- public Object getObject(int index) {
- return new Boolean(get(index) != 0);
- }
-
- /**
- * Get the size requirement (in bytes) for the given number of values.
- */
- @Override
- public int getSizeFromCount(int valueCount) {
- return (int) Math.ceil(valueCount / 8);
- }
-
- @Override
- public int getAllocatedSize() {
- return totalBytes;
- }
-
- public MutableBit getMutable() {
- return (MutableBit)this;
- }
-
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param valueCount The number of values which can be contained within this vector.
- */
- @Override
- public void allocateNew(int valueCount) {
- allocateNew(getSizeFromCount(valueCount), null, valueCount);
- for (int i = 0; i < getSizeFromCount(valueCount); i++) {
- data.setByte(i, 0);
- }
- }
-
- }
-
- /**
- * MutableBit implements a vector of bit-width values. Elements in the vector are accessed
- * by position from the logical start of the vector. Values should be pushed onto the vector
- * sequentially, but may be randomly accessed.
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
- public static class MutableBit extends Bit {
-
- public MutableBit(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- /**
- * Set the bit at the given index to the specified value.
- *
- * @param index position of the bit to set
- * @param value value to set (either 1 or 0)
- */
- public void set(int index, int value) {
- byte currentByte = data.getByte((int)Math.floor(index/8));
- if (value != 0) {
- // true
- currentByte |= (byte) Math.pow(2, (index % 8));
- }
- else if ((currentByte & (byte) Math.pow(2, (index % 8))) == (byte) Math.pow(2, (index % 8))) {
- // false, and bit was previously set
- currentByte -= (byte) Math.pow(2, (index % 8));
- }
- data.setByte((int) Math.floor(index/8), currentByte);
- }
-
- @Override
- public void randomizeData() {
- if (this.data != DeadBuf.DEAD_BUFFER) {
- Random r = new Random();
- for (int i = 0; i < data.capacity() - 1; i++) {
- byte[] bytes = new byte[1];
- r.nextBytes(bytes);
- data.setByte(i, bytes[0]);
- }
- }
- }
- }
-
-<#list types as type>
- <#list type.minor as minor>
- <#if type.major == "Fixed">
-
- /**
- * ${minor.class} implements a vector of fixed width values. Elements in the vector are accessed
- * by position, starting from the logical start of the vector. Values should be pushed onto the
- * vector sequentially, but may be randomly accessed.
- * The width of each element is ${type.width} byte(s)
- * The equivilent Java primitive is '${minor.javaType!type.javaType}'
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
- public static class ${minor.class} extends Base {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}.class);
-
- public ${minor.class}(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param valueCount
- * The number of values which can be contained within this vector.
- */
- public void allocateNew(int valueCount) {
- totalBytes = valueCount * ${type.width};
- allocateNew(totalBytes, allocator.buffer(totalBytes), valueCount);
- }
-
- @Override
- public int getAllocatedSize() {
- return (int) Math.ceil(totalBytes);
- }
-
- /**
- * Get the size requirement (in bytes) for the given number of values. Only accurate
- * for fixed width value vectors.
- */
- @Override
- public int getSizeFromCount(int valueCount) {
- return valueCount * ${type.width};
- }
-
- public Mutable${minor.class} getMutable() {
- return (Mutable${minor.class})this;
- }
-
- <#if (type.width > 8)>
-
- public ${minor.javaType!type.javaType} get(int index) {
- ByteBuf dst = allocator.buffer(${type.width});
- data.getBytes(index * ${type.width}, dst, 0, ${type.width});
- return dst;
- }
-
- @Override
- public Object getObject(int index) {
- ByteBuf dst = allocator.buffer(${type.width});
- data.getBytes(index, dst, 0, ${type.width});
- return dst;
- }
-
- @Override
- public void randomizeData() {
- if (this.data != DeadBuf.DEAD_BUFFER) {
- Random r = new Random();
- for(int i =0; i < data.capacity()-${type.width}; i += ${type.width}){
- byte[] bytes = new byte[${type.width}];
- r.nextBytes(bytes);
- data.setByte(i, bytes[0]);
- }
- }
- }
-
- <#else> <#-- type.width <= 8 -->
-
- public ${minor.javaType!type.javaType} get(int index) {
- return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
- }
-
- public Object getObject(int index) {
- return get(index);
- }
-
- @Override
- public void randomizeData() {
- if (this.data != DeadBuf.DEAD_BUFFER) {
- Random r = new Random();
- for(int i =0; i < data.capacity()-${type.width}; i += ${type.width}){
- data.set${(minor.javaType!type.javaType)?cap_first}(i,
- r.next<#if (type.width >= 4)>${(minor.javaType!type.javaType)?cap_first}
- <#else>Int
- </#if>());
- }
- }
- }
-
- </#if> <#-- type.width -->
- }
-
- /**
- * Mutable${minor.class} implements a mutable vector of fixed width values. Elements in the
- * vector are accessed by position from the logical start of the vector. Values should be pushed
- * onto the vector sequentially, but may be randomly accessed.
- * The width of each element is ${type.width} byte(s)
- * The equivilent Java primitive is '${minor.javaType!type.javaType}'
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
- public static class Mutable${minor.class} extends ${minor.class} {
-
- public Mutable${minor.class}(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- /**
- * Set the element at the given index to the given value. Note that widths smaller than
- * 32 bits are handled by the ByteBuf interface.
- *
- * @param index position of the bit to set
- * @param value value to set
- */
- <#if (type.width > 8)>
- public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
- data.setBytes(index * ${type.width}, value);
- <#else> <#-- type.width <= 8 -->
- public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
- data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value);
- </#if> <#-- type.width -->
- }
- }
-
- <#elseif type.major == "VarLen">
-
- /**
- * ${minor.class} implements a vector of variable width values. Elements in the vector
- * are accessed by position from the logical start of the vector. A fixed width lengthVector
- * is used to convert an element's position to it's offset from the start of the (0-based)
- * ByteBuf. Size is inferred by adjacent elements.
- * The width of each element is ${type.width} byte(s)
- * The equivilent Java primitive is '${minor.javaType!type.javaType}'
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
- public static class ${minor.class} extends Base {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}.class);
-
- protected final MutableUInt${type.width} lengthVector;
-
- public ${minor.class}(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- this.lengthVector = new MutableUInt${type.width}(null, allocator);
- }
-
- public byte[] get(int index) {
- checkArgument(index >= 0);
- int startIdx = 0;
- int size = 0;
- if (index == 0) {
- size = lengthVector.get(1);
- } else {
- startIdx = lengthVector.get(index);
- size = lengthVector.get(index + 1) - startIdx;
- }
- checkState(size >= 0);
- byte[] dst = new byte[size];
- data.getBytes(startIdx, dst, 0, size);
- return dst;
- }
-
- @Override
- public int getAllocatedSize() {
- return lengthVector.getAllocatedSize() + totalBytes;
- }
-
- /**
- * Get the size requirement (in bytes) for the given number of values. Only accurate
- * for fixed width value vectors.
- */
- public int getSizeFromCount(int valueCount) {
- return valueCount * ${type.width};
- }
-
- @Override
- protected void clear() {
- super.clear();
- lengthVector.clear();
- }
-
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param valueCount
- * The number of values which can be contained within this vector.
- */
- public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
- super.allocateNew(totalBytes, sourceBuffer, valueCount);
- lengthVector.allocateNew(valueCount);
- }
-
- @Override
- public ByteBuf[] getBuffers() {
- return new ByteBuf[]{lengthVector.data, data};
- }
-
- public Object getObject(int index) {
- return get(index);
- }
-
- public Mutable${minor.class} getMutable() {
- return (Mutable${minor.class})this;
- }
- }
-
- /**
- * Mutable${minor.class} implements a vector of variable width values. Elements in the vector
- * are accessed by position from the logical start of the vector. A fixed width lengthVector
- * is used to convert an element's position to it's offset from the start of the (0-based)
- * ByteBuf. Size is inferred by adjacent elements.
- * The width of each element is ${type.width} byte(s)
- * The equivilent Java primitive is '${minor.javaType!type.javaType}'
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
- public static class Mutable${minor.class} extends ${minor.class} {
-
- public Mutable${minor.class}(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- }
-
- /**
- * Set the variable length element at the specified index to the supplied byte array.
- *
- * @param index position of the bit to set
- * @param bytes array of bytes to write
- */
- public void set(int index, byte[] bytes) {
- checkArgument(index >= 0);
- if (index == 0) {
- lengthVector.set(0, 0);
- lengthVector.set(1, bytes.length);
- data.setBytes(0, bytes);
- }
- else {
- int currentOffset = lengthVector.get(index);
- // set the end offset of the buffer
- lengthVector.set(index + 1, currentOffset + bytes.length);
- data.setBytes(currentOffset, bytes);
- }
- }
-
- @Override
- public void setRecordCount(int recordCount) {
- super.setRecordCount(recordCount);
- lengthVector.setRecordCount(recordCount);
- }
-
- }
-
- </#if> <#-- type.major -->
-
- /**
- * Nullable${minor.class} implements a vector of values which could be null. Elements in the vector
- * are first checked against a fixed length vector of boolean values. Then the element is retrieved
- * from the base class (if not null).
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
- public static class Nullable${minor.class} extends Mutable${minor.class} {
-
- protected MutableBit bits;
-
- public Nullable${minor.class}(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- bits = new MutableBit(null, allocator);
- }
-
- /**
- * Set the variable length element at the specified index to the supplied byte array.
- *
- * @param index position of the bit to set
- * @param bytes array of bytes to write
- */
- public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
- setNotNull(index);
- super.set(index, value);
- }
-
- /**
- * Get the element at the specified position.
- *
- * @param index position of the value
- * @return value of the element, if not null
- * @throws NullValueException if the value is null
- */
- public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
- if (isNull(index))
- throw new NullValueException(index);
- return super.get(index);
- }
-
- public void setNull(int index) {
- bits.set(index, 0);
- }
-
- private void setNotNull(int index) {
- bits.set(index, 1);
- }
-
- public boolean isNull(int index) {
- return bits.get(index) == 0;
- }
-
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param valueCount The number of values which may be contained by this vector.
- */
- public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
- super.allocateNew(totalBytes, sourceBuffer, valueCount);
- bits.allocateNew(valueCount);
- }
-
- @Override
- public int getAllocatedSize() {
- return bits.getAllocatedSize() + super.getAllocatedSize();
- }
-
- /**
- * Get the size requirement (in bytes) for the given number of values. Only accurate
- * for fixed width value vectors.
- */
- public int getTotalSizeFromCount(int valueCount) {
- return getSizeFromCount(valueCount) + bits.getSizeFromCount(valueCount);
- }
-
- @Override
- public MaterializedField getField() {
- return field;
- }
-
- @Override
- public ByteBuf[] getBuffers() {
- return new ByteBuf[]{bits.data, super.data};
- }
-
- @Override
- public void setRecordCount(int recordCount) {
- super.setRecordCount(recordCount);
- bits.setRecordCount(recordCount);
- }
-
- @Override
- public Object getObject(int index) {
- return isNull(index) ? null : super.getObject(index);
- }
- }
-
- /**
- * Repeated${minor.class} implements a vector with multple values per row (e.g. JSON array or
- * repeated protobuf field). The implementation uses two additional value vectors; one to convert
- * the index offset to the underlying element offset, and another to store the number of values
- * in the vector.
- *
- * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
- */
- public static class Repeated${minor.class} extends Mutable${minor.class} {
-
- private MutableUInt4 countVector; // number of repeated elements in each record
- private MutableUInt4 offsetVector; // offsets to start of each record
-
- public Repeated${minor.class}(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- countVector = new MutableUInt4(null, allocator);
- offsetVector = new MutableUInt4(null, allocator);
- }
-
- public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
- super.allocateNew(totalBytes, sourceBuffer, valueCount);
- countVector.allocateNew(valueCount);
- offsetVector.allocateNew(valueCount);
- }
-
- /**
- * Add an element to the given record index. This is similar to the set() method in other
- * value vectors, except that it permits setting multiple values for a single record.
- *
- * @param index record of the element to add
- * @param value value to add to the given row
- */
- public void add(int index, <#if (type.width > 4)> ${minor.javaType!type.javaType}
- <#elseif type.major == "VarLen"> byte[]
- <#else> int
- </#if> value) {
- countVector.set(index, countVector.get(index) + 1);
- offsetVector.set(index, offsetVector.get(index - 1) + countVector.get(index-1));
- super.set(offsetVector.get(index), value);
- }
-
- /**
- * Get a value for the given record. Each element in the repeated field is accessed by
- * the positionIndex param.
- *
- * @param index record containing the repeated field
- * @param positionIndex position within the repeated field
- * @return element at the given position in the given record
- */
- public <#if type.major == "VarLen">byte[]
- <#else>${minor.javaType!type.javaType}
- </#if> get(int index, int positionIndex) {
-
- assert positionIndex < countVector.get(index);
- return super.get(offsetVector.get(index) + positionIndex);
- }
-
- public MaterializedField getField() {
- return field;
- }
-
- /**
- * Get the size requirement (in bytes) for the given number of values. Only accurate
- * for fixed width value vectors.
- */
- public int getTotalSizeFromCount(int valueCount) {
- return getSizeFromCount(valueCount) +
- countVector.getSizeFromCount(valueCount) +
- offsetVector.getSizeFromCount(valueCount);
- }
-
- /**
- * Get the explicitly specified size of the allocated buffer, if available. Otherwise
- * calculate the size based on width and record count.
- */
- public int getAllocatedSize() {
- return super.getAllocatedSize() +
- countVector.getAllocatedSize() +
- offsetVector.getAllocatedSize();
- }
-
- /**
- * Get the elements at the given index.
- */
- public int getCount(int index) {
- return countVector.get(index);
- }
-
- public void setRecordCount(int recordCount) {
- super.setRecordCount(recordCount);
- offsetVector.setRecordCount(recordCount);
- countVector.setRecordCount(recordCount);
- }
-
- public ByteBuf[] getBuffers() {
- return new ByteBuf[]{countVector.data, offsetVector.data, data};
- }
-
- public Object getObject(int index) {
- return data.slice(index, getSizeFromCount(countVector.get(index)));
- }
-
- }
- </#list>
-</#list>
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
new file mode 100644
index 0000000..954836a
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
@@ -0,0 +1,152 @@
+<@pp.dropOutputFile />
+<#list types as type>
+<#list type.minor as minor>
+
+<#if type.major == "VarLen">
+<@pp.changeOutputFile name="${minor.class}Vector.java" />
+package org.apache.drill.exec.vector;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import io.netty.buffer.ByteBuf;
+
+import java.io.Closeable;
+import java.util.Random;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.SchemaDefProtos;
+import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
+import org.apache.drill.exec.record.DeadBuf;
+import org.apache.drill.exec.record.MaterializedField;
+
+/**
+ * ${minor.class}Vector implements a vector of variable width values. Elements in the vector
+ * are accessed by position from the logical start of the vector. A fixed width lengthVector
+ * is used to convert an element's position to it's offset from the start of the (0-based)
+ * ByteBuf. Size is inferred by adjacent elements.
+ * The width of each element is ${type.width} byte(s)
+ * The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+@SuppressWarnings("unused")
+public final class ${minor.class}Vector extends ValueVector {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
+
+ private final UInt${type.width}Vector lengthVector;
+ private final UInt${type.width}Vector.Mutator lengthVectorMutator;
+
+ public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ this.lengthVector = new UInt${type.width}Vector(null, allocator);
+ this.lengthVectorMutator = lengthVector.getMutator();
+ }
+
+ public byte[] get(int index) {
+ checkArgument(index >= 0);
+ int startIdx = 0;
+ int size = 0;
+ if (index == 0) {
+ size = lengthVector.get(1);
+ } else {
+ startIdx = lengthVector.get(index);
+ size = lengthVector.get(index + 1) - startIdx;
+ }
+ checkState(size >= 0);
+ byte[] dst = new byte[size];
+ data.getBytes(startIdx, dst, 0, size);
+ return dst;
+ }
+
+ @Override
+ public int getAllocatedSize() {
+ return lengthVector.getAllocatedSize() + totalBytes;
+ }
+
+ /**
+ * Get the size requirement (in bytes) for the given number of values. Only accurate
+ * for fixed width value vectors.
+ */
+ public int getSizeFromCount(int valueCount) {
+ return valueCount * ${type.width};
+ }
+
+ @Override
+ protected void clear() {
+ super.clear();
+ lengthVector.clear();
+ }
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount
+ * The number of values which can be contained within this vector.
+ */
+ public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
+ super.allocateNew(totalBytes, sourceBuffer, valueCount);
+ lengthVector.allocateNew(valueCount);
+ }
+
+ @Override
+ public ByteBuf[] getBuffers() {
+ return new ByteBuf[]{lengthVector.data, data};
+ }
+
+ public Object getObject(int index) {
+ return get(index);
+ }
+
+ public Mutator getMutator() {
+ return new Mutator();
+ }
+
+
+ /**
+ * Mutable${minor.class} implements a vector of variable width values. Elements in the vector
+ * are accessed by position from the logical start of the vector. A fixed width lengthVector
+ * is used to convert an element's position to it's offset from the start of the (0-based)
+ * ByteBuf. Size is inferred by adjacent elements.
+ * The width of each element is ${type.width} byte(s)
+ * The equivalent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+ public class Mutator implements ValueVector.Mutator{
+
+ /**
+ * Set the variable length element at the specified index to the supplied byte array.
+ *
+ * @param index position of the bit to set
+ * @param bytes array of bytes to write
+ */
+ public void set(int index, byte[] bytes) {
+ checkArgument(index >= 0);
+ if (index == 0) {
+ lengthVectorMutator.set(0, 0);
+ lengthVectorMutator.set(1, bytes.length);
+ data.setBytes(0, bytes);
+ } else {
+ int currentOffset = lengthVector.get(index);
+ // set the end offset of the buffer
+ lengthVectorMutator.set(index + 1, currentOffset + bytes.length);
+ data.setBytes(currentOffset, bytes);
+ }
+ }
+
+ @Override
+ public void setRecordCount(int recordCount) {
+ ${minor.class}Vector.this.setRecordCount(recordCount);
+ lengthVector.setRecordCount(recordCount);
+ }
+
+ @Override
+ public void randomizeData(){}
+ }
+
+}
+
+
+</#if> <#-- type.major -->
+</#list>
+</#list>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
index 65584db..cd3371d 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
@@ -27,9 +27,9 @@ import org.apache.drill.exec.physical.impl.OutputMutator;
import org.apache.drill.exec.proto.SchemaDefProtos.DataMode;
import org.apache.drill.exec.proto.SchemaDefProtos.MajorType;
import org.apache.drill.exec.record.MaterializedField;
-import org.apache.drill.exec.record.vector.TypeHelper;
-import org.apache.drill.exec.record.vector.ValueVector;
import org.apache.drill.exec.store.RecordReader;
+import org.apache.drill.exec.vector.TypeHelper;
+import org.apache.drill.exec.vector.ValueVector;
public class MockRecordReader implements RecordReader {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockRecordReader.class);
@@ -37,7 +37,7 @@ public class MockRecordReader implements RecordReader {
private OutputMutator output;
private MockScanEntry config;
private FragmentContext context;
- private ValueVector.Base[] valueVectors;
+ private ValueVector[] valueVectors;
private int recordsRead;
public MockRecordReader(FragmentContext context, MockScanEntry config) {
@@ -53,12 +53,12 @@ public class MockRecordReader implements RecordReader {
return x;
}
- private ValueVector.Base getVector(int fieldId, String name, MajorType type, int length) {
+ private ValueVector getVector(int fieldId, String name, MajorType type, int length) {
assert context != null : "Context shouldn't be null.";
if(type.getMode() != DataMode.REQUIRED) throw new UnsupportedOperationException();
MaterializedField f = MaterializedField.create(new SchemaPath(name), fieldId, 0, type);
- ValueVector.Base v;
+ ValueVector v;
v = TypeHelper.getNewVector(f, context.getAllocator());
v.allocateNew(length);
return v;
@@ -70,7 +70,7 @@ public class MockRecordReader implements RecordReader {
try {
this.output = output;
int estimateRowSize = getEstimatedRecordSize(config.getTypes());
- valueVectors = new ValueVector.Base[config.getTypes().length];
+ valueVectors = new ValueVector[config.getTypes().length];
int batchRecordCount = 250000 / estimateRowSize;
for (int i = 0; i < config.getTypes().length; i++) {
@@ -88,10 +88,11 @@ public class MockRecordReader implements RecordReader {
public int next() {
int recordSetSize = Math.min(valueVectors[0].capacity(), this.config.getRecords()- recordsRead);
recordsRead += recordSetSize;
- for(ValueVector.Base v : valueVectors){
+ for(ValueVector v : valueVectors){
logger.debug("MockRecordReader: Generating random data for VV of type " + v.getClass().getName());
v.randomizeData();
- v.setRecordCount(recordSetSize);
+
+ v.getMutator().setRecordCount(recordSetSize);
}
return recordSetSize;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockScanBatchCreator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockScanBatchCreator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockScanBatchCreator.java
index b821d6e..bfc19af 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockScanBatchCreator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockScanBatchCreator.java
@@ -24,10 +24,7 @@ import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.physical.config.MockScanPOP.MockScanEntry;
import org.apache.drill.exec.physical.impl.BatchCreator;
import org.apache.drill.exec.physical.impl.ScanBatch;
-import org.apache.drill.exec.record.BatchSchema;
-import org.apache.drill.exec.record.InvalidValueAccessor;
import org.apache.drill.exec.record.RecordBatch;
-import org.apache.drill.exec.record.vector.ValueVector;
import org.apache.drill.exec.store.RecordReader;
import com.google.common.base.Preconditions;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockScanPOP.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockScanPOP.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockScanPOP.java
index 4a3a606..40227e5 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockScanPOP.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockScanPOP.java
@@ -32,7 +32,7 @@ import org.apache.drill.exec.proto.CoordinationProtos.DrillbitEndpoint;
import org.apache.drill.exec.proto.SchemaDefProtos.DataMode;
import org.apache.drill.exec.proto.SchemaDefProtos.MajorType;
import org.apache.drill.exec.proto.SchemaDefProtos.MinorType;
-import org.apache.drill.exec.record.vector.TypeHelper;
+import org.apache.drill.exec.vector.TypeHelper;
import com.fasterxml.jackson.annotation.JsonCreator;
import com.fasterxml.jackson.annotation.JsonIgnore;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
index b7b7d93..6440d98 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
@@ -22,7 +22,7 @@ import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.InvalidValueAccessor;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.vector.SelectionVector;
-import org.apache.drill.exec.record.vector.ValueVector;
+import org.apache.drill.exec.vector.ValueVector;
public abstract class FilterRecordBatch implements RecordBatch {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FilterRecordBatch.class);
@@ -58,7 +58,7 @@ public abstract class FilterRecordBatch implements RecordBatch {
}
@Override
- public <T extends ValueVector.Base> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
+ public <T extends ValueVector> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java
index e96c1be..b3b9f5f 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java
@@ -18,11 +18,10 @@
package org.apache.drill.exec.physical.impl;
import org.apache.drill.exec.exception.SchemaChangeException;
-import org.apache.drill.exec.record.BatchSchema;
-import org.apache.drill.exec.record.vector.ValueVector;
+import org.apache.drill.exec.vector.ValueVector;
public interface OutputMutator {
public void removeField(int fieldId) throws SchemaChangeException;
- public void addField(int fieldId, ValueVector.Base vector) throws SchemaChangeException ;
+ public void addField(int fieldId, ValueVector vector) throws SchemaChangeException ;
public void setNewSchema() throws SchemaChangeException ;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index 822d828..2f3e1fe 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -17,27 +17,21 @@
******************************************************************************/
package org.apache.drill.exec.physical.impl;
-import io.netty.buffer.ByteBuf;
-
import java.util.Iterator;
-import java.util.List;
import org.apache.drill.common.exceptions.ExecutionSetupException;
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
-import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef;
import org.apache.drill.exec.record.BatchSchema;
import org.apache.drill.exec.record.InvalidValueAccessor;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.SchemaBuilder;
import org.apache.drill.exec.record.WritableBatch;
-import org.apache.drill.exec.record.vector.ValueVector;
import org.apache.drill.exec.store.RecordReader;
+import org.apache.drill.exec.vector.ValueVector;
import com.carrotsearch.hppc.IntObjectOpenHashMap;
import com.carrotsearch.hppc.procedures.IntObjectProcedure;
-import com.google.common.collect.Lists;
/**
* Record batch used for a particular scan. Operators against one or more
@@ -45,7 +39,7 @@ import com.google.common.collect.Lists;
public class ScanBatch implements RecordBatch {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScanBatch.class);
- private IntObjectOpenHashMap<ValueVector.Base> fields = new IntObjectOpenHashMap<ValueVector.Base>();
+ private IntObjectOpenHashMap<ValueVector> fields = new IntObjectOpenHashMap<ValueVector>();
private BatchSchema schema;
private int recordCount;
private boolean schemaChanged = true;
@@ -89,9 +83,9 @@ public class ScanBatch implements RecordBatch {
}
private void releaseAssets() {
- fields.forEach(new IntObjectProcedure<ValueVector.Base>() {
+ fields.forEach(new IntObjectProcedure<ValueVector>() {
@Override
- public void apply(int key, ValueVector.Base value) {
+ public void apply(int key, ValueVector value) {
value.close();
}
});
@@ -99,9 +93,9 @@ public class ScanBatch implements RecordBatch {
@SuppressWarnings("unchecked")
@Override
- public <T extends ValueVector.Base> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
+ public <T extends ValueVector> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
if (fields.containsKey(fieldId)) throw new InvalidValueAccessor(String.format("Unknown value accesor for field id %d."));
- ValueVector.Base vector = this.fields.lget();
+ ValueVector vector = this.fields.lget();
if (vector.getClass().isAssignableFrom(clazz)) {
return (T) vector;
} else {
@@ -143,14 +137,14 @@ public class ScanBatch implements RecordBatch {
public void removeField(int fieldId) throws SchemaChangeException {
schemaChanged();
- ValueVector.Base v = fields.remove(fieldId);
+ ValueVector v = fields.remove(fieldId);
if (v == null) throw new SchemaChangeException("Failure attempting to remove an unknown field.");
v.close();
}
- public void addField(int fieldId, ValueVector.Base vector) {
+ public void addField(int fieldId, ValueVector vector) {
schemaChanged();
- ValueVector.Base v = fields.put(fieldId, vector);
+ ValueVector v = fields.put(fieldId, vector);
vector.getField();
builder.addField(vector.getField());
if (v != null) v.close();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
index be32d1f..fcbd272 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
@@ -17,8 +17,6 @@
******************************************************************************/
package org.apache.drill.exec.physical.impl;
-import java.util.Iterator;
-
import org.apache.drill.exec.exception.SchemaChangeException;
import org.apache.drill.exec.ops.FragmentContext;
import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef;
@@ -29,8 +27,7 @@ import org.apache.drill.exec.record.RawFragmentBatchProvider;
import org.apache.drill.exec.record.RecordBatch;
import org.apache.drill.exec.record.RecordBatchLoader;
import org.apache.drill.exec.record.WritableBatch;
-import org.apache.drill.exec.record.RecordBatch.IterOutcome;
-import org.apache.drill.exec.record.vector.ValueVector;
+import org.apache.drill.exec.vector.ValueVector;
public class WireRecordBatch implements RecordBatch{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(WireRecordBatch.class);
@@ -68,7 +65,7 @@ public class WireRecordBatch implements RecordBatch{
}
@Override
- public <T extends ValueVector.Base> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
+ public <T extends ValueVector> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
return batchLoader.getValueVector(fieldId, clazz);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
index 05b1cc7..b26e742 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/BatchSchema.java
@@ -20,8 +20,6 @@ package org.apache.drill.exec.record;
import java.util.Iterator;
import java.util.List;
-import org.apache.drill.exec.record.vector.ValueVector;
-
public class BatchSchema implements Iterable<MaterializedField> {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BatchSchema.class);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
index 042c40c..c6b7888 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
@@ -18,7 +18,7 @@
package org.apache.drill.exec.record;
import org.apache.drill.exec.ops.FragmentContext;
-import org.apache.drill.exec.record.vector.ValueVector;
+import org.apache.drill.exec.vector.ValueVector;
/**
* A record batch contains a set of field values for a particular range of records. In the case of a record batch
@@ -68,7 +68,7 @@ public interface RecordBatch {
public void kill();
- public abstract <T extends ValueVector.Base> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor;
+ public abstract <T extends ValueVector> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor;
// public abstract void getDictReader(int fieldId, Class<?> clazz) throws InvalidValueAccessor;
//
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
index ea1de73..be43026 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
@@ -27,16 +27,16 @@ import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.SchemaDefProtos.FieldDef;
import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef;
-import org.apache.drill.exec.record.vector.TypeHelper;
-import org.apache.drill.exec.record.vector.ValueVector;
+import org.apache.drill.exec.vector.TypeHelper;
+import org.apache.drill.exec.vector.ValueVector;
import com.carrotsearch.hppc.IntObjectOpenHashMap;
import com.carrotsearch.hppc.cursors.IntObjectCursor;
-public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector.Base>>{
+public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector>>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RecordBatchLoader.class);
- private IntObjectOpenHashMap<ValueVector.Base> vectors = new IntObjectOpenHashMap<ValueVector.Base>();
+ private IntObjectOpenHashMap<ValueVector> vectors = new IntObjectOpenHashMap<ValueVector>();
private final BufferAllocator allocator;
private int recordCount;
private BatchSchema schema;
@@ -61,14 +61,14 @@ public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector.B
this.recordCount = def.getRecordCount();
boolean schemaChanged = false;
- IntObjectOpenHashMap<ValueVector.Base> newVectors = new IntObjectOpenHashMap<ValueVector.Base>();
+ IntObjectOpenHashMap<ValueVector> newVectors = new IntObjectOpenHashMap<ValueVector>();
List<FieldMetadata> fields = def.getFieldList();
int bufOffset = 0;
for (FieldMetadata fmd : fields) {
FieldDef fieldDef = fmd.getDef();
- ValueVector.Base v = vectors.remove(fieldDef.getFieldId());
+ ValueVector v = vectors.remove(fieldDef.getFieldId());
if (v != null) {
if (v.getField().getDef().equals(fieldDef)) {
v.allocateNew(fmd.getBufferLength(), buf.slice(bufOffset, fmd.getBufferLength()), recordCount);
@@ -89,7 +89,7 @@ public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector.B
if(!vectors.isEmpty()){
schemaChanged = true;
- for(IntObjectCursor<ValueVector.Base> cursor : newVectors){
+ for(IntObjectCursor<ValueVector> cursor : newVectors){
cursor.value.close();
}
@@ -98,7 +98,7 @@ public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector.B
if(schemaChanged){
// rebuild the schema.
SchemaBuilder b = BatchSchema.newBuilder();
- for(IntObjectCursor<ValueVector.Base> cursor : newVectors){
+ for(IntObjectCursor<ValueVector> cursor : newVectors){
b.addField(cursor.value.getField());
}
b.setSelectionVector(false);
@@ -110,8 +110,8 @@ public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector.B
}
@SuppressWarnings("unchecked")
- public <T extends ValueVector.Base> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
- ValueVector.Base v = vectors.get(fieldId);
+ public <T extends ValueVector> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
+ ValueVector v = vectors.get(fieldId);
assert v != null;
if (v.getClass() != clazz)
throw new InvalidValueAccessor(String.format(
@@ -130,7 +130,7 @@ public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector.B
}
@Override
- public Iterator<IntObjectCursor<ValueVector.Base>> iterator() {
+ public Iterator<IntObjectCursor<ValueVector>> iterator() {
return this.vectors.iterator();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
index 67c6cb9..a367b6d 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
@@ -23,7 +23,7 @@ import java.util.List;
import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
import org.apache.drill.exec.proto.UserBitShared.RecordBatchDef;
-import org.apache.drill.exec.record.vector.ValueVector;
+import org.apache.drill.exec.vector.ValueVector;
import com.carrotsearch.hppc.IntObjectOpenHashMap;
import com.carrotsearch.hppc.procedures.IntObjectProcedure;
@@ -58,7 +58,7 @@ public class WritableBatch {
return buffers;
}
-// public static WritableBatch get(ValueVector.Base[] vectors){
+// public static WritableBatch get(ValueVector[] vectors){
// WritableCreator c = new WritableCreator();
// for(int i =0; i < vectors.length; i++){
// c.apply(i, vectors[i]);
@@ -67,14 +67,14 @@ public class WritableBatch {
// }
//
- public static WritableBatch get(int recordCount, IntObjectOpenHashMap<ValueVector.Base> fields){
+ public static WritableBatch get(int recordCount, IntObjectOpenHashMap<ValueVector> fields){
WritableCreator creator = new WritableCreator(recordCount);
fields.forEach(creator);
return creator.get();
}
- private static class WritableCreator implements IntObjectProcedure<ValueVector.Base>{
+ private static class WritableCreator implements IntObjectProcedure<ValueVector>{
List<ByteBuf> buffers = Lists.newArrayList();
List<FieldMetadata> metadata = Lists.newArrayList();
@@ -87,7 +87,7 @@ public class WritableBatch {
}
@Override
- public void apply(int key, ValueVector.Base value) {
+ public void apply(int key, ValueVector value) {
metadata.add(value.getMetadata());
for(ByteBuf b : value.getBuffers()){
buffers.add(b);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/36793bb2/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java
index e79a525..02b75ce 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/SelectionVector.java
@@ -17,19 +17,25 @@
******************************************************************************/
package org.apache.drill.exec.record.vector;
-import io.netty.buffer.ByteBufAllocator;
-
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.record.MaterializedField;
/**
* Convenience/Clarification Fixed2 wrapper.
*/
-public class SelectionVector extends ValueVector.UInt2 {
+public class SelectionVector {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(SelectionVector.class);
public SelectionVector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
+
+ }
+
+ public int capacity() {
+ return -1;
+ }
+
+ public void allocateNew(int count) {
+
}
}
[6/7] git commit: Separate allocate and load methods. rename
setRecordCount to setValueCount add setGroupAndValueCount to RepeatedVectors.
add a number of marker/cross-inheritance interfaces.
Posted by ja...@apache.org.
Separate allocate and load methods.
rename setRecordCount to setValueCount
add setGroupAndValueCount to RepeatedVectors.
add a number of marker/cross-inheritance interfaces.
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/9ca9eb9b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/9ca9eb9b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/9ca9eb9b
Branch: refs/heads/execwork
Commit: 9ca9eb9b3d88e86e28d1b688d9cd943e6a7f08df
Parents: 36793bb
Author: Jacques Nadeau <ja...@apache.org>
Authored: Mon Jul 15 10:50:07 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jul 15 13:13:59 2013 -0700
----------------------------------------------------------------------
.../templates/FixedValueVectors.java | 118 +++++---
.../templates/NullableValueVectors.java | 220 ++++++++++-----
.../templates/RepeatedValueVectors.java | 273 ++++++++++++++-----
.../templates/VariableLengthVectors.java | 177 +++++++-----
.../exec/physical/config/MockRecordReader.java | 37 ++-
.../drill/exec/record/MaterializedField.java | 16 ++
.../drill/exec/record/RecordBatchLoader.java | 7 +-
.../apache/drill/exec/record/WritableBatch.java | 3 +-
.../drill/exec/store/JSONRecordReader.java | 4 +-
.../apache/drill/exec/store/VectorHolder.java | 21 +-
.../drill/exec/vector/BaseDataValueVector.java | 47 ++++
.../drill/exec/vector/BaseValueVector.java | 38 +++
.../org/apache/drill/exec/vector/BitVector.java | 112 +++++---
.../apache/drill/exec/vector/ByteHolder.java | 12 +
.../drill/exec/vector/FixedWidthVector.java | 23 ++
.../drill/exec/vector/NonRepeatedMutator.java | 7 +
.../exec/vector/RepeatedFixedWidthVector.java | 22 ++
.../vector/RepeatedVariableWidthVector.java | 24 ++
.../apache/drill/exec/vector/ValueVector.java | 179 ++++--------
.../drill/exec/vector/VariableWidthVector.java | 29 ++
.../apache/drill/exec/work/foreman/Foreman.java | 5 +-
.../physical/impl/TestSimpleFragmentRun.java | 2 +-
.../exec/record/vector/TestValueVector.java | 81 +++---
.../drill/exec/store/JSONRecordReaderTest.java | 2 +-
24 files changed, 995 insertions(+), 464 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java
index 09dd5d8..7583d9f 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/FixedValueVectors.java
@@ -12,6 +12,7 @@ import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
import org.apache.drill.exec.record.DeadBuf;
import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.BaseValueVector;
import org.apache.drill.exec.vector.MsgPack2Vector;
import java.util.Random;
@@ -26,71 +27,100 @@ import java.util.Random;
* NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
*/
@SuppressWarnings("unused")
-public final class ${minor.class}Vector extends ValueVector {
+public final class ${minor.class}Vector extends BaseDataValueVector implements FixedWidthVector{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
+
+ private final Accessor accessor = new Accessor();
+ private final Mutator mutator = new Mutator();
+
public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
super(field, allocator);
}
+ public int getValueCapacity(){
+ return (int) (data.capacity() *1.0 / ${type.width});
+ }
+
+ public Accessor getAccessor(){
+ return accessor;
+ }
+
+ public Mutator getMutator(){
+ return mutator;
+ }
+
/**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
+ * Allocate a new buffer that supports setting at least the provided number of values. May actually be sized bigger depending on underlying buffer rounding size. Must be called prior to using the ValueVector.
* @param valueCount
- * The number of values which can be contained within this vector.
*/
public void allocateNew(int valueCount) {
- totalBytes = valueCount * ${type.width};
- allocateNew(totalBytes, allocator.buffer(totalBytes), valueCount);
+ clear();
+ this.data = allocator.buffer(valueCount * ${type.width});
+ this.data.retain();
+ this.data.readerIndex(0);
}
-
+
@Override
- public int getAllocatedSize() {
- return (int) Math.ceil(totalBytes);
+ public FieldMetadata getMetadata() {
+ return FieldMetadata.newBuilder()
+ .setDef(getField().getDef())
+ .setValueCount(recordCount)
+ .setBufferLength(recordCount * ${type.width})
+ .build();
}
- /**
- * Get the size requirement (in bytes) for the given number of values. Only accurate
- * for fixed width value vectors.
- */
@Override
- public int getSizeFromCount(int valueCount) {
- return valueCount * ${type.width};
+ public int load(int valueCount, ByteBuf buf){
+ clear();
+ this.recordCount = valueCount;
+ int len = recordCount * ${type.width};
+ data = buf.slice(0, len);
+ data.retain();
+ return len;
}
-
- public Mutator getMutator() {
- return new Mutator();
+
+ @Override
+ public void load(FieldMetadata metadata, ByteBuf buffer) {
+ assert this.field.getDef().equals(metadata.getDef());
+ int loaded = load(metadata.getValueCount(), buffer);
+ assert metadata.getBufferLength() == loaded;
}
+
+ public final class Accessor extends BaseValueVector.BaseAccessor{
- <#if (type.width > 8)>
-
- public ${minor.javaType!type.javaType} get(int index) {
- ByteBuf dst = allocator.buffer(${type.width});
- data.getBytes(index * ${type.width}, dst, 0, ${type.width});
- return dst;
- }
+ public int getRecordCount() {
+ return recordCount;
+ }
+
+ <#if (type.width > 8)>
- @Override
- public Object getObject(int index) {
- ByteBuf dst = allocator.buffer(${type.width});
- data.getBytes(index, dst, 0, ${type.width});
- return dst;
- }
+ public ${minor.javaType!type.javaType} get(int index) {
+ ByteBuf dst = allocator.buffer(${type.width});
+ data.getBytes(index * ${type.width}, dst, 0, ${type.width});
+ return dst;
+ }
+ @Override
+ public Object getObject(int index) {
+ ByteBuf dst = allocator.buffer(${type.width});
+ data.getBytes(index, dst, 0, ${type.width});
+ return dst;
+ }
- <#else> <#-- type.width <= 8 -->
+ <#else> <#-- type.width <= 8 -->
- public ${minor.javaType!type.javaType} get(int index) {
- return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
- }
+ public ${minor.javaType!type.javaType} get(int index) {
+ return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+ }
- public Object getObject(int index) {
- return get(index);
- }
+ public Object getObject(int index) {
+ return get(index);
+ }
- </#if> <#-- type.width -->
-
+ </#if> <#-- type.width -->
+ }
/**
* ${minor.class}.Mutator implements a mutable vector of fixed width values. Elements in the
@@ -101,7 +131,7 @@ public final class ${minor.class}Vector extends ValueVector {
*
* NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
*/
- public class Mutator implements ValueVector.Mutator{
+ public final class Mutator extends BaseValueVector.BaseMutator{
private Mutator(){};
/**
@@ -147,9 +177,9 @@ public final class ${minor.class}Vector extends ValueVector {
}
</#if> <#-- type.width -->
- @Override
- public void setRecordCount(int recordCount) {
- ${minor.class}Vector.this.setRecordCount(recordCount);
+ public void setValueCount(int recordCount) {
+ ${minor.class}Vector.this.recordCount = recordCount;
+ data.writerIndex(${type.width} * recordCount);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
index c7de73f..3232f87 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/NullableValueVectors.java
@@ -10,12 +10,15 @@ import io.netty.buffer.ByteBuf;
import java.io.Closeable;
import java.util.Random;
+import java.util.Vector;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.SchemaDefProtos;
import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
import org.apache.drill.exec.record.DeadBuf;
import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.BaseValueVector;
+import org.apache.drill.exec.vector.BitVector;
import org.apache.drill.exec.vector.UInt2Vector;
import org.apache.drill.exec.vector.UInt4Vector;
@@ -27,93 +30,180 @@ import org.apache.drill.exec.vector.UInt4Vector;
* NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
*/
@SuppressWarnings("unused")
-public final class Nullable${minor.class}Vector extends ValueVector {
+public final class Nullable${minor.class}Vector extends BaseValueVector implements <#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector {
+ private int recordCount;
private final BitVector bits;
private final ${minor.class}Vector values;
+ private final Accessor accessor = new Accessor();
+ private final Mutator mutator = new Mutator();
public Nullable${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
super(field, allocator);
- bits = new BitVector(null, allocator);
- values = new ${minor.class}Vector(null, allocator);
+ this.bits = new BitVector(null, allocator);
+ this.values = new ${minor.class}Vector(null, allocator);
}
-
- /**
- * Get the element at the specified position.
- *
- * @param index position of the value
- * @return value of the element, if not null
- * @throws NullValueException if the value is null
- */
- public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
- assert !isNull(index);
- return values.get(index);
+
+ public int getValueCapacity(){
+ return bits.getValueCapacity();
+ }
+
+ @Override
+ public ByteBuf[] getBuffers() {
+ return new ByteBuf[]{bits.data, values.data};
}
-
- public boolean isNull(int index) {
- return bits.get(index) == 0;
+ @Override
+ public void clear() {
+ recordCount = 0;
+ bits.clear();
+ values.clear();
+ }
+
+ int getBufferSize(){
+ return values.getBufferSize() + bits.getBufferSize();
}
- public int isSet(int index){
- return bits.get(index);
+ <#if type.major == "VarLen">
+ @Override
+ public FieldMetadata getMetadata() {
+ return FieldMetadata.newBuilder()
+ .setDef(getField().getDef())
+ .setValueCount(recordCount)
+ .setVarByteLength(values.getVarByteLength())
+ .setBufferLength(getBufferSize())
+ .build();
}
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param valueCount The number of values which may be contained by this vector.
- */
- public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
- values.allocateNew(totalBytes, sourceBuffer, valueCount);
+ @Override
+ public void allocateNew(int totalBytes, int valueCount) {
+ values.allocateNew(totalBytes, valueCount);
bits.allocateNew(valueCount);
+ mutator.reset();
+ accessor.reset();
}
@Override
- public int getAllocatedSize() {
- return bits.getAllocatedSize() + values.getAllocatedSize();
+ public int load(int dataBytes, int valueCount, ByteBuf buf){
+ clear();
+ this.recordCount = valueCount;
+ int loaded = bits.load(valueCount, buf);
+
+ // remove bits part of buffer.
+ buf = buf.slice(loaded, buf.capacity() - loaded);
+ loaded += values.load(dataBytes, valueCount, buf);
+ return loaded;
}
-
- /**
- * Get the size requirement (in bytes) for the given number of values. Only accurate
- * for fixed width value vectors.
- */
- public int getTotalSizeFromCount(int valueCount) {
- return values.getSizeFromCount(valueCount) + bits.getSizeFromCount(valueCount);
+
+ @Override
+ public void load(FieldMetadata metadata, ByteBuf buffer) {
+ assert this.field.getDef().equals(metadata.getDef());
+ int loaded = load(metadata.getVarByteLength(), metadata.getValueCount(), buffer);
+ assert metadata.getBufferLength() == loaded;
}
- public int getSizeFromCount(int valueCount){
- return getTotalSizeFromCount(valueCount);
+ @Override
+ public int getByteCapacity(){
+ return values.getByteCapacity();
}
+ <#else>
@Override
- public MaterializedField getField() {
- return field;
+ public FieldMetadata getMetadata() {
+ return FieldMetadata.newBuilder()
+ .setDef(getField().getDef())
+ .setValueCount(recordCount)
+ .setBufferLength(getBufferSize())
+ .build();
}
-
+
@Override
- public ByteBuf[] getBuffers() {
- return new ByteBuf[]{bits.data, values.data};
+ public void allocateNew(int valueCount) {
+ values.allocateNew(valueCount);
+ bits.allocateNew(valueCount);
+ mutator.reset();
+ accessor.reset();
}
-
-
+
+ @Override
+ public int load(int valueCount, ByteBuf buf){
+ clear();
+ this.recordCount = valueCount;
+ int loaded = bits.load(valueCount, buf);
+
+ // remove bits part of buffer.
+ buf = buf.slice(loaded, buf.capacity() - loaded);
+ loaded += values.load(valueCount, buf);
+ return loaded;
+ }
+
@Override
- public Object getObject(int index) {
- return isNull(index) ? null : values.getObject(index);
+ public void load(FieldMetadata metadata, ByteBuf buffer) {
+ assert this.field.getDef().equals(metadata.getDef());
+ int loaded = load(metadata.getValueCount(), buffer);
+ assert metadata.getBufferLength() == loaded;
+ }
+
+ </#if>
+
+ public Accessor getAccessor(){
+ return accessor;
}
public Mutator getMutator(){
- return new Mutator();
+ return mutator;
+ }
+
+ public ${minor.class}Vector convertToRequiredVector(){
+ ${minor.class}Vector v = new ${minor.class}Vector(getField().getOtherNullableVersion(), allocator);
+ v.data = values.data;
+ v.recordCount = this.recordCount;
+ v.data.retain();
+ clear();
+ return v;
}
- public class Mutator implements ValueVector.Mutator{
+
+
+ public final class Accessor implements ValueVector.Accessor{
- private final BitVector.Mutator bitMutator;
- private final ${minor.class}Vector.Mutator valueMutator;
+ /**
+ * Get the element at the specified position.
+ *
+ * @param index position of the value
+ * @return value of the element, if not null
+ * @throws NullValueException if the value is null
+ */
+ public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
+ assert !isNull(index);
+ return values.getAccessor().get(index);
+ }
+
+ public boolean isNull(int index) {
+ return isSet(index) == 0;
+ }
+
+ public int isSet(int index){
+ return bits.getAccessor().get(index);
+ }
+
+ @Override
+ public Object getObject(int index) {
+ return isNull(index) ? null : values.getAccessor().getObject(index);
+ }
+
+ public int getRecordCount(){
+ return recordCount;
+ }
+
+ public void reset(){}
+ }
+
+ public final class Mutator implements ValueVector.Mutator{
+
+ private int setCount;
private Mutator(){
- bitMutator = bits.getMutator();
- valueMutator = values.getMutator();
}
/**
@@ -123,28 +213,30 @@ public final class Nullable${minor.class}Vector extends ValueVector {
* @param bytes array of bytes to write
*/
public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
- setNotNull(index);
- valueMutator.set(index, value);
- }
-
- public void setNull(int index) {
- bitMutator.set(index, 0);
+ setCount++;
+ bits.getMutator().set(index, 1);
+ values.getMutator().set(index, value);
}
- private void setNotNull(int index) {
- bitMutator.set(index, 1);
+ public void setValueCount(int recordCount) {
+ assert recordCount >= 0;
+ Nullable${minor.class}Vector.this.recordCount = recordCount;
+ values.getMutator().setValueCount(recordCount);
+ bits.getMutator().setValueCount(recordCount);
}
- @Override
- public void setRecordCount(int recordCount) {
- Nullable${minor.class}Vector.this.setRecordCount(recordCount);
- bits.setRecordCount(recordCount);
+ public boolean noNulls(){
+ return recordCount == setCount;
}
public void randomizeData(){
throw new UnsupportedOperationException();
}
+ public void reset(){
+ setCount = 0;
+ }
+
}
}
</#list>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
index 4acc4cc..363e4c8 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/RepeatedValueVectors.java
@@ -7,12 +7,18 @@ import org.apache.drill.exec.vector.UInt4Vector;
<@pp.changeOutputFile name="Repeated${minor.class}Vector.java" />
package org.apache.drill.exec.vector;
+
+
+
+
+
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import io.netty.buffer.ByteBuf;
import java.io.Closeable;
import java.util.Random;
+import java.util.Vector;
import org.apache.drill.exec.memory.BufferAllocator;
import org.apache.drill.exec.proto.SchemaDefProtos;
@@ -30,100 +36,216 @@ import org.apache.drill.exec.record.MaterializedField;
* NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
*/
- public final class Repeated${minor.class}Vector extends ValueVector {
-
- private final UInt2Vector countVector; // number of repeated elements in each record
- private final UInt4Vector offsetVector; // offsets to start of each record
- private final ${minor.class}Vector valuesVector;
+ public final class Repeated${minor.class}Vector extends BaseValueVector implements Repeated<#if type.major == "VarLen">VariableWidth<#else>FixedWidth</#if>Vector {
+ private MaterializedField field;
+
+ private int parentValueCount;
+ private int childValueCount;
+
+ private final UInt2Vector counts; // number of repeated elements in each record
+ private final UInt4Vector offsets; // offsets to start of each record
+ private final ${minor.class}Vector values;
+ private final Mutator mutator = new Mutator();
+ private final Accessor accessor = new Accessor();
+
+
public Repeated${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
super(field, allocator);
- this.countVector = new UInt2Vector(null, allocator);
- this.offsetVector = new UInt4Vector(null, allocator);
- this.valuesVector = new ${minor.class}Vector(null, allocator);
+ this.counts = new UInt2Vector(null, allocator);
+ this.offsets = new UInt4Vector(null, allocator);
+ this.values = new ${minor.class}Vector(null, allocator);
}
- public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
- super.allocateNew(totalBytes, sourceBuffer, valueCount);
- countVector.allocateNew(valueCount);
- offsetVector.allocateNew(valueCount);
+ public int getValueCapacity(){
+ return values.getValueCapacity();
+ }
+
+ int getBufferSize(){
+ return counts.getBufferSize() + offsets.getBufferSize() + values.getBufferSize();
+ }
+
+ <#if type.major == "VarLen">
+ @Override
+ public FieldMetadata getMetadata() {
+ return FieldMetadata.newBuilder()
+ .setDef(getField().getDef())
+ .setGroupCount(this.parentValueCount)
+ .setValueCount(this.childValueCount)
+ .setVarByteLength(values.getVarByteLength())
+ .setBufferLength(getBufferSize())
+ .build();
+ }
+
+ public void allocateNew(int totalBytes, int parentValueCount, int childValueCount) {
+ counts.allocateNew(parentValueCount);
+ offsets.allocateNew(parentValueCount);
+ values.allocateNew(totalBytes, childValueCount);
+ mutator.reset();
+ accessor.reset();
}
+
- /**
- * Get a value for the given record. Each element in the repeated field is accessed by
- * the positionIndex param.
- *
- * @param index record containing the repeated field
- * @param positionIndex position within the repeated field
- * @return element at the given position in the given record
- */
- public <#if type.major == "VarLen">byte[]
- <#else>${minor.javaType!type.javaType}
- </#if> get(int index, int positionIndex) {
-
- assert positionIndex < countVector.get(index);
- return valuesVector.get(offsetVector.get(index) + positionIndex);
+
+ @Override
+ public int load(int dataBytes, int parentValueCount, int childValueCount, ByteBuf buf){
+ clear();
+ this.parentValueCount = parentValueCount;
+ this.childValueCount = childValueCount;
+ int loaded = 0;
+ loaded += counts.load(parentValueCount, buf);
+ loaded += offsets.load(parentValueCount, buf.slice(loaded, buf.capacity() - loaded));
+ loaded += values.load(dataBytes, childValueCount, buf.slice(loaded, buf.capacity() - loaded));
+ return loaded;
}
-
- public MaterializedField getField() {
- return field;
+
+ @Override
+ public void load(FieldMetadata metadata, ByteBuf buffer) {
+ assert this.field.getDef().equals(metadata.getDef());
+ int loaded = load(metadata.getVarByteLength(), metadata.getGroupCount(), metadata.getValueCount(), buffer);
+ assert metadata.getBufferLength() == loaded;
+ }
+
+ public int getByteCapacity(){
+ return values.getByteCapacity();
}
- /**
- * Get the size requirement (in bytes) for the given number of values. Only accurate
- * for fixed width value vectors.
- */
- public int getTotalSizeFromCount(int valueCount) {
- return valuesVector.getSizeFromCount(valueCount) +
- countVector.getSizeFromCount(valueCount) +
- offsetVector.getSizeFromCount(valueCount);
+ <#else>
+
+ @Override
+ public FieldMetadata getMetadata() {
+ return FieldMetadata.newBuilder()
+ .setDef(getField().getDef())
+ .setGroupCount(this.parentValueCount)
+ .setValueCount(this.childValueCount)
+ .setBufferLength(getBufferSize())
+ .build();
}
- public int getSizeFromCount(int valueCount){
- return getTotalSizeFromCount(valueCount);
+ public void allocateNew(int parentValueCount, int childValueCount) {
+ clear();
+ values.allocateNew(childValueCount);
+ counts.allocateNew(parentValueCount);
+ offsets.allocateNew(parentValueCount);
+ mutator.reset();
+ accessor.reset();
}
-
- /**
- * Get the explicitly specified size of the allocated buffer, if available. Otherwise
- * calculate the size based on width and record count.
- */
- public int getAllocatedSize() {
- return valuesVector.getAllocatedSize() +
- countVector.getAllocatedSize() +
- offsetVector.getAllocatedSize();
+
+ public int load(int parentValueCount, int childValueCount, ByteBuf buf){
+ clear();
+ this.parentValueCount = parentValueCount;
+ this.childValueCount = childValueCount;
+ int loaded = 0;
+ loaded += counts.load(parentValueCount, buf);
+ loaded += offsets.load(parentValueCount, buf.slice(loaded, buf.capacity() - loaded));
+ loaded += values.load(childValueCount, buf.slice(loaded, buf.capacity() - loaded));
+ return loaded;
}
-
- /**
- * Get the elements at the given index.
- */
- public int getCount(int index) {
- return countVector.get(index);
+
+ @Override
+ public void load(FieldMetadata metadata, ByteBuf buffer) {
+ assert this.field.getDef().equals(metadata.getDef());
+ int loaded = load(metadata.getGroupCount(), metadata.getValueCount(), buffer);
+ assert metadata.getBufferLength() == loaded;
}
+ </#if>
+
+// /**
+// * Get the size requirement (in bytes) for the given number of values. Only accurate
+// * for fixed width value vectors.
+// */
+// public int getTotalSizeFromCount(int valueCount) {
+// return values.getSizeFromCount(valueCount) +
+// counts.getSizeFromCount(valueCount) +
+// offsets.getSizeFromCount(valueCount);
+// }
+//
+// public int getSizeFromCount(int valueCount){
+// return getTotalSizeFromCount(valueCount);
+// }
+
+// /**
+// * Get the explicitly specified size of the allocated buffer, if available. Otherwise
+// * calculate the size based on width and record count.
+// */
+// public int getAllocatedSize() {
+// return values.getAllocatedSize() +
+// counts.getAllocatedSize() +
+// offsets.getAllocatedSize();
+// }
+
+
public ByteBuf[] getBuffers() {
- return new ByteBuf[]{countVector.data, offsetVector.data, data};
+ return new ByteBuf[]{counts.data, offsets.data, values.data};
}
- public Object getObject(int index) {
- return data.slice(index, getSizeFromCount(countVector.get(index)));
+ public void clear(){
+ counts.clear();
+ offsets.clear();
+ values.clear();
+ parentValueCount = 0;
+ childValueCount = 0;
}
public Mutator getMutator(){
- return new Mutator();
+ return mutator;
+ }
+
+ public Accessor getAccessor(){
+ return accessor;
}
- public class Mutator implements ValueVector.Mutator{
+ public final class Accessor implements ValueVector.Accessor{
+ /**
+ * Get the elements at the given index.
+ */
+ public int getCount(int index) {
+ return counts.getAccessor().get(index);
+ }
+
+ public Object getObject(int index) {
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Get a value for the given record. Each element in the repeated field is accessed by
+ * the positionIndex param.
+ *
+ * @param index record containing the repeated field
+ * @param positionIndex position within the repeated field
+ * @return element at the given position in the given record
+ */
+ public <#if type.major == "VarLen">byte[]
+ <#else>${minor.javaType!type.javaType}
+ </#if> get(int index, int positionIndex) {
+ assert positionIndex < counts.getAccessor().get(index);
+ return values.getAccessor().get(offsets.getAccessor().get(index) + positionIndex);
+ }
+
+ public MaterializedField getField() {
+ return field;
+ }
+
+ public int getGroupCount(){
+ return parentValueCount;
+ }
+
+ public int getValueCount(){
+ return childValueCount;
+ }
- private final UInt2Vector.Mutator countMutator;
- private final ${minor.class}Vector.Mutator valuesMutator;
- private final UInt4Vector.Mutator offsetMutator;
+ public void reset(){
+
+ }
+ }
+
+ public final class Mutator implements ValueVector.Mutator{
+
private Mutator(){
- this.countMutator = countVector.getMutator();
- this.offsetMutator = offsetVector.getMutator();
- this.valuesMutator = valuesVector.getMutator();
}
/**
@@ -137,21 +259,28 @@ import org.apache.drill.exec.record.MaterializedField;
<#elseif type.major == "VarLen"> byte[]
<#else> int
</#if> value) {
- countMutator.set(index, countVector.get(index) + 1);
- offsetMutator.set(index, offsetVector.get(index - 1) + countVector.get(index-1));
- valuesMutator.set(offsetVector.get(index), value);
+ counts.getMutator().set(index, counts.getAccessor().get(index) + 1);
+ offsets.getMutator().set(index, offsets.getAccessor().get(index - 1) + counts.getAccessor().get(index-1));
+ values.getMutator().set(offsets.getAccessor().get(index), value);
}
+
- public void setRecordCount(int recordCount) {
- valuesMutator.setRecordCount(recordCount);
- offsetMutator.setRecordCount(recordCount);
- countMutator.setRecordCount(recordCount);
+ public void setGroupAndValueCount(int groupCount, int valueCount) {
+ parentValueCount = groupCount;
+ childValueCount = valueCount;
+ counts.getMutator().setValueCount(groupCount);
+ offsets.getMutator().setValueCount(groupCount);
+ values.getMutator().setValueCount(valueCount);
}
public void randomizeData(){
throw new UnsupportedOperationException();
}
+ public void reset(){
+
+ }
+
}
}
</#list>
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
index 954836a..c615258 100644
--- a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/VariableLengthVectors.java
@@ -11,6 +11,7 @@ import static com.google.common.base.Preconditions.checkState;
import io.netty.buffer.ByteBuf;
import java.io.Closeable;
+import java.nio.ByteBuffer;
import java.util.Random;
import org.apache.drill.exec.memory.BufferAllocator;
@@ -18,10 +19,11 @@ import org.apache.drill.exec.proto.SchemaDefProtos;
import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
import org.apache.drill.exec.record.DeadBuf;
import org.apache.drill.exec.record.MaterializedField;
+import org.apache.drill.exec.vector.ByteHolder;
/**
* ${minor.class}Vector implements a vector of variable width values. Elements in the vector
- * are accessed by position from the logical start of the vector. A fixed width lengthVector
+ * are accessed by position from the logical start of the vector. A fixed width offsetVector
* is used to convert an element's position to it's offset from the start of the (0-based)
* ByteBuf. Size is inferred by adjacent elements.
* The width of each element is ${type.width} byte(s)
@@ -30,81 +32,125 @@ import org.apache.drill.exec.record.MaterializedField;
* NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
*/
@SuppressWarnings("unused")
-public final class ${minor.class}Vector extends ValueVector {
+public final class ${minor.class}Vector extends BaseDataValueVector implements VariableWidthVector{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}Vector.class);
- private final UInt${type.width}Vector lengthVector;
- private final UInt${type.width}Vector.Mutator lengthVectorMutator;
-
+ private final UInt${type.width}Vector offsetVector;
+ private final Accessor accessor = new Accessor();
+ private final Mutator mutator = new Mutator();
+
public ${minor.class}Vector(MaterializedField field, BufferAllocator allocator) {
super(field, allocator);
- this.lengthVector = new UInt${type.width}Vector(null, allocator);
- this.lengthVectorMutator = lengthVector.getMutator();
+ this.offsetVector = new UInt${type.width}Vector(null, allocator);
}
- public byte[] get(int index) {
- checkArgument(index >= 0);
- int startIdx = 0;
- int size = 0;
- if (index == 0) {
- size = lengthVector.get(1);
- } else {
- startIdx = lengthVector.get(index);
- size = lengthVector.get(index + 1) - startIdx;
- }
- checkState(size >= 0);
- byte[] dst = new byte[size];
- data.getBytes(startIdx, dst, 0, size);
- return dst;
- }
- @Override
- public int getAllocatedSize() {
- return lengthVector.getAllocatedSize() + totalBytes;
+ int getSizeFromCount(int valueCount) {
+ return valueCount * ${type.width};
}
-
+
+ public int getValueCapacity(){
+ return offsetVector.getValueCapacity();
+ }
+
+ public int getByteCapacity(){
+ return data.capacity();
+ }
+
/**
- * Get the size requirement (in bytes) for the given number of values. Only accurate
- * for fixed width value vectors.
+ * Return the number of bytes contained in the current var len byte vector.
+ * @return
*/
- public int getSizeFromCount(int valueCount) {
- return valueCount * ${type.width};
+ public int getVarByteLength(){
+ return offsetVector.getAccessor().get(recordCount);
}
-
+
@Override
- protected void clear() {
- super.clear();
- lengthVector.clear();
+ public FieldMetadata getMetadata() {
+ int len = recordCount * ${type.width} + getVarByteLength();
+ return FieldMetadata.newBuilder()
+ .setDef(getField().getDef())
+ .setValueCount(recordCount)
+ .setVarByteLength(getVarByteLength())
+ .setBufferLength(len)
+ .build();
}
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param valueCount
- * The number of values which can be contained within this vector.
- */
- public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
- super.allocateNew(totalBytes, sourceBuffer, valueCount);
- lengthVector.allocateNew(valueCount);
+ public int load(int dataBytes, int valueCount, ByteBuf buf){
+ this.recordCount = valueCount;
+ int loaded = offsetVector.load(valueCount+1, buf);
+ data = buf.slice(loaded, dataBytes);
+ data.retain();
+ return loaded + dataBytes;
+ }
+
+ @Override
+ public void load(FieldMetadata metadata, ByteBuf buffer) {
+ assert this.field.getDef().equals(metadata.getDef());
+ int loaded = load(metadata.getVarByteLength(), metadata.getValueCount(), buffer);
+ assert metadata.getBufferLength() == loaded;
+ }
+
+ @Override
+ public void clear() {
+ super.clear();
+ offsetVector.clear();
}
@Override
public ByteBuf[] getBuffers() {
- return new ByteBuf[]{lengthVector.data, data};
+ return new ByteBuf[]{offsetVector.data, this.data};
}
-
- public Object getObject(int index) {
- return get(index);
+
+ public void allocateNew(int totalBytes, int valueCount) {
+ clear();
+ assert totalBytes >= 0;
+ data = allocator.buffer(totalBytes);
+ data.retain();
+ data.readerIndex(0);
+ offsetVector.allocateNew(valueCount+1);
}
+ public Accessor getAccessor(){
+ return accessor;
+ }
+
public Mutator getMutator() {
- return new Mutator();
+ return mutator;
}
+ public final class Accessor extends BaseValueVector.BaseAccessor{
+
+ public byte[] get(int index) {
+ assert index >= 0;
+ int startIdx = offsetVector.getAccessor().get(index);
+ int length = offsetVector.getAccessor().get(index + 1) - startIdx;
+ assert length >= 0;
+ byte[] dst = new byte[length];
+ data.getBytes(startIdx, dst, 0, length);
+ return dst;
+ }
+
+ public void get(int index, ByteHolder holder){
+ assert index >= 0;
+ holder.start = offsetVector.getAccessor().get(index);
+ holder.length = offsetVector.getAccessor().get(index + 1) - holder.start;
+ assert holder.length >= 0;
+ holder.buffer = offsetVector.data;
+ }
+
+ public Object getObject(int index) {
+ return get(index);
+ }
+
+ public int getRecordCount() {
+ return recordCount;
+ }
+ }
/**
* Mutable${minor.class} implements a vector of variable width values. Elements in the vector
- * are accessed by position from the logical start of the vector. A fixed width lengthVector
+ * are accessed by position from the logical start of the vector. A fixed width offsetVector
* is used to convert an element's position to it's offset from the start of the (0-based)
* ByteBuf. Size is inferred by adjacent elements.
* The width of each element is ${type.width} byte(s)
@@ -112,7 +158,7 @@ public final class ${minor.class}Vector extends ValueVector {
*
* NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
*/
- public class Mutator implements ValueVector.Mutator{
+ public final class Mutator extends BaseValueVector.BaseMutator{
/**
* Set the variable length element at the specified index to the supplied byte array.
@@ -121,23 +167,24 @@ public final class ${minor.class}Vector extends ValueVector {
* @param bytes array of bytes to write
*/
public void set(int index, byte[] bytes) {
- checkArgument(index >= 0);
- if (index == 0) {
- lengthVectorMutator.set(0, 0);
- lengthVectorMutator.set(1, bytes.length);
- data.setBytes(0, bytes);
- } else {
- int currentOffset = lengthVector.get(index);
- // set the end offset of the buffer
- lengthVectorMutator.set(index + 1, currentOffset + bytes.length);
- data.setBytes(currentOffset, bytes);
- }
+ assert index >= 0;
+ int currentOffset = offsetVector.getAccessor().get(index);
+ offsetVector.getMutator().set(index + 1, currentOffset + bytes.length);
+ data.setBytes(currentOffset, bytes);
}
- @Override
- public void setRecordCount(int recordCount) {
- ${minor.class}Vector.this.setRecordCount(recordCount);
- lengthVector.setRecordCount(recordCount);
+ public void set(int index, int start, int length, ByteBuf buffer){
+ assert index >= 0;
+ int currentOffset = offsetVector.getAccessor().get(index);
+ offsetVector.getMutator().set(index + 1, currentOffset + length);
+ ByteBuf bb = buffer.slice(start, length);
+ data.setBytes(currentOffset, bb);
+ }
+
+ public void setValueCount(int recordCount) {
+ ${minor.class}Vector.this.recordCount = recordCount;
+ data.writerIndex(recordCount * ${type.width});
+ offsetVector.getMutator().setValueCount(recordCount+1);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
index cd3371d..0f4619c 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
@@ -28,8 +28,11 @@ import org.apache.drill.exec.proto.SchemaDefProtos.DataMode;
import org.apache.drill.exec.proto.SchemaDefProtos.MajorType;
import org.apache.drill.exec.record.MaterializedField;
import org.apache.drill.exec.store.RecordReader;
+import org.apache.drill.exec.vector.FixedWidthVector;
+import org.apache.drill.exec.vector.NonRepeatedMutator;
import org.apache.drill.exec.vector.TypeHelper;
import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VariableWidthVector;
public class MockRecordReader implements RecordReader {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MockRecordReader.class);
@@ -39,6 +42,7 @@ public class MockRecordReader implements RecordReader {
private FragmentContext context;
private ValueVector[] valueVectors;
private int recordsRead;
+ private int batchRecordCount;
public MockRecordReader(FragmentContext context, MockScanEntry config) {
this.context = context;
@@ -60,7 +64,14 @@ public class MockRecordReader implements RecordReader {
MaterializedField f = MaterializedField.create(new SchemaPath(name), fieldId, 0, type);
ValueVector v;
v = TypeHelper.getNewVector(f, context.getAllocator());
- v.allocateNew(length);
+ if(v instanceof FixedWidthVector){
+ ((FixedWidthVector)v).allocateNew(length);
+ }else if(v instanceof VariableWidthVector){
+ ((VariableWidthVector)v).allocateNew(50*length, length);
+ }else{
+ throw new UnsupportedOperationException(String.format("Unable to get allocate vector %s", v.getClass().getName()));
+ }
+
return v;
}
@@ -71,7 +82,7 @@ public class MockRecordReader implements RecordReader {
this.output = output;
int estimateRowSize = getEstimatedRecordSize(config.getTypes());
valueVectors = new ValueVector[config.getTypes().length];
- int batchRecordCount = 250000 / estimateRowSize;
+ batchRecordCount = 250000 / estimateRowSize;
for (int i = 0; i < config.getTypes().length; i++) {
valueVectors[i] = getVector(i, config.getTypes()[i].getName(), config.getTypes()[i].getMajorType(), batchRecordCount);
@@ -86,13 +97,29 @@ public class MockRecordReader implements RecordReader {
@Override
public int next() {
- int recordSetSize = Math.min(valueVectors[0].capacity(), this.config.getRecords()- recordsRead);
+
+ int recordSetSize = Math.min(batchRecordCount, this.config.getRecords()- recordsRead);
+
recordsRead += recordSetSize;
for(ValueVector v : valueVectors){
+ if(v instanceof FixedWidthVector){
+ ((FixedWidthVector)v).allocateNew(recordSetSize);
+ }else if(v instanceof VariableWidthVector){
+ ((VariableWidthVector)v).allocateNew(50*recordSetSize, recordSetSize);
+ }else{
+ throw new UnsupportedOperationException();
+ }
+
logger.debug("MockRecordReader: Generating random data for VV of type " + v.getClass().getName());
- v.randomizeData();
+ ValueVector.Mutator m = v.getMutator();
+ m.randomizeData();
+
+ if(m instanceof NonRepeatedMutator){
+ ((NonRepeatedMutator)m).setValueCount(recordSetSize);
+ }else{
+ throw new UnsupportedOperationException();
+ }
- v.getMutator().setRecordCount(recordSetSize);
}
return recordSetSize;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
index d1858f1..05fb576 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
@@ -105,6 +105,22 @@ public class MaterializedField implements Comparable<MaterializedField> {
public DataMode getDataMode() {
return def.getMajorType().getMode();
}
+
+ public MaterializedField getOtherNullableVersion(){
+ MajorType mt = def.getMajorType();
+ DataMode newDataMode = null;
+ switch(mt.getMode()){
+ case OPTIONAL:
+ newDataMode = DataMode.REQUIRED;
+ break;
+ case REQUIRED:
+ newDataMode = DataMode.OPTIONAL;
+ break;
+ default:
+ throw new UnsupportedOperationException();
+ }
+ return new MaterializedField(def.toBuilder().setMajorType(mt.toBuilder().setMode(newDataMode).build()).build());
+ }
public boolean matches(SchemaPath path) {
Iterator<NamePart> iter = def.getNameList().iterator();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
index be43026..a2dbd81 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
@@ -52,7 +52,7 @@ public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector>>
* @param def
* The definition for the record batch.
* @param buf
- * The buffer that holds the data ssociated with the record batch
+ * The buffer that holds the data associated with the record batch
* @return Whether or not the schema changed since the previous load.
* @throws SchemaChangeException
*/
@@ -71,7 +71,8 @@ public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector>>
ValueVector v = vectors.remove(fieldDef.getFieldId());
if (v != null) {
if (v.getField().getDef().equals(fieldDef)) {
- v.allocateNew(fmd.getBufferLength(), buf.slice(bufOffset, fmd.getBufferLength()), recordCount);
+ ValueVector.Mutator m = v.getMutator();
+ v.load(fmd, buf.slice(bufOffset, fmd.getBufferLength()));
newVectors.put(fieldDef.getFieldId(), v);
continue;
} else {
@@ -83,7 +84,7 @@ public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector>>
schemaChanged = true;
MaterializedField m = new MaterializedField(fieldDef);
v = TypeHelper.getNewVector(m, allocator);
- v.allocateNew(fmd.getBufferLength(), buf.slice(bufOffset, fmd.getBufferLength()), recordCount);
+ v.load(fmd, buf.slice(bufOffset, fmd.getBufferLength()));
newVectors.put(fieldDef.getFieldId(), v);
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
index a367b6d..4b97768 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
@@ -93,8 +93,7 @@ public class WritableBatch {
buffers.add(b);
b.retain();
}
- // allocate new buffer to release hold on old buffer.
- value.allocateNew(value.capacity());
+ value.clear();
}
public WritableBatch get(){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
index e637518..07ae20a 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
@@ -422,9 +422,11 @@ public class JSONRecordReader implements RecordReader {
SchemaDefProtos.MajorType type = field.getFieldType();
int fieldId = field.getFieldId();
MaterializedField f = MaterializedField.create(new SchemaPath(field.getFieldName()), fieldId, parentFieldId, type);
+
ValueVector v = TypeHelper.getNewVector(f, allocator);
- v.allocateNew(batchSize);
VectorHolder holder = new VectorHolder(batchSize, v);
+ holder.allocateNew(batchSize);
+
valueVectorMap.put(fieldId, holder);
outputMutator.addField(fieldId, v);
return holder;
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
index fa0cbd5..d594b9e 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
@@ -18,16 +18,20 @@
package org.apache.drill.exec.store;
+import org.apache.drill.exec.vector.FixedWidthVector;
import org.apache.drill.exec.vector.ValueVector;
+import org.apache.drill.exec.vector.VariableWidthVector;
public class VectorHolder {
private int length;
private ValueVector vector;
+ private ValueVector.Mutator mutator;
private int currentLength;
VectorHolder(int length, ValueVector vector) {
this.length = length;
this.vector = vector;
+ this.mutator = vector.getMutator();
}
public ValueVector getValueVector() {
@@ -51,6 +55,21 @@ public class VectorHolder {
public void reset() {
currentLength = 0;
- vector.allocateNew(length);
+ allocateNew(length);
+
+ }
+
+ public void allocateNew(int valueLength){
+ if(vector instanceof FixedWidthVector){
+ ((FixedWidthVector)vector).allocateNew(valueLength);
+ }else if(vector instanceof VariableWidthVector){
+ ((VariableWidthVector)vector).allocateNew(valueLength * 10, valueLength);
+ }else{
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public ValueVector.Mutator getMutator(){
+ return mutator;
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
new file mode 100644
index 0000000..dd2b504
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java
@@ -0,0 +1,47 @@
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.ByteBuf;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
+import org.apache.drill.exec.record.DeadBuf;
+import org.apache.drill.exec.record.MaterializedField;
+
+abstract class BaseDataValueVector extends BaseValueVector{
+
+ protected ByteBuf data = DeadBuf.DEAD_BUFFER;
+ protected int recordCount;
+
+ public BaseDataValueVector(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+
+ }
+
+ /**
+ * Release the underlying ByteBuf and reset the ValueVector
+ */
+ @Override
+ public void clear() {
+ if (data != DeadBuf.DEAD_BUFFER) {
+ data.release();
+ data = DeadBuf.DEAD_BUFFER;
+ recordCount = 0;
+ }
+ }
+
+ @Override
+ public ByteBuf[] getBuffers(){
+ return new ByteBuf[]{data};
+ }
+
+ public int getBufferSize() {
+ return data.writerIndex();
+ }
+
+ @Override
+ public FieldMetadata getMetadata() {
+ return null;
+ }
+
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
new file mode 100644
index 0000000..a8678f5
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java
@@ -0,0 +1,38 @@
+package org.apache.drill.exec.vector;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.record.MaterializedField;
+
+abstract class BaseValueVector implements ValueVector{
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseValueVector.class);
+
+ protected final BufferAllocator allocator;
+ protected final MaterializedField field;
+
+ BaseValueVector(MaterializedField field, BufferAllocator allocator) {
+ this.allocator = allocator;
+ this.field = field;
+ }
+
+ @Override
+ public void close() {
+ clear();
+ }
+
+ @Override
+ public MaterializedField getField() {
+ return field;
+ }
+
+ abstract class BaseAccessor implements ValueVector.Accessor{
+
+
+ public void reset(){}
+ }
+
+ abstract class BaseMutator implements NonRepeatedMutator{
+ public void reset(){}
+ }
+
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
index d18a29d..9d247f5 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java
@@ -1,8 +1,11 @@
package org.apache.drill.exec.vector;
+import io.netty.buffer.ByteBuf;
+
import java.util.Random;
import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
import org.apache.drill.exec.record.DeadBuf;
import org.apache.drill.exec.record.MaterializedField;
/**
@@ -13,64 +16,93 @@ import org.apache.drill.exec.record.MaterializedField;
*
* NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
*/
-public final class BitVector extends ValueVector {
+public final class BitVector extends BaseDataValueVector implements FixedWidthVector{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BitVector.class);
+ private int valueCapacity;
+
public BitVector(MaterializedField field, BufferAllocator allocator) {
super(field, allocator);
}
+ private int getSizeFromCount(int valueCount) {
+ return (int) Math.ceil(valueCount / 8);
+ }
+
/**
- * Get the byte holding the desired bit, then mask all other bits. Iff the result is 0, the
- * bit was not set.
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
*
- * @param index position of the bit in the vector
- * @return 1 if set, otherwise 0
+ * @param valueCount The number of values which can be contained within this vector.
*/
- public int get(int index) {
- // logger.debug("BIT GET: index: {}, byte: {}, mask: {}, masked byte: {}",
- // index,
- // data.getByte((int)Math.floor(index/8)),
- // (int)Math.pow(2, (index % 8)),
- // data.getByte((int)Math.floor(index/8)) & (int)Math.pow(2, (index % 8)));
- return ((data.getByte((int)Math.floor(index/8)) & (int)Math.pow(2, (index % 8))) == 0) ? 0 : 1;
+ public void allocateNew(int valueCount) {
+ clear();
+ valueCapacity = valueCount;
+ int valueSize = getSizeFromCount(valueCount);
+ data = allocator.buffer(valueSize);
+ for (int i = 0; i < getSizeFromCount(valueCount); i++) {
+ data.setByte(i, 0);
+ }
}
-
+
@Override
- public Object getObject(int index) {
- return new Boolean(get(index) != 0);
+ public int load(int valueCount, ByteBuf buf){
+ clear();
+ this.recordCount = valueCount;
+ int len = getSizeFromCount(valueCount);
+ data = buf.slice(0, len);
+ data.retain();
+ return len;
}
-
- /**
- * Get the size requirement (in bytes) for the given number of values.
- */
+
@Override
- public int getSizeFromCount(int valueCount) {
- return (int) Math.ceil(valueCount / 8);
+ public void load(FieldMetadata metadata, ByteBuf buffer) {
+ assert this.field.getDef().equals(metadata.getDef());
+ int loaded = load(metadata.getValueCount(), buffer);
+ assert metadata.getBufferLength() == loaded;
}
-
+
@Override
- public int getAllocatedSize() {
- return totalBytes;
+ public int getValueCapacity() {
+ return valueCapacity;
}
public Mutator getMutator() {
return new Mutator();
}
- /**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param valueCount The number of values which can be contained within this vector.
- */
- @Override
- public void allocateNew(int valueCount) {
- allocateNew(getSizeFromCount(valueCount), null, valueCount);
- for (int i = 0; i < getSizeFromCount(valueCount); i++) {
- data.setByte(i, 0);
- }
+ public Accessor getAccessor(){
+ return new Accessor();
}
+
+
+ public class Accessor extends BaseAccessor{
+ /**
+ * Get the byte holding the desired bit, then mask all other bits. Iff the result is 0, the
+ * bit was not set.
+ *
+ * @param index position of the bit in the vector
+ * @return 1 if set, otherwise 0
+ */
+ public int get(int index) {
+ // logger.debug("BIT GET: index: {}, byte: {}, mask: {}, masked byte: {}",
+ // index,
+ // data.getByte((int)Math.floor(index/8)),
+ // (int)Math.pow(2, (index % 8)),
+ // data.getByte((int)Math.floor(index/8)) & (int)Math.pow(2, (index % 8)));
+ return ((data.getByte((int)Math.floor(index/8)) & (int)Math.pow(2, (index % 8))) == 0) ? 0 : 1;
+ }
+
+ @Override
+ public Object getObject(int index) {
+ return new Boolean(get(index) != 0);
+ }
+
+ public int getRecordCount() {
+ return recordCount;
+ }
+
+ }
/**
* MutableBit implements a vector of bit-width values. Elements in the vector are accessed
@@ -79,7 +111,7 @@ public final class BitVector extends ValueVector {
*
* NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
*/
- public class Mutator implements ValueVector.Mutator{
+ public class Mutator extends BaseMutator{
private Mutator(){}
@@ -102,10 +134,9 @@ public final class BitVector extends ValueVector {
data.setByte((int) Math.floor(index/8), currentByte);
}
-
- @Override
- public void setRecordCount(int recordCount) {
- BitVector.this.setRecordCount(recordCount);
+ public void setValueCount(int recordCount) {
+ BitVector.this.recordCount = recordCount;
+ data.writerIndex(getSizeFromCount(recordCount));
}
@Override
@@ -119,5 +150,6 @@ public final class BitVector extends ValueVector {
}
}
}
+
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ByteHolder.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ByteHolder.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ByteHolder.java
new file mode 100644
index 0000000..45d8019
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ByteHolder.java
@@ -0,0 +1,12 @@
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.ByteBuf;
+
+public class ByteHolder {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ByteHolder.class);
+
+ public ByteBuf buffer;
+ public int start;
+ public int length;
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
new file mode 100644
index 0000000..0e3e3e9
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java
@@ -0,0 +1,23 @@
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.ByteBuf;
+
+public interface FixedWidthVector extends ValueVector{
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param totalBytes Desired size of the underlying data buffer.
+ * @param valueCount Number of values in the vector.
+ */
+ public void allocateNew(int valueCount);
+
+ /**
+ * Load the records in the provided buffer based on the given number of values.
+ * @param valueCount Number of values the buffer contains.
+ * @param buf Incoming buffer.
+ * @return The number of bytes of the buffer that were consumed.
+ */
+ public int load(int valueCount, ByteBuf buf);
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NonRepeatedMutator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NonRepeatedMutator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NonRepeatedMutator.java
new file mode 100644
index 0000000..e9bdcbd
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/NonRepeatedMutator.java
@@ -0,0 +1,7 @@
+package org.apache.drill.exec.vector;
+
+import org.apache.drill.exec.vector.ValueVector.Mutator;
+
+public interface NonRepeatedMutator extends Mutator{
+ public void setValueCount(int recordCount);
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java
new file mode 100644
index 0000000..35261d7
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java
@@ -0,0 +1,22 @@
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.ByteBuf;
+
+public interface RepeatedFixedWidthVector extends ValueVector{
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param parentValueCount Number of separate repeating groupings.
+ * @param childValueCount Number of supported values in the vector.
+ */
+ public void allocateNew(int parentValueCount, int childValueCount);
+
+ /**
+ * Load the records in the provided buffer based on the given number of values.
+ * @param parentValueCount Number of separate repeating groupings.
+ * @param valueCount Number atomic values the buffer contains.
+ * @param buf Incoming buffer.
+ * @return The number of bytes of the buffer that were consumed.
+ */
+ public int load(int parentValueCount, int childValueCount, ByteBuf buf);
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java
new file mode 100644
index 0000000..4f22481
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java
@@ -0,0 +1,24 @@
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.ByteBuf;
+
+public interface RepeatedVariableWidthVector extends ValueVector{
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param totalBytes Desired size of the underlying data buffer.
+ * @param parentValueCount Number of separate repeating groupings.
+ * @param childValueCount Number of supported values in the vector.
+ */
+ public void allocateNew(int totalBytes, int parentValueCount, int childValueCount);
+
+ /**
+ * Load the records in the provided buffer based on the given number of values.
+ * @param dataBytes The number of bytes associated with the data array.
+ * @param parentValueCount Number of separate repeating groupings.
+ * @param childValueCount Number of supported values in the vector.
+ * @param buf Incoming buffer.
+ * @return The number of bytes of the buffer that were consumed.
+ */
+ public int load(int dataBytes, int parentValueCount, int childValueCount, ByteBuf buf);
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
index 718478e..328182b 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java
@@ -21,9 +21,8 @@ import io.netty.buffer.ByteBuf;
import java.io.Closeable;
-import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.SchemaDefProtos.FieldDef;
import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
-import org.apache.drill.exec.record.DeadBuf;
import org.apache.drill.exec.record.MaterializedField;
/**
@@ -31,42 +30,25 @@ import org.apache.drill.exec.record.MaterializedField;
* value vectors. The template approach was chosen due to the lack of multiple inheritence. It
* is also important that all related logic be as efficient as possible.
*/
-public abstract class ValueVector implements Closeable {
+public interface ValueVector extends Closeable {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueVector.class);
+// /**
+// * Get the explicitly specified size of the allocated buffer, if available. Otherwise
+// * calculate the size based on width and record count.
+// */
+// public abstract int getAllocatedSize();
- protected final BufferAllocator allocator;
- protected ByteBuf data = DeadBuf.DEAD_BUFFER;
- protected MaterializedField field;
- protected int recordCount;
- protected int totalBytes;
-
- ValueVector(MaterializedField field, BufferAllocator allocator) {
- this.allocator = allocator;
- this.field = field;
- }
/**
- * Get the explicitly specified size of the allocated buffer, if available. Otherwise
- * calculate the size based on width and record count.
+ * Alternative to clear(). Allows use as closeable in try-with-resources.
*/
- public abstract int getAllocatedSize();
-
+ public void close();
+
/**
- * Get the size requirement (in bytes) for the given number of values. Takes derived
- * type specs into account.
+ * Release the underlying ByteBuf and reset the ValueVector to empty.
*/
- public abstract int getSizeFromCount(int valueCount);
-
- /**
- * Get the Java Object representation of the element at the specified position
- *
- * @param index Index of the value to get
- */
- public abstract Object getObject(int index);
-
+ public void clear();
- public abstract Mutator getMutator();
/**
* Return the underlying buffers associated with this vector. Note that this doesn't impact the
@@ -76,117 +58,72 @@ public abstract class ValueVector implements Closeable {
*
* @return The underlying ByteBuf.
*/
- public ByteBuf[] getBuffers() {
- return new ByteBuf[]{data};
- }
-
- /**
- * Returns the maximum number of values contained within this vector.
- * @return Vector size
- */
- public int capacity() {
- return getRecordCount();
- }
-
+ public abstract ByteBuf[] getBuffers();
+
/**
- * Release supporting resources.
+ * Load the data provided in the buffer. Typically used when deserializing from the wire.
+ * @param metadata Metadata used to decode the incoming buffer.
+ * @param buffer The buffer that contains the ValueVector.
*/
- @Override
- public void close() {
- clear();
- }
+ public void load(FieldMetadata metadata, ByteBuf buffer);
- /**
- * Get information about how this field is materialized.
- * @return
- */
- public MaterializedField getField() {
- return field;
- }
/**
- * Get the number of records allocated for this value vector.
- * @return number of allocated records
+ * Given the current buffer allocation, return the maximum number of values that this buffer can contain.
+ * @return Maximum values buffer can contain. In the case of a Repeated field, this is the number of atoms, not repeated groups.
*/
- public int getRecordCount() {
- return recordCount;
- }
-
+ public int getValueCapacity();
+
/**
- * Get the metadata for this field.
+ * Get information about how this field is materialized.
* @return
*/
- public FieldMetadata getMetadata() {
- int len = 0;
- for(ByteBuf b : getBuffers()){
- len += b.writerIndex();
- }
- return FieldMetadata.newBuilder()
- .setDef(getField().getDef())
- .setValueCount(getRecordCount())
- .setBufferLength(len)
- .build();
- }
-
+ public MaterializedField getField();
+
/**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param totalBytes Optional desired size of the underlying buffer. Specifying 0 will
- * estimate the size based on valueCount.
- * @param sourceBuffer Optional ByteBuf to use for storage (null will allocate automatically).
- * @param valueCount Number of values in the vector.
+ * Get the metadata for this field. Used in serialization
+ * @return FieldMetadata for this field.
*/
- public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
- clear();
- this.recordCount = valueCount;
- this.totalBytes = totalBytes > 0 ? totalBytes : getSizeFromCount(valueCount);
- this.data = (sourceBuffer != null) ? sourceBuffer : allocator.buffer(this.totalBytes);
- this.data.retain();
- data.readerIndex(0);
- }
-
+ public FieldMetadata getMetadata();
+
/**
- * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
- *
- * @param valueCount
- * The number of elements which can be contained within this vector.
+ * Get Accessor to read value vector data.
+ * @return
*/
- public void allocateNew(int valueCount) {
- allocateNew(0, null, valueCount);
- }
-
+ public abstract Accessor getAccessor();
+
/**
- * Release the underlying ByteBuf and reset the ValueVector
+ * Get a Mutator to update this vectors data.
+ * @return
*/
- protected void clear() {
- if (data != DeadBuf.DEAD_BUFFER) {
- data.release();
- data = DeadBuf.DEAD_BUFFER;
- recordCount = 0;
- totalBytes = 0;
- }
- }
+ public abstract Mutator getMutator();
- //public abstract <T extends Mutator> T getMutator();
- /**
- * Define the number of records that are in this value vector.
- * @param recordCount Number of records active in this vector.
- */
- void setRecordCount(int recordCount) {
- data.writerIndex(getSizeFromCount(recordCount));
- this.recordCount = recordCount;
+ public interface Accessor{
+
+// /**
+// * Get the number of records allocated for this value vector.
+// * @return number of allocated records
+// */
+// public int getRecordCount();
+
+ /**
+ * Get the Java Object representation of the element at the specified position. Useful for testing.
+ *
+ * @param index Index of the value to get
+ */
+ public abstract Object getObject(int index);
+
+ public void reset();
}
-
- /**
- * For testing only -- randomize the buffer contents
- */
- public void randomizeData() { }
-
- public static interface Mutator{
+
+
+
+
+ public interface Mutator{
+ public void reset();
public void randomizeData();
- public void setRecordCount(int recordCount);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
new file mode 100644
index 0000000..c26cbab
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java
@@ -0,0 +1,29 @@
+package org.apache.drill.exec.vector;
+
+import io.netty.buffer.ByteBuf;
+
+public interface VariableWidthVector extends ValueVector{
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param totalBytes Desired size of the underlying data buffer.
+ * @param valueCount Number of values in the vector.
+ */
+ public void allocateNew(int totalBytes, int valueCount);
+
+ /**
+ * Provide the maximum amount of variable width bytes that can be stored int his vector.
+ * @return
+ */
+ public int getByteCapacity();
+
+ /**
+ * Load the records in the provided buffer based on the given number of values.
+ * @param dataBytes The number of bytes associated with the data array.
+ * @param valueCount Number of values the buffer contains.
+ * @param buf Incoming buffer.
+ * @return The number of bytes of the buffer that were consumed.
+ */
+ public int load(int dataBytes, int valueCount, ByteBuf buf);
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
index ba103ed..a90382a 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java
@@ -230,8 +230,7 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
}
public QueryResult getResult(UserClientConnection connection, RequestResults req) {
-
- return null;
+ throw new UnsupportedOperationException();
}
@@ -254,7 +253,7 @@ public class Foreman implements Runnable, Closeable, Comparable<Object>{
}
void cleanupAndSendResult(QueryResult result){
- ForemanManagerListener.this.cleanupAndSendResult(result);
+ Foreman.this.cleanupAndSendResult(result);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
index 3fe0622..cac6aa2 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/physical/impl/TestSimpleFragmentRun.java
@@ -86,7 +86,7 @@ public class TestSimpleFragmentRun extends PopUnitTestBase {
} else {
System.out.print("\t");
}
- System.out.print(v.value.getObject(i));
+ System.out.print(v.value.getAccessor().getObject(i));
}
if(!first) System.out.println();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
index ae4f644..6a1f3ad 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/record/vector/TestValueVector.java
@@ -1,7 +1,6 @@
package org.apache.drill.exec.record.vector;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
import java.nio.charset.Charset;
@@ -46,14 +45,14 @@ public class TestValueVector {
m.set(100, 102);
m.set(1022, 103);
m.set(1023, 104);
- assertEquals(100, v.get(0));
- assertEquals(101, v.get(1));
- assertEquals(102, v.get(100));
- assertEquals(103, v.get(1022));
- assertEquals(104, v.get(1023));
+ assertEquals(100, v.getAccessor().get(0));
+ assertEquals(101, v.getAccessor().get(1));
+ assertEquals(102, v.getAccessor().get(100));
+ assertEquals(103, v.getAccessor().get(1022));
+ assertEquals(104, v.getAccessor().get(1023));
// Ensure unallocated space returns 0
- assertEquals(0, v.get(3));
+ assertEquals(0, v.getAccessor().get(3));
}
@Test
@@ -74,7 +73,7 @@ public class TestValueVector {
// Create a new value vector for 1024 integers
NullableVarChar2Vector v = new NullableVarChar2Vector(field, allocator);
NullableVarChar2Vector.Mutator m = v.getMutator();
- v.allocateNew(1024);
+ v.allocateNew(1024*10, 1024);
// Create and set 3 sample strings
String str1 = new String("AAAAA1");
@@ -85,14 +84,14 @@ public class TestValueVector {
m.set(2, str3.getBytes(Charset.forName("UTF-8")));
// Check the sample strings
- assertEquals(str1, new String(v.get(0), Charset.forName("UTF-8")));
- assertEquals(str2, new String(v.get(1), Charset.forName("UTF-8")));
- assertEquals(str3, new String(v.get(2), Charset.forName("UTF-8")));
+ assertEquals(str1, new String(v.getAccessor().get(0), Charset.forName("UTF-8")));
+ assertEquals(str2, new String(v.getAccessor().get(1), Charset.forName("UTF-8")));
+ assertEquals(str3, new String(v.getAccessor().get(2), Charset.forName("UTF-8")));
// Ensure null value throws
boolean b = false;
try {
- v.get(3);
+ v.getAccessor().get(3);
} catch(AssertionError e) {
b = true;
}finally{
@@ -130,17 +129,17 @@ public class TestValueVector {
m.set(100, 102);
m.set(1022, 103);
m.set(1023, 104);
- assertEquals(100, v.get(0));
- assertEquals(101, v.get(1));
- assertEquals(102, v.get(100));
- assertEquals(103, v.get(1022));
- assertEquals(104, v.get(1023));
+ assertEquals(100, v.getAccessor().get(0));
+ assertEquals(101, v.getAccessor().get(1));
+ assertEquals(102, v.getAccessor().get(100));
+ assertEquals(103, v.getAccessor().get(1022));
+ assertEquals(104, v.getAccessor().get(1023));
// Ensure null values throw
{
boolean b = false;
try {
- v.get(3);
+ v.getAccessor().get(3);
} catch(AssertionError e) {
b = true;
}finally{
@@ -155,7 +154,7 @@ public class TestValueVector {
{
boolean b = false;
try {
- v.get(0);
+ v.getAccessor().get(0);
} catch(AssertionError e) {
b = true;
}finally{
@@ -170,18 +169,18 @@ public class TestValueVector {
m.set(100, 102);
m.set(1022, 103);
m.set(1023, 104);
- assertEquals(100, v.get(0));
- assertEquals(101, v.get(1));
- assertEquals(102, v.get(100));
- assertEquals(103, v.get(1022));
- assertEquals(104, v.get(1023));
+ assertEquals(100, v.getAccessor().get(0));
+ assertEquals(101, v.getAccessor().get(1));
+ assertEquals(102, v.getAccessor().get(100));
+ assertEquals(103, v.getAccessor().get(1022));
+ assertEquals(104, v.getAccessor().get(1023));
// Ensure null values throw
{
boolean b = false;
try {
- v.get(3);
+ v.getAccessor().get(3);
} catch(AssertionError e) {
b = true;
}finally{
@@ -219,17 +218,17 @@ public class TestValueVector {
m.set(100, 102.3f);
m.set(1022, 103.4f);
m.set(1023, 104.5f);
- assertEquals(100.1f, v.get(0), 0);
- assertEquals(101.2f, v.get(1), 0);
- assertEquals(102.3f, v.get(100), 0);
- assertEquals(103.4f, v.get(1022), 0);
- assertEquals(104.5f, v.get(1023), 0);
+ assertEquals(100.1f, v.getAccessor().get(0), 0);
+ assertEquals(101.2f, v.getAccessor().get(1), 0);
+ assertEquals(102.3f, v.getAccessor().get(100), 0);
+ assertEquals(103.4f, v.getAccessor().get(1022), 0);
+ assertEquals(104.5f, v.getAccessor().get(1023), 0);
// Ensure null values throw
{
boolean b = false;
try {
- v.get(3);
+ v.getAccessor().get(3);
} catch(AssertionError e) {
b = true;
}finally{
@@ -243,7 +242,7 @@ public class TestValueVector {
{
boolean b = false;
try {
- v.get(0);
+ v.getAccessor().get(0);
} catch(AssertionError e) {
b = true;
}finally{
@@ -279,27 +278,27 @@ public class TestValueVector {
m.set(1, 0);
m.set(100, 0);
m.set(1022, 1);
- assertEquals(1, v.get(0));
- assertEquals(0, v.get(1));
- assertEquals(0, v.get(100));
- assertEquals(1, v.get(1022));
+ assertEquals(1, v.getAccessor().get(0));
+ assertEquals(0, v.getAccessor().get(1));
+ assertEquals(0, v.getAccessor().get(100));
+ assertEquals(1, v.getAccessor().get(1022));
// test setting the same value twice
m.set(0, 1);
m.set(0, 1);
m.set(1, 0);
m.set(1, 0);
- assertEquals(1, v.get(0));
- assertEquals(0, v.get(1));
+ assertEquals(1, v.getAccessor().get(0));
+ assertEquals(0, v.getAccessor().get(1));
// test toggling the values
m.set(0, 0);
m.set(1, 1);
- assertEquals(0, v.get(0));
- assertEquals(1, v.get(1));
+ assertEquals(0, v.getAccessor().get(0));
+ assertEquals(1, v.getAccessor().get(1));
// Ensure unallocated space returns 0
- assertEquals(0, v.get(3));
+ assertEquals(0, v.getAccessor().get(3));
}
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/9ca9eb9b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
index 117414c..4b35313 100644
--- a/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
+++ b/sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
@@ -73,7 +73,7 @@ public class JSONRecordReaderTest {
return;
}
- T val = (T) valueVector.getObject(index);
+ T val = (T) valueVector.getAccessor().getObject(index);
if (val instanceof byte[]) {
assertTrue(Arrays.equals((byte[]) value, (byte[]) val));
} else {
[5/7] git commit: Create new generated value vectors utilizing fmpp.
Includes: - First pass;
integrate build system and some cursory implementations - starting to split
common logic into base class - implement most of varlen value vector
functionality,
Posted by ja...@apache.org.
Create new generated value vectors utilizing fmpp. Includes:
- First pass; integrate build system and some cursory implementations
- starting to split common logic into base class
- implement most of varlen value vector functionality, minor cleanup of tdd tags
- added nullable derived class
- Merge changes from JA, minor format cleanup.
- minor fix and cleanup
- added bit vector, removed widthInBits which also allowed removal of FixedBase ctor
- apply TC's fix for resetAllocation()
- added repeated value vectors
- Hooked up templated ValueVectors to codebase. Removed old ValueVector classes. Cleanup.
- fix repeated get() and add()
- added some value vector tests. fixed bugs in VV and some call sites. generated TypeHelper from FMPP template. removed unused VV methods
- made base immutable, some debugging
- split mutable/immutable basic VV types. minor refactoring
- fix several allocation bugs
- fix various bugs, only JSONRecordReader test is failing
- fix nullable bit value vector
- make bit vectors use ints to represent the bit value
- remove superfluous logging
- fix value vector getters and setter
- comments and cleanup
- temp disable repeated map JSONReader test
- formatting
- whitespace cleanups
Project: http://git-wip-us.apache.org/repos/asf/incubator-drill/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-drill/commit/7075cca1
Tree: http://git-wip-us.apache.org/repos/asf/incubator-drill/tree/7075cca1
Diff: http://git-wip-us.apache.org/repos/asf/incubator-drill/diff/7075cca1
Branch: refs/heads/execwork
Commit: 7075cca1f1be45b876ef846762f13d0780627c3a
Parents: 5a5d07f
Author: Ben Becker <be...@gmail.com>
Authored: Tue Jun 18 17:36:11 2013 -0700
Committer: Jacques Nadeau <ja...@apache.org>
Committed: Mon Jul 15 11:36:32 2013 -0700
----------------------------------------------------------------------
sandbox/prototype/exec/java-exec/pom.xml | 20 +
.../src/main/codegen/ValueVectors/config.fmpp | 3 +
.../ValueVectors/data/ValueVectorTypes.tdd | 106 +++
.../ValueVectors/templates/TypeHelper.java | 119 +++
.../ValueVectors/templates/ValueVector.java | 768 +++++++++++++++++++
.../exec/physical/config/MockRecordReader.java | 15 +-
.../exec/physical/impl/FilterRecordBatch.java | 2 +-
.../drill/exec/physical/impl/OutputMutator.java | 2 +-
.../drill/exec/physical/impl/ScanBatch.java | 16 +-
.../drill/exec/physical/impl/ScreenCreator.java | 2 +-
.../exec/physical/impl/SingleSenderCreator.java | 2 +-
.../exec/physical/impl/WireRecordBatch.java | 4 +-
.../drill/exec/record/MaterializedField.java | 5 -
.../apache/drill/exec/record/RecordBatch.java | 2 +-
.../drill/exec/record/RecordBatchLoader.java | 22 +-
.../apache/drill/exec/record/WritableBatch.java | 9 +-
.../record/vector/AbstractFixedValueVector.java | 77 --
.../drill/exec/record/vector/AnyVector.java | 30 -
.../exec/record/vector/BaseValueVector.java | 157 ----
.../apache/drill/exec/record/vector/Bit.java | 168 ----
.../apache/drill/exec/record/vector/Fixed1.java | 43 --
.../drill/exec/record/vector/Fixed12.java | 35 -
.../drill/exec/record/vector/Fixed16.java | 37 -
.../apache/drill/exec/record/vector/Fixed2.java | 53 --
.../apache/drill/exec/record/vector/Fixed4.java | 59 --
.../apache/drill/exec/record/vector/Fixed8.java | 58 --
.../drill/exec/record/vector/FixedLen.java | 45 --
.../exec/record/vector/NullValueException.java | 9 +
.../drill/exec/record/vector/NullableBit.java | 20 -
.../exec/record/vector/NullableFixed4.java | 44 --
.../exec/record/vector/NullableValueVector.java | 108 ---
.../exec/record/vector/NullableVarLen4.java | 21 -
.../drill/exec/record/vector/RepeatMap.java | 57 --
.../exec/record/vector/SelectionVector.java | 2 +-
.../drill/exec/record/vector/TypeHelper.java | 259 -------
.../drill/exec/record/vector/ValueVector.java | 118 ---
.../drill/exec/record/vector/VarLen1.java | 36 -
.../drill/exec/record/vector/VarLen2.java | 36 -
.../drill/exec/record/vector/VarLen4.java | 48 --
.../exec/record/vector/VariableVector.java | 102 ---
.../java/org/apache/drill/exec/rpc/RpcBus.java | 2 +-
.../drill/exec/store/JSONRecordReader.java | 30 +-
.../apache/drill/exec/store/VectorHolder.java | 6 +-
.../work/AbstractFragmentRunnerListener.java | 3 +-
.../apache/drill/exec/work/FragmentRunner.java | 3 +-
.../physical/impl/TestSimpleFragmentRun.java | 90 ++-
.../exec/record/vector/TestValueVector.java | 252 ++++++
.../drill/exec/store/JSONRecordReaderTest.java | 36 +-
48 files changed, 1400 insertions(+), 1741 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/pom.xml
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/pom.xml b/sandbox/prototype/exec/java-exec/pom.xml
index f5ece33..4e4df95 100644
--- a/sandbox/prototype/exec/java-exec/pom.xml
+++ b/sandbox/prototype/exec/java-exec/pom.xml
@@ -167,6 +167,26 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>com.googlecode.fmpp-maven-plugin</groupId>
+ <artifactId>fmpp-maven-plugin</artifactId>
+ <version>1.0</version>
+ <configuration>
+ <cfgFile>src/main/codegen/ValueVectors/config.fmpp</cfgFile>
+ <outputDirectory>target/generated-sources/org/apache/drill/exec/record/vector</outputDirectory>
+ <templateDirectory>src/main/codegen/ValueVectors/templates</templateDirectory>
+ </configuration>
+ <executions>
+ <execution>
+ <id>generate-sources</id>
+ <phase>generate-sources</phase>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
+
<!-- <plugin> -->
<!-- <groupId>com.github.igor-petruk.protobuf</groupId> -->
<!-- <artifactId>protobuf-maven-plugin</artifactId> -->
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/config.fmpp
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/config.fmpp b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/config.fmpp
new file mode 100644
index 0000000..da05e2d
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/config.fmpp
@@ -0,0 +1,3 @@
+data: {
+ tdd(../data/ValueVectorTypes.tdd)
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/data/ValueVectorTypes.tdd
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/data/ValueVectorTypes.tdd b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/data/ValueVectorTypes.tdd
new file mode 100644
index 0000000..42153b6
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/data/ValueVectorTypes.tdd
@@ -0,0 +1,106 @@
+{
+ types: [
+ {
+ major: "Fixed",
+ width: 1,
+ javaType: "byte",
+ minor: [
+ { class: "TinyInt" },
+ { class: "UInt1" }
+ ]
+ },
+ {
+ major: "Fixed",
+ width: 2,
+ javaType: "short",
+ minor: [
+ { class: "SmallInt" },
+ { class: "UInt2" }
+ ]
+ },
+ {
+ major: "Fixed",
+ width: 4,
+ javaType: "int",
+ minor: [
+ { class: "Int" },
+ { class: "UInt4" },
+ { class: "Decimal4", maxPrecisionDigits: 8, scale: 1, javaType: "float"},
+ { class: "Float4", javaType: "float" },
+ { class: "Date" }
+ ]
+ },
+ {
+ major: "Fixed",
+ width: 8,
+ javaType: "long",
+ minor: [
+ { class: "BigInt" },
+ { class: "UInt8" },
+ { class: "Decimal8", maxPrecisionDigits: 18, scale: 1, javaType: "double"},
+ { class: "Float8", javaType: "double" },
+ { class: "Money", maxPrecisionDigits: 2, scale: 1, javaType: "double" }
+ { class: "Time" },
+ { class: "TimeStamp" },
+ { class: "DateTime" }
+ ]
+ },
+ {
+ major: "Fixed",
+ width: 12,
+ javaType: "ByteBuf",
+ minor: [
+ { class: "Decimal12", maxPrecisionDigits: 28, scale: 1},
+ { class: "TimeTZ" },
+ { class: "Interval" }
+ ]
+ },
+ {
+ major: "Fixed",
+ width: 16,
+ javaType: "ByteBuf",
+ minor: [
+ { class: "Decimal16", maxPrecisionDigits: 37, scale: 1}
+ ]
+ },
+ {
+ major: "VarLen",
+ width: 1,
+ javaType: "byte",
+ minor: [
+ { class: "VarBinary1" },
+ { class: "VarChar1" }
+ ]
+ },
+ {
+ major: "VarLen",
+ width: 2,
+ javaType: "short",
+ minor: [
+ { class: "VarBinary2" },
+ { class: "VarChar2" },
+ { class: "Proto2" },
+ { class: "MsgPack2" }
+ ]
+ },
+ {
+ major: "VarLen",
+ width: 4,
+ javaType: "int",
+ minor: [
+ { class: "VarBinary4" },
+ { class: "VarChar4" }
+ { class: "Proto4" },
+ { class: "MsgPack4" }
+ ]
+ },
+ {
+ major: "Bit",
+ width: 1,
+ javaType: "int",
+ minor: [
+ { class: "Bit" }
+ ]
+ }
+ ]
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/TypeHelper.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/TypeHelper.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/TypeHelper.java
new file mode 100644
index 0000000..8dfd3af
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/TypeHelper.java
@@ -0,0 +1,119 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.record.vector;
+
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.SchemaDefProtos.DataMode;
+import org.apache.drill.exec.proto.SchemaDefProtos.MajorType;
+import org.apache.drill.exec.proto.SchemaDefProtos.MinorType;
+import org.apache.drill.exec.record.MaterializedField;
+
+public class TypeHelper {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TypeHelper.class);
+
+ private static final int WIDTH_ESTIMATE_1 = 10;
+ private static final int WIDTH_ESTIMATE_2 = 50000;
+ private static final int WIDTH_ESTIMATE_4 = 1024*1024;
+
+ public static int getSize(MajorType major) {
+ switch (major.getMinorType()) {
+<#list types as type>
+ <#list type.minor as minor>
+ <#if minor.class != "Bit">
+ case ${minor.class?upper_case}:
+ return ${type.width}<#if minor.class?substring(0, 3) == "Var" ||
+ minor.class?substring(0, 3) == "PRO" ||
+ minor.class?substring(0, 3) == "MSG"> + WIDTH_ESTIMATE_${type.width}</#if>;
+ </#if>
+ </#list>
+</#list>
+ case BOOLEAN: return 1;
+ case FIXEDCHAR: return major.getWidth();
+ case FIXEDBINARY: return major.getWidth();
+ }
+ throw new UnsupportedOperationException();
+ }
+
+ public static Class<?> getValueVectorClass(MinorType type, DataMode mode){
+ switch (type) {
+<#list types as type>
+ <#list type.minor as minor>
+ <#if minor.class == "Bit">
+ case BOOLEAN:
+ switch (mode) {
+ case REQUIRED:
+ return ValueVector.${minor.class}.class;
+ case OPTIONAL:
+ return ValueVector.Nullable${minor.class}.class;
+ case REPEATED:
+ return ValueVector.Repeated${minor.class}.class;
+ }
+ <#else>
+ case ${minor.class?upper_case}:
+ switch (mode) {
+ case REQUIRED:
+ return ValueVector.${minor.class}.class;
+ case OPTIONAL:
+ return ValueVector.Nullable${minor.class}.class;
+ case REPEATED:
+ return ValueVector.Repeated${minor.class}.class;
+ }
+ </#if>
+ </#list>
+</#list>
+ default:
+ break;
+ }
+ throw new UnsupportedOperationException();
+ }
+
+
+ public static ValueVector.Base getNewVector(MaterializedField field, BufferAllocator allocator){
+ MajorType type = field.getType();
+
+ switch (type.getMinorType()) {
+<#list types as type>
+ <#list type.minor as minor>
+ <#if minor.class != "Bit">
+ case ${minor.class?upper_case}:
+ switch (type.getMode()) {
+ case REQUIRED:
+ return new ValueVector.${minor.class}(field, allocator);
+ case OPTIONAL:
+ return new ValueVector.Nullable${minor.class}(field, allocator);
+ case REPEATED:
+ return new ValueVector.Repeated${minor.class}(field, allocator);
+ }
+ </#if>
+ </#list>
+</#list>
+ case BOOLEAN:
+ switch (type.getMode()) {
+ case REQUIRED:
+ return new ValueVector.Bit(field, allocator);
+ case OPTIONAL:
+ return new ValueVector.NullableBit(field, allocator);
+ case REPEATED:
+ return new ValueVector.RepeatedBit(field, allocator);
+ }
+ }
+ // All ValueVector types have been handled.
+ throw new UnsupportedOperationException(type.getMinorType() + " type is not supported. Mode: " + type.getMode());
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueVector.java b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueVector.java
new file mode 100644
index 0000000..e0e895b
--- /dev/null
+++ b/sandbox/prototype/exec/java-exec/src/main/codegen/ValueVectors/templates/ValueVector.java
@@ -0,0 +1,768 @@
+/*******************************************************************************
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.drill.exec.record.vector;
+
+import static com.google.common.base.Preconditions.checkArgument;
+import static com.google.common.base.Preconditions.checkState;
+import io.netty.buffer.ByteBuf;
+import java.io.Closeable;
+import java.util.Random;
+import org.apache.drill.exec.memory.BufferAllocator;
+import org.apache.drill.exec.proto.SchemaDefProtos;
+import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
+import org.apache.drill.exec.record.DeadBuf;
+import org.apache.drill.exec.record.MaterializedField;
+
+/**
+ * ValueVectorTypes defines a set of template-generated classes which implement type-specific
+ * value vectors. The template approach was chosen due to the lack of multiple inheritence. It
+ * is also important that all related logic be as efficient as possible.
+ */
+public class ValueVector {
+
+ /**
+ * ValueVector.Base implements common logic for all immutable value vectors.
+ */
+ public abstract static class Base implements Closeable {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Base.class);
+
+ protected final BufferAllocator allocator;
+ protected ByteBuf data = DeadBuf.DEAD_BUFFER;
+ protected MaterializedField field;
+ protected int recordCount;
+ protected int totalBytes;
+
+ public Base(MaterializedField field, BufferAllocator allocator) {
+ this.allocator = allocator;
+ this.field = field;
+ }
+
+ /**
+ * Get the explicitly specified size of the allocated buffer, if available. Otherwise
+ * calculate the size based on width and record count.
+ */
+ public abstract int getAllocatedSize();
+
+ /**
+ * Get the size requirement (in bytes) for the given number of values. Takes derived
+ * type specs into account.
+ */
+ public abstract int getSizeFromCount(int valueCount);
+
+ /**
+ * Get the Java Object representation of the element at the specified position
+ *
+ * @param index Index of the value to get
+ */
+ public abstract Object getObject(int index);
+
+ /**
+ * Return the underlying buffers associated with this vector. Note that this doesn't impact the
+ * reference counts for this buffer so it only should be used for in-context access. Also note
+ * that this buffer changes regularly thus external classes shouldn't hold a reference to
+ * it (unless they change it).
+ *
+ * @return The underlying ByteBuf.
+ */
+ public ByteBuf[] getBuffers() {
+ return new ByteBuf[]{data};
+ }
+
+ /**
+ * Returns the maximum number of values contained within this vector.
+ * @return Vector size
+ */
+ public int capacity() {
+ return getRecordCount();
+ }
+
+ /**
+ * Release supporting resources.
+ */
+ @Override
+ public void close() {
+ clear();
+ }
+
+ /**
+ * Get information about how this field is materialized.
+ * @return
+ */
+ public MaterializedField getField() {
+ return field;
+ }
+
+ /**
+ * Get the number of records allocated for this value vector.
+ * @return number of allocated records
+ */
+ public int getRecordCount() {
+ return recordCount;
+ }
+
+ /**
+ * Get the metadata for this field.
+ * @return
+ */
+ public FieldMetadata getMetadata() {
+ int len = 0;
+ for(ByteBuf b : getBuffers()){
+ len += b.writerIndex();
+ }
+ return FieldMetadata.newBuilder()
+ .setDef(getField().getDef())
+ .setValueCount(getRecordCount())
+ .setBufferLength(len)
+ .build();
+ }
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param totalBytes Optional desired size of the underlying buffer. Specifying 0 will
+ * estimate the size based on valueCount.
+ * @param sourceBuffer Optional ByteBuf to use for storage (null will allocate automatically).
+ * @param valueCount Number of values in the vector.
+ */
+ public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
+ clear();
+ this.recordCount = valueCount;
+ this.totalBytes = totalBytes > 0 ? totalBytes : getSizeFromCount(valueCount);
+ this.data = (sourceBuffer != null) ? sourceBuffer : allocator.buffer(this.totalBytes);
+ this.data.retain();
+ data.readerIndex(0);
+ }
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount
+ * The number of elements which can be contained within this vector.
+ */
+ public void allocateNew(int valueCount) {
+ allocateNew(0, null, valueCount);
+ }
+
+ /**
+ * Release the underlying ByteBuf and reset the ValueVector
+ */
+ protected void clear() {
+ if (data != DeadBuf.DEAD_BUFFER) {
+ data.release();
+ data = DeadBuf.DEAD_BUFFER;
+ recordCount = 0;
+ totalBytes = 0;
+ }
+ }
+
+ /**
+ * Define the number of records that are in this value vector.
+ * @param recordCount Number of records active in this vector.
+ */
+ public void setRecordCount(int recordCount) {
+ data.writerIndex(getSizeFromCount(recordCount));
+ this.recordCount = recordCount;
+ }
+
+ /**
+ * For testing only -- randomize the buffer contents
+ */
+ public void randomizeData() { }
+
+ }
+
+ /**
+ * Bit implements a vector of bit-width values. Elements in the vector are accessed
+ * by position from the logical start of the vector.
+ * The width of each element is 1 bit.
+ * The equivilent Java primitive is an int containing the value '0' or '1'.
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+ public static class Bit extends Base {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Bit.class);
+
+ public Bit(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ }
+
+ /**
+ * Get the byte holding the desired bit, then mask all other bits. Iff the result is 0, the
+ * bit was not set.
+ *
+ * @param index position of the bit in the vector
+ * @return 1 if set, otherwise 0
+ */
+ public int get(int index) {
+ // logger.debug("BIT GET: index: {}, byte: {}, mask: {}, masked byte: {}",
+ // index,
+ // data.getByte((int)Math.floor(index/8)),
+ // (int)Math.pow(2, (index % 8)),
+ // data.getByte((int)Math.floor(index/8)) & (int)Math.pow(2, (index % 8)));
+ return ((data.getByte((int)Math.floor(index/8)) & (int)Math.pow(2, (index % 8))) == 0) ? 0 : 1;
+ }
+
+ @Override
+ public Object getObject(int index) {
+ return new Boolean(get(index) != 0);
+ }
+
+ /**
+ * Get the size requirement (in bytes) for the given number of values.
+ */
+ @Override
+ public int getSizeFromCount(int valueCount) {
+ return (int) Math.ceil(valueCount / 8);
+ }
+
+ @Override
+ public int getAllocatedSize() {
+ return totalBytes;
+ }
+
+ public MutableBit getMutable() {
+ return (MutableBit)this;
+ }
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount The number of values which can be contained within this vector.
+ */
+ @Override
+ public void allocateNew(int valueCount) {
+ allocateNew(getSizeFromCount(valueCount), null, valueCount);
+ for (int i = 0; i < getSizeFromCount(valueCount); i++) {
+ data.setByte(i, 0);
+ }
+ }
+
+ }
+
+ /**
+ * MutableBit implements a vector of bit-width values. Elements in the vector are accessed
+ * by position from the logical start of the vector. Values should be pushed onto the vector
+ * sequentially, but may be randomly accessed.
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+ public static class MutableBit extends Bit {
+
+ public MutableBit(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ }
+
+ /**
+ * Set the bit at the given index to the specified value.
+ *
+ * @param index position of the bit to set
+ * @param value value to set (either 1 or 0)
+ */
+ public void set(int index, int value) {
+ byte currentByte = data.getByte((int)Math.floor(index/8));
+ if (value != 0) {
+ // true
+ currentByte |= (byte) Math.pow(2, (index % 8));
+ }
+ else if ((currentByte & (byte) Math.pow(2, (index % 8))) == (byte) Math.pow(2, (index % 8))) {
+ // false, and bit was previously set
+ currentByte -= (byte) Math.pow(2, (index % 8));
+ }
+ data.setByte((int) Math.floor(index/8), currentByte);
+ }
+
+ @Override
+ public void randomizeData() {
+ if (this.data != DeadBuf.DEAD_BUFFER) {
+ Random r = new Random();
+ for (int i = 0; i < data.capacity() - 1; i++) {
+ byte[] bytes = new byte[1];
+ r.nextBytes(bytes);
+ data.setByte(i, bytes[0]);
+ }
+ }
+ }
+ }
+
+<#list types as type>
+ <#list type.minor as minor>
+ <#if type.major == "Fixed">
+
+ /**
+ * ${minor.class} implements a vector of fixed width values. Elements in the vector are accessed
+ * by position, starting from the logical start of the vector. Values should be pushed onto the
+ * vector sequentially, but may be randomly accessed.
+ * The width of each element is ${type.width} byte(s)
+ * The equivilent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+ public static class ${minor.class} extends Base {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}.class);
+
+ public ${minor.class}(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ }
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount
+ * The number of values which can be contained within this vector.
+ */
+ public void allocateNew(int valueCount) {
+ totalBytes = valueCount * ${type.width};
+ allocateNew(totalBytes, allocator.buffer(totalBytes), valueCount);
+ }
+
+ @Override
+ public int getAllocatedSize() {
+ return (int) Math.ceil(totalBytes);
+ }
+
+ /**
+ * Get the size requirement (in bytes) for the given number of values. Only accurate
+ * for fixed width value vectors.
+ */
+ @Override
+ public int getSizeFromCount(int valueCount) {
+ return valueCount * ${type.width};
+ }
+
+ public Mutable${minor.class} getMutable() {
+ return (Mutable${minor.class})this;
+ }
+
+ <#if (type.width > 8)>
+
+ public ${minor.javaType!type.javaType} get(int index) {
+ ByteBuf dst = allocator.buffer(${type.width});
+ data.getBytes(index * ${type.width}, dst, 0, ${type.width});
+ return dst;
+ }
+
+ @Override
+ public Object getObject(int index) {
+ ByteBuf dst = allocator.buffer(${type.width});
+ data.getBytes(index, dst, 0, ${type.width});
+ return dst;
+ }
+
+ @Override
+ public void randomizeData() {
+ if (this.data != DeadBuf.DEAD_BUFFER) {
+ Random r = new Random();
+ for(int i =0; i < data.capacity()-${type.width}; i += ${type.width}){
+ byte[] bytes = new byte[${type.width}];
+ r.nextBytes(bytes);
+ data.setByte(i, bytes[0]);
+ }
+ }
+ }
+
+ <#else> <#-- type.width <= 8 -->
+
+ public ${minor.javaType!type.javaType} get(int index) {
+ return data.get${(minor.javaType!type.javaType)?cap_first}(index * ${type.width});
+ }
+
+ public Object getObject(int index) {
+ return get(index);
+ }
+
+ @Override
+ public void randomizeData() {
+ if (this.data != DeadBuf.DEAD_BUFFER) {
+ Random r = new Random();
+ for(int i =0; i < data.capacity()-${type.width}; i += ${type.width}){
+ data.set${(minor.javaType!type.javaType)?cap_first}(i,
+ r.next<#if (type.width >= 4)>${(minor.javaType!type.javaType)?cap_first}
+ <#else>Int
+ </#if>());
+ }
+ }
+ }
+
+ </#if> <#-- type.width -->
+ }
+
+ /**
+ * Mutable${minor.class} implements a mutable vector of fixed width values. Elements in the
+ * vector are accessed by position from the logical start of the vector. Values should be pushed
+ * onto the vector sequentially, but may be randomly accessed.
+ * The width of each element is ${type.width} byte(s)
+ * The equivilent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+ public static class Mutable${minor.class} extends ${minor.class} {
+
+ public Mutable${minor.class}(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ }
+
+ /**
+ * Set the element at the given index to the given value. Note that widths smaller than
+ * 32 bits are handled by the ByteBuf interface.
+ *
+ * @param index position of the bit to set
+ * @param value value to set
+ */
+ <#if (type.width > 8)>
+ public void set(int index, <#if (type.width > 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+ data.setBytes(index * ${type.width}, value);
+ <#else> <#-- type.width <= 8 -->
+ public void set(int index, <#if (type.width >= 4)>${minor.javaType!type.javaType}<#else>int</#if> value) {
+ data.set${(minor.javaType!type.javaType)?cap_first}(index * ${type.width}, value);
+ </#if> <#-- type.width -->
+ }
+ }
+
+ <#elseif type.major == "VarLen">
+
+ /**
+ * ${minor.class} implements a vector of variable width values. Elements in the vector
+ * are accessed by position from the logical start of the vector. A fixed width lengthVector
+ * is used to convert an element's position to it's offset from the start of the (0-based)
+ * ByteBuf. Size is inferred by adjacent elements.
+ * The width of each element is ${type.width} byte(s)
+ * The equivilent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+ public static class ${minor.class} extends Base {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(${minor.class}.class);
+
+ protected final MutableUInt${type.width} lengthVector;
+
+ public ${minor.class}(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ this.lengthVector = new MutableUInt${type.width}(null, allocator);
+ }
+
+ public byte[] get(int index) {
+ checkArgument(index >= 0);
+ int startIdx = 0;
+ int size = 0;
+ if (index == 0) {
+ size = lengthVector.get(1);
+ } else {
+ startIdx = lengthVector.get(index);
+ size = lengthVector.get(index + 1) - startIdx;
+ }
+ checkState(size >= 0);
+ byte[] dst = new byte[size];
+ data.getBytes(startIdx, dst, 0, size);
+ return dst;
+ }
+
+ @Override
+ public int getAllocatedSize() {
+ return lengthVector.getAllocatedSize() + totalBytes;
+ }
+
+ /**
+ * Get the size requirement (in bytes) for the given number of values. Only accurate
+ * for fixed width value vectors.
+ */
+ public int getSizeFromCount(int valueCount) {
+ return valueCount * ${type.width};
+ }
+
+ @Override
+ protected void clear() {
+ super.clear();
+ lengthVector.clear();
+ }
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount
+ * The number of values which can be contained within this vector.
+ */
+ public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
+ super.allocateNew(totalBytes, sourceBuffer, valueCount);
+ lengthVector.allocateNew(valueCount);
+ }
+
+ @Override
+ public ByteBuf[] getBuffers() {
+ return new ByteBuf[]{lengthVector.data, data};
+ }
+
+ public Object getObject(int index) {
+ return get(index);
+ }
+
+ public Mutable${minor.class} getMutable() {
+ return (Mutable${minor.class})this;
+ }
+ }
+
+ /**
+ * Mutable${minor.class} implements a vector of variable width values. Elements in the vector
+ * are accessed by position from the logical start of the vector. A fixed width lengthVector
+ * is used to convert an element's position to it's offset from the start of the (0-based)
+ * ByteBuf. Size is inferred by adjacent elements.
+ * The width of each element is ${type.width} byte(s)
+ * The equivilent Java primitive is '${minor.javaType!type.javaType}'
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+ public static class Mutable${minor.class} extends ${minor.class} {
+
+ public Mutable${minor.class}(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ }
+
+ /**
+ * Set the variable length element at the specified index to the supplied byte array.
+ *
+ * @param index position of the bit to set
+ * @param bytes array of bytes to write
+ */
+ public void set(int index, byte[] bytes) {
+ checkArgument(index >= 0);
+ if (index == 0) {
+ lengthVector.set(0, 0);
+ lengthVector.set(1, bytes.length);
+ data.setBytes(0, bytes);
+ }
+ else {
+ int currentOffset = lengthVector.get(index);
+ // set the end offset of the buffer
+ lengthVector.set(index + 1, currentOffset + bytes.length);
+ data.setBytes(currentOffset, bytes);
+ }
+ }
+
+ @Override
+ public void setRecordCount(int recordCount) {
+ super.setRecordCount(recordCount);
+ lengthVector.setRecordCount(recordCount);
+ }
+
+ }
+
+ </#if> <#-- type.major -->
+
+ /**
+ * Nullable${minor.class} implements a vector of values which could be null. Elements in the vector
+ * are first checked against a fixed length vector of boolean values. Then the element is retrieved
+ * from the base class (if not null).
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+ public static class Nullable${minor.class} extends Mutable${minor.class} {
+
+ protected MutableBit bits;
+
+ public Nullable${minor.class}(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ bits = new MutableBit(null, allocator);
+ }
+
+ /**
+ * Set the variable length element at the specified index to the supplied byte array.
+ *
+ * @param index position of the bit to set
+ * @param bytes array of bytes to write
+ */
+ public void set(int index, <#if type.major == "VarLen">byte[]<#elseif (type.width < 4)>int<#else>${minor.javaType!type.javaType}</#if> value) {
+ setNotNull(index);
+ super.set(index, value);
+ }
+
+ /**
+ * Get the element at the specified position.
+ *
+ * @param index position of the value
+ * @return value of the element, if not null
+ * @throws NullValueException if the value is null
+ */
+ public <#if type.major == "VarLen">byte[]<#else>${minor.javaType!type.javaType}</#if> get(int index) {
+ if (isNull(index))
+ throw new NullValueException(index);
+ return super.get(index);
+ }
+
+ public void setNull(int index) {
+ bits.set(index, 0);
+ }
+
+ private void setNotNull(int index) {
+ bits.set(index, 1);
+ }
+
+ public boolean isNull(int index) {
+ return bits.get(index) == 0;
+ }
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount The number of values which may be contained by this vector.
+ */
+ public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
+ super.allocateNew(totalBytes, sourceBuffer, valueCount);
+ bits.allocateNew(valueCount);
+ }
+
+ @Override
+ public int getAllocatedSize() {
+ return bits.getAllocatedSize() + super.getAllocatedSize();
+ }
+
+ /**
+ * Get the size requirement (in bytes) for the given number of values. Only accurate
+ * for fixed width value vectors.
+ */
+ public int getTotalSizeFromCount(int valueCount) {
+ return getSizeFromCount(valueCount) + bits.getSizeFromCount(valueCount);
+ }
+
+ @Override
+ public MaterializedField getField() {
+ return field;
+ }
+
+ @Override
+ public ByteBuf[] getBuffers() {
+ return new ByteBuf[]{bits.data, super.data};
+ }
+
+ @Override
+ public void setRecordCount(int recordCount) {
+ super.setRecordCount(recordCount);
+ bits.setRecordCount(recordCount);
+ }
+
+ @Override
+ public Object getObject(int index) {
+ return isNull(index) ? null : super.getObject(index);
+ }
+ }
+
+ /**
+ * Repeated${minor.class} implements a vector with multple values per row (e.g. JSON array or
+ * repeated protobuf field). The implementation uses two additional value vectors; one to convert
+ * the index offset to the underlying element offset, and another to store the number of values
+ * in the vector.
+ *
+ * NB: this class is automatically generated from ValueVectorTypes.tdd using FreeMarker.
+ */
+ public static class Repeated${minor.class} extends Mutable${minor.class} {
+
+ private MutableUInt4 countVector; // number of repeated elements in each record
+ private MutableUInt4 offsetVector; // offsets to start of each record
+
+ public Repeated${minor.class}(MaterializedField field, BufferAllocator allocator) {
+ super(field, allocator);
+ countVector = new MutableUInt4(null, allocator);
+ offsetVector = new MutableUInt4(null, allocator);
+ }
+
+ public void allocateNew(int totalBytes, ByteBuf sourceBuffer, int valueCount) {
+ super.allocateNew(totalBytes, sourceBuffer, valueCount);
+ countVector.allocateNew(valueCount);
+ offsetVector.allocateNew(valueCount);
+ }
+
+ /**
+ * Add an element to the given record index. This is similar to the set() method in other
+ * value vectors, except that it permits setting multiple values for a single record.
+ *
+ * @param index record of the element to add
+ * @param value value to add to the given row
+ */
+ public void add(int index, <#if (type.width > 4)> ${minor.javaType!type.javaType}
+ <#elseif type.major == "VarLen"> byte[]
+ <#else> int
+ </#if> value) {
+ countVector.set(index, countVector.get(index) + 1);
+ offsetVector.set(index, offsetVector.get(index - 1) + countVector.get(index-1));
+ super.set(offsetVector.get(index), value);
+ }
+
+ /**
+ * Get a value for the given record. Each element in the repeated field is accessed by
+ * the positionIndex param.
+ *
+ * @param index record containing the repeated field
+ * @param positionIndex position within the repeated field
+ * @return element at the given position in the given record
+ */
+ public <#if type.major == "VarLen">byte[]
+ <#else>${minor.javaType!type.javaType}
+ </#if> get(int index, int positionIndex) {
+
+ assert positionIndex < countVector.get(index);
+ return super.get(offsetVector.get(index) + positionIndex);
+ }
+
+ public MaterializedField getField() {
+ return field;
+ }
+
+ /**
+ * Get the size requirement (in bytes) for the given number of values. Only accurate
+ * for fixed width value vectors.
+ */
+ public int getTotalSizeFromCount(int valueCount) {
+ return getSizeFromCount(valueCount) +
+ countVector.getSizeFromCount(valueCount) +
+ offsetVector.getSizeFromCount(valueCount);
+ }
+
+ /**
+ * Get the explicitly specified size of the allocated buffer, if available. Otherwise
+ * calculate the size based on width and record count.
+ */
+ public int getAllocatedSize() {
+ return super.getAllocatedSize() +
+ countVector.getAllocatedSize() +
+ offsetVector.getAllocatedSize();
+ }
+
+ /**
+ * Get the elements at the given index.
+ */
+ public int getCount(int index) {
+ return countVector.get(index);
+ }
+
+ public void setRecordCount(int recordCount) {
+ super.setRecordCount(recordCount);
+ offsetVector.setRecordCount(recordCount);
+ countVector.setRecordCount(recordCount);
+ }
+
+ public ByteBuf[] getBuffers() {
+ return new ByteBuf[]{countVector.data, offsetVector.data, data};
+ }
+
+ public Object getObject(int index) {
+ return data.slice(index, getSizeFromCount(countVector.get(index)));
+ }
+
+ }
+ </#list>
+</#list>
+}
+
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
index 6a1eba4..65584db 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/config/MockRecordReader.java
@@ -37,7 +37,7 @@ public class MockRecordReader implements RecordReader {
private OutputMutator output;
private MockScanEntry config;
private FragmentContext context;
- private ValueVector<?>[] valueVectors;
+ private ValueVector.Base[] valueVectors;
private int recordsRead;
public MockRecordReader(FragmentContext context, MockScanEntry config) {
@@ -53,16 +53,14 @@ public class MockRecordReader implements RecordReader {
return x;
}
- private ValueVector<?> getVector(int fieldId, String name, MajorType type, int length) {
+ private ValueVector.Base getVector(int fieldId, String name, MajorType type, int length) {
assert context != null : "Context shouldn't be null.";
-
if(type.getMode() != DataMode.REQUIRED) throw new UnsupportedOperationException();
MaterializedField f = MaterializedField.create(new SchemaPath(name), fieldId, 0, type);
- ValueVector<?> v;
+ ValueVector.Base v;
v = TypeHelper.getNewVector(f, context.getAllocator());
v.allocateNew(length);
-
return v;
}
@@ -72,7 +70,7 @@ public class MockRecordReader implements RecordReader {
try {
this.output = output;
int estimateRowSize = getEstimatedRecordSize(config.getTypes());
- valueVectors = new ValueVector<?>[config.getTypes().length];
+ valueVectors = new ValueVector.Base[config.getTypes().length];
int batchRecordCount = 250000 / estimateRowSize;
for (int i = 0; i < config.getTypes().length; i++) {
@@ -90,7 +88,8 @@ public class MockRecordReader implements RecordReader {
public int next() {
int recordSetSize = Math.min(valueVectors[0].capacity(), this.config.getRecords()- recordsRead);
recordsRead += recordSetSize;
- for(ValueVector<?> v : valueVectors){
+ for(ValueVector.Base v : valueVectors){
+ logger.debug("MockRecordReader: Generating random data for VV of type " + v.getClass().getName());
v.randomizeData();
v.setRecordCount(recordSetSize);
}
@@ -103,7 +102,7 @@ public class MockRecordReader implements RecordReader {
try {
output.removeField(valueVectors[i].getField().getFieldId());
} catch (SchemaChangeException e) {
- logger.warn("Failure while trying tremove field.", e);
+ logger.warn("Failure while trying to remove field.", e);
}
valueVectors[i].close();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
index 6592ca1..b7b7d93 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/FilterRecordBatch.java
@@ -58,7 +58,7 @@ public abstract class FilterRecordBatch implements RecordBatch {
}
@Override
- public <T extends ValueVector<T>> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
+ public <T extends ValueVector.Base> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
return null;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java
index ce0cf66..e96c1be 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/OutputMutator.java
@@ -23,6 +23,6 @@ import org.apache.drill.exec.record.vector.ValueVector;
public interface OutputMutator {
public void removeField(int fieldId) throws SchemaChangeException;
- public void addField(int fieldId, ValueVector<?> vector) throws SchemaChangeException ;
+ public void addField(int fieldId, ValueVector.Base vector) throws SchemaChangeException ;
public void setNewSchema() throws SchemaChangeException ;
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
index 33c1e29..822d828 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScanBatch.java
@@ -45,7 +45,7 @@ import com.google.common.collect.Lists;
public class ScanBatch implements RecordBatch {
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ScanBatch.class);
- private IntObjectOpenHashMap<ValueVector<?>> fields = new IntObjectOpenHashMap<ValueVector<?>>();
+ private IntObjectOpenHashMap<ValueVector.Base> fields = new IntObjectOpenHashMap<ValueVector.Base>();
private BatchSchema schema;
private int recordCount;
private boolean schemaChanged = true;
@@ -89,9 +89,9 @@ public class ScanBatch implements RecordBatch {
}
private void releaseAssets() {
- fields.forEach(new IntObjectProcedure<ValueVector<?>>() {
+ fields.forEach(new IntObjectProcedure<ValueVector.Base>() {
@Override
- public void apply(int key, ValueVector<?> value) {
+ public void apply(int key, ValueVector.Base value) {
value.close();
}
});
@@ -99,9 +99,9 @@ public class ScanBatch implements RecordBatch {
@SuppressWarnings("unchecked")
@Override
- public <T extends ValueVector<T>> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
+ public <T extends ValueVector.Base> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
if (fields.containsKey(fieldId)) throw new InvalidValueAccessor(String.format("Unknown value accesor for field id %d."));
- ValueVector<?> vector = this.fields.lget();
+ ValueVector.Base vector = this.fields.lget();
if (vector.getClass().isAssignableFrom(clazz)) {
return (T) vector;
} else {
@@ -143,14 +143,14 @@ public class ScanBatch implements RecordBatch {
public void removeField(int fieldId) throws SchemaChangeException {
schemaChanged();
- ValueVector<?> v = fields.remove(fieldId);
+ ValueVector.Base v = fields.remove(fieldId);
if (v == null) throw new SchemaChangeException("Failure attempting to remove an unknown field.");
v.close();
}
- public void addField(int fieldId, ValueVector<?> vector) {
+ public void addField(int fieldId, ValueVector.Base vector) {
schemaChanged();
- ValueVector<?> v = fields.put(fieldId, vector);
+ ValueVector.Base v = fields.put(fieldId, vector);
vector.getField();
builder.addField(vector.getField());
if (v != null) v.close();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
index c20538d..3819036 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/ScreenCreator.java
@@ -73,7 +73,7 @@ public class ScreenCreator implements RootCreator<Screen>{
stop();
return false;
}
-
+
IterOutcome outcome = incoming.next();
logger.debug("Screen Outcome {}", outcome);
switch(outcome){
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java
index b7d4c7e..a2b9865 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/SingleSenderCreator.java
@@ -103,7 +103,7 @@ public class SingleSenderCreator implements RootCreator<SingleSender>{
@Override
public void success(Ack value) {
if(value.getOk()) return;
-
+
logger.error("Downstream fragment was not accepted. Stopping future sends.");
// if we didn't get ack ok, we'll need to kill the query.
context.fail(new RpcException("A downstream fragment batch wasn't accepted. This fragment thus fails."));
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
index b41b0cd..be32d1f 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/physical/impl/WireRecordBatch.java
@@ -68,7 +68,7 @@ public class WireRecordBatch implements RecordBatch{
}
@Override
- public <T extends ValueVector<T>> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
+ public <T extends ValueVector.Base> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
return batchLoader.getValueVector(fieldId, clazz);
}
@@ -76,7 +76,7 @@ public class WireRecordBatch implements RecordBatch{
public IterOutcome next() {
RawFragmentBatch batch = fragProvider.getNext();
try{
- if(batch == null) return IterOutcome.NONE;
+ if (batch == null) return IterOutcome.NONE;
logger.debug("Next received batch {}", batch);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
index 09427ef..d1858f1 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/MaterializedField.java
@@ -28,7 +28,6 @@ import org.apache.drill.exec.proto.SchemaDefProtos.FieldDef;
import org.apache.drill.exec.proto.SchemaDefProtos.MajorType;
import org.apache.drill.exec.proto.SchemaDefProtos.NamePart;
import org.apache.drill.exec.proto.SchemaDefProtos.NamePart.Type;
-import org.apache.drill.exec.record.vector.TypeHelper;
public class MaterializedField implements Comparable<MaterializedField> {
private final FieldDef def;
@@ -107,10 +106,6 @@ public class MaterializedField implements Comparable<MaterializedField> {
return def.getMajorType().getMode();
}
- public Class<?> getValueClass() {
- return TypeHelper.getValueVectorClass(getType().getMinorType(), getDataMode());
- }
-
public boolean matches(SchemaPath path) {
Iterator<NamePart> iter = def.getNameList().iterator();
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
index 3e4ded2..042c40c 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatch.java
@@ -68,7 +68,7 @@ public interface RecordBatch {
public void kill();
- public abstract <T extends ValueVector<T>> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor;
+ public abstract <T extends ValueVector.Base> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor;
// public abstract void getDictReader(int fieldId, Class<?> clazz) throws InvalidValueAccessor;
//
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
index d990198..ea1de73 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/RecordBatchLoader.java
@@ -33,10 +33,10 @@ import org.apache.drill.exec.record.vector.ValueVector;
import com.carrotsearch.hppc.IntObjectOpenHashMap;
import com.carrotsearch.hppc.cursors.IntObjectCursor;
-public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector<?>>>{
+public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector.Base>>{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RecordBatchLoader.class);
- private IntObjectOpenHashMap<ValueVector<?>> vectors = new IntObjectOpenHashMap<ValueVector<?>>();
+ private IntObjectOpenHashMap<ValueVector.Base> vectors = new IntObjectOpenHashMap<ValueVector.Base>();
private final BufferAllocator allocator;
private int recordCount;
private BatchSchema schema;
@@ -61,17 +61,17 @@ public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector<?
this.recordCount = def.getRecordCount();
boolean schemaChanged = false;
- IntObjectOpenHashMap<ValueVector<?>> newVectors = new IntObjectOpenHashMap<ValueVector<?>>();
+ IntObjectOpenHashMap<ValueVector.Base> newVectors = new IntObjectOpenHashMap<ValueVector.Base>();
List<FieldMetadata> fields = def.getFieldList();
int bufOffset = 0;
for (FieldMetadata fmd : fields) {
FieldDef fieldDef = fmd.getDef();
- ValueVector<?> v = vectors.remove(fieldDef.getFieldId());
+ ValueVector.Base v = vectors.remove(fieldDef.getFieldId());
if (v != null) {
if (v.getField().getDef().equals(fieldDef)) {
- v.setTo(fmd, buf.slice(bufOffset, fmd.getBufferLength()));
+ v.allocateNew(fmd.getBufferLength(), buf.slice(bufOffset, fmd.getBufferLength()), recordCount);
newVectors.put(fieldDef.getFieldId(), v);
continue;
} else {
@@ -83,13 +83,13 @@ public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector<?
schemaChanged = true;
MaterializedField m = new MaterializedField(fieldDef);
v = TypeHelper.getNewVector(m, allocator);
- v.setTo(fmd, buf.slice(bufOffset, fmd.getBufferLength()));
+ v.allocateNew(fmd.getBufferLength(), buf.slice(bufOffset, fmd.getBufferLength()), recordCount);
newVectors.put(fieldDef.getFieldId(), v);
}
if(!vectors.isEmpty()){
schemaChanged = true;
- for(IntObjectCursor<ValueVector<?>> cursor : newVectors){
+ for(IntObjectCursor<ValueVector.Base> cursor : newVectors){
cursor.value.close();
}
@@ -98,7 +98,7 @@ public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector<?
if(schemaChanged){
// rebuild the schema.
SchemaBuilder b = BatchSchema.newBuilder();
- for(IntObjectCursor<ValueVector<?>> cursor : newVectors){
+ for(IntObjectCursor<ValueVector.Base> cursor : newVectors){
b.addField(cursor.value.getField());
}
b.setSelectionVector(false);
@@ -110,8 +110,8 @@ public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector<?
}
@SuppressWarnings("unchecked")
- public <T extends ValueVector<T>> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
- ValueVector<?> v = vectors.get(fieldId);
+ public <T extends ValueVector.Base> T getValueVector(int fieldId, Class<T> clazz) throws InvalidValueAccessor {
+ ValueVector.Base v = vectors.get(fieldId);
assert v != null;
if (v.getClass() != clazz)
throw new InvalidValueAccessor(String.format(
@@ -130,7 +130,7 @@ public class RecordBatchLoader implements Iterable<IntObjectCursor<ValueVector<?
}
@Override
- public Iterator<IntObjectCursor<ValueVector<?>>> iterator() {
+ public Iterator<IntObjectCursor<ValueVector.Base>> iterator() {
return this.vectors.iterator();
}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
index 788c731..67c6cb9 100644
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
+++ b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/WritableBatch.java
@@ -58,7 +58,7 @@ public class WritableBatch {
return buffers;
}
-// public static WritableBatch get(ValueVector<?>[] vectors){
+// public static WritableBatch get(ValueVector.Base[] vectors){
// WritableCreator c = new WritableCreator();
// for(int i =0; i < vectors.length; i++){
// c.apply(i, vectors[i]);
@@ -67,14 +67,14 @@ public class WritableBatch {
// }
//
- public static WritableBatch get(int recordCount, IntObjectOpenHashMap<ValueVector<?>> fields){
+ public static WritableBatch get(int recordCount, IntObjectOpenHashMap<ValueVector.Base> fields){
WritableCreator creator = new WritableCreator(recordCount);
fields.forEach(creator);
return creator.get();
}
- private static class WritableCreator implements IntObjectProcedure<ValueVector<?>>{
+ private static class WritableCreator implements IntObjectProcedure<ValueVector.Base>{
List<ByteBuf> buffers = Lists.newArrayList();
List<FieldMetadata> metadata = Lists.newArrayList();
@@ -87,7 +87,7 @@ public class WritableBatch {
}
@Override
- public void apply(int key, ValueVector<?> value) {
+ public void apply(int key, ValueVector.Base value) {
metadata.add(value.getMetadata());
for(ByteBuf b : value.getBuffers()){
buffers.add(b);
@@ -97,7 +97,6 @@ public class WritableBatch {
value.allocateNew(value.capacity());
}
-
public WritableBatch get(){
RecordBatchDef batchDef = RecordBatchDef.newBuilder().addAllField(metadata).setRecordCount(recordCount).build();
WritableBatch b = new WritableBatch(batchDef, buffers);
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/AbstractFixedValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/AbstractFixedValueVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/AbstractFixedValueVector.java
deleted file mode 100644
index 8748285..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/AbstractFixedValueVector.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import io.netty.buffer.ByteBuf;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
-import org.apache.drill.exec.record.MaterializedField;
-
-/**
- * Abstract class that fixed value vectors are derived from.
- */
-abstract class AbstractFixedValueVector<T extends AbstractFixedValueVector<T>> extends BaseValueVector<T> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractFixedValueVector.class);
-
- protected final int widthInBits;
-
- protected int longWords = 0;
-
- public AbstractFixedValueVector(MaterializedField field, BufferAllocator allocator, int widthInBits) {
- super(field, allocator);
- this.widthInBits = widthInBits;
- }
-
- public int getWidthInBits() {
- return widthInBits;
- }
-
- @Override
- protected int getAllocationSize(int valueCount) {
- return (int) Math.ceil(valueCount*widthInBits*1.0/8);
- }
-
- @Override
- protected void childResetAllocation(int valueCount, ByteBuf buf) {
- this.longWords = valueCount/8;
- }
-
- @Override
- protected void childCloneMetadata(T other) {
- other.longWords = this.longWords;
- }
-
- @Override
- protected void childClear() {
- longWords = 0;
- }
-
- @Override
- public void setRecordCount(int recordCount) {
- this.data.writerIndex(recordCount*(widthInBits/8));
- super.setRecordCount(recordCount);
- }
-
-
-
-
-
-
-}
-
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/AnyVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/AnyVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/AnyVector.java
deleted file mode 100644
index 6becfcd..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/AnyVector.java
+++ /dev/null
@@ -1,30 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-/**
- * A Vector that holds each value with type information. In the case of complex types, an any vector will inline the
- * complex type within the value space. The complex value will be stored in
- */
-public class AnyVector {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AnyVector.class);
-
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/BaseValueVector.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/BaseValueVector.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/BaseValueVector.java
deleted file mode 100644
index 69cd628..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/BaseValueVector.java
+++ /dev/null
@@ -1,157 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import io.netty.buffer.ByteBuf;
-
-import java.util.Random;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.proto.UserBitShared.FieldMetadata;
-import org.apache.drill.exec.record.DeadBuf;
-import org.apache.drill.exec.record.MaterializedField;
-
-public abstract class BaseValueVector<T extends BaseValueVector<T>> implements ValueVector<T>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(BaseValueVector.class);
-
- protected final BufferAllocator allocator;
- protected ByteBuf data = DeadBuf.DEAD_BUFFER;
- protected int maxValueCount = 0;
- protected final MaterializedField field;
- private int recordCount;
-
- public BaseValueVector(MaterializedField field, BufferAllocator allocator) {
- this.allocator = allocator;
- this.field = field;
- }
-
- public final void allocateNew(int valueCount){
- int allocationSize = getAllocationSize(valueCount);
- ByteBuf newBuf = allocator.buffer(allocationSize);
- resetAllocation(valueCount, newBuf);
- }
-
- protected abstract int getAllocationSize(int maxValueCount);
- protected abstract void childResetAllocation(int valueCount, ByteBuf buf);
- protected abstract void childCloneMetadata(T other);
- protected abstract void childClear();
-
- /**
- * Update the current buffer allocation utilize the provided allocation.
- * @param maxValueCount
- * @param buf
- */
- protected final void resetAllocation(int maxValueCount, ByteBuf buf){
- clear();
- buf.retain();
- this.maxValueCount = maxValueCount;
- this.data = buf;
- childResetAllocation(maxValueCount, buf);
- }
-
- public final void cloneMetadata(T other){
- other.maxValueCount = this.maxValueCount;
- }
-
-
- @Override
- public final void cloneInto(T vector) {
- vector.allocateNew(maxValueCount);
- data.writeBytes(vector.data);
- cloneMetadata(vector);
- childResetAllocation(maxValueCount, vector.data);
- }
-
- @Override
- public final void transferTo(T vector) {
- vector.data = this.data;
- cloneMetadata(vector);
- childResetAllocation(maxValueCount, data);
- clear();
- }
-
- protected final void clear(){
- if(this.data != DeadBuf.DEAD_BUFFER){
- this.data.release();
- this.data = DeadBuf.DEAD_BUFFER;
- this.maxValueCount = 0;
- }
- childClear();
- }
-
- /**
- * Give the length of the value vector in bytes.
- *
- * @return
- */
- public int capacity() {
- return maxValueCount;
- }
-
- @Override
- public void close() {
- clear();
- }
-
- @Override
- public ByteBuf[] getBuffers() {
- return new ByteBuf[]{data};
- }
-
- public MaterializedField getField(){
- return field;
- }
-
-
- public int getRecordCount() {
- return recordCount;
- }
-
- public void setRecordCount(int recordCount) {
- this.recordCount = recordCount;
- }
-
- @Override
- public FieldMetadata getMetadata() {
- int len = 0;
- for(ByteBuf b : getBuffers()){
- len += b.writerIndex();
- }
- return FieldMetadata.newBuilder().setDef(getField().getDef()).setValueCount(getRecordCount()).setBufferLength(len).build();
- }
-
- @Override
- public void setTo(FieldMetadata metadata, ByteBuf data) {
-// logger.debug("Updating value vector to {}, {}", metadata, data);
- clear();
- resetAllocation(metadata.getValueCount(), data);
- }
-
- @Override
- public void randomizeData() {
- if(this.data != DeadBuf.DEAD_BUFFER){
- Random r = new Random();
- for(int i =0; i < data.capacity()-8; i+=8){
- data.setLong(i, r.nextLong());
- }
- }
-
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Bit.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Bit.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Bit.java
deleted file mode 100644
index 533e3bd..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Bit.java
+++ /dev/null
@@ -1,168 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.common.expression.types.DataType;
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.physical.RecordField.ValueMode;
-import org.apache.drill.exec.record.MaterializedField;
-import org.apache.hadoop.io.SequenceFile;
-
-/**
- * Describes a vector which holds a number of true/false values.
- */
-public class Bit extends AbstractFixedValueVector<Bit> {
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Bit.class);
-
- public Bit(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator, 1);
- }
-
-
-// /** Returns true or false for the specified bit index.
-// * The index should be less than the OpenBitSet size
-// */
-// public boolean get(int index) {
-// assert index >= 0 && index < this.valueCount;
-// int i = index >> 3; // div 8
-// // signed shift will keep a negative index and force an
-// // array-index-out-of-bounds-exception, removing the need for an explicit check.
-// int bit = index & 0x3f; // mod 64
-// long bitmask = 1L << bit;
-// return (data.getLong(i) & bitmask) != 0;
-// }
-
- public int getBit(int index) {
-
- assert index >= 0 && index < this.maxValueCount;
- int i = 8*(index >> 6); // div 8
- int bit = index & 0x3f; // mod 64
- return ((int) (data.getLong(i) >>> bit)) & 0x01;
- }
-
- /** Sets the bit at the specified index.
- * The index should be less than the OpenBitSet size.
- */
- public void set(int index) {
- assert index >= 0 && index < this.maxValueCount;
- int wordNum = index >> 3;
- int bit = index & 0x3f;
- long bitmask = 1L << bit;
- data.setLong(wordNum, data.getLong(wordNum) | bitmask);
- }
-
- public void clear(int index) {
- assert index >= 0 && index < this.maxValueCount;
- int wordNum = index >> 3;
- int bit = index & 0x03f;
- long bitmask = 1L << bit;
- data.setLong(wordNum, data.getLong(wordNum) & ~bitmask);
- }
-
-
-
- /** Clears a range of bits. Clearing past the end does not change the size of the set.
- *
- * @param startBitIndex lower index
- * @param lastBitIndex one-past the last bit to clear
- */
- private void clear2(int startBitIndex, int lastBitIndex) {
- if (lastBitIndex <= startBitIndex) return;
-
- int firstWordStart = (startBitIndex>>3);
- if (firstWordStart >= this.longWords) return;
-
- // since endIndex is one past the end, this is index of the last
- // word to be changed.
- int lastWordStart = ((lastBitIndex-1)>>3);
-
- long startmask = -1L << startBitIndex;
- long endmask = -1L >>> -lastBitIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
-
- // invert masks since we are clearing
- startmask = ~startmask;
- endmask = ~endmask;
-
- if (firstWordStart == lastWordStart) {
- data.setLong(firstWordStart, data.getLong(firstWordStart) & (startmask | endmask));
- return;
- }
- data.setLong(firstWordStart, data.getLong(firstWordStart) & startmask);
-
- int middle = Math.min(this.longWords, lastWordStart);
-
- for(int i =firstWordStart+8; i < middle; i += 8){
- data.setLong(i, 0L);
- }
- if (lastWordStart < this.longWords) {
- data.setLong(lastWordStart, data.getLong(lastWordStart) & endmask);
- }
- }
-
- public void setAllFalse(){
- clear(0, maxValueCount);
- }
-
-
- public void clear(int startIndex, int endIndex) {
- if (endIndex <= startIndex) return;
-
- int startWord = (startIndex >> 6);
- if (startWord >= longWords) return;
-
- // since endIndex is one past the end, this is index of the last
- // word to be changed.
- int endWord = ((endIndex - 1) >> 6);
-
- long startmask = -1L << startIndex;
- long endmask = -1L >>> -endIndex; // 64-(endIndex&0x3f) is the same as -endIndex due to wrap
-
- // invert masks since we are clearing
- startmask = ~startmask;
- endmask = ~endmask;
-
- int startWordPos = startWord * 8;
- if (startWord == endWord) {
- data.setLong(startWordPos, data.getLong(startWordPos) & (startmask | endmask));
- return;
- }
-
- int endWordPos = endWord * 8;
-
- data.setLong(startWordPos, data.getLong(startWordPos) & startmask);
-
- int middle = Math.min(longWords, endWord)*8;
-
-
- for(int i =startWordPos+8; i < middle; i += 8){
- data.setLong(i, 0L);
- }
-
- if (endWordPos < startWordPos) {
- data.setLong(endWordPos, data.getLong(endWordPos) & endmask);
- }
- }
-
-
- @Override
- public Object getObject(int index) {
- return this.getBit(index);
- }
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed1.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed1.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed1.java
deleted file mode 100644
index 82c86d1..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed1.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-public class Fixed1 extends AbstractFixedValueVector<Fixed1>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Fixed1.class);
-
- public Fixed1(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator, 8);
- }
-
- public void setByte(int index, byte b){
- data.setByte(index, b);
- }
-
- public byte getByte(int index){
- return data.getByte(index);
- }
-
- @Override
- public Object getObject(int index) {
- return getByte(index);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed12.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed12.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed12.java
deleted file mode 100644
index c5f641a..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed12.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-public class Fixed12 extends AbstractFixedValueVector<Fixed12>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Fixed12.class);
-
- public Fixed12(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator, 12*8);
- }
-
-
- @Override
- public Object getObject(int index) {
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed16.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed16.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed16.java
deleted file mode 100644
index 649832b..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed16.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-public class Fixed16 extends AbstractFixedValueVector<Fixed16>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Fixed16.class);
-
- public Fixed16(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator, 16*8);
- }
-
- @Override
- public Object getObject(int index) {
- return null;
- }
-
-
-
-}
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/7075cca1/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed2.java
----------------------------------------------------------------------
diff --git a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed2.java b/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed2.java
deleted file mode 100644
index bd0e313..0000000
--- a/sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/Fixed2.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/*******************************************************************************
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.drill.exec.record.vector;
-
-import org.apache.drill.exec.memory.BufferAllocator;
-import org.apache.drill.exec.record.MaterializedField;
-
-public class Fixed2 extends AbstractFixedValueVector<Fixed2>{
- static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(Fixed2.class);
-
- public Fixed2(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator, 2*8);
- }
-
- public final void setSmallInt(int index, short value){
- index*=2;
- data.setShort(index, value);
- }
-
- public final short getSmallInt(int index){
- index*=2;
- return data.getShort(index);
- }
-
- public final void setUInt2(int index, short value){
- setSmallInt(index, value);
- }
-
- public final short getUInt2(int index){
- return getSmallInt(index);
- }
-
- @Override
- public Object getObject(int index) {
- return getSmallInt(index);
- }
-
-}