You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by ja...@apache.org on 2016/02/17 13:39:40 UTC
[05/17] arrow git commit: ARROW-1: Initial Arrow Code Commit
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
new file mode 100644
index 0000000..61ce285
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueHolderHelper.java
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.math.BigDecimal;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.holders.BigIntHolder;
+import org.apache.arrow.vector.holders.BitHolder;
+import org.apache.arrow.vector.holders.DateHolder;
+import org.apache.arrow.vector.holders.Decimal18Holder;
+import org.apache.arrow.vector.holders.Decimal28SparseHolder;
+import org.apache.arrow.vector.holders.Decimal38SparseHolder;
+import org.apache.arrow.vector.holders.Decimal9Holder;
+import org.apache.arrow.vector.holders.Float4Holder;
+import org.apache.arrow.vector.holders.Float8Holder;
+import org.apache.arrow.vector.holders.IntHolder;
+import org.apache.arrow.vector.holders.IntervalDayHolder;
+import org.apache.arrow.vector.holders.IntervalYearHolder;
+import org.apache.arrow.vector.holders.NullableBitHolder;
+import org.apache.arrow.vector.holders.TimeHolder;
+import org.apache.arrow.vector.holders.TimeStampHolder;
+import org.apache.arrow.vector.holders.VarCharHolder;
+import org.apache.arrow.vector.util.DecimalUtility;
+
+import com.google.common.base.Charsets;
+
+
+public class ValueHolderHelper {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueHolderHelper.class);
+
+ public static IntHolder getIntHolder(int value) {
+ IntHolder holder = new IntHolder();
+ holder.value = value;
+
+ return holder;
+ }
+
+ public static BigIntHolder getBigIntHolder(long value) {
+ BigIntHolder holder = new BigIntHolder();
+ holder.value = value;
+
+ return holder;
+ }
+
+ public static Float4Holder getFloat4Holder(float value) {
+ Float4Holder holder = new Float4Holder();
+ holder.value = value;
+
+ return holder;
+ }
+
+ public static Float8Holder getFloat8Holder(double value) {
+ Float8Holder holder = new Float8Holder();
+ holder.value = value;
+
+ return holder;
+ }
+
+ public static DateHolder getDateHolder(long value) {
+ DateHolder holder = new DateHolder();
+ holder.value = value;
+ return holder;
+ }
+
+ public static TimeHolder getTimeHolder(int value) {
+ TimeHolder holder = new TimeHolder();
+ holder.value = value;
+ return holder;
+ }
+
+ public static TimeStampHolder getTimeStampHolder(long value) {
+ TimeStampHolder holder = new TimeStampHolder();
+ holder.value = value;
+ return holder;
+ }
+
+ public static BitHolder getBitHolder(int value) {
+ BitHolder holder = new BitHolder();
+ holder.value = value;
+
+ return holder;
+ }
+
+ public static NullableBitHolder getNullableBitHolder(boolean isNull, int value) {
+ NullableBitHolder holder = new NullableBitHolder();
+ holder.isSet = isNull? 0 : 1;
+ if (! isNull) {
+ holder.value = value;
+ }
+
+ return holder;
+ }
+
+ public static VarCharHolder getVarCharHolder(ArrowBuf buf, String s){
+ VarCharHolder vch = new VarCharHolder();
+
+ byte[] b = s.getBytes(Charsets.UTF_8);
+ vch.start = 0;
+ vch.end = b.length;
+ vch.buffer = buf.reallocIfNeeded(b.length);
+ vch.buffer.setBytes(0, b);
+ return vch;
+ }
+
+ public static VarCharHolder getVarCharHolder(BufferAllocator a, String s){
+ VarCharHolder vch = new VarCharHolder();
+
+ byte[] b = s.getBytes(Charsets.UTF_8);
+ vch.start = 0;
+ vch.end = b.length;
+ vch.buffer = a.buffer(b.length); //
+ vch.buffer.setBytes(0, b);
+ return vch;
+ }
+
+
+ public static IntervalYearHolder getIntervalYearHolder(int intervalYear) {
+ IntervalYearHolder holder = new IntervalYearHolder();
+
+ holder.value = intervalYear;
+ return holder;
+ }
+
+ public static IntervalDayHolder getIntervalDayHolder(int days, int millis) {
+ IntervalDayHolder dch = new IntervalDayHolder();
+
+ dch.days = days;
+ dch.milliseconds = millis;
+ return dch;
+ }
+
+ public static Decimal9Holder getDecimal9Holder(int decimal, int scale, int precision) {
+ Decimal9Holder dch = new Decimal9Holder();
+
+ dch.scale = scale;
+ dch.precision = precision;
+ dch.value = decimal;
+
+ return dch;
+ }
+
+ public static Decimal18Holder getDecimal18Holder(long decimal, int scale, int precision) {
+ Decimal18Holder dch = new Decimal18Holder();
+
+ dch.scale = scale;
+ dch.precision = precision;
+ dch.value = decimal;
+
+ return dch;
+ }
+
+ public static Decimal28SparseHolder getDecimal28Holder(ArrowBuf buf, String decimal) {
+
+ Decimal28SparseHolder dch = new Decimal28SparseHolder();
+
+ BigDecimal bigDecimal = new BigDecimal(decimal);
+
+ dch.scale = bigDecimal.scale();
+ dch.precision = bigDecimal.precision();
+ Decimal28SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
+ dch.start = 0;
+ dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.INTEGER_SIZE);
+ DecimalUtility
+ .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
+
+ return dch;
+ }
+
+ public static Decimal38SparseHolder getDecimal38Holder(ArrowBuf buf, String decimal) {
+
+ Decimal38SparseHolder dch = new Decimal38SparseHolder();
+
+ BigDecimal bigDecimal = new BigDecimal(decimal);
+
+ dch.scale = bigDecimal.scale();
+ dch.precision = bigDecimal.precision();
+ Decimal38SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer);
+ dch.start = 0;
+ dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.INTEGER_SIZE);
+ DecimalUtility
+ .getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits);
+
+ return dch;
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
new file mode 100644
index 0000000..c05f0e7
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -0,0 +1,222 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import java.io.Closeable;
+
+import io.netty.buffer.ArrowBuf;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.util.TransferPair;
+
+/**
+ * An abstraction that is used to store a sequence of values in an individual column.
+ *
+ * A {@link ValueVector value vector} stores underlying data in-memory in a columnar fashion that is compact and
+ * efficient. The column whose data is stored, is referred by {@link #getField()}.
+ *
+ * A vector when instantiated, relies on a {@link org.apache.drill.exec.record.DeadBuf dead buffer}. It is important
+ * that vector is allocated before attempting to read or write.
+ *
+ * There are a few "rules" around vectors:
+ *
+ * <ul>
+ * <li>values need to be written in order (e.g. index 0, 1, 2, 5)</li>
+ * <li>null vectors start with all values as null before writing anything</li>
+ * <li>for variable width types, the offset vector should be all zeros before writing</li>
+ * <li>you must call setValueCount before a vector can be read</li>
+ * <li>you should never write to a vector once it has been read.</li>
+ * </ul>
+ *
+ * Please note that the current implementation doesn't enfore those rules, hence we may find few places that
+ * deviate from these rules (e.g. offset vectors in Variable Length and Repeated vector)
+ *
+ * This interface "should" strive to guarantee this order of operation:
+ * <blockquote>
+ * allocate > mutate > setvaluecount > access > clear (or allocate to start the process over).
+ * </blockquote>
+ */
+public interface ValueVector extends Closeable, Iterable<ValueVector> {
+ /**
+ * Allocate new buffers. ValueVector implements logic to determine how much to allocate.
+ * @throws OutOfMemoryException Thrown if no memory can be allocated.
+ */
+ void allocateNew() throws OutOfMemoryException;
+
+ /**
+ * Allocates new buffers. ValueVector implements logic to determine how much to allocate.
+ * @return Returns true if allocation was succesful.
+ */
+ boolean allocateNewSafe();
+
+ BufferAllocator getAllocator();
+
+ /**
+ * Set the initial record capacity
+ * @param numRecords
+ */
+ void setInitialCapacity(int numRecords);
+
+ /**
+ * Returns the maximum number of values that can be stored in this vector instance.
+ */
+ int getValueCapacity();
+
+ /**
+ * Alternative to clear(). Allows use as an AutoCloseable in try-with-resources.
+ */
+ @Override
+ void close();
+
+ /**
+ * Release the underlying DrillBuf and reset the ValueVector to empty.
+ */
+ void clear();
+
+ /**
+ * Get information about how this field is materialized.
+ */
+ MaterializedField getField();
+
+ /**
+ * Returns a {@link org.apache.arrow.vector.util.TransferPair transfer pair}, creating a new target vector of
+ * the same type.
+ */
+ TransferPair getTransferPair(BufferAllocator allocator);
+
+ TransferPair getTransferPair(String ref, BufferAllocator allocator);
+
+ /**
+ * Returns a new {@link org.apache.arrow.vector.util.TransferPair transfer pair} that is used to transfer underlying
+ * buffers into the target vector.
+ */
+ TransferPair makeTransferPair(ValueVector target);
+
+ /**
+ * Returns an {@link org.apache.arrow.vector.ValueVector.Accessor accessor} that is used to read from this vector
+ * instance.
+ */
+ Accessor getAccessor();
+
+ /**
+ * Returns an {@link org.apache.arrow.vector.ValueVector.Mutator mutator} that is used to write to this vector
+ * instance.
+ */
+ Mutator getMutator();
+
+ /**
+ * Returns a {@link org.apache.arrow.vector.complex.reader.FieldReader field reader} that supports reading values
+ * from this vector.
+ */
+ FieldReader getReader();
+
+ /**
+ * Get the metadata for this field. Used in serialization
+ *
+ * @return FieldMetadata for this field.
+ */
+// SerializedField getMetadata();
+
+ /**
+ * Returns the number of bytes that is used by this vector instance.
+ */
+ int getBufferSize();
+
+ /**
+ * Returns the number of bytes that is used by this vector if it holds the given number
+ * of values. The result will be the same as if Mutator.setValueCount() were called, followed
+ * by calling getBufferSize(), but without any of the closing side-effects that setValueCount()
+ * implies wrt finishing off the population of a vector. Some operations might wish to use
+ * this to determine how much memory has been used by a vector so far, even though it is
+ * not finished being populated.
+ *
+ * @param valueCount the number of values to assume this vector contains
+ * @return the buffer size if this vector is holding valueCount values
+ */
+ int getBufferSizeFor(int valueCount);
+
+ /**
+ * Return the underlying buffers associated with this vector. Note that this doesn't impact the reference counts for
+ * this buffer so it only should be used for in-context access. Also note that this buffer changes regularly thus
+ * external classes shouldn't hold a reference to it (unless they change it).
+ * @param clear Whether to clear vector before returning; the buffers will still be refcounted;
+ * but the returned array will be the only reference to them
+ *
+ * @return The underlying {@link io.netty.buffer.ArrowBuf buffers} that is used by this vector instance.
+ */
+ ArrowBuf[] getBuffers(boolean clear);
+
+ /**
+ * Load the data provided in the buffer. Typically used when deserializing from the wire.
+ *
+ * @param metadata
+ * Metadata used to decode the incoming buffer.
+ * @param buffer
+ * The buffer that contains the ValueVector.
+ */
+// void load(SerializedField metadata, DrillBuf buffer);
+
+ /**
+ * An abstraction that is used to read from this vector instance.
+ */
+ interface Accessor {
+ /**
+ * Get the Java Object representation of the element at the specified position. Useful for testing.
+ *
+ * @param index
+ * Index of the value to get
+ */
+ Object getObject(int index);
+
+ /**
+ * Returns the number of values that is stored in this vector.
+ */
+ int getValueCount();
+
+ /**
+ * Returns true if the value at the given index is null, false otherwise.
+ */
+ boolean isNull(int index);
+ }
+
+ /**
+ * An abstractiong that is used to write into this vector instance.
+ */
+ interface Mutator {
+ /**
+ * Sets the number of values that is stored in this vector to the given value count.
+ *
+ * @param valueCount value count to set.
+ */
+ void setValueCount(int valueCount);
+
+ /**
+ * Resets the mutator to pristine state.
+ */
+ void reset();
+
+ /**
+ * @deprecated this has nothing to do with value vector abstraction and should be removed.
+ */
+ @Deprecated
+ void generateTestData(int values);
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
new file mode 100644
index 0000000..e227bb4
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VariableWidthVector.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+public interface VariableWidthVector extends ValueVector{
+
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param totalBytes Desired size of the underlying data buffer.
+ * @param valueCount Number of values in the vector.
+ */
+ void allocateNew(int totalBytes, int valueCount);
+
+ /**
+ * Provide the maximum amount of variable width bytes that can be stored int his vector.
+ * @return
+ */
+ int getByteCapacity();
+
+ VariableWidthMutator getMutator();
+
+ VariableWidthAccessor getAccessor();
+
+ interface VariableWidthAccessor extends Accessor {
+ int getValueLength(int index);
+ }
+
+ int getCurrentSizeInBytes();
+
+ interface VariableWidthMutator extends Mutator {
+ void setValueLengthSafe(int index, int length);
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
new file mode 100644
index 0000000..fdad99a
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorDescriptor.java
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import java.util.Collection;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.MajorType;
+
+public class VectorDescriptor {
+ private static final String DEFAULT_NAME = "NONE";
+
+ private final MaterializedField field;
+
+ public VectorDescriptor(final MajorType type) {
+ this(DEFAULT_NAME, type);
+ }
+
+ public VectorDescriptor(final String name, final MajorType type) {
+ this(MaterializedField.create(name, type));
+ }
+
+ public VectorDescriptor(final MaterializedField field) {
+ this.field = Preconditions.checkNotNull(field, "field cannot be null");
+ }
+
+ public MaterializedField getField() {
+ return field;
+ }
+
+ public MajorType getType() {
+ return field.getType();
+ }
+
+ public String getName() {
+ return field.getLastName();
+ }
+
+ public Collection<MaterializedField> getChildren() {
+ return field.getChildren();
+ }
+
+ public boolean hasName() {
+ return getName() != DEFAULT_NAME;
+ }
+
+ public VectorDescriptor withName(final String name) {
+ return new VectorDescriptor(field.withPath(name));
+ }
+
+ public VectorDescriptor withType(final MajorType type) {
+ return new VectorDescriptor(field.withType(type));
+ }
+
+ public static VectorDescriptor create(final String name, final MajorType type) {
+ return new VectorDescriptor(name, type);
+ }
+
+ public static VectorDescriptor create(final MajorType type) {
+ return new VectorDescriptor(type);
+ }
+
+ public static VectorDescriptor create(final MaterializedField field) {
+ return new VectorDescriptor(field);
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java b/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java
new file mode 100644
index 0000000..055857e
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/VectorTrimmer.java
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ArrowBuf;
+
+public class VectorTrimmer {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VectorTrimmer.class);
+
+ public static void trim(ByteBuf data, int idx) {
+ data.writerIndex(idx);
+ if (data instanceof ArrowBuf) {
+ // data.capacity(idx);
+ data.writerIndex(idx);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
new file mode 100644
index 0000000..78de870
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -0,0 +1,181 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Iterator;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.complex.impl.NullReader;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.TransferPair;
+
+import com.google.common.collect.Iterators;
+
+public class ZeroVector implements ValueVector {
+ public final static ZeroVector INSTANCE = new ZeroVector();
+
+ private final MaterializedField field = MaterializedField.create("[DEFAULT]", Types.required(MinorType.LATE));
+
+ private final TransferPair defaultPair = new TransferPair() {
+ @Override
+ public void transfer() { }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) { }
+
+ @Override
+ public ValueVector getTo() {
+ return ZeroVector.this;
+ }
+
+ @Override
+ public void copyValueSafe(int from, int to) { }
+ };
+
+ private final Accessor defaultAccessor = new Accessor() {
+ @Override
+ public Object getObject(int index) {
+ return null;
+ }
+
+ @Override
+ public int getValueCount() {
+ return 0;
+ }
+
+ @Override
+ public boolean isNull(int index) {
+ return true;
+ }
+ };
+
+ private final Mutator defaultMutator = new Mutator() {
+ @Override
+ public void setValueCount(int valueCount) { }
+
+ @Override
+ public void reset() { }
+
+ @Override
+ public void generateTestData(int values) { }
+ };
+
+ public ZeroVector() { }
+
+ @Override
+ public void close() { }
+
+ @Override
+ public void clear() { }
+
+ @Override
+ public MaterializedField getField() {
+ return field;
+ }
+
+ @Override
+ public TransferPair getTransferPair(BufferAllocator allocator) {
+ return defaultPair;
+ }
+
+// @Override
+// public UserBitShared.SerializedField getMetadata() {
+// return getField()
+// .getAsBuilder()
+// .setBufferLength(getBufferSize())
+// .setValueCount(getAccessor().getValueCount())
+// .build();
+// }
+
+ @Override
+ public Iterator iterator() {
+ return Iterators.emptyIterator();
+ }
+
+ @Override
+ public int getBufferSize() {
+ return 0;
+ }
+
+ @Override
+ public int getBufferSizeFor(final int valueCount) {
+ return 0;
+ }
+
+ @Override
+ public ArrowBuf[] getBuffers(boolean clear) {
+ return new ArrowBuf[0];
+ }
+
+ @Override
+ public void allocateNew() throws OutOfMemoryException {
+ allocateNewSafe();
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ return true;
+ }
+
+ @Override
+ public BufferAllocator getAllocator() {
+ throw new UnsupportedOperationException("Tried to get allocator from ZeroVector");
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) { }
+
+ @Override
+ public int getValueCapacity() {
+ return 0;
+ }
+
+ @Override
+ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+ return defaultPair;
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector target) {
+ return defaultPair;
+ }
+
+ @Override
+ public Accessor getAccessor() {
+ return defaultAccessor;
+ }
+
+ @Override
+ public Mutator getMutator() {
+ return defaultMutator;
+ }
+
+ @Override
+ public FieldReader getReader() {
+ return NullReader.INSTANCE;
+ }
+
+// @Override
+// public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
new file mode 100644
index 0000000..c671c9e
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractContainerVector.java
@@ -0,0 +1,143 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import java.util.Collection;
+
+import javax.annotation.Nullable;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.CallBack;
+
+import com.google.common.base.Function;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Sets;
+
+/**
+ * Base class for composite vectors.
+ *
+ * This class implements common functionality of composite vectors.
+ */
+public abstract class AbstractContainerVector implements ValueVector {
+ static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
+
+ protected MaterializedField field;
+ protected final BufferAllocator allocator;
+ protected final CallBack callBack;
+
+ protected AbstractContainerVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+ this.field = Preconditions.checkNotNull(field);
+ this.allocator = allocator;
+ this.callBack = callBack;
+ }
+
+ @Override
+ public void allocateNew() throws OutOfMemoryException {
+ if (!allocateNewSafe()) {
+ throw new OutOfMemoryException();
+ }
+ }
+
+ public BufferAllocator getAllocator() {
+ return allocator;
+ }
+
+ /**
+ * Returns the field definition of this instance.
+ */
+ @Override
+ public MaterializedField getField() {
+ return field;
+ }
+
+ /**
+ * Returns a {@link org.apache.arrow.vector.ValueVector} corresponding to the given field name if exists or null.
+ */
+ public ValueVector getChild(String name) {
+ return getChild(name, ValueVector.class);
+ }
+
+ /**
+ * Returns a sequence of field names in the order that they show up in the schema.
+ */
+ protected Collection<String> getChildFieldNames() {
+ return Sets.newLinkedHashSet(Iterables.transform(field.getChildren(), new Function<MaterializedField, String>() {
+ @Nullable
+ @Override
+ public String apply(MaterializedField field) {
+ return Preconditions.checkNotNull(field).getLastName();
+ }
+ }));
+ }
+
+ /**
+ * Clears out all underlying child vectors.
+ */
+ @Override
+ public void close() {
+ for (ValueVector vector:(Iterable<ValueVector>)this) {
+ vector.close();
+ }
+ }
+
+ protected <T extends ValueVector> T typeify(ValueVector v, Class<T> clazz) {
+ if (clazz.isAssignableFrom(v.getClass())) {
+ return (T) v;
+ }
+ throw new IllegalStateException(String.format("Vector requested [%s] was different than type stored [%s]. Drill doesn't yet support hetergenous types.", clazz.getSimpleName(), v.getClass().getSimpleName()));
+ }
+
+ MajorType getLastPathType() {
+ if((this.getField().getType().getMinorType() == MinorType.LIST &&
+ this.getField().getType().getMode() == DataMode.REPEATED)) { // Use Repeated scalar type instead of Required List.
+ VectorWithOrdinal vord = getChildVectorWithOrdinal(null);
+ ValueVector v = vord.vector;
+ if (! (v instanceof AbstractContainerVector)) {
+ return v.getField().getType();
+ }
+ } else if (this.getField().getType().getMinorType() == MinorType.MAP &&
+ this.getField().getType().getMode() == DataMode.REPEATED) { // Use Required Map
+ return new MajorType(MinorType.MAP, DataMode.REQUIRED);
+ }
+
+ return this.getField().getType();
+ }
+
+ protected boolean supportsDirectRead() {
+ return false;
+ }
+
+ // return the number of child vectors
+ public abstract int size();
+
+ // add a new vector with the input MajorType or return the existing vector if we already added one with the same type
+ public abstract <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz);
+
+ // return the child vector with the input name
+ public abstract <T extends ValueVector> T getChild(String name, Class<T> clazz);
+
+ // return the child vector's ordinal in the composite container
+ public abstract VectorWithOrdinal getChildVectorWithOrdinal(String name);
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
new file mode 100644
index 0000000..d4189b2
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -0,0 +1,278 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.MapWithOrdinal;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+/*
+ * Base class for MapVectors. Currently used by RepeatedMapVector and MapVector
+ */
+public abstract class AbstractMapVector extends AbstractContainerVector {
+ private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractContainerVector.class);
+
+ // Maintains a map with key as field name and value is the vector itself
+ private final MapWithOrdinal<String, ValueVector> vectors = new MapWithOrdinal<>();
+
+ protected AbstractMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+ super(field.clone(), allocator, callBack);
+ MaterializedField clonedField = field.clone();
+ // create the hierarchy of the child vectors based on the materialized field
+ for (MaterializedField child : clonedField.getChildren()) {
+ if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
+ final String fieldName = child.getLastName();
+ final ValueVector v = BasicTypeHelper.getNewVector(child, allocator, callBack);
+ putVector(fieldName, v);
+ }
+ }
+ }
+
+ @Override
+ public void close() {
+ for(final ValueVector valueVector : vectors.values()) {
+ valueVector.close();
+ }
+ vectors.clear();
+
+ super.close();
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ /* boolean to keep track if all the memory allocation were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ for (final ValueVector v : vectors.values()) {
+ if (!v.allocateNewSafe()) {
+ return false;
+ }
+ }
+ success = true;
+ } finally {
+ if (!success) {
+ clear();
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
+ * {@link org.apache.arrow.vector.ValueVector}.
+ *
+ * Execution takes place in the following order:
+ * <ul>
+ * <li>
+ * if field is new, create and insert a new vector of desired type.
+ * </li>
+ * <li>
+ * if field exists and existing vector is of desired vector type, return the vector.
+ * </li>
+ * <li>
+ * if field exists and null filled, clear the existing vector; create and insert a new vector of desired type.
+ * </li>
+ * <li>
+ * otherwise, throw an {@link java.lang.IllegalStateException}
+ * </li>
+ * </ul>
+ *
+ * @param name name of the field
+ * @param type type of the field
+ * @param clazz class of expected vector type
+ * @param <T> class type of expected vector type
+ * @throws java.lang.IllegalStateException raised if there is a hard schema change
+ *
+ * @return resultant {@link org.apache.arrow.vector.ValueVector}
+ */
+ @Override
+ public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
+ final ValueVector existing = getChild(name);
+ boolean create = false;
+ if (existing == null) {
+ create = true;
+ } else if (clazz.isAssignableFrom(existing.getClass())) {
+ return (T) existing;
+ } else if (nullFilled(existing)) {
+ existing.clear();
+ create = true;
+ }
+ if (create) {
+ final T vector = (T) BasicTypeHelper.getNewVector(name, allocator, type, callBack);
+ putChild(name, vector);
+ if (callBack!=null) {
+ callBack.doWork();
+ }
+ return vector;
+ }
+ final String message = "Drill does not support schema change yet. Existing[%s] and desired[%s] vector types mismatch";
+ throw new IllegalStateException(String.format(message, existing.getClass().getSimpleName(), clazz.getSimpleName()));
+ }
+
+ private boolean nullFilled(ValueVector vector) {
+ for (int r = 0; r < vector.getAccessor().getValueCount(); r++) {
+ if (!vector.getAccessor().isNull(r)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Returns a {@link org.apache.arrow.vector.ValueVector} corresponding to the given ordinal identifier.
+ */
+ public ValueVector getChildByOrdinal(int id) {
+ return vectors.getByOrdinal(id);
+ }
+
+ /**
+ * Returns a {@link org.apache.arrow.vector.ValueVector} instance of subtype of <T> corresponding to the given
+ * field name if exists or null.
+ */
+ @Override
+ public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
+ final ValueVector v = vectors.get(name.toLowerCase());
+ if (v == null) {
+ return null;
+ }
+ return typeify(v, clazz);
+ }
+
+ /**
+ * Inserts the vector with the given name if it does not exist else replaces it with the new value.
+ *
+ * Note that this method does not enforce any vector type check nor throws a schema change exception.
+ */
+ protected void putChild(String name, ValueVector vector) {
+ putVector(name, vector);
+ field.addChild(vector.getField());
+ }
+
+ /**
+ * Inserts the input vector into the map if it does not exist, replaces if it exists already
+ * @param name field name
+ * @param vector vector to be inserted
+ */
+ protected void putVector(String name, ValueVector vector) {
+ final ValueVector old = vectors.put(
+ Preconditions.checkNotNull(name, "field name cannot be null").toLowerCase(),
+ Preconditions.checkNotNull(vector, "vector cannot be null")
+ );
+ if (old != null && old != vector) {
+ logger.debug("Field [{}] mutated from [{}] to [{}]", name, old.getClass().getSimpleName(),
+ vector.getClass().getSimpleName());
+ }
+ }
+
+ /**
+ * Returns a sequence of underlying child vectors.
+ */
+ protected Collection<ValueVector> getChildren() {
+ return vectors.values();
+ }
+
+ /**
+ * Returns the number of underlying child vectors.
+ */
+ @Override
+ public int size() {
+ return vectors.size();
+ }
+
+ @Override
+ public Iterator<ValueVector> iterator() {
+ return vectors.values().iterator();
+ }
+
+ /**
+ * Returns a list of scalar child vectors recursing the entire vector hierarchy.
+ */
+ public List<ValueVector> getPrimitiveVectors() {
+ final List<ValueVector> primitiveVectors = Lists.newArrayList();
+ for (final ValueVector v : vectors.values()) {
+ if (v instanceof AbstractMapVector) {
+ AbstractMapVector mapVector = (AbstractMapVector) v;
+ primitiveVectors.addAll(mapVector.getPrimitiveVectors());
+ } else {
+ primitiveVectors.add(v);
+ }
+ }
+ return primitiveVectors;
+ }
+
+ /**
+ * Returns a vector with its corresponding ordinal mapping if field exists or null.
+ */
+ @Override
+ public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
+ final int ordinal = vectors.getOrdinal(name.toLowerCase());
+ if (ordinal < 0) {
+ return null;
+ }
+ final ValueVector vector = vectors.getByOrdinal(ordinal);
+ return new VectorWithOrdinal(vector, ordinal);
+ }
+
+ @Override
+ public ArrowBuf[] getBuffers(boolean clear) {
+ final List<ArrowBuf> buffers = Lists.newArrayList();
+
+ for (final ValueVector vector : vectors.values()) {
+ for (final ArrowBuf buf : vector.getBuffers(false)) {
+ buffers.add(buf);
+ if (clear) {
+ buf.retain(1);
+ }
+ }
+ if (clear) {
+ vector.clear();
+ }
+ }
+
+ return buffers.toArray(new ArrowBuf[buffers.size()]);
+ }
+
+ @Override
+ public int getBufferSize() {
+ int actualBufSize = 0 ;
+
+ for (final ValueVector v : vectors.values()) {
+ for (final ArrowBuf buf : v.getBuffers(false)) {
+ actualBufSize += buf.writerIndex();
+ }
+ }
+ return actualBufSize;
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
new file mode 100644
index 0000000..6518897
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -0,0 +1,260 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Collections;
+import java.util.Iterator;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorDescriptor;
+import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
+
+ public final static ValueVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
+ public final static String OFFSETS_VECTOR_NAME = "$offsets$";
+ public final static String DATA_VECTOR_NAME = "$data$";
+
+ public final static MaterializedField OFFSETS_FIELD =
+ MaterializedField.create(OFFSETS_VECTOR_NAME, new MajorType(MinorType.UINT4, DataMode.REQUIRED));
+
+ protected final UInt4Vector offsets;
+ protected ValueVector vector;
+
+ protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator) {
+ this(field, allocator, DEFAULT_DATA_VECTOR);
+ }
+
+ protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) {
+ super(field, allocator);
+ this.offsets = new UInt4Vector(OFFSETS_FIELD, allocator);
+ this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ /* boolean to keep track if all the memory allocation were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ if (!offsets.allocateNewSafe()) {
+ return false;
+ }
+ success = vector.allocateNewSafe();
+ } finally {
+ if (!success) {
+ clear();
+ }
+ }
+ offsets.zeroVector();
+ return success;
+ }
+
+
+ @Override
+ public UInt4Vector getOffsetVector() {
+ return offsets;
+ }
+
+ @Override
+ public ValueVector getDataVector() {
+ return vector;
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) {
+ offsets.setInitialCapacity(numRecords + 1);
+ vector.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
+ }
+
+ @Override
+ public int getValueCapacity() {
+ final int offsetValueCapacity = Math.max(offsets.getValueCapacity() - 1, 0);
+ if (vector == DEFAULT_DATA_VECTOR) {
+ return offsetValueCapacity;
+ }
+ return Math.min(vector.getValueCapacity(), offsetValueCapacity);
+ }
+
+// @Override
+// protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
+// return super.getMetadataBuilder()
+// .addChild(offsets.getMetadata())
+// .addChild(vector.getMetadata());
+// }
+
+ @Override
+ public int getBufferSize() {
+ if (getAccessor().getValueCount() == 0) {
+ return 0;
+ }
+ return offsets.getBufferSize() + vector.getBufferSize();
+ }
+
+ @Override
+ public int getBufferSizeFor(int valueCount) {
+ if (valueCount == 0) {
+ return 0;
+ }
+
+ return offsets.getBufferSizeFor(valueCount + 1) + vector.getBufferSizeFor(valueCount);
+ }
+
+ @Override
+ public Iterator<ValueVector> iterator() {
+ return Collections.singleton(getDataVector()).iterator();
+ }
+
+ @Override
+ public void clear() {
+ offsets.clear();
+ vector.clear();
+ super.clear();
+ }
+
+ @Override
+ public ArrowBuf[] getBuffers(boolean clear) {
+ final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), vector.getBuffers(false), ArrowBuf.class);
+ if (clear) {
+ for (ArrowBuf buffer:buffers) {
+ buffer.retain();
+ }
+ clear();
+ }
+ return buffers;
+ }
+
+// @Override
+// public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
+// final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
+// offsets.load(offsetMetadata, buffer);
+//
+// final UserBitShared.SerializedField vectorMetadata = metadata.getChild(1);
+// if (getDataVector() == DEFAULT_DATA_VECTOR) {
+// addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
+// }
+//
+// final int offsetLength = offsetMetadata.getBufferLength();
+// final int vectorLength = vectorMetadata.getBufferLength();
+// vector.load(vectorMetadata, buffer.slice(offsetLength, vectorLength));
+// }
+
+ /**
+ * Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
+ *
+ * @see {@link ContainerVectorLike#size}
+ */
+ @Override
+ public int size() {
+ return vector == DEFAULT_DATA_VECTOR ? 0:1;
+ }
+
+ @Override
+ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+ boolean created = false;
+ if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != MinorType.LATE) {
+ final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField();
+ vector = BasicTypeHelper.getNewVector(field, allocator);
+ // returned vector must have the same field
+ assert field.equals(vector.getField());
+ getField().addChild(field);
+ created = true;
+ }
+
+ final MajorType actual = vector.getField().getType();
+ if (!actual.equals(descriptor.getType())) {
+ final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
+ descriptor.getType(), actual);
+ throw new SchemaChangeRuntimeException(msg);
+ }
+
+ return new AddOrGetResult<>((T)vector, created);
+ }
+
+ protected void replaceDataVector(ValueVector v) {
+ vector.clear();
+ vector = v;
+ }
+
+ public abstract class BaseRepeatedAccessor extends BaseValueVector.BaseAccessor implements RepeatedAccessor {
+
+ @Override
+ public int getValueCount() {
+ return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
+ }
+
+ @Override
+ public int getInnerValueCount() {
+ return vector.getAccessor().getValueCount();
+ }
+
+ @Override
+ public int getInnerValueCountAt(int index) {
+ return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
+ }
+
+ @Override
+ public boolean isNull(int index) {
+ return false;
+ }
+
+ @Override
+ public boolean isEmpty(int index) {
+ return false;
+ }
+ }
+
+ public abstract class BaseRepeatedMutator extends BaseValueVector.BaseMutator implements RepeatedMutator {
+
+ @Override
+ public void startNewValue(int index) {
+ while (offsets.getValueCapacity() <= index) {
+ offsets.reAlloc();
+ }
+ offsets.getMutator().setSafe(index+1, offsets.getAccessor().get(index));
+ setValueCount(index+1);
+ }
+
+ @Override
+ public void setValueCount(int valueCount) {
+ // TODO: populate offset end points
+ offsets.getMutator().setValueCount(valueCount == 0 ? 0 : valueCount+1);
+ final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
+ vector.getMutator().setValueCount(childValueCount);
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
new file mode 100644
index 0000000..e50b0d0
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
@@ -0,0 +1,43 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorDescriptor;
+
+/**
+ * A mix-in used for introducing container vector-like behaviour.
+ */
+public interface ContainerVectorLike {
+
+ /**
+ * Creates and adds a child vector if none with the same name exists, else returns the vector instance.
+ *
+ * @param descriptor vector descriptor
+ * @return result of operation wrapping vector corresponding to the given descriptor and whether it's newly created
+ * @throws org.apache.drill.common.exceptions.DrillRuntimeException
+ * if schema change is not permissible between the given and existing data vector types.
+ */
+ <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor);
+
+ /**
+ * Returns the number of child vectors in this container vector-like instance.
+ */
+ int size();
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
new file mode 100644
index 0000000..df69975
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/EmptyValuePopulator.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import org.apache.arrow.vector.UInt4Vector;
+
+import com.google.common.base.Preconditions;
+
+/**
+ * A helper class that is used to track and populate empty values in repeated value vectors.
+ */
+public class EmptyValuePopulator {
+ private final UInt4Vector offsets;
+
+ public EmptyValuePopulator(UInt4Vector offsets) {
+ this.offsets = Preconditions.checkNotNull(offsets, "offsets cannot be null");
+ }
+
+ /**
+ * Marks all values since the last set as empty. The last set value is obtained from underlying offsets vector.
+ *
+ * @param lastIndex the last index (inclusive) in the offsets vector until which empty population takes place
+ * @throws java.lang.IndexOutOfBoundsException if lastIndex is negative or greater than offsets capacity.
+ */
+ public void populate(int lastIndex) {
+ if (lastIndex < 0) {
+ throw new IndexOutOfBoundsException("index cannot be negative");
+ }
+ final UInt4Vector.Accessor accessor = offsets.getAccessor();
+ final UInt4Vector.Mutator mutator = offsets.getMutator();
+ final int lastSet = Math.max(accessor.getValueCount() - 1, 0);
+ final int previousEnd = accessor.get(lastSet);//0 ? 0 : accessor.get(lastSet);
+ for (int i = lastSet; i < lastIndex; i++) {
+ mutator.setSafe(i + 1, previousEnd);
+ }
+ mutator.setValueCount(lastIndex+1);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
new file mode 100644
index 0000000..8387c9e
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -0,0 +1,321 @@
+/*******************************************************************************
+
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ ******************************************************************************/
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.List;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.OutOfMemoryException;
+import org.apache.arrow.vector.AddOrGetResult;
+import org.apache.arrow.vector.UInt1Vector;
+import org.apache.arrow.vector.UInt4Vector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.VectorDescriptor;
+import org.apache.arrow.vector.ZeroVector;
+import org.apache.arrow.vector.complex.impl.ComplexCopier;
+import org.apache.arrow.vector.complex.impl.UnionListReader;
+import org.apache.arrow.vector.complex.impl.UnionListWriter;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.complex.writer.FieldWriter;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringArrayList;
+import org.apache.arrow.vector.util.TransferPair;
+
+import com.google.common.collect.ObjectArrays;
+
+public class ListVector extends BaseRepeatedValueVector {
+
+ private UInt4Vector offsets;
+ private final UInt1Vector bits;
+ private Mutator mutator = new Mutator();
+ private Accessor accessor = new Accessor();
+ private UnionListWriter writer;
+ private UnionListReader reader;
+ private CallBack callBack;
+
+ public ListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
+ super(field, allocator);
+ this.bits = new UInt1Vector(MaterializedField.create("$bits$", new MajorType(MinorType.UINT1, DataMode.REQUIRED)), allocator);
+ offsets = getOffsetVector();
+ this.field.addChild(getDataVector().getField());
+ this.writer = new UnionListWriter(this);
+ this.reader = new UnionListReader(this);
+ this.callBack = callBack;
+ }
+
+ public UnionListWriter getWriter() {
+ return writer;
+ }
+
+ @Override
+ public void allocateNew() throws OutOfMemoryException {
+ super.allocateNewSafe();
+ }
+
+ public void transferTo(ListVector target) {
+ offsets.makeTransferPair(target.offsets).transfer();
+ bits.makeTransferPair(target.bits).transfer();
+ if (target.getDataVector() instanceof ZeroVector) {
+ target.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+ }
+ getDataVector().makeTransferPair(target.getDataVector()).transfer();
+ }
+
+ public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
+ copyFrom(inIndex, outIndex, from);
+ }
+
+ public void copyFrom(int inIndex, int outIndex, ListVector from) {
+ FieldReader in = from.getReader();
+ in.setPosition(inIndex);
+ FieldWriter out = getWriter();
+ out.setPosition(outIndex);
+ ComplexCopier.copy(in, out);
+ }
+
+ @Override
+ public ValueVector getDataVector() {
+ return vector;
+ }
+
+ @Override
+ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+ return new TransferImpl(field.withPath(ref), allocator);
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector target) {
+ return new TransferImpl((ListVector) target);
+ }
+
+ private class TransferImpl implements TransferPair {
+
+ ListVector to;
+
+ public TransferImpl(MaterializedField field, BufferAllocator allocator) {
+ to = new ListVector(field, allocator, null);
+ to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+ }
+
+ public TransferImpl(ListVector to) {
+ this.to = to;
+ to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+ }
+
+ @Override
+ public void transfer() {
+ transferTo(to);
+ }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+ to.allocateNew();
+ for (int i = 0; i < length; i++) {
+ copyValueSafe(startIndex + i, i);
+ }
+ }
+
+ @Override
+ public ValueVector getTo() {
+ return to;
+ }
+
+ @Override
+ public void copyValueSafe(int from, int to) {
+ this.to.copyFrom(from, to, ListVector.this);
+ }
+ }
+
+ @Override
+ public Accessor getAccessor() {
+ return accessor;
+ }
+
+ @Override
+ public Mutator getMutator() {
+ return mutator;
+ }
+
+ @Override
+ public FieldReader getReader() {
+ return reader;
+ }
+
+ @Override
+ public boolean allocateNewSafe() {
+ /* boolean to keep track if all the memory allocation were successful
+ * Used in the case of composite vectors when we need to allocate multiple
+ * buffers for multiple vectors. If one of the allocations failed we need to
+ * clear all the memory that we allocated
+ */
+ boolean success = false;
+ try {
+ if (!offsets.allocateNewSafe()) {
+ return false;
+ }
+ success = vector.allocateNewSafe();
+ success = success && bits.allocateNewSafe();
+ } finally {
+ if (!success) {
+ clear();
+ }
+ }
+ if (success) {
+ offsets.zeroVector();
+ bits.zeroVector();
+ }
+ return success;
+ }
+
+// @Override
+// protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
+// return getField().getAsBuilder()
+// .setValueCount(getAccessor().getValueCount())
+// .setBufferLength(getBufferSize())
+// .addChild(offsets.getMetadata())
+// .addChild(bits.getMetadata())
+// .addChild(vector.getMetadata());
+// }
+ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+ AddOrGetResult<T> result = super.addOrGetVector(descriptor);
+ reader = new UnionListReader(this);
+ return result;
+ }
+
+ @Override
+ public int getBufferSize() {
+ if (getAccessor().getValueCount() == 0) {
+ return 0;
+ }
+ return offsets.getBufferSize() + bits.getBufferSize() + vector.getBufferSize();
+ }
+
+ @Override
+ public void clear() {
+ offsets.clear();
+ vector.clear();
+ bits.clear();
+ lastSet = 0;
+ super.clear();
+ }
+
+ @Override
+ public ArrowBuf[] getBuffers(boolean clear) {
+ final ArrowBuf[] buffers = ObjectArrays.concat(offsets.getBuffers(false), ObjectArrays.concat(bits.getBuffers(false),
+ vector.getBuffers(false), ArrowBuf.class), ArrowBuf.class);
+ if (clear) {
+ for (ArrowBuf buffer:buffers) {
+ buffer.retain();
+ }
+ clear();
+ }
+ return buffers;
+ }
+
+// @Override
+// public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
+// final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
+// offsets.load(offsetMetadata, buffer);
+//
+// final int offsetLength = offsetMetadata.getBufferLength();
+// final UserBitShared.SerializedField bitMetadata = metadata.getChild(1);
+// final int bitLength = bitMetadata.getBufferLength();
+// bits.load(bitMetadata, buffer.slice(offsetLength, bitLength));
+//
+// final UserBitShared.SerializedField vectorMetadata = metadata.getChild(2);
+// if (getDataVector() == DEFAULT_DATA_VECTOR) {
+// addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
+// }
+//
+// final int vectorLength = vectorMetadata.getBufferLength();
+// vector.load(vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength));
+// }
+
+ public UnionVector promoteToUnion() {
+ MaterializedField newField = MaterializedField.create(getField().getPath(), new MajorType(MinorType.UNION, DataMode.OPTIONAL));
+ UnionVector vector = new UnionVector(newField, allocator, null);
+ replaceDataVector(vector);
+ reader = new UnionListReader(this);
+ return vector;
+ }
+
+ private int lastSet;
+
+ public class Accessor extends BaseRepeatedAccessor {
+
+ @Override
+ public Object getObject(int index) {
+ if (isNull(index)) {
+ return null;
+ }
+ final List<Object> vals = new JsonStringArrayList<>();
+ final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
+ final int start = offsetsAccessor.get(index);
+ final int end = offsetsAccessor.get(index + 1);
+ final ValueVector.Accessor valuesAccessor = getDataVector().getAccessor();
+ for(int i = start; i < end; i++) {
+ vals.add(valuesAccessor.getObject(i));
+ }
+ return vals;
+ }
+
+ @Override
+ public boolean isNull(int index) {
+ return bits.getAccessor().get(index) == 0;
+ }
+ }
+
+ public class Mutator extends BaseRepeatedMutator {
+ public void setNotNull(int index) {
+ bits.getMutator().setSafe(index, 1);
+ lastSet = index + 1;
+ }
+
+ @Override
+ public void startNewValue(int index) {
+ for (int i = lastSet; i <= index; i++) {
+ offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i));
+ }
+ setNotNull(index);
+ lastSet = index + 1;
+ }
+
+ @Override
+ public void setValueCount(int valueCount) {
+ // TODO: populate offset end points
+ if (valueCount == 0) {
+ offsets.getMutator().setValueCount(0);
+ } else {
+ for (int i = lastSet; i < valueCount; i++) {
+ offsets.getMutator().setSafe(i + 1, offsets.getAccessor().get(i));
+ }
+ offsets.getMutator().setValueCount(valueCount + 1);
+ }
+ final int childValueCount = valueCount == 0 ? 0 : offsets.getAccessor().get(valueCount);
+ vector.getMutator().setValueCount(childValueCount);
+ bits.getMutator().setValueCount(valueCount);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
new file mode 100644
index 0000000..1bbce73
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -0,0 +1,374 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+import io.netty.buffer.ArrowBuf;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.Map;
+
+import javax.annotation.Nullable;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.vector.BaseValueVector;
+import org.apache.arrow.vector.ValueVector;
+import org.apache.arrow.vector.complex.RepeatedMapVector.MapSingleCopier;
+import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
+import org.apache.arrow.vector.complex.reader.FieldReader;
+import org.apache.arrow.vector.holders.ComplexHolder;
+import org.apache.arrow.vector.types.MaterializedField;
+import org.apache.arrow.vector.types.Types.DataMode;
+import org.apache.arrow.vector.types.Types.MajorType;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.CallBack;
+import org.apache.arrow.vector.util.JsonStringHashMap;
+import org.apache.arrow.vector.util.TransferPair;
+
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Ordering;
+import com.google.common.primitives.Ints;
+
+public class MapVector extends AbstractMapVector {
+ //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
+
+ public final static MajorType TYPE = new MajorType(MinorType.MAP, DataMode.OPTIONAL);
+
+ private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this);
+ private final Accessor accessor = new Accessor();
+ private final Mutator mutator = new Mutator();
+ private int valueCount;
+
+ public MapVector(String path, BufferAllocator allocator, CallBack callBack){
+ this(MaterializedField.create(path, TYPE), allocator, callBack);
+ }
+
+ public MapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
+ super(field, allocator, callBack);
+ }
+
+ @Override
+ public FieldReader getReader() {
+ //return new SingleMapReaderImpl(MapVector.this);
+ return reader;
+ }
+
+ transient private MapTransferPair ephPair;
+ transient private MapSingleCopier ephPair2;
+
+ public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
+ if(ephPair == null || ephPair.from != from) {
+ ephPair = (MapTransferPair) from.makeTransferPair(this);
+ }
+ ephPair.copyValueSafe(fromIndex, thisIndex);
+ }
+
+ public void copyFromSafe(int fromSubIndex, int thisIndex, RepeatedMapVector from) {
+ if(ephPair2 == null || ephPair2.from != from) {
+ ephPair2 = from.makeSingularCopier(this);
+ }
+ ephPair2.copySafe(fromSubIndex, thisIndex);
+ }
+
+ @Override
+ protected boolean supportsDirectRead() {
+ return true;
+ }
+
+ public Iterator<String> fieldNameIterator() {
+ return getChildFieldNames().iterator();
+ }
+
+ @Override
+ public void setInitialCapacity(int numRecords) {
+ for (final ValueVector v : (Iterable<ValueVector>) this) {
+ v.setInitialCapacity(numRecords);
+ }
+ }
+
+ @Override
+ public int getBufferSize() {
+ if (valueCount == 0 || size() == 0) {
+ return 0;
+ }
+ long buffer = 0;
+ for (final ValueVector v : (Iterable<ValueVector>)this) {
+ buffer += v.getBufferSize();
+ }
+
+ return (int) buffer;
+ }
+
+ @Override
+ public int getBufferSizeFor(final int valueCount) {
+ if (valueCount == 0) {
+ return 0;
+ }
+
+ long bufferSize = 0;
+ for (final ValueVector v : (Iterable<ValueVector>) this) {
+ bufferSize += v.getBufferSizeFor(valueCount);
+ }
+
+ return (int) bufferSize;
+ }
+
+ @Override
+ public ArrowBuf[] getBuffers(boolean clear) {
+ int expectedSize = getBufferSize();
+ int actualSize = super.getBufferSize();
+
+ Preconditions.checkArgument(expectedSize == actualSize);
+ return super.getBuffers(clear);
+ }
+
+ @Override
+ public TransferPair getTransferPair(BufferAllocator allocator) {
+ return new MapTransferPair(this, getField().getPath(), allocator);
+ }
+
+ @Override
+ public TransferPair makeTransferPair(ValueVector to) {
+ return new MapTransferPair(this, (MapVector) to);
+ }
+
+ @Override
+ public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
+ return new MapTransferPair(this, ref, allocator);
+ }
+
+ protected static class MapTransferPair implements TransferPair{
+ private final TransferPair[] pairs;
+ private final MapVector from;
+ private final MapVector to;
+
+ public MapTransferPair(MapVector from, String path, BufferAllocator allocator) {
+ this(from, new MapVector(MaterializedField.create(path, TYPE), allocator, from.callBack), false);
+ }
+
+ public MapTransferPair(MapVector from, MapVector to) {
+ this(from, to, true);
+ }
+
+ protected MapTransferPair(MapVector from, MapVector to, boolean allocate) {
+ this.from = from;
+ this.to = to;
+ this.pairs = new TransferPair[from.size()];
+ this.to.ephPair = null;
+ this.to.ephPair2 = null;
+
+ int i = 0;
+ ValueVector vector;
+ for (String child:from.getChildFieldNames()) {
+ int preSize = to.size();
+ vector = from.getChild(child);
+ if (vector == null) {
+ continue;
+ }
+ //DRILL-1872: we add the child fields for the vector, looking up the field by name. For a map vector,
+ // the child fields may be nested fields of the top level child. For example if the structure
+ // of a child field is oa.oab.oabc then we add oa, then add oab to oa then oabc to oab.
+ // But the children member of a Materialized field is a HashSet. If the fields are added in the
+ // children HashSet, and the hashCode of the Materialized field includes the hash code of the
+ // children, the hashCode value of oa changes *after* the field has been added to the HashSet.
+ // (This is similar to what happens in ScanBatch where the children cannot be added till they are
+ // read). To take care of this, we ensure that the hashCode of the MaterializedField does not
+ // include the hashCode of the children but is based only on MaterializedField$key.
+ final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+ if (allocate && to.size() != preSize) {
+ newVector.allocateNew();
+ }
+ pairs[i++] = vector.makeTransferPair(newVector);
+ }
+ }
+
+ @Override
+ public void transfer() {
+ for (final TransferPair p : pairs) {
+ p.transfer();
+ }
+ to.valueCount = from.valueCount;
+ from.clear();
+ }
+
+ @Override
+ public ValueVector getTo() {
+ return to;
+ }
+
+ @Override
+ public void copyValueSafe(int from, int to) {
+ for (TransferPair p : pairs) {
+ p.copyValueSafe(from, to);
+ }
+ }
+
+ @Override
+ public void splitAndTransfer(int startIndex, int length) {
+ for (TransferPair p : pairs) {
+ p.splitAndTransfer(startIndex, length);
+ }
+ to.getMutator().setValueCount(length);
+ }
+ }
+
+ @Override
+ public int getValueCapacity() {
+ if (size() == 0) {
+ return 0;
+ }
+
+ final Ordering<ValueVector> natural = new Ordering<ValueVector>() {
+ @Override
+ public int compare(@Nullable ValueVector left, @Nullable ValueVector right) {
+ return Ints.compare(
+ Preconditions.checkNotNull(left).getValueCapacity(),
+ Preconditions.checkNotNull(right).getValueCapacity()
+ );
+ }
+ };
+
+ return natural.min(getChildren()).getValueCapacity();
+ }
+
+ @Override
+ public Accessor getAccessor() {
+ return accessor;
+ }
+
+// @Override
+// public void load(SerializedField metadata, DrillBuf buf) {
+// final List<SerializedField> fields = metadata.getChildList();
+// valueCount = metadata.getValueCount();
+//
+// int bufOffset = 0;
+// for (final SerializedField child : fields) {
+// final MaterializedField fieldDef = SerializedFieldHelper.create(child);
+//
+// ValueVector vector = getChild(fieldDef.getLastName());
+// if (vector == null) {
+// if we arrive here, we didn't have a matching vector.
+// vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
+// putChild(fieldDef.getLastName(), vector);
+// }
+// if (child.getValueCount() == 0) {
+// vector.clear();
+// } else {
+// vector.load(child, buf.slice(bufOffset, child.getBufferLength()));
+// }
+// bufOffset += child.getBufferLength();
+// }
+//
+// assert bufOffset == buf.capacity();
+// }
+//
+// @Override
+// public SerializedField getMetadata() {
+// SerializedField.Builder b = getField() //
+// .getAsBuilder() //
+// .setBufferLength(getBufferSize()) //
+// .setValueCount(valueCount);
+//
+//
+// for(ValueVector v : getChildren()) {
+// b.addChild(v.getMetadata());
+// }
+// return b.build();
+// }
+
+ @Override
+ public Mutator getMutator() {
+ return mutator;
+ }
+
+ public class Accessor extends BaseValueVector.BaseAccessor {
+
+ @Override
+ public Object getObject(int index) {
+ Map<String, Object> vv = new JsonStringHashMap<>();
+ for (String child:getChildFieldNames()) {
+ ValueVector v = getChild(child);
+ // TODO(DRILL-4001): Resolve this hack:
+ // The index/value count check in the following if statement is a hack
+ // to work around the current fact that RecordBatchLoader.load and
+ // MapVector.load leave child vectors with a length of zero (as opposed
+ // to matching the lengths of siblings and the parent map vector)
+ // because they don't remove (or set the lengths of) vectors from
+ // previous batches that aren't in the current batch.
+ if (v != null && index < v.getAccessor().getValueCount()) {
+ Object value = v.getAccessor().getObject(index);
+ if (value != null) {
+ vv.put(child, value);
+ }
+ }
+ }
+ return vv;
+ }
+
+ public void get(int index, ComplexHolder holder) {
+ reader.setPosition(index);
+ holder.reader = reader;
+ }
+
+ @Override
+ public int getValueCount() {
+ return valueCount;
+ }
+ }
+
+ public ValueVector getVectorById(int id) {
+ return getChildByOrdinal(id);
+ }
+
+ public class Mutator extends BaseValueVector.BaseMutator {
+
+ @Override
+ public void setValueCount(int valueCount) {
+ for (final ValueVector v : getChildren()) {
+ v.getMutator().setValueCount(valueCount);
+ }
+ MapVector.this.valueCount = valueCount;
+ }
+
+ @Override
+ public void reset() { }
+
+ @Override
+ public void generateTestData(int values) { }
+ }
+
+ @Override
+ public void clear() {
+ for (final ValueVector v : getChildren()) {
+ v.clear();
+ }
+ valueCount = 0;
+ }
+
+ @Override
+ public void close() {
+ final Collection<ValueVector> vectors = getChildren();
+ for (final ValueVector v : vectors) {
+ v.close();
+ }
+ vectors.clear();
+ valueCount = 0;
+
+ super.close();
+ }
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
new file mode 100644
index 0000000..9345118
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/Positionable.java
@@ -0,0 +1,22 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+public interface Positionable {
+ public void setPosition(int index);
+}
http://git-wip-us.apache.org/repos/asf/arrow/blob/fa5f0299/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java
new file mode 100644
index 0000000..23850bc
--- /dev/null
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedFixedWidthVectorLike.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector.complex;
+
+/**
+ * A {@link org.apache.arrow.vector.ValueVector} mix-in that can be used in conjunction with
+ * {@link RepeatedValueVector} subtypes.
+ */
+public interface RepeatedFixedWidthVectorLike {
+ /**
+ * Allocate a new memory space for this vector. Must be called prior to using the ValueVector.
+ *
+ * @param valueCount Number of separate repeating groupings.
+ * @param innerValueCount Number of supported values in the vector.
+ */
+ void allocateNew(int valueCount, int innerValueCount);
+
+ /**
+ * Load the records in the provided buffer based on the given number of values.
+ * @param valueCount Number of separate repeating groupings.
+ * @param innerValueCount Number atomic values the buffer contains.
+ * @param buf Incoming buffer.
+ * @return The number of bytes of the buffer that were consumed.
+ */
+}