You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by sm...@apache.org on 2016/08/18 23:30:36 UTC
[3/5] arrow git commit: ARROW-259: Use Flatbuffer Field type instead
of MaterializedField
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index de6ae82..5964f80 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -17,17 +17,17 @@
*/
package org.apache.arrow.vector.complex;
+import com.google.common.collect.ImmutableList;
import io.netty.buffer.ArrowBuf;
-import java.util.Collection;
+import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
+import org.apache.arrow.flatbuf.Field;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.MapWithOrdinal;
@@ -43,17 +43,8 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
// Maintains a map with key as field name and value is the vector itself
private final MapWithOrdinal<String, ValueVector> vectors = new MapWithOrdinal<>();
- protected AbstractMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
- super(field.clone(), allocator, callBack);
- MaterializedField clonedField = field.clone();
- // create the hierarchy of the child vectors based on the materialized field
- for (MaterializedField child : clonedField.getChildren()) {
- if (!child.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
- final String fieldName = child.getLastName();
- final ValueVector v = BasicTypeHelper.getNewVector(child, allocator, callBack);
- putVector(fieldName, v);
- }
- }
+ protected AbstractMapVector(String name, BufferAllocator allocator, CallBack callBack) {
+ super(name, allocator, callBack);
}
@Override
@@ -109,8 +100,8 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
* </li>
* </ul>
*
- * @param name name of the field
- * @param type type of the field
+ * @param name the name of the field
+ * @param minorType the minorType for the vector
* @param clazz class of expected vector type
* @param <T> class type of expected vector type
* @throws java.lang.IllegalStateException raised if there is a hard schema change
@@ -118,7 +109,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
* @return resultant {@link org.apache.arrow.vector.ValueVector}
*/
@Override
- public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
+ public <T extends ValueVector> T addOrGet(String name, MinorType minorType, Class<T> clazz, int... precisionScale) {
final ValueVector existing = getChild(name);
boolean create = false;
if (existing == null) {
@@ -130,7 +121,7 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
create = true;
}
if (create) {
- final T vector = (T) BasicTypeHelper.getNewVector(name, allocator, type, callBack);
+ final T vector = (T) minorType.getNewVector(name, allocator, callBack, precisionScale);
putChild(name, vector);
if (callBack!=null) {
callBack.doWork();
@@ -177,7 +168,6 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
*/
protected void putChild(String name, ValueVector vector) {
putVector(name, vector);
- field.addChild(vector.getField());
}
/**
@@ -199,8 +189,21 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
/**
* Returns a sequence of underlying child vectors.
*/
- protected Collection<ValueVector> getChildren() {
- return vectors.values();
+ protected List<ValueVector> getChildren() {
+ int size = vectors.size();
+ List<ValueVector> children = new ArrayList<>();
+ for (int i = 0; i < size; i++) {
+ children.add(vectors.getByOrdinal(i));
+ }
+ return children;
+ }
+
+ protected List<String> getChildFieldNames() {
+ ImmutableList.Builder<String> builder = ImmutableList.builder();
+ for (ValueVector child : getChildren()) {
+ builder.add(child.getField().getName());
+ }
+ return builder.build();
}
/**
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 6518897..4226274 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -22,22 +22,18 @@ import io.netty.buffer.ArrowBuf;
import java.util.Collections;
import java.util.Iterator;
+import org.apache.arrow.flatbuf.Type;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.AddOrGetResult;
import org.apache.arrow.vector.BaseValueVector;
import org.apache.arrow.vector.UInt4Vector;
import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
import org.apache.arrow.vector.ZeroVector;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
-import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
import com.google.common.base.Preconditions;
import com.google.common.collect.ObjectArrays;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
@@ -45,19 +41,16 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
public final static String OFFSETS_VECTOR_NAME = "$offsets$";
public final static String DATA_VECTOR_NAME = "$data$";
- public final static MaterializedField OFFSETS_FIELD =
- MaterializedField.create(OFFSETS_VECTOR_NAME, new MajorType(MinorType.UINT4, DataMode.REQUIRED));
-
protected final UInt4Vector offsets;
protected ValueVector vector;
- protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator) {
- this(field, allocator, DEFAULT_DATA_VECTOR);
+ protected BaseRepeatedValueVector(String name, BufferAllocator allocator) {
+ this(name, allocator, DEFAULT_DATA_VECTOR);
}
- protected BaseRepeatedValueVector(MaterializedField field, BufferAllocator allocator, ValueVector vector) {
- super(field, allocator);
- this.offsets = new UInt4Vector(OFFSETS_FIELD, allocator);
+ protected BaseRepeatedValueVector(String name, BufferAllocator allocator, ValueVector vector) {
+ super(name, allocator);
+ this.offsets = new UInt4Vector(OFFSETS_VECTOR_NAME, allocator);
this.vector = Preconditions.checkNotNull(vector, "data vector cannot be null");
}
@@ -109,13 +102,6 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
return Math.min(vector.getValueCapacity(), offsetValueCapacity);
}
-// @Override
-// protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
-// return super.getMetadataBuilder()
-// .addChild(offsets.getMetadata())
-// .addChild(vector.getMetadata());
-// }
-
@Override
public int getBufferSize() {
if (getAccessor().getValueCount() == 0) {
@@ -157,47 +143,24 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
return buffers;
}
-// @Override
-// public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
-// final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
-// offsets.load(offsetMetadata, buffer);
-//
-// final UserBitShared.SerializedField vectorMetadata = metadata.getChild(1);
-// if (getDataVector() == DEFAULT_DATA_VECTOR) {
-// addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
-// }
-//
-// final int offsetLength = offsetMetadata.getBufferLength();
-// final int vectorLength = vectorMetadata.getBufferLength();
-// vector.load(vectorMetadata, buffer.slice(offsetLength, vectorLength));
-// }
-
/**
* Returns 1 if inner vector is explicitly set via #addOrGetVector else 0
- *
- * @see {@link ContainerVectorLike#size}
*/
- @Override
public int size() {
return vector == DEFAULT_DATA_VECTOR ? 0:1;
}
- @Override
- public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
+ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType) {
boolean created = false;
- if (vector == DEFAULT_DATA_VECTOR && descriptor.getType().getMinorType() != MinorType.LATE) {
- final MaterializedField field = descriptor.withName(DATA_VECTOR_NAME).getField();
- vector = BasicTypeHelper.getNewVector(field, allocator);
+ if (vector instanceof ZeroVector) {
+ vector = minorType.getNewVector(DATA_VECTOR_NAME, allocator, null);
// returned vector must have the same field
- assert field.equals(vector.getField());
- getField().addChild(field);
created = true;
}
- final MajorType actual = vector.getField().getType();
- if (!actual.equals(descriptor.getType())) {
+ if (vector.getField().getType().getTypeType() != minorType.getType().getTypeType()) {
final String msg = String.format("Inner vector type mismatch. Requested type: [%s], actual type: [%s]",
- descriptor.getType(), actual);
+ Type.name(minorType.getType().getTypeType()), Type.name(vector.getField().getType().getTypeType()));
throw new SchemaChangeRuntimeException(msg);
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
deleted file mode 100644
index 655b55a..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ContainerVectorLike.java
+++ /dev/null
@@ -1,43 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.complex;
-
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
-
-/**
- * A mix-in used for introducing container vector-like behaviour.
- */
-public interface ContainerVectorLike {
-
- /**
- * Creates and adds a child vector if none with the same name exists, else returns the vector instance.
- *
- * @param descriptor vector descriptor
- * @return result of operation wrapping vector corresponding to the given descriptor and whether it's newly created
- * @throws org.apache.arrow.vector.util.SchemaChangeRuntimeException
- * if schema change is not permissible between the given and existing data vector types.
- */
- <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor);
-
- /**
- * Returns the number of child vectors in this container vector-like instance.
- */
- int size();
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 3e60c76..c6c6b09 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -18,6 +18,8 @@
******************************************************************************/
package org.apache.arrow.vector.complex;
+import com.google.common.collect.ImmutableList;
+import com.google.flatbuffers.FlatBufferBuilder;
import io.netty.buffer.ArrowBuf;
import java.util.List;
@@ -28,17 +30,14 @@ import org.apache.arrow.vector.AddOrGetResult;
import org.apache.arrow.vector.UInt1Vector;
import org.apache.arrow.vector.UInt4Vector;
import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
import org.apache.arrow.vector.ZeroVector;
import org.apache.arrow.vector.complex.impl.ComplexCopier;
import org.apache.arrow.vector.complex.impl.UnionListReader;
import org.apache.arrow.vector.complex.impl.UnionListWriter;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.complex.writer.FieldWriter;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringArrayList;
import org.apache.arrow.vector.util.TransferPair;
@@ -55,11 +54,10 @@ public class ListVector extends BaseRepeatedValueVector {
private UnionListReader reader;
private CallBack callBack;
- public ListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
- super(field, allocator);
- this.bits = new UInt1Vector(MaterializedField.create("$bits$", new MajorType(MinorType.UINT1, DataMode.REQUIRED)), allocator);
+ public ListVector(String name, BufferAllocator allocator, CallBack callBack) {
+ super(name, allocator);
+ this.bits = new UInt1Vector("$bits$", allocator);
offsets = getOffsetVector();
- this.field.addChild(getDataVector().getField());
this.writer = new UnionListWriter(this);
this.reader = new UnionListReader(this);
this.callBack = callBack;
@@ -75,15 +73,6 @@ public class ListVector extends BaseRepeatedValueVector {
bits.allocateNewSafe();
}
- public void transferTo(ListVector target) {
- offsets.makeTransferPair(target.offsets).transfer();
- bits.makeTransferPair(target.bits).transfer();
- if (target.getDataVector() instanceof ZeroVector) {
- target.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
- }
- getDataVector().makeTransferPair(target.getDataVector()).transfer();
- }
-
public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
copyFrom(inIndex, outIndex, from);
}
@@ -103,7 +92,7 @@ public class ListVector extends BaseRepeatedValueVector {
@Override
public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- return new TransferImpl(field.withPath(ref), allocator);
+ return new TransferImpl(ref, allocator);
}
@Override
@@ -114,20 +103,28 @@ public class ListVector extends BaseRepeatedValueVector {
private class TransferImpl implements TransferPair {
ListVector to;
+ TransferPair pairs[] = new TransferPair[3];
- public TransferImpl(MaterializedField field, BufferAllocator allocator) {
- to = new ListVector(field, allocator, null);
- to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+ public TransferImpl(String name, BufferAllocator allocator) {
+ this(new ListVector(name, allocator, null));
}
public TransferImpl(ListVector to) {
this.to = to;
- to.addOrGetVector(new VectorDescriptor(vector.getField().getType()));
+ to.addOrGetVector(vector.getMinorType());
+ pairs[0] = offsets.makeTransferPair(to.offsets);
+ pairs[1] = bits.makeTransferPair(to.bits);
+ if (to.getDataVector() instanceof ZeroVector) {
+ to.addOrGetVector(vector.getMinorType());
+ }
+ pairs[2] = getDataVector().makeTransferPair(to.getDataVector());
}
@Override
public void transfer() {
- transferTo(to);
+ for (TransferPair pair : pairs) {
+ pair.transfer();
+ }
}
@Override
@@ -190,17 +187,8 @@ public class ListVector extends BaseRepeatedValueVector {
return success;
}
-// @Override
-// protected UserBitShared.SerializedField.Builder getMetadataBuilder() {
-// return getField().getAsBuilder()
-// .setValueCount(getAccessor().getValueCount())
-// .setBufferLength(getBufferSize())
-// .addChild(offsets.getMetadata())
-// .addChild(bits.getMetadata())
-// .addChild(vector.getMetadata());
-// }
- public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
- AddOrGetResult<T> result = super.addOrGetVector(descriptor);
+ public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(MinorType minorType) {
+ AddOrGetResult<T> result = super.addOrGetVector(minorType);
reader = new UnionListReader(this);
return result;
}
@@ -214,6 +202,17 @@ public class ListVector extends BaseRepeatedValueVector {
}
@Override
+ public Field getField() {
+ return new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.List(),
+ ImmutableList.of(getDataVector().getField()));
+ }
+
+ @Override
+ public MinorType getMinorType() {
+ return MinorType.LIST;
+ }
+
+ @Override
public void clear() {
offsets.clear();
vector.clear();
@@ -235,28 +234,8 @@ public class ListVector extends BaseRepeatedValueVector {
return buffers;
}
-// @Override
-// public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) {
-// final UserBitShared.SerializedField offsetMetadata = metadata.getChild(0);
-// offsets.load(offsetMetadata, buffer);
-//
-// final int offsetLength = offsetMetadata.getBufferLength();
-// final UserBitShared.SerializedField bitMetadata = metadata.getChild(1);
-// final int bitLength = bitMetadata.getBufferLength();
-// bits.load(bitMetadata, buffer.slice(offsetLength, bitLength));
-//
-// final UserBitShared.SerializedField vectorMetadata = metadata.getChild(2);
-// if (getDataVector() == DEFAULT_DATA_VECTOR) {
-// addOrGetVector(VectorDescriptor.create(vectorMetadata.getMajorType()));
-// }
-//
-// final int vectorLength = vectorMetadata.getBufferLength();
-// vector.load(vectorMetadata, buffer.slice(offsetLength + bitLength, vectorLength));
-// }
-
public UnionVector promoteToUnion() {
- MaterializedField newField = MaterializedField.create(getField().getPath(), new MajorType(MinorType.UNION, DataMode.OPTIONAL));
- UnionVector vector = new UnionVector(newField, allocator, null);
+ UnionVector vector = new UnionVector(name, allocator, null);
replaceDataVector(vector);
reader = new UnionListReader(this);
return vector;
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
index cc0953a..0cb613e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/MapVector.java
@@ -19,8 +19,10 @@ package org.apache.arrow.vector.complex;
import io.netty.buffer.ArrowBuf;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
@@ -28,14 +30,13 @@ import javax.annotation.Nullable;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.BaseValueVector;
import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.complex.RepeatedMapVector.MapSingleCopier;
import org.apache.arrow.vector.complex.impl.SingleMapReaderImpl;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.ArrowType;
+import org.apache.arrow.vector.types.pojo.ArrowType.Tuple;
+import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.JsonStringHashMap;
import org.apache.arrow.vector.util.TransferPair;
@@ -47,19 +48,13 @@ import com.google.common.primitives.Ints;
public class MapVector extends AbstractMapVector {
//private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(MapVector.class);
- public final static MajorType TYPE = new MajorType(MinorType.MAP, DataMode.OPTIONAL);
-
private final SingleMapReaderImpl reader = new SingleMapReaderImpl(MapVector.this);
private final Accessor accessor = new Accessor();
private final Mutator mutator = new Mutator();
int valueCount;
- public MapVector(String path, BufferAllocator allocator, CallBack callBack){
- this(MaterializedField.create(path, TYPE), allocator, callBack);
- }
-
- public MapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
- super(field, allocator, callBack);
+ public MapVector(String name, BufferAllocator allocator, CallBack callBack){
+ super(name, allocator, callBack);
}
@Override
@@ -69,7 +64,6 @@ public class MapVector extends AbstractMapVector {
}
transient private MapTransferPair ephPair;
- transient private MapSingleCopier ephPair2;
public void copyFromSafe(int fromIndex, int thisIndex, MapVector from) {
if(ephPair == null || ephPair.from != from) {
@@ -78,13 +72,6 @@ public class MapVector extends AbstractMapVector {
ephPair.copyValueSafe(fromIndex, thisIndex);
}
- public void copyFromSafe(int fromSubIndex, int thisIndex, RepeatedMapVector from) {
- if(ephPair2 == null || ephPair2.from != from) {
- ephPair2 = from.makeSingularCopier(this);
- }
- ephPair2.copySafe(fromSubIndex, thisIndex);
- }
-
@Override
protected boolean supportsDirectRead() {
return true;
@@ -139,7 +126,7 @@ public class MapVector extends AbstractMapVector {
@Override
public TransferPair getTransferPair(BufferAllocator allocator) {
- return new MapTransferPair(this, getField().getPath(), allocator);
+ return new MapTransferPair(this, name, allocator);
}
@Override
@@ -157,8 +144,8 @@ public class MapVector extends AbstractMapVector {
private final MapVector from;
private final MapVector to;
- public MapTransferPair(MapVector from, String path, BufferAllocator allocator) {
- this(from, new MapVector(MaterializedField.create(path, TYPE), allocator, from.callBack), false);
+ public MapTransferPair(MapVector from, String name, BufferAllocator allocator) {
+ this(from, new MapVector(name, allocator, from.callBack), false);
}
public MapTransferPair(MapVector from, MapVector to) {
@@ -170,7 +157,6 @@ public class MapVector extends AbstractMapVector {
this.to = to;
this.pairs = new TransferPair[from.size()];
this.to.ephPair = null;
- this.to.ephPair2 = null;
int i = 0;
ValueVector vector;
@@ -189,7 +175,7 @@ public class MapVector extends AbstractMapVector {
// (This is similar to what happens in ScanBatch where the children cannot be added till they are
// read). To take care of this, we ensure that the hashCode of the MaterializedField does not
// include the hashCode of the children but is based only on MaterializedField$key.
- final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
+ final ValueVector newVector = to.addOrGet(child, vector.getMinorType(), vector.getClass());
if (allocate && to.size() != preSize) {
newVector.allocateNew();
}
@@ -251,46 +237,6 @@ public class MapVector extends AbstractMapVector {
return accessor;
}
-// @Override
-// public void load(SerializedField metadata, DrillBuf buf) {
-// final List<SerializedField> fields = metadata.getChildList();
-// valueCount = metadata.getValueCount();
-//
-// int bufOffset = 0;
-// for (final SerializedField child : fields) {
-// final MaterializedField fieldDef = SerializedFieldHelper.create(child);
-//
-// ValueVector vector = getChild(fieldDef.getLastName());
-// if (vector == null) {
-// if we arrive here, we didn't have a matching vector.
-// vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
-// putChild(fieldDef.getLastName(), vector);
-// }
-// if (child.getValueCount() == 0) {
-// vector.clear();
-// } else {
-// vector.load(child, buf.slice(bufOffset, child.getBufferLength()));
-// }
-// bufOffset += child.getBufferLength();
-// }
-//
-// assert bufOffset == buf.capacity();
-// }
-//
-// @Override
-// public SerializedField getMetadata() {
-// SerializedField.Builder b = getField() //
-// .getAsBuilder() //
-// .setBufferLength(getBufferSize()) //
-// .setValueCount(valueCount);
-//
-//
-// for(ValueVector v : getChildren()) {
-// b.addChild(v.getMetadata());
-// }
-// return b.build();
-// }
-
@Override
public Mutator getMutator() {
return mutator;
@@ -303,13 +249,6 @@ public class MapVector extends AbstractMapVector {
Map<String, Object> vv = new JsonStringHashMap<>();
for (String child:getChildFieldNames()) {
ValueVector v = getChild(child);
- // TODO(DRILL-4001): Resolve this hack:
- // The index/value count check in the following if statement is a hack
- // to work around the current fact that RecordBatchLoader.load and
- // MapVector.load leave child vectors with a length of zero (as opposed
- // to matching the lengths of siblings and the parent map vector)
- // because they don't remove (or set the lengths of) vectors from
- // previous batches that aren't in the current batch.
if (v != null && index < v.getAccessor().getValueCount()) {
Object value = v.getAccessor().getObject(index);
if (value != null) {
@@ -361,6 +300,20 @@ public class MapVector extends AbstractMapVector {
}
@Override
+ public Field getField() {
+ List<Field> children = new ArrayList<>();
+ for (ValueVector child : getChildren()) {
+ children.add(child.getField());
+ }
+ return new Field(name, false, Tuple.INSTANCE, children);
+ }
+
+ @Override
+ public MinorType getMinorType() {
+ return MinorType.MAP;
+ }
+
+ @Override
public void close() {
final Collection<ValueVector> vectors = getChildren();
for (final ValueVector v : vectors) {
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
deleted file mode 100644
index f337f9c..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedListVector.java
+++ /dev/null
@@ -1,427 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.complex;
-
-import io.netty.buffer.ArrowBuf;
-
-import java.util.Iterator;
-import java.util.List;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.UInt4Vector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
-import org.apache.arrow.vector.complex.impl.NullReader;
-import org.apache.arrow.vector.complex.impl.RepeatedListReaderImpl;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.holders.RepeatedListHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringArrayList;
-import org.apache.arrow.vector.util.TransferPair;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-public class RepeatedListVector extends AbstractContainerVector
- implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
-
- public final static MajorType TYPE = new MajorType(MinorType.LIST, DataMode.REPEATED);
- private final RepeatedListReaderImpl reader = new RepeatedListReaderImpl(null, this);
- final DelegateRepeatedVector delegate;
-
- protected static class DelegateRepeatedVector extends BaseRepeatedValueVector {
-
- private final RepeatedListAccessor accessor = new RepeatedListAccessor();
- private final RepeatedListMutator mutator = new RepeatedListMutator();
- private final EmptyValuePopulator emptyPopulator;
- private transient DelegateTransferPair ephPair;
-
- public class RepeatedListAccessor extends BaseRepeatedValueVector.BaseRepeatedAccessor {
-
- @Override
- public Object getObject(int index) {
- final List<Object> list = new JsonStringArrayList<>();
- final int start = offsets.getAccessor().get(index);
- final int until = offsets.getAccessor().get(index+1);
- for (int i = start; i < until; i++) {
- list.add(vector.getAccessor().getObject(i));
- }
- return list;
- }
-
- public void get(int index, RepeatedListHolder holder) {
- assert index <= getValueCapacity();
- holder.start = getOffsetVector().getAccessor().get(index);
- holder.end = getOffsetVector().getAccessor().get(index+1);
- }
-
- public void get(int index, ComplexHolder holder) {
- final FieldReader reader = getReader();
- reader.setPosition(index);
- holder.reader = reader;
- }
-
- public void get(int index, int arrayIndex, ComplexHolder holder) {
- final RepeatedListHolder listHolder = new RepeatedListHolder();
- get(index, listHolder);
- int offset = listHolder.start + arrayIndex;
- if (offset >= listHolder.end) {
- holder.reader = NullReader.INSTANCE;
- } else {
- FieldReader r = getDataVector().getReader();
- r.setPosition(offset);
- holder.reader = r;
- }
- }
- }
-
- public class RepeatedListMutator extends BaseRepeatedValueVector.BaseRepeatedMutator {
-
- public int add(int index) {
- final int curEnd = getOffsetVector().getAccessor().get(index+1);
- getOffsetVector().getMutator().setSafe(index + 1, curEnd + 1);
- return curEnd;
- }
-
- @Override
- public void startNewValue(int index) {
- emptyPopulator.populate(index+1);
- super.startNewValue(index);
- }
-
- @Override
- public void setValueCount(int valueCount) {
- emptyPopulator.populate(valueCount);
- super.setValueCount(valueCount);
- }
- }
-
-
- public class DelegateTransferPair implements TransferPair {
- private final DelegateRepeatedVector target;
- private final TransferPair[] children;
-
- public DelegateTransferPair(DelegateRepeatedVector target) {
- this.target = Preconditions.checkNotNull(target);
- if (target.getDataVector() == DEFAULT_DATA_VECTOR) {
- target.addOrGetVector(VectorDescriptor.create(getDataVector().getField()));
- target.getDataVector().allocateNew();
- }
- this.children = new TransferPair[] {
- getOffsetVector().makeTransferPair(target.getOffsetVector()),
- getDataVector().makeTransferPair(target.getDataVector())
- };
- }
-
- @Override
- public void transfer() {
- for (TransferPair child:children) {
- child.transfer();
- }
- }
-
- @Override
- public ValueVector getTo() {
- return target;
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- target.allocateNew();
- for (int i = 0; i < length; i++) {
- copyValueSafe(startIndex + i, i);
- }
- }
-
- @Override
- public void copyValueSafe(int srcIndex, int destIndex) {
- final RepeatedListHolder holder = new RepeatedListHolder();
- getAccessor().get(srcIndex, holder);
- target.emptyPopulator.populate(destIndex+1);
- final TransferPair vectorTransfer = children[1];
- int newIndex = target.getOffsetVector().getAccessor().get(destIndex);
- //todo: make this a bulk copy.
- for (int i = holder.start; i < holder.end; i++, newIndex++) {
- vectorTransfer.copyValueSafe(i, newIndex);
- }
- target.getOffsetVector().getMutator().setSafe(destIndex + 1, newIndex);
- }
- }
-
- public DelegateRepeatedVector(String path, BufferAllocator allocator) {
- this(MaterializedField.create(path, TYPE), allocator);
- }
-
- public DelegateRepeatedVector(MaterializedField field, BufferAllocator allocator) {
- super(field, allocator);
- emptyPopulator = new EmptyValuePopulator(getOffsetVector());
- }
-
- @Override
- public void allocateNew() throws OutOfMemoryException {
- if (!allocateNewSafe()) {
- throw new OutOfMemoryException();
- }
- }
-
- @Override
- public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- return makeTransferPair(new DelegateRepeatedVector(ref, allocator));
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector target) {
- return new DelegateTransferPair(DelegateRepeatedVector.class.cast(target));
- }
-
- @Override
- public RepeatedListAccessor getAccessor() {
- return accessor;
- }
-
- @Override
- public RepeatedListMutator getMutator() {
- return mutator;
- }
-
- @Override
- public FieldReader getReader() {
- throw new UnsupportedOperationException();
- }
-
- public void copyFromSafe(int fromIndex, int thisIndex, DelegateRepeatedVector from) {
- if(ephPair == null || ephPair.target != from) {
- ephPair = DelegateTransferPair.class.cast(from.makeTransferPair(this));
- }
- ephPair.copyValueSafe(fromIndex, thisIndex);
- }
-
- }
-
- protected class RepeatedListTransferPair implements TransferPair {
- private final TransferPair delegate;
-
- public RepeatedListTransferPair(TransferPair delegate) {
- this.delegate = delegate;
- }
-
- public void transfer() {
- delegate.transfer();
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- delegate.splitAndTransfer(startIndex, length);
- }
-
- @Override
- public ValueVector getTo() {
- final DelegateRepeatedVector delegateVector = DelegateRepeatedVector.class.cast(delegate.getTo());
- return new RepeatedListVector(getField(), allocator, callBack, delegateVector);
- }
-
- @Override
- public void copyValueSafe(int from, int to) {
- delegate.copyValueSafe(from, to);
- }
- }
-
- public RepeatedListVector(String path, BufferAllocator allocator, CallBack callBack) {
- this(MaterializedField.create(path, TYPE), allocator, callBack);
- }
-
- public RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack) {
- this(field, allocator, callBack, new DelegateRepeatedVector(field, allocator));
- }
-
- protected RepeatedListVector(MaterializedField field, BufferAllocator allocator, CallBack callBack, DelegateRepeatedVector delegate) {
- super(field, allocator, callBack);
- this.delegate = Preconditions.checkNotNull(delegate);
-
- final List<MaterializedField> children = Lists.newArrayList(field.getChildren());
- final int childSize = children.size();
- assert childSize < 3;
- final boolean hasChild = childSize > 0;
- if (hasChild) {
- // the last field is data field
- final MaterializedField child = children.get(childSize-1);
- addOrGetVector(VectorDescriptor.create(child));
- }
- }
-
-
- @Override
- public RepeatedListReaderImpl getReader() {
- return reader;
- }
-
- @Override
- public DelegateRepeatedVector.RepeatedListAccessor getAccessor() {
- return delegate.getAccessor();
- }
-
- @Override
- public DelegateRepeatedVector.RepeatedListMutator getMutator() {
- return delegate.getMutator();
- }
-
- @Override
- public UInt4Vector getOffsetVector() {
- return delegate.getOffsetVector();
- }
-
- @Override
- public ValueVector getDataVector() {
- return delegate.getDataVector();
- }
-
- @Override
- public void allocateNew() throws OutOfMemoryException {
- delegate.allocateNew();
- }
-
- @Override
- public boolean allocateNewSafe() {
- return delegate.allocateNewSafe();
- }
-
- @Override
- public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
- final AddOrGetResult<T> result = delegate.addOrGetVector(descriptor);
- if (result.isCreated() && callBack != null) {
- callBack.doWork();
- }
- return result;
- }
-
- @Override
- public int size() {
- return delegate.size();
- }
-
- @Override
- public int getBufferSize() {
- return delegate.getBufferSize();
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- return delegate.getBufferSizeFor(valueCount);
- }
-
- @Override
- public void close() {
- delegate.close();
- }
-
- @Override
- public void clear() {
- delegate.clear();
- }
-
- @Override
- public TransferPair getTransferPair(BufferAllocator allocator) {
- return new RepeatedListTransferPair(delegate.getTransferPair(allocator));
- }
-
- @Override
- public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- return new RepeatedListTransferPair(delegate.getTransferPair(ref, allocator));
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- final RepeatedListVector target = RepeatedListVector.class.cast(to);
- return new RepeatedListTransferPair(delegate.makeTransferPair(target.delegate));
- }
-
- @Override
- public int getValueCapacity() {
- return delegate.getValueCapacity();
- }
-
- @Override
- public ArrowBuf[] getBuffers(boolean clear) {
- return delegate.getBuffers(clear);
- }
-
-
-// @Override
-// public void load(SerializedField metadata, DrillBuf buf) {
-// delegate.load(metadata, buf);
-// }
-
-// @Override
-// public SerializedField getMetadata() {
-// return delegate.getMetadata();
-// }
-
- @Override
- public Iterator<ValueVector> iterator() {
- return delegate.iterator();
- }
-
- @Override
- public void setInitialCapacity(int numRecords) {
- delegate.setInitialCapacity(numRecords);
- }
-
- /**
- * @deprecated
- * prefer using {@link #addOrGetVector(org.apache.arrow.vector.VectorDescriptor)} instead.
- */
- @Override
- public <T extends ValueVector> T addOrGet(String name, MajorType type, Class<T> clazz) {
- final AddOrGetResult<T> result = addOrGetVector(VectorDescriptor.create(type));
- return result.getVector();
- }
-
- @Override
- public <T extends ValueVector> T getChild(String name, Class<T> clazz) {
- if (name != null) {
- return null;
- }
- return typeify(delegate.getDataVector(), clazz);
- }
-
- @Override
- public void allocateNew(int valueCount, int innerValueCount) {
- clear();
- getOffsetVector().allocateNew(valueCount + 1);
- getMutator().reset();
- }
-
- @Override
- public VectorWithOrdinal getChildVectorWithOrdinal(String name) {
- if (name != null) {
- return null;
- }
- return new VectorWithOrdinal(delegate.getDataVector(), 0);
- }
-
- public void copyFromSafe(int fromIndex, int thisIndex, RepeatedListVector from) {
- delegate.copyFromSafe(fromIndex, thisIndex, from.delegate);
- }
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
deleted file mode 100644
index 686414e..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedMapVector.java
+++ /dev/null
@@ -1,584 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.arrow.vector.complex;
-
-import io.netty.buffer.ArrowBuf;
-
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.arrow.memory.BufferAllocator;
-import org.apache.arrow.memory.OutOfMemoryException;
-import org.apache.arrow.vector.AddOrGetResult;
-import org.apache.arrow.vector.AllocationHelper;
-import org.apache.arrow.vector.UInt4Vector;
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
-import org.apache.arrow.vector.complex.impl.NullReader;
-import org.apache.arrow.vector.complex.impl.RepeatedMapReaderImpl;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.holders.ComplexHolder;
-import org.apache.arrow.vector.holders.RepeatedMapHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.CallBack;
-import org.apache.arrow.vector.util.JsonStringArrayList;
-import org.apache.arrow.vector.util.TransferPair;
-import org.apache.commons.lang3.ArrayUtils;
-
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Maps;
-
-public class RepeatedMapVector extends AbstractMapVector
- implements RepeatedValueVector, RepeatedFixedWidthVectorLike {
- //private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(RepeatedMapVector.class);
-
- public final static MajorType TYPE = new MajorType(MinorType.MAP, DataMode.REPEATED);
-
- final UInt4Vector offsets; // offsets to start of each record (considering record indices are 0-indexed)
- private final RepeatedMapReaderImpl reader = new RepeatedMapReaderImpl(RepeatedMapVector.this);
- private final RepeatedMapAccessor accessor = new RepeatedMapAccessor();
- private final Mutator mutator = new Mutator();
- private final EmptyValuePopulator emptyPopulator;
-
- public RepeatedMapVector(MaterializedField field, BufferAllocator allocator, CallBack callBack){
- super(field, allocator, callBack);
- this.offsets = new UInt4Vector(BaseRepeatedValueVector.OFFSETS_FIELD, allocator);
- this.emptyPopulator = new EmptyValuePopulator(offsets);
- }
-
- @Override
- public UInt4Vector getOffsetVector() {
- return offsets;
- }
-
- @Override
- public ValueVector getDataVector() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public <T extends ValueVector> AddOrGetResult<T> addOrGetVector(VectorDescriptor descriptor) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void setInitialCapacity(int numRecords) {
- offsets.setInitialCapacity(numRecords + 1);
- for(final ValueVector v : (Iterable<ValueVector>) this) {
- v.setInitialCapacity(numRecords * RepeatedValueVector.DEFAULT_REPEAT_PER_RECORD);
- }
- }
-
- @Override
- public RepeatedMapReaderImpl getReader() {
- return reader;
- }
-
- @Override
- public void allocateNew(int groupCount, int innerValueCount) {
- clear();
- try {
- offsets.allocateNew(groupCount + 1);
- for (ValueVector v : getChildren()) {
- AllocationHelper.allocatePrecomputedChildCount(v, groupCount, 50, innerValueCount);
- }
- } catch (OutOfMemoryException e){
- clear();
- throw e;
- }
- offsets.zeroVector();
- mutator.reset();
- }
-
- public Iterator<String> fieldNameIterator() {
- return getChildFieldNames().iterator();
- }
-
- @Override
- public List<ValueVector> getPrimitiveVectors() {
- final List<ValueVector> primitiveVectors = super.getPrimitiveVectors();
- primitiveVectors.add(offsets);
- return primitiveVectors;
- }
-
- @Override
- public int getBufferSize() {
- if (getAccessor().getValueCount() == 0) {
- return 0;
- }
- long bufferSize = offsets.getBufferSize();
- for (final ValueVector v : (Iterable<ValueVector>) this) {
- bufferSize += v.getBufferSize();
- }
- return (int) bufferSize;
- }
-
- @Override
- public int getBufferSizeFor(final int valueCount) {
- if (valueCount == 0) {
- return 0;
- }
-
- long bufferSize = 0;
- for (final ValueVector v : (Iterable<ValueVector>) this) {
- bufferSize += v.getBufferSizeFor(valueCount);
- }
-
- return (int) bufferSize;
- }
-
- @Override
- public void close() {
- offsets.close();
- super.close();
- }
-
- @Override
- public TransferPair getTransferPair(BufferAllocator allocator) {
- return new RepeatedMapTransferPair(this, getField().getPath(), allocator);
- }
-
- @Override
- public TransferPair makeTransferPair(ValueVector to) {
- return new RepeatedMapTransferPair(this, (RepeatedMapVector)to);
- }
-
- MapSingleCopier makeSingularCopier(MapVector to) {
- return new MapSingleCopier(this, to);
- }
-
- protected static class MapSingleCopier {
- private final TransferPair[] pairs;
- public final RepeatedMapVector from;
-
- public MapSingleCopier(RepeatedMapVector from, MapVector to) {
- this.from = from;
- this.pairs = new TransferPair[from.size()];
-
- int i = 0;
- ValueVector vector;
- for (final String child:from.getChildFieldNames()) {
- int preSize = to.size();
- vector = from.getChild(child);
- if (vector == null) {
- continue;
- }
- final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
- if (to.size() != preSize) {
- newVector.allocateNew();
- }
- pairs[i++] = vector.makeTransferPair(newVector);
- }
- }
-
- public void copySafe(int fromSubIndex, int toIndex) {
- for (TransferPair p : pairs) {
- p.copyValueSafe(fromSubIndex, toIndex);
- }
- }
- }
-
- public TransferPair getTransferPairToSingleMap(String reference, BufferAllocator allocator) {
- return new SingleMapTransferPair(this, reference, allocator);
- }
-
- @Override
- public TransferPair getTransferPair(String ref, BufferAllocator allocator) {
- return new RepeatedMapTransferPair(this, ref, allocator);
- }
-
- @Override
- public boolean allocateNewSafe() {
- /* boolean to keep track if all the memory allocation were successful
- * Used in the case of composite vectors when we need to allocate multiple
- * buffers for multiple vectors. If one of the allocations failed we need to
- * clear all the memory that we allocated
- */
- boolean success = false;
- try {
- if (!offsets.allocateNewSafe()) {
- return false;
- }
- success = super.allocateNewSafe();
- } finally {
- if (!success) {
- clear();
- }
- }
- offsets.zeroVector();
- return success;
- }
-
- protected static class SingleMapTransferPair implements TransferPair {
- private final TransferPair[] pairs;
- private final RepeatedMapVector from;
- private final MapVector to;
- private static final MajorType MAP_TYPE = new MajorType(MinorType.MAP, DataMode.REQUIRED);
-
- public SingleMapTransferPair(RepeatedMapVector from, String path, BufferAllocator allocator) {
- this(from, new MapVector(MaterializedField.create(path, MAP_TYPE), allocator, from.callBack), false);
- }
-
- public SingleMapTransferPair(RepeatedMapVector from, MapVector to) {
- this(from, to, true);
- }
-
- public SingleMapTransferPair(RepeatedMapVector from, MapVector to, boolean allocate) {
- this.from = from;
- this.to = to;
- this.pairs = new TransferPair[from.size()];
- int i = 0;
- ValueVector vector;
- for (final String child : from.getChildFieldNames()) {
- int preSize = to.size();
- vector = from.getChild(child);
- if (vector == null) {
- continue;
- }
- final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
- if (allocate && to.size() != preSize) {
- newVector.allocateNew();
- }
- pairs[i++] = vector.makeTransferPair(newVector);
- }
- }
-
-
- @Override
- public void transfer() {
- for (TransferPair p : pairs) {
- p.transfer();
- }
- to.getMutator().setValueCount(from.getAccessor().getValueCount());
- from.clear();
- }
-
- @Override
- public ValueVector getTo() {
- return to;
- }
-
- @Override
- public void copyValueSafe(int from, int to) {
- for (TransferPair p : pairs) {
- p.copyValueSafe(from, to);
- }
- }
-
- @Override
- public void splitAndTransfer(int startIndex, int length) {
- for (TransferPair p : pairs) {
- p.splitAndTransfer(startIndex, length);
- }
- to.getMutator().setValueCount(length);
- }
- }
-
- private static class RepeatedMapTransferPair implements TransferPair{
-
- private final TransferPair[] pairs;
- private final RepeatedMapVector to;
- private final RepeatedMapVector from;
-
- public RepeatedMapTransferPair(RepeatedMapVector from, String path, BufferAllocator allocator) {
- this(from, new RepeatedMapVector(MaterializedField.create(path, TYPE), allocator, from.callBack), false);
- }
-
- public RepeatedMapTransferPair(RepeatedMapVector from, RepeatedMapVector to) {
- this(from, to, true);
- }
-
- public RepeatedMapTransferPair(RepeatedMapVector from, RepeatedMapVector to, boolean allocate) {
- this.from = from;
- this.to = to;
- this.pairs = new TransferPair[from.size()];
- this.to.ephPair = null;
-
- int i = 0;
- ValueVector vector;
- for (final String child : from.getChildFieldNames()) {
- final int preSize = to.size();
- vector = from.getChild(child);
- if (vector == null) {
- continue;
- }
-
- final ValueVector newVector = to.addOrGet(child, vector.getField().getType(), vector.getClass());
- if (to.size() != preSize) {
- newVector.allocateNew();
- }
-
- pairs[i++] = vector.makeTransferPair(newVector);
- }
- }
-
- @Override
- public void transfer() {
- from.offsets.transferTo(to.offsets);
- for (TransferPair p : pairs) {
- p.transfer();
- }
- from.clear();
- }
-
- @Override
- public ValueVector getTo() {
- return to;
- }
-
- @Override
- public void copyValueSafe(int srcIndex, int destIndex) {
- RepeatedMapHolder holder = new RepeatedMapHolder();
- from.getAccessor().get(srcIndex, holder);
- to.emptyPopulator.populate(destIndex + 1);
- int newIndex = to.offsets.getAccessor().get(destIndex);
- //todo: make these bulk copies
- for (int i = holder.start; i < holder.end; i++, newIndex++) {
- for (TransferPair p : pairs) {
- p.copyValueSafe(i, newIndex);
- }
- }
- to.offsets.getMutator().setSafe(destIndex + 1, newIndex);
- }
-
- @Override
- public void splitAndTransfer(final int groupStart, final int groups) {
- final UInt4Vector.Accessor a = from.offsets.getAccessor();
- final UInt4Vector.Mutator m = to.offsets.getMutator();
-
- final int startPos = a.get(groupStart);
- final int endPos = a.get(groupStart + groups);
- final int valuesToCopy = endPos - startPos;
-
- to.offsets.clear();
- to.offsets.allocateNew(groups + 1);
-
- int normalizedPos;
- for (int i = 0; i < groups + 1; i++) {
- normalizedPos = a.get(groupStart + i) - startPos;
- m.set(i, normalizedPos);
- }
-
- m.setValueCount(groups + 1);
- to.emptyPopulator.populate(groups);
-
- for (final TransferPair p : pairs) {
- p.splitAndTransfer(startPos, valuesToCopy);
- }
- }
- }
-
-
- transient private RepeatedMapTransferPair ephPair;
-
- public void copyFromSafe(int fromIndex, int thisIndex, RepeatedMapVector from) {
- if (ephPair == null || ephPair.from != from) {
- ephPair = (RepeatedMapTransferPair) from.makeTransferPair(this);
- }
- ephPair.copyValueSafe(fromIndex, thisIndex);
- }
-
- @Override
- public int getValueCapacity() {
- return Math.max(offsets.getValueCapacity() - 1, 0);
- }
-
- @Override
- public RepeatedMapAccessor getAccessor() {
- return accessor;
- }
-
- @Override
- public ArrowBuf[] getBuffers(boolean clear) {
- final int expectedBufferSize = getBufferSize();
- final int actualBufferSize = super.getBufferSize();
-
- Preconditions.checkArgument(expectedBufferSize == actualBufferSize + offsets.getBufferSize());
- return ArrayUtils.addAll(offsets.getBuffers(clear), super.getBuffers(clear));
- }
-
-
-// @Override
-// public void load(SerializedField metadata, DrillBuf buffer) {
-// final List<SerializedField> children = metadata.getChildList();
-//
-// final SerializedField offsetField = children.get(0);
-// offsets.load(offsetField, buffer);
-// int bufOffset = offsetField.getBufferLength();
-//
-// for (int i = 1; i < children.size(); i++) {
-// final SerializedField child = children.get(i);
-// final MaterializedField fieldDef = SerializedFieldHelper.create(child);
-// ValueVector vector = getChild(fieldDef.getLastName());
-// if (vector == null) {
- // if we arrive here, we didn't have a matching vector.
-// vector = BasicTypeHelper.getNewVector(fieldDef, allocator);
-// putChild(fieldDef.getLastName(), vector);
-// }
-// final int vectorLength = child.getBufferLength();
-// vector.load(child, buffer.slice(bufOffset, vectorLength));
-// bufOffset += vectorLength;
-// }
-//
-// assert bufOffset == buffer.capacity();
-// }
-//
-//
-// @Override
-// public SerializedField getMetadata() {
-// SerializedField.Builder builder = getField() //
-// .getAsBuilder() //
-// .setBufferLength(getBufferSize()) //
- // while we don't need to actually read this on load, we need it to make sure we don't skip deserialization of this vector
-// .setValueCount(accessor.getValueCount());
-// builder.addChild(offsets.getMetadata());
-// for (final ValueVector child : getChildren()) {
-// builder.addChild(child.getMetadata());
-// }
-// return builder.build();
-// }
-
- @Override
- public Mutator getMutator() {
- return mutator;
- }
-
- public class RepeatedMapAccessor implements RepeatedAccessor {
- @Override
- public Object getObject(int index) {
- final List<Object> list = new JsonStringArrayList<>();
- final int end = offsets.getAccessor().get(index+1);
- String fieldName;
- for (int i = offsets.getAccessor().get(index); i < end; i++) {
- final Map<String, Object> vv = Maps.newLinkedHashMap();
- for (final MaterializedField field : getField().getChildren()) {
- if (!field.equals(BaseRepeatedValueVector.OFFSETS_FIELD)) {
- fieldName = field.getLastName();
- final Object value = getChild(fieldName).getAccessor().getObject(i);
- if (value != null) {
- vv.put(fieldName, value);
- }
- }
- }
- list.add(vv);
- }
- return list;
- }
-
- @Override
- public int getValueCount() {
- return Math.max(offsets.getAccessor().getValueCount() - 1, 0);
- }
-
- @Override
- public int getInnerValueCount() {
- final int valueCount = getValueCount();
- if (valueCount == 0) {
- return 0;
- }
- return offsets.getAccessor().get(valueCount);
- }
-
- @Override
- public int getInnerValueCountAt(int index) {
- return offsets.getAccessor().get(index+1) - offsets.getAccessor().get(index);
- }
-
- @Override
- public boolean isEmpty(int index) {
- return false;
- }
-
- @Override
- public boolean isNull(int index) {
- return false;
- }
-
- public void get(int index, RepeatedMapHolder holder) {
- assert index < getValueCapacity() :
- String.format("Attempted to access index %d when value capacity is %d",
- index, getValueCapacity());
- final UInt4Vector.Accessor offsetsAccessor = offsets.getAccessor();
- holder.start = offsetsAccessor.get(index);
- holder.end = offsetsAccessor.get(index + 1);
- }
-
- public void get(int index, ComplexHolder holder) {
- final FieldReader reader = getReader();
- reader.setPosition(index);
- holder.reader = reader;
- }
-
- public void get(int index, int arrayIndex, ComplexHolder holder) {
- final RepeatedMapHolder h = new RepeatedMapHolder();
- get(index, h);
- final int offset = h.start + arrayIndex;
-
- if (offset >= h.end) {
- holder.reader = NullReader.INSTANCE;
- } else {
- reader.setSinglePosition(index, arrayIndex);
- holder.reader = reader;
- }
- }
- }
-
- public class Mutator implements RepeatedMutator {
- @Override
- public void startNewValue(int index) {
- emptyPopulator.populate(index + 1);
- offsets.getMutator().setSafe(index + 1, offsets.getAccessor().get(index));
- }
-
- @Override
- public void setValueCount(int topLevelValueCount) {
- emptyPopulator.populate(topLevelValueCount);
- offsets.getMutator().setValueCount(topLevelValueCount == 0 ? 0 : topLevelValueCount + 1);
- int childValueCount = offsets.getAccessor().get(topLevelValueCount);
- for (final ValueVector v : getChildren()) {
- v.getMutator().setValueCount(childValueCount);
- }
- }
-
- @Override
- public void reset() {}
-
- @Override
- public void generateTestData(int values) {}
-
- public int add(int index) {
- final int prevEnd = offsets.getAccessor().get(index + 1);
- offsets.getMutator().setSafe(index + 1, prevEnd + 1);
- return prevEnd;
- }
- }
-
- @Override
- public void clear() {
- getMutator().reset();
-
- offsets.clear();
- for(final ValueVector vector : getChildren()) {
- vector.clear();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
index 99c0a0a..54db393 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/RepeatedValueVector.java
@@ -28,7 +28,7 @@ import org.apache.arrow.vector.ValueVector;
* uses the offset vector to determine the sequence of cells pertaining to an individual value.
*
*/
-public interface RepeatedValueVector extends ValueVector, ContainerVectorLike {
+public interface RepeatedValueVector extends ValueVector {
final static int DEFAULT_REPEAT_PER_RECORD = 5;
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
index 264e241..259a954 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseReader.java
@@ -19,20 +19,20 @@ package org.apache.arrow.vector.complex.impl;
import java.util.Iterator;
+import com.google.flatbuffers.FlatBufferBuilder;
+import org.apache.arrow.flatbuf.Type;
+import org.apache.arrow.flatbuf.Union;
+import org.apache.arrow.flatbuf.UnionMode;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.arrow.vector.complex.writer.FieldWriter;
import org.apache.arrow.vector.holders.UnionHolder;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.Field;
abstract class AbstractBaseReader implements FieldReader{
static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseReader.class);
- private static final MajorType LATE_BIND_TYPE = new MajorType(MinorType.LATE, DataMode.OPTIONAL);
private int index;
@@ -58,15 +58,6 @@ abstract class AbstractBaseReader implements FieldReader{
throw new IllegalStateException("The current reader doesn't support reading as a map.");
}
- public MajorType getType(){
- throw new IllegalStateException("The current reader doesn't support getting type information.");
- }
-
- @Override
- public MaterializedField getField() {
- return MaterializedField.create("unknown", LATE_BIND_TYPE);
- }
-
@Override
public boolean next() {
throw new IllegalStateException("The current reader doesn't support getting next information.");
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
index 4e1e103..e6cf098 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/AbstractBaseWriter.java
@@ -23,25 +23,11 @@ import org.apache.arrow.vector.complex.writer.FieldWriter;
abstract class AbstractBaseWriter implements FieldWriter {
//private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(AbstractBaseWriter.class);
- final FieldWriter parent;
private int index;
- public AbstractBaseWriter(FieldWriter parent) {
- this.parent = parent;
- }
-
@Override
public String toString() {
- return super.toString() + "[index = " + index + ", parent = " + parent + "]";
- }
-
- @Override
- public FieldWriter getParent() {
- return parent;
- }
-
- public boolean isRoot() {
- return parent == null;
+ return super.toString() + "[index = " + index + "]";
}
int idx() {
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
index 4e2051f..4d2adfb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/ComplexWriterImpl.java
@@ -17,20 +17,20 @@
*/
package org.apache.arrow.vector.complex.impl;
+import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.MapVector;
import org.apache.arrow.vector.complex.StateTool;
import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.Types.MinorType;
import com.google.common.base.Preconditions;
+import org.apache.arrow.vector.types.pojo.Field;
public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWriter {
// private static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ComplexWriterImpl.class);
private SingleMapWriter mapRoot;
- private SingleListWriter listRoot;
+ private UnionListWriter listRoot;
private final MapVector container;
Mode mode = Mode.INIT;
@@ -40,7 +40,6 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
private enum Mode { INIT, MAP, LIST };
public ComplexWriterImpl(String name, MapVector container, boolean unionEnabled){
- super(null);
this.name = name;
this.container = container;
this.unionEnabled = unionEnabled;
@@ -51,7 +50,7 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
}
@Override
- public MaterializedField getField() {
+ public Field getField() {
return container.getField();
}
@@ -123,7 +122,7 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
case INIT:
MapVector map = (MapVector) container;
- mapRoot = new SingleMapWriter(map, this, unionEnabled);
+ mapRoot = new SingleMapWriter(map);
mapRoot.setPosition(idx());
mode = Mode.MAP;
break;
@@ -143,8 +142,8 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
switch(mode){
case INIT:
- MapVector map = container.addOrGet(name, Types.required(MinorType.MAP), MapVector.class);
- mapRoot = new SingleMapWriter(map, this, unionEnabled);
+ MapVector map = container.addOrGet(name, MinorType.MAP, MapVector.class);
+ mapRoot = new SingleMapWriter(map);
mapRoot.setPosition(idx());
mode = Mode.MAP;
break;
@@ -174,7 +173,12 @@ public class ComplexWriterImpl extends AbstractFieldWriter implements ComplexWri
switch(mode){
case INIT:
- listRoot = new SingleListWriter(name, container, this);
+ int vectorCount = container.size();
+ ListVector listVector = container.addOrGet(name, MinorType.LIST, ListVector.class);
+ if (container.size() > vectorCount) {
+ listVector.allocateNew();
+ }
+ listRoot = new UnionListWriter(listVector);
listRoot.setPosition(idx());
mode = Mode.LIST;
break;
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
index 462ec9d..586b128 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/PromotableWriter.java
@@ -17,20 +17,14 @@
*/
package org.apache.arrow.vector.complex.impl;
-import java.lang.reflect.Constructor;
-
import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.VectorDescriptor;
import org.apache.arrow.vector.ZeroVector;
import org.apache.arrow.vector.complex.AbstractMapVector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.UnionVector;
import org.apache.arrow.vector.complex.writer.FieldWriter;
-import org.apache.arrow.vector.types.MaterializedField;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
import org.apache.arrow.vector.types.Types.MinorType;
-import org.apache.arrow.vector.util.BasicTypeHelper;
+import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.TransferPair;
/**
@@ -56,14 +50,12 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
private FieldWriter writer;
public PromotableWriter(ValueVector v, AbstractMapVector parentContainer) {
- super(null);
this.parentContainer = parentContainer;
this.listVector = null;
init(v);
}
public PromotableWriter(ValueVector v, ListVector listVector) {
- super(null);
this.listVector = listVector;
this.parentContainer = null;
init(v);
@@ -84,30 +76,8 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
private void setWriter(ValueVector v) {
state = State.SINGLE;
vector = v;
- type = v.getField().getType().getMinorType();
- Class<?> writerClass = BasicTypeHelper
- .getWriterImpl(v.getField().getType().getMinorType(), v.getField().getDataMode());
- if (writerClass.equals(SingleListWriter.class)) {
- writerClass = UnionListWriter.class;
- }
- Class<?> vectorClass = BasicTypeHelper.getValueVectorClass(v.getField().getType().getMinorType(), v.getField()
- .getDataMode());
- try {
- Constructor<?> constructor = null;
- for (Constructor<?> c : writerClass.getConstructors()) {
- if (c.getParameterTypes().length == 3) {
- constructor = c;
- }
- }
- if (constructor == null) {
- constructor = writerClass.getConstructor(vectorClass, AbstractFieldWriter.class);
- writer = (FieldWriter) constructor.newInstance(vector, null);
- } else {
- writer = (FieldWriter) constructor.newInstance(vector, null, true);
- }
- } catch (ReflectiveOperationException e) {
- throw new RuntimeException(e);
- }
+ type = v.getMinorType();
+ writer = type.getNewFieldWriter(vector);
}
@Override
@@ -129,7 +99,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
if (type == null) {
return null;
}
- ValueVector v = listVector.addOrGetVector(new VectorDescriptor(new MajorType(type, DataMode.OPTIONAL))).getVector();
+ ValueVector v = listVector.addOrGetVector(type).getVector();
v.allocateNew();
setWriter(v);
writer.setPosition(position);
@@ -150,11 +120,11 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
}
private FieldWriter promoteToUnion() {
- String name = vector.getField().getLastName();
- TransferPair tp = vector.getTransferPair(vector.getField().getType().getMinorType().name().toLowerCase(), vector.getAllocator());
+ String name = vector.getField().getName();
+ TransferPair tp = vector.getTransferPair(vector.getMinorType().name().toLowerCase(), vector.getAllocator());
tp.transfer();
if (parentContainer != null) {
- unionVector = parentContainer.addOrGet(name, new MajorType(MinorType.UNION, DataMode.OPTIONAL), UnionVector.class);
+ unionVector = parentContainer.addOrGet(name, MinorType.UNION, UnionVector.class);
unionVector.allocateNew();
} else if (listVector != null) {
unionVector = listVector.promoteToUnion();
@@ -163,7 +133,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
writer = new UnionWriter(unionVector);
writer.setPosition(idx());
for (int i = 0; i < idx(); i++) {
- unionVector.getMutator().setType(i, vector.getField().getType().getMinorType());
+ unionVector.getMutator().setType(i, vector.getMinorType());
}
vector = null;
state = State.UNION;
@@ -181,7 +151,7 @@ public class PromotableWriter extends AbstractPromotableFieldWriter {
}
@Override
- public MaterializedField getField() {
+ public Field getField() {
return getWriter().getField();
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
deleted file mode 100644
index dd1a152..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedListReaderImpl.java
+++ /dev/null
@@ -1,145 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.arrow.vector.complex.impl;
-
-
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.complex.RepeatedListVector;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.holders.RepeatedListHolder;
-import org.apache.arrow.vector.types.Types.DataMode;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-
-public class RepeatedListReaderImpl extends AbstractFieldReader{
- private static final int NO_VALUES = Integer.MAX_VALUE - 1;
- private static final MajorType TYPE = new MajorType(MinorType.LIST, DataMode.REPEATED);
- private final String name;
- private final RepeatedListVector container;
- private FieldReader reader;
-
- public RepeatedListReaderImpl(String name, RepeatedListVector container) {
- super();
- this.name = name;
- this.container = container;
- }
-
- @Override
- public MajorType getType() {
- return TYPE;
- }
-
- @Override
- public void copyAsValue(ListWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- RepeatedListWriter impl = (RepeatedListWriter) writer;
- impl.container.copyFromSafe(idx(), impl.idx(), container);
- }
-
- @Override
- public void copyAsField(String name, MapWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- RepeatedListWriter impl = (RepeatedListWriter) writer.list(name);
- impl.container.copyFromSafe(idx(), impl.idx(), container);
- }
-
- private int currentOffset;
- private int maxOffset;
-
- @Override
- public void reset() {
- super.reset();
- currentOffset = 0;
- maxOffset = 0;
- if (reader != null) {
- reader.reset();
- }
- reader = null;
- }
-
- @Override
- public int size() {
- return maxOffset - currentOffset;
- }
-
- @Override
- public void setPosition(int index) {
- if (index < 0 || index == NO_VALUES) {
- currentOffset = NO_VALUES;
- return;
- }
-
- super.setPosition(index);
- RepeatedListHolder h = new RepeatedListHolder();
- container.getAccessor().get(index, h);
- if (h.start == h.end) {
- currentOffset = NO_VALUES;
- } else {
- currentOffset = h.start-1;
- maxOffset = h.end;
- if(reader != null) {
- reader.setPosition(currentOffset);
- }
- }
- }
-
- @Override
- public boolean next() {
- if (currentOffset +1 < maxOffset) {
- currentOffset++;
- if (reader != null) {
- reader.setPosition(currentOffset);
- }
- return true;
- } else {
- currentOffset = NO_VALUES;
- return false;
- }
- }
-
- @Override
- public Object readObject() {
- return container.getAccessor().getObject(idx());
- }
-
- @Override
- public FieldReader reader() {
- if (reader == null) {
- ValueVector child = container.getChild(name);
- if (child == null) {
- reader = NullReader.INSTANCE;
- } else {
- reader = child.getReader();
- }
- reader.setPosition(currentOffset);
- }
- return reader;
- }
-
- public boolean isSet() {
- return true;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
deleted file mode 100644
index 09a831d..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/RepeatedMapReaderImpl.java
+++ /dev/null
@@ -1,192 +0,0 @@
-/*******************************************************************************
-
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- ******************************************************************************/
-package org.apache.arrow.vector.complex.impl;
-
-import java.util.Map;
-
-import org.apache.arrow.vector.ValueVector;
-import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.holders.RepeatedMapHolder;
-import org.apache.arrow.vector.types.Types.MajorType;
-
-import com.google.common.collect.Maps;
-
-@SuppressWarnings("unused")
-public class RepeatedMapReaderImpl extends AbstractFieldReader{
- private static final int NO_VALUES = Integer.MAX_VALUE - 1;
-
- private final RepeatedMapVector vector;
- private final Map<String, FieldReader> fields = Maps.newHashMap();
-
- public RepeatedMapReaderImpl(RepeatedMapVector vector) {
- this.vector = vector;
- }
-
- private void setChildrenPosition(int index) {
- for (FieldReader r : fields.values()) {
- r.setPosition(index);
- }
- }
-
- @Override
- public FieldReader reader(String name) {
- FieldReader reader = fields.get(name);
- if (reader == null) {
- ValueVector child = vector.getChild(name);
- if (child == null) {
- reader = NullReader.INSTANCE;
- } else {
- reader = child.getReader();
- }
- fields.put(name, reader);
- reader.setPosition(currentOffset);
- }
- return reader;
- }
-
- @Override
- public FieldReader reader() {
- if (currentOffset == NO_VALUES) {
- return NullReader.INSTANCE;
- }
-
- setChildrenPosition(currentOffset);
- return new SingleLikeRepeatedMapReaderImpl(vector, this);
- }
-
- private int currentOffset;
- private int maxOffset;
-
- @Override
- public void reset() {
- super.reset();
- currentOffset = 0;
- maxOffset = 0;
- for (FieldReader reader:fields.values()) {
- reader.reset();
- }
- fields.clear();
- }
-
- @Override
- public int size() {
- if (isNull()) {
- return 0;
- }
- return maxOffset - (currentOffset < 0 ? 0 : currentOffset);
- }
-
- @Override
- public void setPosition(int index) {
- if (index < 0 || index == NO_VALUES) {
- currentOffset = NO_VALUES;
- return;
- }
-
- super.setPosition(index);
- RepeatedMapHolder h = new RepeatedMapHolder();
- vector.getAccessor().get(index, h);
- if (h.start == h.end) {
- currentOffset = NO_VALUES;
- } else {
- currentOffset = h.start-1;
- maxOffset = h.end;
- setChildrenPosition(currentOffset);
- }
- }
-
- public void setSinglePosition(int index, int childIndex) {
- super.setPosition(index);
- RepeatedMapHolder h = new RepeatedMapHolder();
- vector.getAccessor().get(index, h);
- if (h.start == h.end) {
- currentOffset = NO_VALUES;
- } else {
- int singleOffset = h.start + childIndex;
- assert singleOffset < h.end;
- currentOffset = singleOffset;
- maxOffset = singleOffset + 1;
- setChildrenPosition(singleOffset);
- }
- }
-
- @Override
- public boolean next() {
- if (currentOffset +1 < maxOffset) {
- setChildrenPosition(++currentOffset);
- return true;
- } else {
- currentOffset = NO_VALUES;
- return false;
- }
- }
-
- public boolean isNull() {
- return currentOffset == NO_VALUES;
- }
-
- @Override
- public Object readObject() {
- return vector.getAccessor().getObject(idx());
- }
-
- @Override
- public MajorType getType() {
- return vector.getField().getType();
- }
-
- @Override
- public java.util.Iterator<String> iterator() {
- return vector.fieldNameIterator();
- }
-
- @Override
- public boolean isSet() {
- return true;
- }
-
- @Override
- public void copyAsValue(MapWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- RepeatedMapWriter impl = (RepeatedMapWriter) writer;
- impl.container.copyFromSafe(idx(), impl.idx(), vector);
- }
-
- public void copyAsValueSingle(MapWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- SingleMapWriter impl = (SingleMapWriter) writer;
- impl.container.copyFromSafe(currentOffset, impl.idx(), vector);
- }
-
- @Override
- public void copyAsField(String name, MapWriter writer) {
- if (currentOffset == NO_VALUES) {
- return;
- }
- RepeatedMapWriter impl = (RepeatedMapWriter) writer.map(name);
- impl.container.copyFromSafe(idx(), impl.idx(), vector);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/arrow/blob/e7e399db/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
deleted file mode 100644
index 086d26e..0000000
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/impl/SingleLikeRepeatedMapReaderImpl.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.arrow.vector.complex.impl;
-
-import java.util.Iterator;
-
-import org.apache.arrow.vector.complex.RepeatedMapVector;
-import org.apache.arrow.vector.complex.reader.FieldReader;
-import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
-import org.apache.arrow.vector.types.Types;
-import org.apache.arrow.vector.types.Types.MajorType;
-import org.apache.arrow.vector.types.Types.MinorType;
-
-public class SingleLikeRepeatedMapReaderImpl extends AbstractFieldReader{
-
- private RepeatedMapReaderImpl delegate;
-
- public SingleLikeRepeatedMapReaderImpl(RepeatedMapVector vector, FieldReader delegate) {
- this.delegate = (RepeatedMapReaderImpl) delegate;
- }
-
- @Override
- public int size() {
- throw new UnsupportedOperationException("You can't call size on a single map reader.");
- }
-
- @Override
- public boolean next() {
- throw new UnsupportedOperationException("You can't call next on a single map reader.");
- }
-
- @Override
- public MajorType getType() {
- return Types.required(MinorType.MAP);
- }
-
-
- @Override
- public void copyAsValue(MapWriter writer) {
- delegate.copyAsValueSingle(writer);
- }
-
- public void copyAsValueSingle(MapWriter writer){
- delegate.copyAsValueSingle(writer);
- }
-
- @Override
- public FieldReader reader(String name) {
- return delegate.reader(name);
- }
-
- @Override
- public void setPosition(int index) {
- delegate.setPosition(index);
- }
-
- @Override
- public Object readObject() {
- return delegate.readObject();
- }
-
- @Override
- public Iterator<String> iterator() {
- return delegate.iterator();
- }
-
- @Override
- public boolean isSet() {
- return ! delegate.isNull();
- }
-
-
-}