You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/09/20 12:01:54 UTC
arrow git commit: ARROW-1553: [JAVA] Implement setInitialCapacity for
MapWriter
Repository: arrow
Updated Branches:
refs/heads/master 2551050b3 -> 903d03b0b
ARROW-1553: [JAVA] Implement setInitialCapacity for MapWriter
Implement setInitialCapacity for MapWriter and pass on this capacity during lazy creation of child vectors
cc @jacques-n , @StevenMPhillips
Author: siddharth <si...@dremio.com>
Closes #1113 from siddharthteotia/ARROW-1553 and squashes the following commits:
5a759be7 [siddharth] ARROW-1553: Implement setInitialCapacity for MapWriter and pass on this capacity during lazy creation of child vectors
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/903d03b0
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/903d03b0
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/903d03b0
Branch: refs/heads/master
Commit: 903d03b0bab82fe6b8724d98cccedf582e2b4f86
Parents: 2551050
Author: siddharth <si...@dremio.com>
Authored: Wed Sep 20 08:01:49 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Wed Sep 20 08:01:49 2017 -0400
----------------------------------------------------------------------
.../src/main/codegen/templates/MapWriters.java | 10 +++
.../complex/writer/TestComplexWriter.java | 88 ++++++++++++++++++++
2 files changed, 98 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/903d03b0/java/vector/src/main/codegen/templates/MapWriters.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/MapWriters.java b/java/vector/src/main/codegen/templates/MapWriters.java
index 14cc08d..b89f914 100644
--- a/java/vector/src/main/codegen/templates/MapWriters.java
+++ b/java/vector/src/main/codegen/templates/MapWriters.java
@@ -47,6 +47,7 @@ import com.google.common.collect.Maps;
public class ${mode}MapWriter extends AbstractFieldWriter {
protected final ${containerClass} container;
+ private int initialCapacity;
private final Map<String, FieldWriter> fields = Maps.newHashMap();
public ${mode}MapWriter(${containerClass} container) {
<#if mode == "Single">
@@ -55,6 +56,7 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
}
</#if>
this.container = container;
+ this.initialCapacity = 0;
for (Field child : container.getField().getChildren()) {
MinorType minorType = Types.getMinorTypeForArrowType(child.getType());
switch (minorType) {
@@ -101,6 +103,11 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
return container.getValueCapacity();
}
+ public void setInitialCapacity(int initialCapacity) {
+ this.initialCapacity = initialCapacity;
+ container.setInitialCapacity(initialCapacity);
+ }
+
@Override
public boolean isEmptyMap() {
return 0 == container.size();
@@ -248,6 +255,9 @@ public class ${mode}MapWriter extends AbstractFieldWriter {
writer = new PromotableWriter(v, container, getNullableMapWriterFactory());
vector = v;
if (currentVector == null || currentVector != vector) {
+ if(this.initialCapacity > 0) {
+ vector.setInitialCapacity(this.initialCapacity);
+ }
vector.allocateNewSafe();
}
writer.setPosition(idx());
http://git-wip-us.apache.org/repos/asf/arrow/blob/903d03b0/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index f81cd55..856d607 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -28,6 +28,10 @@ import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.SchemaChangeCallBack;
+import org.apache.arrow.vector.NullableFloat8Vector;
+import org.apache.arrow.vector.NullableFloat4Vector;
+import org.apache.arrow.vector.NullableBigIntVector;
+import org.apache.arrow.vector.NullableIntVector;
import org.apache.arrow.vector.complex.ListVector;
import org.apache.arrow.vector.complex.MapVector;
import org.apache.arrow.vector.complex.NullableMapVector;
@@ -38,6 +42,11 @@ import org.apache.arrow.vector.complex.impl.UnionListReader;
import org.apache.arrow.vector.complex.impl.UnionListWriter;
import org.apache.arrow.vector.complex.impl.UnionReader;
import org.apache.arrow.vector.complex.impl.UnionWriter;
+import org.apache.arrow.vector.complex.impl.SingleMapWriter;
+import org.apache.arrow.vector.complex.reader.IntReader;
+import org.apache.arrow.vector.complex.reader.Float8Reader;
+import org.apache.arrow.vector.complex.reader.Float4Reader;
+import org.apache.arrow.vector.complex.reader.BigIntReader;
import org.apache.arrow.vector.complex.reader.BaseReader.MapReader;
import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
@@ -834,4 +843,83 @@ public class TestComplexWriter {
innerMap = (JsonStringHashMap<?, ?>) object.get(3);
assertEquals(2, innerMap.get("a"));
}
+
+ @Test
+ public void testSingleMapWriter1() {
+ /* initialize a SingleMapWriter with empty MapVector and then lazily
+ * create all vectors with expected initialCapacity.
+ */
+ MapVector parent = MapVector.empty("parent", allocator);
+ SingleMapWriter singleMapWriter = new SingleMapWriter(parent);
+
+ int initialCapacity = 1024;
+ singleMapWriter.setInitialCapacity(initialCapacity);
+
+ IntWriter intWriter = singleMapWriter.integer("intField");
+ BigIntWriter bigIntWriter = singleMapWriter.bigInt("bigIntField");
+ Float4Writer float4Writer = singleMapWriter.float4("float4Field");
+ Float8Writer float8Writer = singleMapWriter.float8("float8Field");
+ ListWriter listWriter = singleMapWriter.list("listField");
+
+ int intValue = 100;
+ long bigIntValue = 10000;
+ float float4Value = 100.5f;
+ double float8Value = 100.375;
+
+ for (int i = 0; i < initialCapacity; i++) {
+ singleMapWriter.start();
+
+ intWriter.writeInt(intValue + i);
+ bigIntWriter.writeBigInt(bigIntValue + (long)i);
+ float4Writer.writeFloat4(float4Value + (float)i);
+ float8Writer.writeFloat8(float8Value + (double)i);
+
+ listWriter.setPosition(i);
+ listWriter.startList();
+ listWriter.integer().writeInt(intValue + i);
+ listWriter.integer().writeInt(intValue + i + 1);
+ listWriter.integer().writeInt(intValue + i + 2);
+ listWriter.integer().writeInt(intValue + i + 3);
+ listWriter.endList();
+
+ singleMapWriter.end();
+ }
+
+ NullableIntVector intVector = (NullableIntVector)parent.getChild("intField");
+ NullableBigIntVector bigIntVector = (NullableBigIntVector)parent.getChild("bigIntField");
+ NullableFloat4Vector float4Vector = (NullableFloat4Vector)parent.getChild("float4Field");
+ NullableFloat8Vector float8Vector = (NullableFloat8Vector)parent.getChild("float8Field");
+
+ assertEquals(initialCapacity, singleMapWriter.getValueCapacity());
+ assertEquals(initialCapacity, intVector.getValueCapacity());
+ assertEquals(initialCapacity, bigIntVector.getValueCapacity());
+ assertEquals(initialCapacity, float4Vector.getValueCapacity());
+ assertEquals(initialCapacity, float8Vector.getValueCapacity());
+
+ MapReader singleMapReader = new SingleMapReaderImpl(parent);
+
+ IntReader intReader = singleMapReader.reader("intField");
+ BigIntReader bigIntReader = singleMapReader.reader("bigIntField");
+ Float4Reader float4Reader = singleMapReader.reader("float4Field");
+ Float8Reader float8Reader = singleMapReader.reader("float8Field");
+ UnionListReader listReader = (UnionListReader)singleMapReader.reader("listField");
+
+ for (int i = 0; i < initialCapacity; i++) {
+ intReader.setPosition(i);
+ bigIntReader.setPosition(i);
+ float4Reader.setPosition(i);
+ float8Reader.setPosition(i);
+ listReader.setPosition(i);
+
+ assertEquals(intValue + i, intReader.readInteger().intValue());
+ assertEquals(bigIntValue + (long)i, bigIntReader.readLong().longValue());
+ assertEquals(float4Value + (float)i, float4Reader.readFloat().floatValue(), 0);
+ assertEquals(float8Value + (double)i, float8Reader.readDouble().doubleValue(), 0);
+
+ for (int j = 0; j < 4; j++) {
+ listReader.next();
+ assertEquals(intValue + i + j, listReader.reader().readInteger().intValue());
+ }
+ }
+ }
}