You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/04/14 20:37:38 UTC
arrow git commit: ARROW-815 [Java] Exposing reAlloc for ValueVector
Repository: arrow
Updated Branches:
refs/heads/master 01114d831 -> b6033378c
ARROW-815 [Java] Exposing reAlloc for ValueVector
Author: Emilio Lahr-Vivaz <el...@ccri.com>
Closes #534 from elahrvivaz/ARROW-815 and squashes the following commits:
cf14944 [Emilio Lahr-Vivaz] unit test
45fa773 [Emilio Lahr-Vivaz] ARROW-815 [Java] Exposing reAlloc for ValueVector
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/b6033378
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/b6033378
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/b6033378
Branch: refs/heads/master
Commit: b6033378c2533ed7b396f111cc5aed10450907fb
Parents: 01114d8
Author: Emilio Lahr-Vivaz <el...@ccri.com>
Authored: Fri Apr 14 16:37:25 2017 -0400
Committer: Wes McKinney <we...@twosigma.com>
Committed: Fri Apr 14 16:37:25 2017 -0400
----------------------------------------------------------------------
.../codegen/templates/NullableValueVectors.java | 6 +
.../src/main/codegen/templates/UnionVector.java | 6 +
.../org/apache/arrow/vector/ValueVector.java | 6 +
.../org/apache/arrow/vector/ZeroVector.java | 3 +
.../arrow/vector/complex/AbstractMapVector.java | 7 +
.../vector/complex/BaseRepeatedValueVector.java | 14 +-
.../apache/arrow/vector/complex/ListVector.java | 6 +
.../arrow/vector/complex/NullableMapVector.java | 7 +
.../apache/arrow/vector/TestVectorReAlloc.java | 144 +++++++++++++++++++
9 files changed, 194 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index e5257ce..acee6cb 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -242,6 +242,12 @@ public final class ${className} extends BaseDataValueVector implements <#if type
return success;
}
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ values.reAlloc();
+ }
+
<#if type.major == "VarLen">
@Override
public void allocateNew(int totalBytes, int valueCount) {
http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/codegen/templates/UnionVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/UnionVector.java b/java/vector/src/main/codegen/templates/UnionVector.java
index 797b293..d70cbae 100644
--- a/java/vector/src/main/codegen/templates/UnionVector.java
+++ b/java/vector/src/main/codegen/templates/UnionVector.java
@@ -215,6 +215,12 @@ public class UnionVector implements FieldVector {
}
@Override
+ public void reAlloc() {
+ internalMap.reAlloc();
+ typeVector.reAlloc();
+ }
+
+ @Override
public void setInitialCapacity(int numRecords) {
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
index 8e35398..685b0be 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ValueVector.java
@@ -68,6 +68,12 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> {
*/
boolean allocateNewSafe();
+ /**
+ * Allocate new buffer with double capacity, and copy data into the new buffer.
+ * Replace vector's buffer with new buffer, and release old one
+ */
+ void reAlloc();
+
BufferAllocator getAllocator();
/**
http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
index 73f858e..e48788c 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/ZeroVector.java
@@ -143,6 +143,9 @@ public class ZeroVector implements FieldVector {
}
@Override
+ public void reAlloc() {}
+
+ @Override
public BufferAllocator getAllocator() {
throw new UnsupportedOperationException("Tried to get allocator from ZeroVector");
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
index dc833ed..15e8a5b 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/AbstractMapVector.java
@@ -81,6 +81,13 @@ public abstract class AbstractMapVector extends AbstractContainerVector {
return true;
}
+ @Override
+ public void reAlloc() {
+ for (final ValueVector v: vectors.values()) {
+ v.reAlloc();
+ }
+ }
+
/**
* Adds a new field with the given parameters or replaces the existing one and consequently returns the resultant
* {@link org.apache.arrow.vector.ValueVector}.
http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
index 6b240c0..da221e3 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/BaseRepeatedValueVector.java
@@ -20,6 +20,10 @@ package org.apache.arrow.vector.complex;
import java.util.Collections;
import java.util.Iterator;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.ObjectArrays;
+
+import io.netty.buffer.ArrowBuf;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.vector.AddOrGetResult;
import org.apache.arrow.vector.BaseValueVector;
@@ -31,11 +35,6 @@ import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.arrow.vector.util.CallBack;
import org.apache.arrow.vector.util.SchemaChangeRuntimeException;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.ObjectArrays;
-
-import io.netty.buffer.ArrowBuf;
-
public abstract class BaseRepeatedValueVector extends BaseValueVector implements RepeatedValueVector {
public final static FieldVector DEFAULT_DATA_VECTOR = ZeroVector.INSTANCE;
@@ -79,6 +78,11 @@ public abstract class BaseRepeatedValueVector extends BaseValueVector implements
return success;
}
+ @Override
+ public void reAlloc() {
+ offsets.reAlloc();
+ vector.reAlloc();
+ }
@Override
public UInt4Vector getOffsetVector() {
http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
index 0461a8d..63235df 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/ListVector.java
@@ -124,6 +124,12 @@ public class ListVector extends BaseRepeatedValueVector implements FieldVector {
bits.allocateNewSafe();
}
+ @Override
+ public void reAlloc() {
+ super.reAlloc();
+ bits.reAlloc();
+ }
+
public void copyFromSafe(int inIndex, int outIndex, ListVector from) {
copyFrom(inIndex, outIndex, from);
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
index 71fee67..647ab28 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/complex/NullableMapVector.java
@@ -197,6 +197,13 @@ public class NullableMapVector extends MapVector implements FieldVector {
bits.zeroVector();
return success;
}
+
+ @Override
+ public void reAlloc() {
+ bits.reAlloc();
+ super.reAlloc();
+ }
+
public final class Accessor extends MapVector.Accessor {
final BitVector.Accessor bAccessor = bits.getAccessor();
http://git-wip-us.apache.org/repos/asf/arrow/blob/b6033378/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
new file mode 100644
index 0000000..a7c35b6
--- /dev/null
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.arrow.vector;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.arrow.memory.BufferAllocator;
+import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.complex.ListVector;
+import org.apache.arrow.vector.complex.NullableMapVector;
+import org.apache.arrow.vector.types.Types.MinorType;
+import org.apache.arrow.vector.types.pojo.FieldType;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+
+public class TestVectorReAlloc {
+
+ private BufferAllocator allocator;
+
+ @Before
+ public void init() {
+ allocator = new RootAllocator(Long.MAX_VALUE);
+ }
+
+ @After
+ public void terminate() throws Exception {
+ allocator.close();
+ }
+
+ @Test
+ public void testFixedType() {
+ try (final UInt4Vector vector = new UInt4Vector("", allocator)) {
+ final UInt4Vector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, 0);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+
+ m.set(512, 100);
+ assertEquals(100, vector.getAccessor().get(512));
+ }
+ }
+
+ @Test
+ public void testNullableType() {
+ try (final NullableVarCharVector vector = new NullableVarCharVector("", allocator)) {
+ final NullableVarCharVector.Mutator m = vector.getMutator();
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1023, vector.getValueCapacity()); // note: size - 1 for some reason...
+
+ m.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ assertEquals("foo", new String(vector.getAccessor().get(512), StandardCharsets.UTF_8));
+ }
+ }
+
+ @Test
+ public void testListType() {
+ try (final ListVector vector = new ListVector("", allocator, null)) {
+ vector.addOrGetVector(FieldType.nullable(MinorType.INT.getType()));
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(1023, vector.getValueCapacity()); // TODO this doubles for some reason...
+
+ try {
+ vector.getOffsetVector().getAccessor().get(2014);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(2047, vector.getValueCapacity()); // note: size - 1
+ assertEquals(0, vector.getOffsetVector().getAccessor().get(2014));
+ }
+ }
+
+ @Test
+ public void testMapType() {
+ try (final NullableMapVector vector = new NullableMapVector("", allocator, null)) {
+ vector.addOrGet("", FieldType.nullable(MinorType.INT.getType()), NullableIntVector.class);
+
+ vector.setInitialCapacity(512);
+ vector.allocateNew();
+
+ assertEquals(512, vector.getValueCapacity());
+
+ try {
+ vector.getAccessor().getObject(513);
+ Assert.fail("Expected out of bounds exception");
+ } catch (Exception e) {
+ // ok
+ }
+
+ vector.reAlloc();
+ assertEquals(1024, vector.getValueCapacity());
+ assertNull(vector.getAccessor().getObject(513));
+ }
+ }
+}