You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by si...@apache.org on 2019/01/09 03:41:06 UTC
[arrow] branch master updated: ARROW-4147: [Java] reduce heap usage
for varwidth vectors (#3298)
This is an automated email from the ASF dual-hosted git repository.
siddteotia pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new bfe6865 ARROW-4147: [Java] reduce heap usage for varwidth vectors (#3298)
bfe6865 is described below
commit bfe6865ba8087a46bd7665679e48af3a77987cef
Author: Pindikura Ravindra <ra...@dremio.com>
AuthorDate: Wed Jan 9 09:11:01 2019 +0530
ARROW-4147: [Java] reduce heap usage for varwidth vectors (#3298)
* ARROW-4147: reduce heap usage for varwidth vectors
- some code reorg to avoid duplication
- changed the default initial alloc from 4096 to 3970
* ARROW-4147: [Java] Address review comments
* ARROW-4147: remove check on width to be <= 16:
* ARROW-4147: allow initial valueCount to be 0.
* ARROW-4147: Fix incorrect comment on initial alloc
---
.../apache/arrow/vector/BaseFixedWidthVector.java | 127 ++---
.../org/apache/arrow/vector/BaseValueVector.java | 99 +++-
.../arrow/vector/BaseVariableWidthVector.java | 165 +++---
.../java/org/apache/arrow/vector/BitVector.java | 5 +-
.../arrow/vector/TestBufferOwnershipTransfer.java | 9 +-
.../java/org/apache/arrow/vector/TestCopyFrom.java | 569 +++++++++++----------
.../org/apache/arrow/vector/TestValueVector.java | 435 +++++++++-------
.../org/apache/arrow/vector/TestVectorReAlloc.java | 23 +-
.../vector/complex/writer/TestComplexWriter.java | 15 +-
9 files changed, 799 insertions(+), 648 deletions(-)
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
index f69a9d1..f3c2837 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseFixedWidthVector.java
@@ -22,7 +22,6 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
-import org.apache.arrow.memory.BaseAllocator;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.OutOfMemoryException;
import org.apache.arrow.vector.ipc.message.ArrowFieldNode;
@@ -43,8 +42,7 @@ public abstract class BaseFixedWidthVector extends BaseValueVector
implements FixedWidthVector, FieldVector, VectorDefinitionSetter {
private final int typeWidth;
- protected int valueAllocationSizeInBytes;
- protected int validityAllocationSizeInBytes;
+ protected int initialValueAllocation;
protected final Field field;
private int allocationMonitor;
@@ -61,14 +59,7 @@ public abstract class BaseFixedWidthVector extends BaseValueVector
allocationMonitor = 0;
validityBuffer = allocator.getEmpty();
valueBuffer = allocator.getEmpty();
- if (typeWidth > 0) {
- valueAllocationSizeInBytes = INITIAL_VALUE_ALLOCATION * typeWidth;
- validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
- } else {
- /* specialized handling for BitVector */
- valueAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
- validityAllocationSizeInBytes = valueAllocationSizeInBytes;
- }
+ initialValueAllocation = INITIAL_VALUE_ALLOCATION;
}
@@ -159,12 +150,8 @@ public abstract class BaseFixedWidthVector extends BaseValueVector
*/
@Override
public void setInitialCapacity(int valueCount) {
- final long size = (long) valueCount * typeWidth;
- if (size > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
- }
- valueAllocationSizeInBytes = (int) size;
- validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount);
+ computeAndCheckBufferSize(valueCount);
+ initialValueAllocation = valueCount;
}
/**
@@ -267,18 +254,13 @@ public abstract class BaseFixedWidthVector extends BaseValueVector
*/
@Override
public boolean allocateNewSafe() {
- long curAllocationSizeValue = valueAllocationSizeInBytes;
- long curAllocationSizeValidity = validityAllocationSizeInBytes;
-
- if (align(curAllocationSizeValue) + curAllocationSizeValidity > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Requested amount of memory exceeds limit");
- }
+ computeAndCheckBufferSize(initialValueAllocation);
/* we are doing a new allocation -- release the current buffers */
clear();
try {
- allocateBytes(curAllocationSizeValue, curAllocationSizeValidity);
+ allocateBytes(initialValueAllocation);
} catch (Exception e) {
clear();
return false;
@@ -295,22 +277,13 @@ public abstract class BaseFixedWidthVector extends BaseValueVector
* @throws org.apache.arrow.memory.OutOfMemoryException on error
*/
public void allocateNew(int valueCount) {
- long valueBufferSize = valueCount * typeWidth;
- long validityBufferSize = getValidityBufferSizeFromCount(valueCount);
- if (typeWidth == 0) {
- /* specialized handling for BitVector */
- valueBufferSize = validityBufferSize;
- }
-
- if (align(valueBufferSize) + validityBufferSize > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
- }
+ computeAndCheckBufferSize(valueCount);
/* we are doing a new allocation -- release the current buffers */
clear();
try {
- allocateBytes(valueBufferSize, validityBufferSize);
+ allocateBytes(valueCount);
} catch (Exception e) {
clear();
throw e;
@@ -318,10 +291,16 @@ public abstract class BaseFixedWidthVector extends BaseValueVector
}
/*
- * align to a 8-byte value.
+ * Compute the buffer size required for 'valueCount', and check if it's within bounds.
*/
- private long align(long size) {
- return ((size + 7) / 8) * 8;
+ private long computeAndCheckBufferSize(int valueCount) {
+ final long size = computeCombinedBufferSize(valueCount, typeWidth);
+ if (size > MAX_ALLOCATION_SIZE) {
+ throw new OversizedAllocationException("Memory required for vector capacity " +
+ valueCount +
+ " is (" + size + "), which is more than max allowed (" + MAX_ALLOCATION_SIZE + ")");
+ }
+ return size;
}
/**
@@ -333,25 +312,11 @@ public abstract class BaseFixedWidthVector extends BaseValueVector
* within the bounds of max allocation allowed and any other error
* conditions.
*/
- private void allocateBytes(final long valueBufferSize, final long validityBufferSize) {
- int valueBufferSlice = (int)align(valueBufferSize);
- int validityBufferSlice = (int)validityBufferSize;
-
- /* allocate combined buffer */
- ArrowBuf buffer = allocator.buffer(valueBufferSlice + validityBufferSlice);
-
- valueAllocationSizeInBytes = valueBufferSlice;
- valueBuffer = buffer.slice(0, valueBufferSlice);
- valueBuffer.retain();
- valueBuffer.readerIndex(0);
-
- validityAllocationSizeInBytes = validityBufferSlice;
- validityBuffer = buffer.slice(valueBufferSlice, validityBufferSlice);
- validityBuffer.retain();
- validityBuffer.readerIndex(0);
+ private void allocateBytes(int valueCount) {
+ DataAndValidityBuffers buffers = allocFixedDataAndValidityBufs(valueCount, typeWidth);
+ valueBuffer = buffers.getDataBuf();
+ validityBuffer = buffers.getValidityBuf();
zeroVector();
-
- buffer.release();
}
/**
@@ -363,7 +328,6 @@ public abstract class BaseFixedWidthVector extends BaseValueVector
private void allocateValidityBuffer(final int validityBufferSize) {
validityBuffer = allocator.buffer(validityBufferSize);
validityBuffer.readerIndex(0);
- validityAllocationSizeInBytes = validityBufferSize;
}
/**
@@ -439,50 +403,28 @@ public abstract class BaseFixedWidthVector extends BaseValueVector
*/
@Override
public void reAlloc() {
- int valueBaseSize = Integer.max(valueBuffer.capacity(), valueAllocationSizeInBytes);
- long newValueBufferSlice = align(valueBaseSize * 2L);
- long newValidityBufferSlice;
- if (typeWidth > 0) {
- long targetValueBufferSize = align(BaseAllocator.nextPowerOfTwo(newValueBufferSlice));
- long targetValueCount = targetValueBufferSize / typeWidth;
- targetValueBufferSize -= getValidityBufferSizeFromCount((int) targetValueCount);
- if (newValueBufferSlice < targetValueBufferSize) {
- newValueBufferSlice = targetValueBufferSize;
+ int targetValueCount = getValueCapacity() * 2;
+ if (targetValueCount == 0) {
+ if (initialValueAllocation > 0) {
+ targetValueCount = initialValueAllocation * 2;
+ } else {
+ targetValueCount = INITIAL_VALUE_ALLOCATION * 2;
}
-
- newValidityBufferSlice = getValidityBufferSizeFromCount((int)(newValueBufferSlice / typeWidth));
- } else {
- newValidityBufferSlice = newValueBufferSlice;
- }
-
- long newAllocationSize = newValueBufferSlice + newValidityBufferSlice;
- assert newAllocationSize >= 1;
-
- if (newAllocationSize > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Unable to expand the buffer");
}
+ computeAndCheckBufferSize(targetValueCount);
- final ArrowBuf newBuffer = allocator.buffer((int) newAllocationSize);
- final ArrowBuf newValueBuffer = newBuffer.slice(0, (int)newValueBufferSlice);
+ DataAndValidityBuffers buffers = allocFixedDataAndValidityBufs(targetValueCount, typeWidth);
+ final ArrowBuf newValueBuffer = buffers.getDataBuf();
newValueBuffer.setBytes(0, valueBuffer, 0, valueBuffer.capacity());
- newValueBuffer.setZero(valueBuffer.capacity(), (int)newValueBufferSlice - valueBuffer.capacity());
- newValueBuffer.retain();
- newValueBuffer.readerIndex(0);
+ newValueBuffer.setZero(valueBuffer.capacity(), newValueBuffer.capacity() - valueBuffer.capacity());
valueBuffer.release();
valueBuffer = newValueBuffer;
- valueAllocationSizeInBytes = (int)newValueBufferSlice;
- final ArrowBuf newValidityBuffer = newBuffer.slice((int)newValueBufferSlice,
- (int)newValidityBufferSlice);
+ final ArrowBuf newValidityBuffer = buffers.getValidityBuf();
newValidityBuffer.setBytes(0, validityBuffer, 0, validityBuffer.capacity());
- newValidityBuffer.setZero(validityBuffer.capacity(), (int)newValidityBufferSlice - validityBuffer.capacity());
- newValidityBuffer.retain();
- newValidityBuffer.readerIndex(0);
+ newValidityBuffer.setZero(validityBuffer.capacity(), newValidityBuffer.capacity() - validityBuffer.capacity());
validityBuffer.release();
validityBuffer = newValidityBuffer;
- validityAllocationSizeInBytes = (int)newValidityBufferSlice;
-
- newBuffer.release();
}
@Override
@@ -535,9 +477,6 @@ public abstract class BaseFixedWidthVector extends BaseValueVector
valueBuffer = dataBuffer.retain(allocator);
valueCount = fieldNode.getLength();
-
- valueAllocationSizeInBytes = valueBuffer.capacity();
- validityAllocationSizeInBytes = validityBuffer.capacity();
}
/**
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
index 4cbf4be..4e014bb 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseValueVector.java
@@ -20,6 +20,7 @@ package org.apache.arrow.vector;
import java.util.Collections;
import java.util.Iterator;
+import org.apache.arrow.memory.BaseAllocator;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.util.TransferPair;
@@ -33,7 +34,14 @@ public abstract class BaseValueVector implements ValueVector {
public static final String MAX_ALLOCATION_SIZE_PROPERTY = "arrow.vector.max_allocation_bytes";
public static final int MAX_ALLOCATION_SIZE = Integer.getInteger(MAX_ALLOCATION_SIZE_PROPERTY, Integer.MAX_VALUE);
- public static final int INITIAL_VALUE_ALLOCATION = 4096;
+ /*
+ * For all fixed width vectors, the value and validity buffers are sliced from a single buffer.
+ * Similarly, for variable width vectors, the offsets and validity buffers are sliced from a
+ * single buffer. To ensure the single buffer is power-of-2 size, the initial value allocation
+ * should be less than power-of-2. For IntVectors, this comes to 3970*4 (15880) for the data
+ * buffer and 504 bytes for the validity buffer, totalling to 16384 (2^16).
+ */
+ public static final int INITIAL_VALUE_ALLOCATION = 3970;
protected final BufferAllocator allocator;
protected final String name;
@@ -98,5 +106,94 @@ public abstract class BaseValueVector implements ValueVector {
protected static int getValidityBufferSizeFromCount(final int valueCount) {
return (int) Math.ceil(valueCount / 8.0);
}
+
+ /* round up to the next multiple of 8 */
+ private static long roundUp8(long size) {
+ return ((size + 7) / 8) * 8;
+ }
+
+ protected long computeCombinedBufferSize(int valueCount, int typeWidth) {
+ Preconditions.checkArgument(valueCount >= 0, "valueCount must be >= 0");
+ Preconditions.checkArgument(typeWidth >= 0, "typeWidth must be >= 0");
+
+ // compute size of validity buffer.
+ long bufferSize = roundUp8(getValidityBufferSizeFromCount(valueCount));
+
+ // add the size of the value buffer.
+ if (typeWidth == 0) {
+ // for boolean type, value-buffer and validity-buffer are of same size.
+ bufferSize *= 2;
+ } else {
+ bufferSize += roundUp8(valueCount * typeWidth);
+ }
+ return BaseAllocator.nextPowerOfTwo(bufferSize);
+ }
+
+ class DataAndValidityBuffers {
+ private ArrowBuf dataBuf;
+ private ArrowBuf validityBuf;
+
+ DataAndValidityBuffers(ArrowBuf dataBuf, ArrowBuf validityBuf) {
+ this.dataBuf = dataBuf;
+ this.validityBuf = validityBuf;
+ }
+
+ public ArrowBuf getDataBuf() {
+ return dataBuf;
+ }
+
+ public ArrowBuf getValidityBuf() {
+ return validityBuf;
+ }
+
+ }
+
+ protected DataAndValidityBuffers allocFixedDataAndValidityBufs(int valueCount, int typeWidth) {
+ long bufferSize = computeCombinedBufferSize(valueCount, typeWidth);
+ assert bufferSize < MAX_ALLOCATION_SIZE;
+
+ int validityBufferSize;
+ int dataBufferSize;
+ if (typeWidth == 0) {
+ validityBufferSize = dataBufferSize = (int) (bufferSize / 2);
+ } else {
+ // Due to roundup to power-of-2 allocation, the bufferSize could be greater than the
+ // requested size. Utilize the allocated buffer fully.;
+ int actualCount = (int) ((bufferSize * 8.0) / (8 * typeWidth + 1));
+ do {
+ validityBufferSize = (int) roundUp8(getValidityBufferSizeFromCount(actualCount));
+ dataBufferSize = (int) roundUp8(actualCount * typeWidth);
+ if (validityBufferSize + dataBufferSize <= bufferSize) {
+ break;
+ }
+ --actualCount;
+ } while (true);
+ }
+
+
+ /* allocate combined buffer */
+ ArrowBuf combinedBuffer = allocator.buffer((int) bufferSize);
+
+ /* slice into requested lengths */
+ ArrowBuf dataBuf = null;
+ ArrowBuf validityBuf = null;
+ int bufferOffset = 0;
+ for (int numBuffers = 0; numBuffers < 2; ++numBuffers) {
+ int len = (numBuffers == 0 ? dataBufferSize : validityBufferSize);
+ ArrowBuf buf = combinedBuffer.slice(bufferOffset, len);
+ buf.retain();
+ buf.readerIndex(0);
+ buf.writerIndex(0);
+
+ bufferOffset += len;
+ if (numBuffers == 0) {
+ dataBuf = buf;
+ } else {
+ validityBuf = buf;
+ }
+ }
+ combinedBuffer.release();
+ return new DataAndValidityBuffers(dataBuf, validityBuf);
+ }
}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
index 390dfe9..ac148a2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthVector.java
@@ -38,10 +38,8 @@ public abstract class BaseVariableWidthVector extends BaseValueVector
implements VariableWidthVector, FieldVector, VectorDefinitionSetter {
private static final int DEFAULT_RECORD_BYTE_COUNT = 8;
private static final int INITIAL_BYTE_COUNT = INITIAL_VALUE_ALLOCATION * DEFAULT_RECORD_BYTE_COUNT;
-
- private int valueAllocationSizeInBytes;
- private int validityAllocationSizeInBytes;
- private int offsetAllocationSizeInBytes;
+ private int initialValueAllocation;
+ private int initialValueAllocationSizeInBytes;
/* protected members */
public static final int OFFSET_WIDTH = 4; /* 4 byte unsigned int to track offsets */
@@ -57,9 +55,9 @@ public abstract class BaseVariableWidthVector extends BaseValueVector
public BaseVariableWidthVector(final String name, final BufferAllocator allocator,
FieldType fieldType) {
super(name, allocator);
- valueAllocationSizeInBytes = INITIAL_BYTE_COUNT;
- validityAllocationSizeInBytes = getValidityBufferSizeFromCount(INITIAL_VALUE_ALLOCATION);
- offsetAllocationSizeInBytes = (INITIAL_VALUE_ALLOCATION) * OFFSET_WIDTH;
+ initialValueAllocationSizeInBytes = INITIAL_BYTE_COUNT;
+ // -1 because we require one extra slot for the offset array.
+ initialValueAllocation = INITIAL_VALUE_ALLOCATION - 1;
field = new Field(name, fieldType, null);
valueCount = 0;
lastSet = -1;
@@ -155,15 +153,10 @@ public abstract class BaseVariableWidthVector extends BaseValueVector
@Override
public void setInitialCapacity(int valueCount) {
final long size = (long) valueCount * DEFAULT_RECORD_BYTE_COUNT;
- if (size > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
- }
- valueAllocationSizeInBytes = (int) size;
- validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount);
- /* to track the end offset of last data element in vector, we need
- * an additional slot in offset buffer.
- */
- offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH;
+ checkDataBufferSize(size);
+ computeAndCheckOffsetsBufferSize(valueCount);
+ initialValueAllocationSizeInBytes = (int) size;
+ initialValueAllocation = valueCount;
}
/**
@@ -175,17 +168,10 @@ public abstract class BaseVariableWidthVector extends BaseValueVector
@Override
public void setInitialCapacity(int valueCount, double density) {
long size = Math.max((long)(valueCount * density), 1L);
-
- if (size > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
- }
-
- valueAllocationSizeInBytes = (int) size;
- validityAllocationSizeInBytes = getValidityBufferSizeFromCount(valueCount);
- /* to track the end offset of last data element in vector, we need
- * an additional slot in offset buffer.
- */
- offsetAllocationSizeInBytes = (valueCount + 1) * OFFSET_WIDTH;
+ checkDataBufferSize(size);
+ computeAndCheckOffsetsBufferSize(valueCount);
+ initialValueAllocationSizeInBytes = (int) size;
+ initialValueAllocation = valueCount;
}
/**
@@ -376,20 +362,14 @@ public abstract class BaseVariableWidthVector extends BaseValueVector
*/
@Override
public boolean allocateNewSafe() {
- long curAllocationSizeValue = valueAllocationSizeInBytes;
- long curAllocationSizeValidity = validityAllocationSizeInBytes;
- long curAllocationSizeOffset = offsetAllocationSizeInBytes;
-
- if (curAllocationSizeValue > MAX_ALLOCATION_SIZE ||
- curAllocationSizeOffset > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Requested amount of memory exceeds limit");
- }
+ checkDataBufferSize(initialValueAllocationSizeInBytes);
+ computeAndCheckOffsetsBufferSize(initialValueAllocation);
/* we are doing a new allocation -- release the current buffers */
clear();
try {
- allocateBytes(curAllocationSizeValue, curAllocationSizeValidity, curAllocationSizeOffset);
+ allocateBytes(initialValueAllocationSizeInBytes, initialValueAllocation);
} catch (Exception e) {
clear();
return false;
@@ -409,35 +389,59 @@ public abstract class BaseVariableWidthVector extends BaseValueVector
@Override
public void allocateNew(int totalBytes, int valueCount) {
assert totalBytes >= 0;
- final int offsetBufferSize = (valueCount + 1) * OFFSET_WIDTH;
- final int validityBufferSize = getValidityBufferSizeFromCount(valueCount);
- if (totalBytes > MAX_ALLOCATION_SIZE ||
- offsetBufferSize > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Requested amount of memory exceeds limit");
- }
+ checkDataBufferSize(totalBytes);
+ computeAndCheckOffsetsBufferSize(valueCount);
/* we are doing a new allocation -- release the current buffers */
clear();
try {
- allocateBytes(totalBytes, validityBufferSize, offsetBufferSize);
+ allocateBytes(totalBytes, valueCount);
} catch (Exception e) {
clear();
throw e;
}
}
+ /* Check if the data buffer size is within bounds. */
+ private void checkDataBufferSize(long size) {
+ if (size > MAX_ALLOCATION_SIZE) {
+ throw new OversizedAllocationException("Memory required for vector " +
+ " is (" + size + "), which is more than max allowed (" + MAX_ALLOCATION_SIZE + ")");
+ }
+ }
+
+ /*
+ * Compute the buffer size required for 'valueCount' offsets and validity, and check if it's
+ * within bounds.
+ */
+ private long computeAndCheckOffsetsBufferSize(int valueCount) {
+ /* to track the end offset of last data element in vector, we need
+ * an additional slot in offset buffer.
+ */
+ final long size = computeCombinedBufferSize(valueCount + 1, OFFSET_WIDTH);
+ if (size > MAX_ALLOCATION_SIZE) {
+ throw new OversizedAllocationException("Memory required for vector capacity " +
+ valueCount +
+ " is (" + size + "), which is more than max allowed (" + MAX_ALLOCATION_SIZE + ")");
+ }
+ return size;
+ }
+
/* allocate the inner buffers */
- private void allocateBytes(final long valueBufferSize, final long validityBufferSize,
- final long offsetBufferSize) {
+ private void allocateBytes(final int valueBufferSize, final int valueCount) {
/* allocate data buffer */
- int curSize = (int) valueBufferSize;
+ int curSize = valueBufferSize;
valueBuffer = allocator.buffer(curSize);
valueBuffer.readerIndex(0);
- valueAllocationSizeInBytes = curSize;
- allocateValidityBuffer(validityBufferSize);
- allocateOffsetBuffer(offsetBufferSize);
+
+ /* allocate offset buffer and validity buffer */
+ DataAndValidityBuffers buffers = allocFixedDataAndValidityBufs(valueCount + 1, OFFSET_WIDTH);
+ offsetBuffer = buffers.getDataBuf();
+ validityBuffer = buffers.getValidityBuf();
+ initOffsetBuffer();
+ initValidityBuffer();
}
/* allocate offset buffer */
@@ -445,7 +449,6 @@ public abstract class BaseVariableWidthVector extends BaseValueVector
final int curSize = (int) size;
offsetBuffer = allocator.buffer(curSize);
offsetBuffer.readerIndex(0);
- offsetAllocationSizeInBytes = curSize;
initOffsetBuffer();
}
@@ -454,7 +457,6 @@ public abstract class BaseVariableWidthVector extends BaseValueVector
final int curSize = (int) size;
validityBuffer = allocator.buffer(curSize);
validityBuffer.readerIndex(0);
- validityAllocationSizeInBytes = curSize;
initValidityBuffer();
}
@@ -476,7 +478,7 @@ public abstract class BaseVariableWidthVector extends BaseValueVector
* @throws OutOfMemoryException if the internal memory allocation fails
*/
public void reallocDataBuffer() {
- long baseSize = valueAllocationSizeInBytes;
+ long baseSize = initialValueAllocationSizeInBytes;
final int currentBufferCapacity = valueBuffer.capacity();
if (baseSize < (long) currentBufferCapacity) {
@@ -487,15 +489,12 @@ public abstract class BaseVariableWidthVector extends BaseValueVector
newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
assert newAllocationSize >= 1;
- if (newAllocationSize > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Unable to expand the buffer");
- }
+ checkDataBufferSize(newAllocationSize);
final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
newBuf.setBytes(0, valueBuffer, 0, currentBufferCapacity);
valueBuffer.release();
valueBuffer = newBuf;
- valueAllocationSizeInBytes = (int) newAllocationSize;
}
/**
@@ -522,40 +521,28 @@ public abstract class BaseVariableWidthVector extends BaseValueVector
* @throws OutOfMemoryException if the internal memory allocation fails
*/
public void reallocValidityAndOffsetBuffers() {
- offsetBuffer = reallocBufferHelper(offsetBuffer, true);
- validityBuffer = reallocBufferHelper(validityBuffer, false);
- }
-
- /* helper method to realloc a particular buffer. returns the allocated buffer */
- private ArrowBuf reallocBufferHelper(ArrowBuf buffer, final boolean offsetBuffer) {
- final int currentBufferCapacity = buffer.capacity();
- long baseSize = (offsetBuffer ? offsetAllocationSizeInBytes
- : validityAllocationSizeInBytes);
-
- if (baseSize < (long) currentBufferCapacity) {
- baseSize = (long) currentBufferCapacity;
- }
-
- long newAllocationSize = baseSize * 2L;
- newAllocationSize = BaseAllocator.nextPowerOfTwo(newAllocationSize);
- assert newAllocationSize >= 1;
-
- if (newAllocationSize > MAX_ALLOCATION_SIZE) {
- throw new OversizedAllocationException("Unable to expand the buffer");
+ int targetOffsetCount = (offsetBuffer.capacity() / OFFSET_WIDTH) * 2;
+ if (targetOffsetCount == 0) {
+ if (initialValueAllocation > 0) {
+ targetOffsetCount = 2 * (initialValueAllocation + 1);
+ } else {
+ targetOffsetCount = 2 * (INITIAL_VALUE_ALLOCATION + 1);
+ }
}
+ computeAndCheckOffsetsBufferSize(targetOffsetCount);
- final ArrowBuf newBuf = allocator.buffer((int) newAllocationSize);
- newBuf.setBytes(0, buffer, 0, currentBufferCapacity);
- newBuf.setZero(currentBufferCapacity, newBuf.capacity() - currentBufferCapacity);
- buffer.release(1);
- buffer = newBuf;
- if (offsetBuffer) {
- offsetAllocationSizeInBytes = (int) newAllocationSize;
- } else {
- validityAllocationSizeInBytes = (int) newAllocationSize;
- }
+ DataAndValidityBuffers buffers = allocFixedDataAndValidityBufs(targetOffsetCount, OFFSET_WIDTH);
+ final ArrowBuf newOffsetBuffer = buffers.getDataBuf();
+ newOffsetBuffer.setBytes(0, offsetBuffer, 0, offsetBuffer.capacity());
+ newOffsetBuffer.setZero(offsetBuffer.capacity(), newOffsetBuffer.capacity() - offsetBuffer.capacity());
+ offsetBuffer.release();
+ offsetBuffer = newOffsetBuffer;
- return buffer;
+ final ArrowBuf newValidityBuffer = buffers.getValidityBuf();
+ newValidityBuffer.setBytes(0, validityBuffer, 0, validityBuffer.capacity());
+ newValidityBuffer.setZero(validityBuffer.capacity(), newValidityBuffer.capacity() - validityBuffer.capacity());
+ validityBuffer.release();
+ validityBuffer = newValidityBuffer;
}
/**
@@ -919,7 +906,7 @@ public abstract class BaseVariableWidthVector extends BaseValueVector
@Override
public void setIndexDefined(int index) {
while (index >= getValidityBufferValueCapacity()) {
- validityBuffer = reallocBufferHelper(validityBuffer, false);
+ reallocValidityAndOffsetBuffers();
}
BitVectorHelper.setValidityBitToOne(validityBuffer, index);
}
@@ -1072,7 +1059,7 @@ public abstract class BaseVariableWidthVector extends BaseValueVector
*/
public void setNull(int index) {
while (index >= getValidityBufferValueCapacity()) {
- validityBuffer = reallocBufferHelper(validityBuffer, false);
+ reallocValidityAndOffsetBuffers();
}
BitVectorHelper.setValidityBit(validityBuffer, index, 0);
}
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 7aac28c..c6c9642 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -91,11 +91,10 @@ public class BitVector extends BaseFixedWidthVector {
@Override
public void setInitialCapacity(int valueCount) {
final int size = getValidityBufferSizeFromCount(valueCount);
- if (size > MAX_ALLOCATION_SIZE) {
+ if (size * 2 > MAX_ALLOCATION_SIZE) {
throw new OversizedAllocationException("Requested amount of memory is more than max allowed");
}
- valueAllocationSizeInBytes = size;
- validityAllocationSizeInBytes = size;
+ initialValueAllocation = valueCount;
}
/**
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
index 9165343..a407166 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestBufferOwnershipTransfer.java
@@ -51,7 +51,7 @@ public class TestBufferOwnershipTransfer {
}
@Test
- public void testTransferVariableidth() {
+ public void testTransferVariableWidth() {
BufferAllocator allocator = new RootAllocator(Integer.MAX_VALUE);
BufferAllocator childAllocator1 = allocator.newChildAllocator("child1", 100000, 100000);
BufferAllocator childAllocator2 = allocator.newChildAllocator("child2", 100000, 100000);
@@ -62,15 +62,12 @@ public class TestBufferOwnershipTransfer {
v1.setValueCount(4001);
VarCharVector v2 = new VarCharVector("v2", childAllocator2);
+ long memoryBeforeTransfer = childAllocator1.getAllocatedMemory();
v1.makeTransferPair(v2).transfer();
assertEquals(0, childAllocator1.getAllocatedMemory());
- int expectedValueVector = 4096 * 8;
- int expectedOffsetVector = 4096 * 4;
- int expectedBitVector = 512;
- int expected = expectedBitVector + expectedOffsetVector + expectedValueVector;
- assertEquals(expected, childAllocator2.getAllocatedMemory());
+ assertEquals(memoryBeforeTransfer, childAllocator2.getAllocatedMemory());
}
private static class Pointer<T> {
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java
index f7d3ddb..b10db95 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestCopyFrom.java
@@ -69,14 +69,16 @@ public class TestCopyFrom {
@Test /* NullableVarChar */
public void testCopyFromWithNulls() {
- try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator);
- final VarCharVector vector2 =
- newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
+ try (final VarCharVector vector =
+ newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator);
+ final VarCharVector vector2 =
+ newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
vector.allocateNew();
- int capacity = vector.getValueCapacity();
- assertEquals(4095, capacity);
+ assertTrue(vector.getValueCapacity() >= 1);
+ assertEquals(0, vector.getValueCount());
+ int initialCapacity = vector.getValueCapacity();
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if (i % 3 == 0) {
continue;
}
@@ -85,43 +87,53 @@ public class TestCopyFrom {
}
/* NO reAlloc() should have happened in setSafe() */
- capacity = vector.getValueCapacity();
- assertEquals(4095, capacity);
+ int capacity = vector.getValueCapacity();
+ assertEquals(initialCapacity, capacity);
- vector.setValueCount(4095);
+ vector.setValueCount(initialCapacity);
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if (i % 3 == 0) {
assertNull(vector.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i, Integer.toString(i), vector.getObject(i).toString());
+ assertEquals(
+ "unexpected value at index: " + i,
+ Integer.toString(i),
+ vector.getObject(i).toString());
}
}
+ vector2.setInitialCapacity(initialCapacity);
vector2.allocateNew();
capacity = vector2.getValueCapacity();
- assertEquals(4095, capacity);
+ assertEquals(initialCapacity, capacity);
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector2.copyFromSafe(i, i, vector);
if (i % 3 == 0) {
assertNull(vector2.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString());
+ assertEquals(
+ "unexpected value at index: " + i,
+ Integer.toString(i),
+ vector2.getObject(i).toString());
}
}
/* NO reAlloc() should have happened in copyFrom */
capacity = vector2.getValueCapacity();
- assertEquals(4095, capacity);
+ assertEquals(initialCapacity, capacity);
- vector2.setValueCount(4095);
+ vector2.setValueCount(initialCapacity);
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if (i % 3 == 0) {
assertNull(vector2.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString());
+ assertEquals(
+ "unexpected value at index: " + i,
+ Integer.toString(i),
+ vector2.getObject(i).toString());
}
}
}
@@ -129,14 +141,16 @@ public class TestCopyFrom {
@Test /* NullableVarChar */
public void testCopyFromWithNulls1() {
- try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator);
- final VarCharVector vector2 =
- newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
+ try (final VarCharVector vector =
+ newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator);
+ final VarCharVector vector2 =
+ newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
vector.allocateNew();
- int capacity = vector.getValueCapacity();
- assertEquals(4095, capacity);
+ assertTrue(vector.getValueCapacity() >= 1);
+ assertEquals(0, vector.getValueCount());
+ int initialCapacity = vector.getValueCapacity();
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if (i % 3 == 0) {
continue;
}
@@ -145,47 +159,57 @@ public class TestCopyFrom {
}
/* NO reAlloc() should have happened in setSafe() */
- capacity = vector.getValueCapacity();
- assertEquals(4095, capacity);
+ int capacity = vector.getValueCapacity();
+ assertEquals(initialCapacity, capacity);
- vector.setValueCount(4095);
+ vector.setValueCount(initialCapacity);
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if (i % 3 == 0) {
assertNull(vector.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i, Integer.toString(i), vector.getObject(i).toString());
+ assertEquals(
+ "unexpected value at index: " + i,
+ Integer.toString(i),
+ vector.getObject(i).toString());
}
}
/* set lesser initial capacity than actually needed
* to trigger reallocs in copyFromSafe()
*/
- vector2.allocateNew(1024 * 10, 1024);
+ vector2.allocateNew((initialCapacity / 4) * 10, initialCapacity / 4);
capacity = vector2.getValueCapacity();
- assertEquals(1024, capacity);
+ assertTrue(capacity >= initialCapacity / 4);
+ assertTrue(capacity < initialCapacity / 2);
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector2.copyFromSafe(i, i, vector);
if (i % 3 == 0) {
assertNull(vector2.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString());
+ assertEquals(
+ "unexpected value at index: " + i,
+ Integer.toString(i),
+ vector2.getObject(i).toString());
}
}
/* 2 reAllocs should have happened in copyFromSafe() */
capacity = vector2.getValueCapacity();
- assertEquals(4096, capacity);
+ assertTrue(capacity >= initialCapacity);
- vector2.setValueCount(4095);
+ vector2.setValueCount(initialCapacity);
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if (i % 3 == 0) {
assertNull(vector2.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i, Integer.toString(i), vector2.getObject(i).toString());
+ assertEquals(
+ "unexpected value at index: " + i,
+ Integer.toString(i),
+ vector2.getObject(i).toString());
}
}
}
@@ -194,28 +218,29 @@ public class TestCopyFrom {
@Test /* IntVector */
public void testCopyFromWithNulls2() {
try (final IntVector vector1 = new IntVector(EMPTY_SCHEMA_PATH, allocator);
- final IntVector vector2 = new IntVector(EMPTY_SCHEMA_PATH, allocator)) {
+ final IntVector vector2 = new IntVector(EMPTY_SCHEMA_PATH, allocator)) {
vector1.allocateNew();
- assertEquals(4096, vector1.getValueCapacity());
+ assertTrue(vector1.getValueCapacity() >= vector1.initialValueAllocation);
assertEquals(0, vector1.getValueCount());
+ int initialCapacity = vector1.getValueCapacity();
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
continue;
}
vector1.setSafe(i, 1000 + i);
}
- vector1.setValueCount(4096);
+ vector1.setValueCount(initialCapacity);
/* No realloc should have happened in setSafe or
* setValueCount
*/
- assertEquals(4096, vector1.getValueCapacity());
- assertEquals(4096, vector1.getValueCount());
+ assertEquals(initialCapacity, vector1.getValueCapacity());
+ assertEquals(initialCapacity, vector1.getValueCount());
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
assertNull(vector1.getObject(i));
} else {
@@ -226,23 +251,24 @@ public class TestCopyFrom {
/* set lesser initial capacity than actually needed
* to trigger reallocs in copyFromSafe()
*/
- vector2.allocateNew(1024);
- assertEquals(1024, vector2.getValueCapacity());
+ vector2.allocateNew(initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() < initialCapacity / 2);
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector2.copyFromSafe(i, i, vector1);
}
/* 2 realloc should have happened in copyFromSafe() */
- assertEquals(4096, vector2.getValueCapacity());
- vector2.setValueCount(8192);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity);
+ vector2.setValueCount(initialCapacity * 2);
/* setValueCount() should have done another realloc */
- assertEquals(8192, vector2.getValueCount());
- assertEquals(8192, vector2.getValueCapacity());
+ assertEquals(initialCapacity * 2, vector2.getValueCount());
+ assertTrue(vector2.getValueCapacity() >= initialCapacity * 2);
/* check vector data after copy and realloc */
- for (int i = 0; i < 8192; i++) {
- if (((i & 1) == 0) || (i >= 4096)) {
+ for (int i = 0; i < initialCapacity * 2; i++) {
+ if (((i & 1) == 0) || (i >= initialCapacity)) {
assertNull(vector2.getObject(i));
} else {
assertEquals("unexpected value at index: " + i, 1000 + i, vector2.get(i));
@@ -254,60 +280,60 @@ public class TestCopyFrom {
@Test /* BigIntVector */
public void testCopyFromWithNulls3() {
try (final BigIntVector vector1 = new BigIntVector(EMPTY_SCHEMA_PATH, allocator);
- final BigIntVector vector2 = new BigIntVector(EMPTY_SCHEMA_PATH, allocator)) {
+ final BigIntVector vector2 = new BigIntVector(EMPTY_SCHEMA_PATH, allocator)) {
vector1.allocateNew();
- assertEquals(4096, vector1.getValueCapacity());
+ assertTrue(vector1.getValueCapacity() >= vector1.initialValueAllocation);
assertEquals(0, vector1.getValueCount());
+ int initialCapacity = vector1.getValueCapacity();
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
continue;
}
- vector1.setSafe(i, 10000000000L + (long)i);
+ vector1.setSafe(i, 10000000000L + (long) i);
}
- vector1.setValueCount(4096);
+ vector1.setValueCount(initialCapacity);
/* No realloc should have happened in setSafe or
* setValueCount
*/
- assertEquals(4096, vector1.getValueCapacity());
- assertEquals(4096, vector1.getValueCount());
+ assertEquals(initialCapacity, vector1.getValueCapacity());
+ assertEquals(initialCapacity, vector1.getValueCount());
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
assertNull(vector1.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i,
- 10000000000L + (long)i, vector1.get(i));
+ assertEquals("unexpected value at index: " + i, 10000000000L + (long) i, vector1.get(i));
}
}
/* set lesser initial capacity than actually needed
* to trigger reallocs in copyFromSafe()
*/
- vector2.allocateNew(1024);
- assertEquals(1024, vector2.getValueCapacity());
+ vector2.allocateNew(initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() < initialCapacity / 2);
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector2.copyFromSafe(i, i, vector1);
}
/* 2 realloc should have happened in copyFromSafe() */
- assertEquals(4096, vector2.getValueCapacity());
- vector2.setValueCount(8192);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity);
+ vector2.setValueCount(initialCapacity * 2);
/* setValueCount() should have done another realloc */
- assertEquals(8192, vector2.getValueCount());
- assertEquals(8192, vector2.getValueCapacity());
+ assertEquals(initialCapacity * 2, vector2.getValueCount());
+ assertTrue(vector2.getValueCapacity() >= initialCapacity * 2);
/* check vector data after copy and realloc */
- for (int i = 0; i < 8192; i++) {
- if (((i & 1) == 0) || (i >= 4096)) {
+ for (int i = 0; i < initialCapacity * 2; i++) {
+ if (((i & 1) == 0) || (i >= initialCapacity)) {
assertNull(vector2.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i,
- 10000000000L + (long)i, vector2.get(i));
+ assertEquals("unexpected value at index: " + i, 10000000000L + (long) i, vector2.get(i));
}
}
}
@@ -316,8 +342,9 @@ public class TestCopyFrom {
@Test /* BitVector */
public void testCopyFromWithNulls4() {
try (final BitVector vector1 = new BitVector(EMPTY_SCHEMA_PATH, allocator);
- final BitVector vector2 = new BitVector(EMPTY_SCHEMA_PATH, allocator)) {
+ final BitVector vector2 = new BitVector(EMPTY_SCHEMA_PATH, allocator)) {
+ vector1.setInitialCapacity(4096);
vector1.allocateNew();
assertEquals(4096, vector1.getValueCapacity());
assertEquals(0, vector1.getValueCount());
@@ -394,60 +421,60 @@ public class TestCopyFrom {
@Test /* Float4Vector */
public void testCopyFromWithNulls5() {
try (final Float4Vector vector1 = new Float4Vector(EMPTY_SCHEMA_PATH, allocator);
- final Float4Vector vector2 = new Float4Vector(EMPTY_SCHEMA_PATH, allocator)) {
+ final Float4Vector vector2 = new Float4Vector(EMPTY_SCHEMA_PATH, allocator)) {
vector1.allocateNew();
- assertEquals(4096, vector1.getValueCapacity());
+ assertTrue(vector1.getValueCapacity() >= vector1.initialValueAllocation);
assertEquals(0, vector1.getValueCount());
+ int initialCapacity = vector1.getValueCapacity();
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
continue;
}
- vector1.setSafe(i, 100.25f + (float)i);
+ vector1.setSafe(i, 100.25f + (float) i);
}
- vector1.setValueCount(4096);
+ vector1.setValueCount(initialCapacity);
/* No realloc should have happened in setSafe or
* setValueCount
*/
- assertEquals(4096, vector1.getValueCapacity());
- assertEquals(4096, vector1.getValueCount());
+ assertEquals(initialCapacity, vector1.getValueCapacity());
+ assertEquals(initialCapacity, vector1.getValueCount());
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
assertNull(vector1.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i,
- 100.25f + (float)i, vector1.get(i), 0);
+ assertEquals("unexpected value at index: " + i, 100.25f + (float) i, vector1.get(i), 0);
}
}
/* set lesser initial capacity than actually needed
* to trigger reallocs in copyFromSafe()
*/
- vector2.allocateNew(1024);
- assertEquals(1024, vector2.getValueCapacity());
+ vector2.allocateNew(initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() < initialCapacity / 2);
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector2.copyFromSafe(i, i, vector1);
}
/* 2 realloc should have happened in copyFromSafe() */
- assertEquals(4096, vector2.getValueCapacity());
- vector2.setValueCount(8192);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity);
+ vector2.setValueCount(initialCapacity * 2);
/* setValueCount() should have done another realloc */
- assertEquals(8192, vector2.getValueCount());
- assertEquals(8192, vector2.getValueCapacity());
+ assertEquals(initialCapacity * 2, vector2.getValueCount());
+ assertTrue(vector2.getValueCapacity() >= initialCapacity * 2);
/* check vector data after copy and realloc */
- for (int i = 0; i < 8192; i++) {
- if (((i & 1) == 0) || (i >= 4096)) {
+ for (int i = 0; i < initialCapacity * 2; i++) {
+ if (((i & 1) == 0) || (i >= initialCapacity)) {
assertNull(vector2.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i,
- 100.25f + i * 1.0f, vector2.get(i), 0);
+ assertEquals("unexpected value at index: " + i, 100.25f + i * 1.0f, vector2.get(i), 0);
}
}
}
@@ -456,60 +483,62 @@ public class TestCopyFrom {
@Test /* Float8Vector */
public void testCopyFromWithNulls6() {
try (final Float8Vector vector1 = new Float8Vector(EMPTY_SCHEMA_PATH, allocator);
- final Float8Vector vector2 = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) {
+ final Float8Vector vector2 = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) {
vector1.allocateNew();
- assertEquals(4096, vector1.getValueCapacity());
+ assertTrue(vector1.getValueCapacity() >= vector1.initialValueAllocation);
assertEquals(0, vector1.getValueCount());
+ int initialCapacity = vector1.getValueCapacity();
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
continue;
}
vector1.setSafe(i, 123456.7865 + (double) i);
}
- vector1.setValueCount(4096);
+ vector1.setValueCount(initialCapacity);
/* No realloc should have happened in setSafe or
* setValueCount
*/
- assertEquals(4096, vector1.getValueCapacity());
- assertEquals(4096, vector1.getValueCount());
+ assertEquals(initialCapacity, vector1.getValueCapacity());
+ assertEquals(initialCapacity, vector1.getValueCount());
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
assertNull(vector1.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i,
- 123456.7865 + (double) i, vector1.get(i), 0);
+ assertEquals(
+ "unexpected value at index: " + i, 123456.7865 + (double) i, vector1.get(i), 0);
}
}
/* set lesser initial capacity than actually needed
* to trigger reallocs in copyFromSafe()
*/
- vector2.allocateNew(1024);
- assertEquals(1024, vector2.getValueCapacity());
+ vector2.allocateNew(initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() < initialCapacity / 2);
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector2.copyFromSafe(i, i, vector1);
}
/* 2 realloc should have happened in copyFromSafe() */
- assertEquals(4096, vector2.getValueCapacity());
- vector2.setValueCount(8192);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity);
+ vector2.setValueCount(initialCapacity * 2);
/* setValueCount() should have done another realloc */
- assertEquals(8192, vector2.getValueCount());
- assertEquals(8192, vector2.getValueCapacity());
+ assertEquals(initialCapacity * 2, vector2.getValueCount());
+ assertTrue(vector2.getValueCapacity() >= initialCapacity * 2);
/* check vector data after copy and realloc */
- for (int i = 0; i < 8192; i++) {
- if (((i & 1) == 0) || (i >= 4096)) {
+ for (int i = 0; i < initialCapacity * 2; i++) {
+ if (((i & 1) == 0) || (i >= initialCapacity)) {
assertNull(vector2.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i,
- 123456.7865 + (double) i, vector2.get(i), 0);
+ assertEquals(
+ "unexpected value at index: " + i, 123456.7865 + (double) i, vector2.get(i), 0);
}
}
}
@@ -518,30 +547,31 @@ public class TestCopyFrom {
@Test /* IntervalDayVector */
public void testCopyFromWithNulls7() {
try (final IntervalDayVector vector1 = new IntervalDayVector(EMPTY_SCHEMA_PATH, allocator);
- final IntervalDayVector vector2 = new IntervalDayVector(EMPTY_SCHEMA_PATH, allocator)) {
+ final IntervalDayVector vector2 = new IntervalDayVector(EMPTY_SCHEMA_PATH, allocator)) {
vector1.allocateNew();
- assertEquals(4096, vector1.getValueCapacity());
+ assertTrue(vector1.getValueCapacity() >= vector1.initialValueAllocation);
assertEquals(0, vector1.getValueCount());
+ int initialCapacity = vector1.getValueCapacity();
final int days = 10;
final int milliseconds = 10000;
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
continue;
}
vector1.setSafe(i, days + i, milliseconds + i);
}
- vector1.setValueCount(4096);
+ vector1.setValueCount(initialCapacity);
/* No realloc should have happened in setSafe or
* setValueCount
*/
- assertEquals(4096, vector1.getValueCapacity());
- assertEquals(4096, vector1.getValueCount());
+ assertEquals(initialCapacity, vector1.getValueCapacity());
+ assertEquals(initialCapacity, vector1.getValueCount());
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
assertNull(vector1.getObject(i));
} else {
@@ -554,23 +584,24 @@ public class TestCopyFrom {
/* set lesser initial capacity than actually needed
* to trigger reallocs in copyFromSafe()
*/
- vector2.allocateNew(1024);
- assertEquals(1024, vector2.getValueCapacity());
+ vector2.allocateNew(initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() < initialCapacity / 2);
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector2.copyFromSafe(i, i, vector1);
}
/* 2 realloc should have happened in copyFromSafe() */
- assertEquals(4096, vector2.getValueCapacity());
- vector2.setValueCount(8192);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity);
+ vector2.setValueCount(initialCapacity * 2);
/* setValueCount() should have done another realloc */
- assertEquals(8192, vector2.getValueCount());
- assertEquals(8192, vector2.getValueCapacity());
+ assertEquals(initialCapacity * 2, vector2.getValueCount());
+ assertTrue(vector2.getValueCapacity() >= initialCapacity * 2);
/* check vector data after copy and realloc */
- for (int i = 0; i < 8192; i++) {
- if (((i & 1) == 0) || (i >= 4096)) {
+ for (int i = 0; i < initialCapacity * 2; i++) {
+ if (((i & 1) == 0) || (i >= initialCapacity)) {
assertNull(vector2.getObject(i));
} else {
final Period p = vector2.getObject(i);
@@ -584,15 +615,16 @@ public class TestCopyFrom {
@Test /* IntervalYearVector */
public void testCopyFromWithNulls8() {
try (final IntervalYearVector vector1 = new IntervalYearVector(EMPTY_SCHEMA_PATH, allocator);
- final IntervalYearVector vector2 = new IntervalYearVector(EMPTY_SCHEMA_PATH, allocator)) {
+ final IntervalYearVector vector2 = new IntervalYearVector(EMPTY_SCHEMA_PATH, allocator)) {
vector1.allocateNew();
- assertEquals(4096, vector1.getValueCapacity());
+ assertTrue(vector1.getValueCapacity() >= vector1.initialValueAllocation);
assertEquals(0, vector1.getValueCount());
+ int initialCapacity = vector1.getValueCapacity();
final int interval = 30; /* 2 years 6 months */
- final Period[] periods = new Period[4096];
- for (int i = 0; i < 4096; i++) {
+ final Period[] periods = new Period[4096];
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
continue;
}
@@ -600,18 +632,19 @@ public class TestCopyFrom {
final Period p = new Period();
final int years = (interval + i) / org.apache.arrow.vector.util.DateUtility.yearsToMonths;
final int months = (interval + i) % org.apache.arrow.vector.util.DateUtility.yearsToMonths;
- periods[i] = p.plusYears(years).plusMonths(months);;
+ periods[i] = p.plusYears(years).plusMonths(months);
+ ;
}
- vector1.setValueCount(4096);
+ vector1.setValueCount(initialCapacity);
/* No realloc should have happened in setSafe or
* setValueCount
*/
- assertEquals(4096, vector1.getValueCapacity());
- assertEquals(4096, vector1.getValueCount());
+ assertEquals(initialCapacity, vector1.getValueCapacity());
+ assertEquals(initialCapacity, vector1.getValueCount());
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
assertNull(vector1.getObject(i));
} else {
@@ -624,23 +657,24 @@ public class TestCopyFrom {
/* set lesser initial capacity than actually needed
* to trigger reallocs in copyFromSafe()
*/
- vector2.allocateNew(1024);
- assertEquals(1024, vector2.getValueCapacity());
+ vector2.allocateNew(initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() < initialCapacity / 2);
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector2.copyFromSafe(i, i, vector1);
}
/* 2 realloc should have happened in copyFromSafe() */
- assertEquals(4096, vector2.getValueCapacity());
- vector2.setValueCount(8192);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity);
+ vector2.setValueCount(initialCapacity * 2);
/* setValueCount() should have done another realloc */
- assertEquals(8192, vector2.getValueCount());
- assertEquals(8192, vector2.getValueCapacity());
+ assertEquals(initialCapacity * 2, vector2.getValueCount());
+ assertTrue(vector2.getValueCapacity() >= initialCapacity * 2);
/* check vector data after copy and realloc */
- for (int i = 0; i < 8192; i++) {
- if (((i & 1) == 0) || (i >= 4096)) {
+ for (int i = 0; i < initialCapacity * 2; i++) {
+ if (((i & 1) == 0) || (i >= initialCapacity)) {
assertNull(vector2.getObject(i));
} else {
final Period p = vector2.getObject(i);
@@ -653,61 +687,61 @@ public class TestCopyFrom {
@Test /* SmallIntVector */
public void testCopyFromWithNulls9() {
try (final SmallIntVector vector1 = new SmallIntVector(EMPTY_SCHEMA_PATH, allocator);
- final SmallIntVector vector2 = new SmallIntVector(EMPTY_SCHEMA_PATH, allocator)) {
+ final SmallIntVector vector2 = new SmallIntVector(EMPTY_SCHEMA_PATH, allocator)) {
vector1.allocateNew();
- assertEquals(4096, vector1.getValueCapacity());
+ assertTrue(vector1.getValueCapacity() >= vector1.initialValueAllocation);
assertEquals(0, vector1.getValueCount());
+ int initialCapacity = vector1.getValueCapacity();
final short val = 1000;
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
continue;
}
- vector1.setSafe(i, val + (short)i);
+ vector1.setSafe(i, val + (short) i);
}
- vector1.setValueCount(4096);
+ vector1.setValueCount(initialCapacity);
/* No realloc should have happened in setSafe or
* setValueCount
*/
- assertEquals(4096, vector1.getValueCapacity());
- assertEquals(4096, vector1.getValueCount());
+ assertEquals(initialCapacity, vector1.getValueCapacity());
+ assertEquals(initialCapacity, vector1.getValueCount());
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
assertNull(vector1.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i,
- val + (short)i, vector1.get(i));
+ assertEquals("unexpected value at index: " + i, val + (short) i, vector1.get(i));
}
}
/* set lesser initial capacity than actually needed
* to trigger reallocs in copyFromSafe()
*/
- vector2.allocateNew(1024);
- assertEquals(1024, vector2.getValueCapacity());
+ vector2.allocateNew(initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() < initialCapacity / 2);
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector2.copyFromSafe(i, i, vector1);
}
/* 2 realloc should have happened in copyFromSafe() */
- assertEquals(4096, vector2.getValueCapacity());
- vector2.setValueCount(8192);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity);
+ vector2.setValueCount(initialCapacity * 2);
/* setValueCount() should have done another realloc */
- assertEquals(8192, vector2.getValueCount());
- assertEquals(8192, vector2.getValueCapacity());
+ assertEquals(initialCapacity * 2, vector2.getValueCount());
+ assertTrue(vector2.getValueCapacity() >= initialCapacity * 2);
/* check vector data after copy and realloc */
- for (int i = 0; i < 8192; i++) {
- if (((i & 1) == 0) || (i >= 4096)) {
+ for (int i = 0; i < initialCapacity * 2; i++) {
+ if (((i & 1) == 0) || (i >= initialCapacity)) {
assertNull(vector2.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i,
- val + (short)i, vector2.get(i));
+ assertEquals("unexpected value at index: " + i, val + (short) i, vector2.get(i));
}
}
}
@@ -716,61 +750,61 @@ public class TestCopyFrom {
@Test /* TimeMicroVector */
public void testCopyFromWithNulls10() {
try (final TimeMicroVector vector1 = new TimeMicroVector(EMPTY_SCHEMA_PATH, allocator);
- final TimeMicroVector vector2 = new TimeMicroVector(EMPTY_SCHEMA_PATH, allocator)) {
+ final TimeMicroVector vector2 = new TimeMicroVector(EMPTY_SCHEMA_PATH, allocator)) {
vector1.allocateNew();
- assertEquals(4096, vector1.getValueCapacity());
+ assertTrue(vector1.getValueCapacity() >= vector1.initialValueAllocation);
assertEquals(0, vector1.getValueCount());
+ int initialCapacity = vector1.getValueCapacity();
final long val = 100485765432L;
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
continue;
}
- vector1.setSafe(i, val + (long)i);
+ vector1.setSafe(i, val + (long) i);
}
- vector1.setValueCount(4096);
+ vector1.setValueCount(initialCapacity);
/* No realloc should have happened in setSafe or
* setValueCount
*/
- assertEquals(4096, vector1.getValueCapacity());
- assertEquals(4096, vector1.getValueCount());
+ assertEquals(initialCapacity, vector1.getValueCapacity());
+ assertEquals(initialCapacity, vector1.getValueCount());
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
assertNull(vector1.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i,
- val + (long)i, vector1.get(i));
+ assertEquals("unexpected value at index: " + i, val + (long) i, vector1.get(i));
}
}
/* set lesser initial capacity than actually needed
* to trigger reallocs in copyFromSafe()
*/
- vector2.allocateNew(1024);
- assertEquals(1024, vector2.getValueCapacity());
+ vector2.allocateNew(initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() < initialCapacity / 2);
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector2.copyFromSafe(i, i, vector1);
}
/* 2 realloc should have happened in copyFromSafe() */
- assertEquals(4096, vector2.getValueCapacity());
- vector2.setValueCount(8192);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity);
+ vector2.setValueCount(initialCapacity * 2);
/* setValueCount() should have done another realloc */
- assertEquals(8192, vector2.getValueCount());
- assertEquals(8192, vector2.getValueCapacity());
+ assertEquals(initialCapacity * 2, vector2.getValueCount());
+ assertTrue(vector2.getValueCapacity() >= initialCapacity * 2);
/* check vector data after copy and realloc */
- for (int i = 0; i < 8192; i++) {
- if (((i & 1) == 0) || (i >= 4096)) {
+ for (int i = 0; i < initialCapacity * 2; i++) {
+ if (((i & 1) == 0) || (i >= initialCapacity)) {
assertNull(vector2.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i,
- val + (long) i, vector2.get(i));
+ assertEquals("unexpected value at index: " + i, val + (long) i, vector2.get(i));
}
}
}
@@ -779,61 +813,61 @@ public class TestCopyFrom {
@Test /* TimeMilliVector */
public void testCopyFromWithNulls11() {
try (final TimeMilliVector vector1 = new TimeMilliVector(EMPTY_SCHEMA_PATH, allocator);
- final TimeMilliVector vector2 = new TimeMilliVector(EMPTY_SCHEMA_PATH, allocator)) {
+ final TimeMilliVector vector2 = new TimeMilliVector(EMPTY_SCHEMA_PATH, allocator)) {
vector1.allocateNew();
- assertEquals(4096, vector1.getValueCapacity());
+ assertTrue(vector1.getValueCapacity() >= vector1.initialValueAllocation);
assertEquals(0, vector1.getValueCount());
+ int initialCapacity = vector1.getValueCapacity();
final int val = 1000;
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
continue;
}
vector1.setSafe(i, val + i);
}
- vector1.setValueCount(4096);
+ vector1.setValueCount(initialCapacity);
/* No realloc should have happened in setSafe or
* setValueCount
*/
- assertEquals(4096, vector1.getValueCapacity());
- assertEquals(4096, vector1.getValueCount());
+ assertEquals(initialCapacity, vector1.getValueCapacity());
+ assertEquals(initialCapacity, vector1.getValueCount());
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
assertNull(vector1.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i,
- val + i, vector1.get(i));
+ assertEquals("unexpected value at index: " + i, val + i, vector1.get(i));
}
}
/* set lesser initial capacity than actually needed
* to trigger reallocs in copyFromSafe()
*/
- vector2.allocateNew(1024);
- assertEquals(1024, vector2.getValueCapacity());
+ vector2.allocateNew(initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() < initialCapacity / 2);
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector2.copyFromSafe(i, i, vector1);
}
/* 2 realloc should have happened in copyFromSafe() */
- assertEquals(4096, vector2.getValueCapacity());
- vector2.setValueCount(8192);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity);
+ vector2.setValueCount(initialCapacity * 2);
/* setValueCount() should have done another realloc */
- assertEquals(8192, vector2.getValueCount());
- assertEquals(8192, vector2.getValueCapacity());
+ assertEquals(initialCapacity * 2, vector2.getValueCount());
+ assertTrue(vector2.getValueCapacity() >= initialCapacity * 2);
/* check vector data after copy and realloc */
- for (int i = 0; i < 8192; i++) {
- if (((i & 1) == 0) || (i >= 4096)) {
+ for (int i = 0; i < initialCapacity * 2; i++) {
+ if (((i & 1) == 0) || (i >= initialCapacity)) {
assertNull(vector2.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i,
- val + i, vector2.get(i));
+ assertEquals("unexpected value at index: " + i, val + i, vector2.get(i));
}
}
}
@@ -842,14 +876,15 @@ public class TestCopyFrom {
@Test /* TinyIntVector */
public void testCopyFromWithNulls12() {
try (final TinyIntVector vector1 = new TinyIntVector(EMPTY_SCHEMA_PATH, allocator);
- final TinyIntVector vector2 = new TinyIntVector(EMPTY_SCHEMA_PATH, allocator)) {
+ final TinyIntVector vector2 = new TinyIntVector(EMPTY_SCHEMA_PATH, allocator)) {
vector1.allocateNew();
- assertEquals(4096, vector1.getValueCapacity());
+ assertTrue(vector1.getValueCapacity() >= vector1.initialValueAllocation);
assertEquals(0, vector1.getValueCount());
+ int initialCapacity = vector1.getValueCapacity();
byte val = -128;
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
continue;
}
@@ -857,16 +892,16 @@ public class TestCopyFrom {
val++;
}
- vector1.setValueCount(4096);
+ vector1.setValueCount(initialCapacity);
/* No realloc should have happened in setSafe or
* setValueCount
*/
- assertEquals(4096, vector1.getValueCapacity());
- assertEquals(4096, vector1.getValueCount());
+ assertEquals(initialCapacity, vector1.getValueCapacity());
+ assertEquals(initialCapacity, vector1.getValueCount());
val = -128;
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
assertNull(vector1.getObject(i));
} else {
@@ -878,24 +913,24 @@ public class TestCopyFrom {
/* set lesser initial capacity than actually needed
* to trigger reallocs in copyFromSafe()
*/
- vector2.allocateNew(1024);
- assertEquals(1024, vector2.getValueCapacity());
+ vector2.allocateNew(initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity / 4);
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector2.copyFromSafe(i, i, vector1);
}
/* 2 realloc should have happened in copyFromSafe() */
- assertEquals(4096, vector2.getValueCapacity());
- vector2.setValueCount(8192);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity);
+ vector2.setValueCount(initialCapacity * 2);
/* setValueCount() should have done another realloc */
- assertEquals(8192, vector2.getValueCount());
- assertEquals(8192, vector2.getValueCapacity());
+ assertEquals(initialCapacity * 2, vector2.getValueCount());
+ assertTrue(vector2.getValueCapacity() >= initialCapacity * 2);
/* check vector data after copy and realloc */
val = -128;
- for (int i = 0; i < 8192; i++) {
- if (((i & 1) == 0) || (i >= 4096)) {
+ for (int i = 0; i < initialCapacity * 2; i++) {
+ if (((i & 1) == 0) || (i >= initialCapacity)) {
assertNull(vector2.getObject(i));
} else {
assertEquals("unexpected value at index: " + i, val, vector2.get(i));
@@ -908,32 +943,33 @@ public class TestCopyFrom {
@Test /* DecimalVector */
public void testCopyFromWithNulls13() {
try (final DecimalVector vector1 = new DecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16);
- final DecimalVector vector2 = new DecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16)) {
+ final DecimalVector vector2 = new DecimalVector(EMPTY_SCHEMA_PATH, allocator, 30, 16)) {
vector1.allocateNew();
- assertEquals(4096, vector1.getValueCapacity());
+ assertTrue(vector1.getValueCapacity() >= vector1.initialValueAllocation);
assertEquals(0, vector1.getValueCount());
+ int initialCapacity = vector1.getValueCapacity();
final double baseValue = 104567897654.876543654;
final BigDecimal[] decimals = new BigDecimal[4096];
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
continue;
}
- BigDecimal decimal = new BigDecimal(baseValue + (double)i);
+ BigDecimal decimal = new BigDecimal(baseValue + (double) i);
vector1.setSafe(i, decimal);
decimals[i] = decimal;
}
- vector1.setValueCount(4096);
+ vector1.setValueCount(initialCapacity);
/* No realloc should have happened in setSafe or
* setValueCount
*/
- assertEquals(4096, vector1.getValueCapacity());
- assertEquals(4096, vector1.getValueCount());
+ assertEquals(initialCapacity, vector1.getValueCapacity());
+ assertEquals(initialCapacity, vector1.getValueCount());
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
assertNull(vector1.getObject(i));
} else {
@@ -945,23 +981,24 @@ public class TestCopyFrom {
/* set lesser initial capacity than actually needed
* to trigger reallocs in copyFromSafe()
*/
- vector2.allocateNew(1024);
- assertEquals(1024, vector2.getValueCapacity());
+ vector2.allocateNew(initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() < initialCapacity / 2);
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector2.copyFromSafe(i, i, vector1);
}
/* 2 realloc should have happened in copyFromSafe() */
- assertEquals(4096, vector2.getValueCapacity());
- vector2.setValueCount(8192);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity);
+ vector2.setValueCount(initialCapacity * 2);
/* setValueCount() should have done another realloc */
- assertEquals(8192, vector2.getValueCount());
- assertEquals(8192, vector2.getValueCapacity());
+ assertEquals(initialCapacity * 2, vector2.getValueCount());
+ assertTrue(vector2.getValueCapacity() >= initialCapacity * 2);
/* check vector data after copy and realloc */
- for (int i = 0; i < 8192; i++) {
- if (((i & 1) == 0) || (i >= 4096)) {
+ for (int i = 0; i < initialCapacity * 2; i++) {
+ if (((i & 1) == 0) || (i >= initialCapacity)) {
assertNull(vector2.getObject(i));
} else {
final BigDecimal decimal = vector2.getObject(i);
@@ -974,61 +1011,61 @@ public class TestCopyFrom {
@Test /* TimeStampVector */
public void testCopyFromWithNulls14() {
try (final TimeStampVector vector1 = new TimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator);
- final TimeStampVector vector2 = new TimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator)) {
+ final TimeStampVector vector2 = new TimeStampMicroVector(EMPTY_SCHEMA_PATH, allocator)) {
vector1.allocateNew();
- assertEquals(4096, vector1.getValueCapacity());
+ assertTrue(vector1.getValueCapacity() >= vector1.initialValueAllocation);
assertEquals(0, vector1.getValueCount());
+ int initialCapacity = vector1.getValueCapacity();
final long val = 20145678912L;
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
continue;
}
- vector1.setSafe(i, val + (long)i);
+ vector1.setSafe(i, val + (long) i);
}
- vector1.setValueCount(4096);
+ vector1.setValueCount(initialCapacity);
/* No realloc should have happened in setSafe or
* setValueCount
*/
- assertEquals(4096, vector1.getValueCapacity());
- assertEquals(4096, vector1.getValueCount());
+ assertEquals(initialCapacity, vector1.getValueCapacity());
+ assertEquals(initialCapacity, vector1.getValueCount());
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
if ((i & 1) == 0) {
assertNull(vector1.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i,
- val + (long)i, vector1.get(i));
+ assertEquals("unexpected value at index: " + i, val + (long) i, vector1.get(i));
}
}
/* set lesser initial capacity than actually needed
* to trigger reallocs in copyFromSafe()
*/
- vector2.allocateNew(1024);
- assertEquals(1024, vector2.getValueCapacity());
+ vector2.allocateNew(initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity / 4);
+ assertTrue(vector2.getValueCapacity() < initialCapacity / 2);
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector2.copyFromSafe(i, i, vector1);
}
/* 2 realloc should have happened in copyFromSafe() */
- assertEquals(4096, vector2.getValueCapacity());
- vector2.setValueCount(8192);
+ assertTrue(vector2.getValueCapacity() >= initialCapacity);
+ vector2.setValueCount(initialCapacity * 2);
/* setValueCount() should have done another realloc */
- assertEquals(8192, vector2.getValueCount());
- assertEquals(8192, vector2.getValueCapacity());
+ assertEquals(initialCapacity * 2, vector2.getValueCount());
+ assertTrue(vector2.getValueCapacity() >= initialCapacity * 2);
/* check vector data after copy and realloc */
- for (int i = 0; i < 8192; i++) {
- if (((i & 1) == 0) || (i >= 4096)) {
+ for (int i = 0; i < initialCapacity * 2; i++) {
+ if (((i & 1) == 0) || (i >= initialCapacity)) {
assertNull(vector2.getObject(i));
} else {
- assertEquals("unexpected value at index: " + i,
- val + (long) i, vector2.get(i));
+ assertEquals("unexpected value at index: " + i, val + (long) i, vector2.get(i));
}
}
}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index 4772a86..30fe23c 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -32,6 +32,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import org.apache.arrow.memory.BaseAllocator;
import org.apache.arrow.memory.BufferAllocator;
import org.apache.arrow.memory.RootAllocator;
import org.apache.arrow.vector.ipc.message.ArrowRecordBatch;
@@ -68,8 +69,8 @@ public class TestValueVector {
private static final byte[] STR5 = "EEE5".getBytes(utf8Charset);
private static final byte[] STR6 = "FFFFF6".getBytes(utf8Charset);
private static final int MAX_VALUE_COUNT =
- Integer.getInteger("arrow.vector.max_allocation_bytes", Integer.MAX_VALUE) / 4;
- private static final int MAX_VALUE_COUNT_8BYTE = MAX_VALUE_COUNT / 2;
+ (int)(Integer.getInteger("arrow.vector.max_allocation_bytes", Integer.MAX_VALUE) / 7);
+ private static final int MAX_VALUE_COUNT_8BYTE = (int)(MAX_VALUE_COUNT / 2);
@After
public void terminate() throws Exception {
@@ -108,7 +109,7 @@ public class TestValueVector {
vector.allocateNew(1024);
initialCapacity = vector.getValueCapacity();
- assertEquals(1024, initialCapacity);
+ assertTrue(initialCapacity >= 1024);
// Put and set a few values
vector.setSafe(0, 100);
@@ -124,7 +125,7 @@ public class TestValueVector {
assertEquals(104, vector.get(1023));
try {
- vector.set(1024, 10000);
+ vector.set(initialCapacity, 10000);
} catch (IndexOutOfBoundsException ie) {
error = true;
} finally {
@@ -133,7 +134,7 @@ public class TestValueVector {
}
try {
- vector.get(1024);
+ vector.get(initialCapacity);
} catch (IndexOutOfBoundsException ie) {
error = true;
} finally {
@@ -142,10 +143,10 @@ public class TestValueVector {
}
/* this should trigger a realloc() */
- vector.setSafe(1024, 10000);
+ vector.setSafe(initialCapacity, 10000);
/* underlying buffer should now be able to store double the number of values */
- assertEquals(initialCapacity * 2, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= 2 * initialCapacity);
/* check vector data after realloc */
assertEquals(100, vector.get(0));
@@ -153,16 +154,17 @@ public class TestValueVector {
assertEquals(102, vector.get(100));
assertEquals(103, vector.get(1022));
assertEquals(104, vector.get(1023));
- assertEquals(10000, vector.get(1024));
+ assertEquals(10000, vector.get(initialCapacity));
/* reset the vector */
+ int capacityBeforeReset = vector.getValueCapacity();
vector.reset();
/* capacity shouldn't change after reset */
- assertEquals(initialCapacity * 2, vector.getValueCapacity());
+ assertEquals(capacityBeforeReset, vector.getValueCapacity());
/* vector data should have been zeroed out */
- for (int i = 0; i < (initialCapacity * 2); i++) {
+ for (int i = 0; i < capacityBeforeReset; i++) {
// TODO: test vector.get(i) is 0 after unsafe get added
assertEquals("non-zero data not expected at index: " + i, true, vector.isNull(i));
}
@@ -180,7 +182,7 @@ public class TestValueVector {
intVector.setInitialCapacity(MAX_VALUE_COUNT);
try {
- intVector.setInitialCapacity(MAX_VALUE_COUNT + 1);
+ intVector.setInitialCapacity(MAX_VALUE_COUNT * 2);
} catch (OversizedAllocationException oe) {
error = true;
} finally {
@@ -195,17 +197,18 @@ public class TestValueVector {
/* allocate 64 bytes (16 * 4) */
intVector.allocateNew();
/* underlying buffer should be able to store 16 values */
- assertEquals(initialCapacity, intVector.getValueCapacity());
+ assertTrue(intVector.getValueCapacity() >= initialCapacity);
+ initialCapacity = intVector.getValueCapacity();
/* populate the vector */
int j = 1;
- for (int i = 0; i < 16; i += 2) {
+ for (int i = 0; i < initialCapacity; i += 2) {
intVector.set(i, j);
j++;
}
try {
- intVector.set(16, 9);
+ intVector.set(initialCapacity, j);
} catch (IndexOutOfBoundsException ie) {
error = true;
} finally {
@@ -215,13 +218,13 @@ public class TestValueVector {
/* check vector contents */
j = 1;
- for (int i = 0; i < 16; i += 2) {
+ for (int i = 0; i < initialCapacity; i += 2) {
assertEquals("unexpected value at index: " + i, j, intVector.get(i));
j++;
}
try {
- intVector.get(16);
+ intVector.get(initialCapacity);
} catch (IndexOutOfBoundsException ie) {
error = true;
} finally {
@@ -230,26 +233,27 @@ public class TestValueVector {
}
/* this should trigger a realloc() */
- intVector.setSafe(16, 9);
+ intVector.setSafe(initialCapacity, j);
/* underlying buffer should now be able to store double the number of values */
- assertEquals(initialCapacity * 2, intVector.getValueCapacity());
+ assertTrue(intVector.getValueCapacity() >= initialCapacity * 2);
/* vector data should still be intact after realloc */
j = 1;
- for (int i = 0; i <= 16; i += 2) {
+ for (int i = 0; i <= initialCapacity; i += 2) {
assertEquals("unexpected value at index: " + i, j, intVector.get(i));
j++;
}
/* reset the vector */
+ int capacityBeforeRealloc = intVector.getValueCapacity();
intVector.reset();
/* capacity shouldn't change after reset */
- assertEquals(initialCapacity * 2, intVector.getValueCapacity());
+ assertEquals(capacityBeforeRealloc, intVector.getValueCapacity());
/* vector data should have been zeroed out */
- for (int i = 0; i < (initialCapacity * 2); i++) {
+ for (int i = 0; i < capacityBeforeRealloc; i++) {
assertEquals("non-zero data not expected at index: " + i, true, intVector.isNull(i));
}
}
@@ -266,7 +270,7 @@ public class TestValueVector {
floatVector.setInitialCapacity(MAX_VALUE_COUNT);
try {
- floatVector.setInitialCapacity(MAX_VALUE_COUNT + 1);
+ floatVector.setInitialCapacity(MAX_VALUE_COUNT * 2);
} catch (OversizedAllocationException oe) {
error = true;
} finally {
@@ -281,7 +285,8 @@ public class TestValueVector {
/* allocate 64 bytes (16 * 4) */
floatVector.allocateNew();
/* underlying buffer should be able to store 16 values */
- assertEquals(initialCapacity, floatVector.getValueCapacity());
+ assertTrue(floatVector.getValueCapacity() >= initialCapacity);
+ initialCapacity = floatVector.getValueCapacity();
floatVector.zeroVector();
@@ -296,7 +301,7 @@ public class TestValueVector {
floatVector.set(14, 8.5f);
try {
- floatVector.set(16, 9.5f);
+ floatVector.set(initialCapacity, 9.5f);
} catch (IndexOutOfBoundsException ie) {
error = true;
} finally {
@@ -315,7 +320,7 @@ public class TestValueVector {
assertEquals(8.5f, floatVector.get(14), 0);
try {
- floatVector.get(16);
+ floatVector.get(initialCapacity);
} catch (IndexOutOfBoundsException ie) {
error = true;
} finally {
@@ -324,10 +329,10 @@ public class TestValueVector {
}
/* this should trigger a realloc() */
- floatVector.setSafe(16, 9.5f);
+ floatVector.setSafe(initialCapacity, 9.5f);
/* underlying buffer should now be able to store double the number of values */
- assertEquals(initialCapacity * 2, floatVector.getValueCapacity());
+ assertTrue(floatVector.getValueCapacity() >= initialCapacity * 2);
/* vector data should still be intact after realloc */
assertEquals(1.5f, floatVector.get(0), 0);
@@ -338,16 +343,17 @@ public class TestValueVector {
assertEquals(6.6f, floatVector.get(10), 0);
assertEquals(7.8f, floatVector.get(12), 0);
assertEquals(8.5f, floatVector.get(14), 0);
- assertEquals(9.5f, floatVector.get(16), 0);
+ assertEquals(9.5f, floatVector.get(initialCapacity), 0);
/* reset the vector */
+ int capacityBeforeReset = floatVector.getValueCapacity();
floatVector.reset();
/* capacity shouldn't change after reset */
- assertEquals(initialCapacity * 2, floatVector.getValueCapacity());
+ assertEquals(capacityBeforeReset, floatVector.getValueCapacity());
/* vector data should be zeroed out */
- for (int i = 0; i < (initialCapacity * 2); i++) {
+ for (int i = 0; i < capacityBeforeReset; i++) {
assertEquals("non-zero data not expected at index: " + i, true, floatVector.isNull(i));
}
}
@@ -364,7 +370,7 @@ public class TestValueVector {
floatVector.setInitialCapacity(MAX_VALUE_COUNT_8BYTE);
try {
- floatVector.setInitialCapacity(MAX_VALUE_COUNT_8BYTE + 1);
+ floatVector.setInitialCapacity(MAX_VALUE_COUNT_8BYTE * 2);
} catch (OversizedAllocationException oe) {
error = true;
} finally {
@@ -379,7 +385,8 @@ public class TestValueVector {
/* allocate 128 bytes (16 * 8) */
floatVector.allocateNew();
/* underlying buffer should be able to store 16 values */
- assertEquals(initialCapacity, floatVector.getValueCapacity());
+ assertTrue(floatVector.getValueCapacity() >= initialCapacity);
+ initialCapacity = floatVector.getValueCapacity();
/* populate the vector */
floatVector.set(0, 1.55);
@@ -392,7 +399,7 @@ public class TestValueVector {
floatVector.set(14, 8.56);
try {
- floatVector.set(16, 9.53);
+ floatVector.set(initialCapacity, 9.53);
} catch (IndexOutOfBoundsException ie) {
error = true;
} finally {
@@ -411,7 +418,7 @@ public class TestValueVector {
assertEquals(8.56, floatVector.get(14), 0);
try {
- floatVector.get(16);
+ floatVector.get(initialCapacity);
} catch (IndexOutOfBoundsException ie) {
error = true;
} finally {
@@ -420,10 +427,10 @@ public class TestValueVector {
}
/* this should trigger a realloc() */
- floatVector.setSafe(16, 9.53);
+ floatVector.setSafe(initialCapacity, 9.53);
/* underlying buffer should now be able to store double the number of values */
- assertEquals(initialCapacity * 2, floatVector.getValueCapacity());
+ assertTrue(floatVector.getValueCapacity() >= initialCapacity * 2);
/* vector data should still be intact after realloc */
assertEquals(1.55, floatVector.get(0), 0);
@@ -434,16 +441,17 @@ public class TestValueVector {
assertEquals(6.67, floatVector.get(10), 0);
assertEquals(7.87, floatVector.get(12), 0);
assertEquals(8.56, floatVector.get(14), 0);
- assertEquals(9.53, floatVector.get(16), 0);
+ assertEquals(9.53, floatVector.get(initialCapacity), 0);
/* reset the vector */
+ int capacityBeforeReset = floatVector.getValueCapacity();
floatVector.reset();
/* capacity shouldn't change after reset */
- assertEquals(initialCapacity * 2, floatVector.getValueCapacity());
+ assertEquals(capacityBeforeReset, floatVector.getValueCapacity());
/* vector data should be zeroed out */
- for (int i = 0; i < (initialCapacity * 2); i++) {
+ for (int i = 0; i < capacityBeforeReset; i++) {
assertEquals("non-zero data not expected at index: " + i, true, floatVector.isNull(i));
}
}
@@ -463,36 +471,37 @@ public class TestValueVector {
assertEquals(0, vector.getValueCapacity());
vector.allocateNew();
- assertEquals(initialCapacity, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= initialCapacity);
+ initialCapacity = vector.getValueCapacity();
// Put and set a few values
vector.set(0, 100);
vector.set(1, 101);
vector.set(100, 102);
- vector.set(1022, 103);
- vector.set(1023, 104);
+ vector.set(initialCapacity - 2, 103);
+ vector.set(initialCapacity - 1, 104);
/* check vector contents */
assertEquals(100, vector.get(0));
assertEquals(101, vector.get(1));
assertEquals(102, vector.get(100));
- assertEquals(103, vector.get(1022));
- assertEquals(104, vector.get(1023));
+ assertEquals(103, vector.get(initialCapacity - 2));
+ assertEquals(104, vector.get(initialCapacity - 1));
int val = 0;
/* check unset bits/null values */
- for (int i = 2, j = 101; i <= 99 || j <= 1021; i++, j++) {
+ for (int i = 2, j = 101; i <= 99 || j <= initialCapacity - 3; i++, j++) {
if (i <= 99) {
assertTrue(vector.isNull(i));
}
- if (j <= 1021) {
+ if (j <= initialCapacity - 3) {
assertTrue(vector.isNull(j));
}
}
try {
- vector.set(1024, 10000);
+ vector.set(initialCapacity, 10000);
} catch (IndexOutOfBoundsException ie) {
error = true;
} finally {
@@ -501,7 +510,7 @@ public class TestValueVector {
}
try {
- vector.get(1024);
+ vector.get(initialCapacity);
} catch (IndexOutOfBoundsException ie) {
error = true;
} finally {
@@ -510,39 +519,40 @@ public class TestValueVector {
}
/* should trigger a realloc of the underlying bitvector and valuevector */
- vector.setSafe(1024, 10000);
+ vector.setSafe(initialCapacity, 10000);
/* check new capacity */
- assertEquals(initialCapacity * 2, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= initialCapacity * 2);
/* vector contents should still be intact after realloc */
assertEquals(100, vector.get(0));
assertEquals(101, vector.get(1));
assertEquals(102, vector.get(100));
- assertEquals(103, vector.get(1022));
- assertEquals(104, vector.get(1023));
- assertEquals(10000, vector.get(1024));
+ assertEquals(103, vector.get(initialCapacity - 2));
+ assertEquals(104, vector.get(initialCapacity - 1));
+ assertEquals(10000, vector.get(initialCapacity));
val = 0;
/* check unset bits/null values */
- for (int i = 2, j = 101; i < 99 || j < 1021; i++, j++) {
+ for (int i = 2, j = 101; i < 99 || j < initialCapacity - 3; i++, j++) {
if (i <= 99) {
assertTrue(vector.isNull(i));
}
- if (j <= 1021) {
+ if (j <= initialCapacity - 3) {
assertTrue(vector.isNull(j));
}
}
/* reset the vector */
+ int capacityBeforeReset = vector.getValueCapacity();
vector.reset();
/* capacity shouldn't change after reset */
- assertEquals(initialCapacity * 2, vector.getValueCapacity());
+ assertEquals(capacityBeforeReset, vector.getValueCapacity());
/* vector data should be zeroed out */
- for (int i = 0; i < (initialCapacity * 2); i++) {
+ for (int i = 0; i < capacityBeforeReset; i++) {
assertTrue("non-null data not expected at index: " + i, vector.isNull(i));
}
}
@@ -560,7 +570,8 @@ public class TestValueVector {
assertEquals(0, vector.getValueCapacity());
vector.allocateNew();
- assertEquals(initialCapacity, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= initialCapacity);
+ initialCapacity = vector.getValueCapacity();
/* populate the vector */
vector.set(0, 100.5f);
@@ -573,7 +584,7 @@ public class TestValueVector {
vector.set(14, 89.5f);
try {
- vector.set(16, 90.5f);
+ vector.set(initialCapacity, 90.5f);
} catch (IndexOutOfBoundsException ie) {
error = true;
} finally {
@@ -600,7 +611,7 @@ public class TestValueVector {
assertTrue(vector.isNull(15));
try {
- vector.get(16);
+ vector.get(initialCapacity);
} catch (IndexOutOfBoundsException ie) {
error = true;
} finally {
@@ -609,10 +620,10 @@ public class TestValueVector {
}
/* this should trigger a realloc() */
- vector.setSafe(16, 90.5f);
+ vector.setSafe(initialCapacity, 90.5f);
/* underlying buffer should now be able to store double the number of values */
- assertEquals(initialCapacity * 2, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= 2 * initialCapacity);
/* vector data should still be intact after realloc */
assertEquals(100.5f, vector.get(0), 0);
@@ -633,13 +644,14 @@ public class TestValueVector {
assertTrue(vector.isNull(15));
/* reset the vector */
+ int capacityBeforeReset = vector.getValueCapacity();
vector.reset();
/* capacity shouldn't change after reset */
- assertEquals(initialCapacity * 2, vector.getValueCapacity());
+ assertEquals(capacityBeforeReset, vector.getValueCapacity());
/* vector data should be zeroed out */
- for (int i = 0; i < (initialCapacity * 2); i++) {
+ for (int i = 0; i < capacityBeforeReset; i++) {
assertTrue("non-null data not expected at index: " + i, vector.isNull(i));
}
}
@@ -656,8 +668,9 @@ public class TestValueVector {
assertEquals(0, vector.getValueCapacity());
/* allocate space for 4KB data (1024 * 4) */
vector.allocateNew(initialCapacity);
- /* underlying buffer should be able to store 16 values */
- assertEquals(initialCapacity, vector.getValueCapacity());
+ /* underlying buffer should be able to store 1024 values */
+ assertTrue(vector.getValueCapacity() >= initialCapacity);
+ initialCapacity = vector.getValueCapacity();
vector.set(0, 1);
vector.set(1, 2);
@@ -687,7 +700,7 @@ public class TestValueVector {
ArrowBuf validityVectorBuf = buffers.get(0);
/* bitvector tracks 1024 integers --> 1024 bits --> 128 bytes */
- assertEquals(128, validityVectorBuf.readableBytes());
+ assertTrue(validityVectorBuf.readableBytes() >= 128);
assertEquals(3, validityVectorBuf.getByte(0)); // 1st and second bit defined
for (int i = 1; i < 12; i++) {
assertEquals(0, validityVectorBuf.getByte(i)); // nothing defined until 100
@@ -699,15 +712,15 @@ public class TestValueVector {
assertEquals(-64, validityVectorBuf.getByte(127)); // 1022nd and 1023rd bit defined
/* this should trigger a realloc() */
- vector.setSafe(1024, 6);
+ vector.setSafe(initialCapacity, 6);
/* underlying buffer should now be able to store double the number of values */
- assertEquals(initialCapacity * 2, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= 2 * initialCapacity);
/* vector data should still be intact after realloc */
j = 1;
for (int i = 0; i < (initialCapacity * 2); i++) {
- if ((i > 1024) || (i >= 2 && i <= 99) || (i >= 101 && i <= 1021)) {
+ if ((i > 1023 && i != initialCapacity) || (i >= 2 && i <= 99) || (i >= 101 && i <= 1021)) {
assertTrue("non-null data not expected at index: " + i, vector.isNull(i));
} else {
assertFalse("null data not expected at index: " + i, vector.isNull(i));
@@ -717,19 +730,20 @@ public class TestValueVector {
}
/* reset the vector */
+ int capacityBeforeReset = vector.getValueCapacity();
vector.reset();
/* capacity shouldn't change after reset */
- assertEquals(initialCapacity * 2, vector.getValueCapacity());
+ assertEquals(capacityBeforeReset, vector.getValueCapacity());
/* vector data should have been zeroed out */
- for (int i = 0; i < (initialCapacity * 2); i++) {
+ for (int i = 0; i < capacityBeforeReset; i++) {
assertTrue("non-null data not expected at index: " + i, vector.isNull(i));
}
- vector.allocateNew(4096);
+ vector.allocateNew(initialCapacity * 4);
// vector has been erased
- for (int i = 0; i < 4096; i++) {
+ for (int i = 0; i < initialCapacity * 4; i++) {
assertTrue("non-null data not expected at index: " + i, vector.isNull(i));
}
}
@@ -764,7 +778,7 @@ public class TestValueVector {
}
vector.setSafe(valueCapacity, 20000000);
- assertEquals(valueCapacity * 2, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= valueCapacity * 2);
for (int i = 0; i < vector.getValueCapacity(); i++) {
if (i == valueCapacity) {
@@ -795,14 +809,15 @@ public class TestValueVector {
}
}
- vector.setSafe((valueCapacity * 2) + 1000, 400000000);
- assertEquals(valueCapacity * 4, vector.getValueCapacity());
+ int valueCapacityBeforeRealloc = vector.getValueCapacity();
+ vector.setSafe(valueCapacityBeforeRealloc + 1000, 400000000);
+ assertTrue(vector.getValueCapacity() >= valueCapacity * 4);
for (int i = 0; i < vector.getValueCapacity(); i++) {
- if (i == (valueCapacity * 2 + 1000)) {
+ if (i == (valueCapacityBeforeRealloc + 1000)) {
assertFalse("unexpected null value at index: " + i, vector.isNull(i));
assertEquals("unexpected value at index: " + i, 400000000, vector.get(i));
- } else if (i < valueCapacity * 2 && (i % 2) == 0) {
+ } else if (i < valueCapacityBeforeRealloc && (i % 2) == 0) {
assertFalse("unexpected null value at index: " + i, vector.isNull(i));
assertEquals("unexpected value at index: " + i, baseValue + i, vector.get(i));
} else {
@@ -811,13 +826,14 @@ public class TestValueVector {
}
/* reset the vector */
+ int valueCapacityBeforeReset = vector.getValueCapacity();
vector.reset();
/* capacity shouldn't change after reset */
- assertEquals(valueCapacity * 4, vector.getValueCapacity());
+ assertEquals(valueCapacityBeforeReset, vector.getValueCapacity());
/* vector data should be zeroed out */
- for (int i = 0; i < (valueCapacity * 4); i++) {
+ for (int i = 0; i < valueCapacityBeforeReset; i++) {
assertTrue("non-null data not expected at index: " + i, vector.isNull(i));
}
}
@@ -936,52 +952,56 @@ public class TestValueVector {
@Test /* Float8Vector */
public void testReallocAfterVectorTransfer1() {
try (final Float8Vector vector = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) {
- final int initialDefaultCapacity = 4096;
+ int initialCapacity = 4096;
boolean error = false;
/* use the default capacity; 4096*8 => 32KB */
+ vector.setInitialCapacity(initialCapacity);
vector.allocateNew();
- assertEquals(initialDefaultCapacity, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= initialCapacity);
+ initialCapacity = vector.getValueCapacity();
double baseValue = 100.375;
- for (int i = 0; i < initialDefaultCapacity; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector.setSafe(i, baseValue + (double)i);
}
/* the above setSafe calls should not have triggered a realloc as
* we are within the capacity. check the vector contents
*/
- assertEquals(initialDefaultCapacity, vector.getValueCapacity());
+ assertEquals(initialCapacity, vector.getValueCapacity());
- for (int i = 0; i < initialDefaultCapacity; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
double value = vector.get(i);
assertEquals(baseValue + (double)i, value, 0);
}
/* this should trigger a realloc */
- vector.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity);
- assertEquals(initialDefaultCapacity * 2, vector.getValueCapacity());
+ vector.setSafe(initialCapacity, baseValue + (double)initialCapacity);
+ assertTrue(vector.getValueCapacity() >= initialCapacity * 2);
+ int capacityAfterRealloc1 = vector.getValueCapacity();
- for (int i = initialDefaultCapacity + 1; i < (initialDefaultCapacity * 2); i++) {
+ for (int i = initialCapacity + 1; i < capacityAfterRealloc1; i++) {
vector.setSafe(i, baseValue + (double)i);
}
- for (int i = 0; i < (initialDefaultCapacity * 2); i++) {
+ for (int i = 0; i < capacityAfterRealloc1; i++) {
double value = vector.get(i);
assertEquals(baseValue + (double)i, value, 0);
}
/* this should trigger a realloc */
- vector.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2));
- assertEquals(initialDefaultCapacity * 4, vector.getValueCapacity());
+ vector.setSafe(capacityAfterRealloc1, baseValue + (double)(capacityAfterRealloc1));
+ assertTrue(vector.getValueCapacity() >= initialCapacity * 4);
+ int capacityAfterRealloc2 = vector.getValueCapacity();
- for (int i = (initialDefaultCapacity * 2) + 1; i < (initialDefaultCapacity * 4); i++) {
+ for (int i = capacityAfterRealloc1 + 1; i < capacityAfterRealloc2; i++) {
vector.setSafe(i, baseValue + (double)i);
}
- for (int i = 0; i < (initialDefaultCapacity * 4); i++) {
+ for (int i = 0; i < capacityAfterRealloc2; i++) {
double value = vector.get(i);
assertEquals(baseValue + (double)i, value, 0);
}
@@ -997,10 +1017,10 @@ public class TestValueVector {
/* now let's realloc the toVector */
toVector.reAlloc();
- assertEquals(initialDefaultCapacity * 8, toVector.getValueCapacity());
+ assertTrue(toVector.getValueCapacity() >= initialCapacity * 8);
- for (int i = 0; i < (initialDefaultCapacity * 8); i++) {
- if (i < (initialDefaultCapacity * 4)) {
+ for (int i = 0; i < toVector.getValueCapacity(); i++) {
+ if (i < capacityAfterRealloc2) {
assertEquals(baseValue + (double)i, toVector.get(i), 0);
} else {
assertTrue(toVector.isNull(i));
@@ -1014,51 +1034,53 @@ public class TestValueVector {
@Test /* Float8Vector */
public void testReallocAfterVectorTransfer2() {
try (final Float8Vector vector = new Float8Vector(EMPTY_SCHEMA_PATH, allocator)) {
- final int initialDefaultCapacity = 4096;
+ int initialCapacity = 4096;
boolean error = false;
- vector.allocateNew(initialDefaultCapacity);
-
- assertEquals(initialDefaultCapacity, vector.getValueCapacity());
+ vector.allocateNew(initialCapacity);
+ assertTrue(vector.getValueCapacity() >= initialCapacity);
+ initialCapacity = vector.getValueCapacity();
double baseValue = 100.375;
- for (int i = 0; i < initialDefaultCapacity; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
vector.setSafe(i, baseValue + (double)i);
}
/* the above setSafe calls should not have triggered a realloc as
* we are within the capacity. check the vector contents
*/
- assertEquals(initialDefaultCapacity, vector.getValueCapacity());
+ assertEquals(initialCapacity, vector.getValueCapacity());
- for (int i = 0; i < initialDefaultCapacity; i++) {
+ for (int i = 0; i < initialCapacity; i++) {
double value = vector.get(i);
assertEquals(baseValue + (double)i, value, 0);
}
/* this should trigger a realloc */
- vector.setSafe(initialDefaultCapacity, baseValue + (double)initialDefaultCapacity);
- assertEquals(initialDefaultCapacity * 2, vector.getValueCapacity());
+ vector.setSafe(initialCapacity, baseValue + (double)initialCapacity);
+ assertTrue(vector.getValueCapacity() >= initialCapacity * 2);
+ int capacityAfterRealloc1 = vector.getValueCapacity();
- for (int i = initialDefaultCapacity + 1; i < (initialDefaultCapacity * 2); i++) {
+ for (int i = initialCapacity + 1; i < capacityAfterRealloc1; i++) {
vector.setSafe(i, baseValue + (double)i);
}
- for (int i = 0; i < (initialDefaultCapacity * 2); i++) {
+ for (int i = 0; i < capacityAfterRealloc1; i++) {
double value = vector.get(i);
assertEquals(baseValue + (double)i, value, 0);
}
/* this should trigger a realloc */
- vector.setSafe(initialDefaultCapacity * 2, baseValue + (double)(initialDefaultCapacity * 2));
- assertEquals(initialDefaultCapacity * 4, vector.getValueCapacity());
+ vector.setSafe(capacityAfterRealloc1, baseValue + (double)(capacityAfterRealloc1));
+ assertTrue(vector.getValueCapacity() >= initialCapacity * 4);
+ int capacityAfterRealloc2 = vector.getValueCapacity();
- for (int i = (initialDefaultCapacity * 2) + 1; i < (initialDefaultCapacity * 4); i++) {
+ for (int i = capacityAfterRealloc1 + 1; i < capacityAfterRealloc2; i++) {
vector.setSafe(i, baseValue + (double)i);
}
- for (int i = 0; i < (initialDefaultCapacity * 4); i++) {
+ for (int i = 0; i < capacityAfterRealloc2; i++) {
double value = vector.get(i);
assertEquals(baseValue + (double)i, value, 0);
}
@@ -1073,7 +1095,7 @@ public class TestValueVector {
Float8Vector toVector = (Float8Vector)transferPair.getTo();
/* check toVector contents before realloc */
- for (int i = 0; i < (initialDefaultCapacity * 4); i++) {
+ for (int i = 0; i < toVector.getValueCapacity(); i++) {
assertFalse("unexpected null value at index: " + i, toVector.isNull(i));
double value = toVector.get(i);
assertEquals("unexpected value at index: " + i, baseValue + (double)i, value, 0);
@@ -1081,10 +1103,10 @@ public class TestValueVector {
/* now let's realloc the toVector and check contents again */
toVector.reAlloc();
- assertEquals(initialDefaultCapacity * 8, toVector.getValueCapacity());
+ assertTrue(toVector.getValueCapacity() >= initialCapacity * 8);
- for (int i = 0; i < (initialDefaultCapacity * 8); i++) {
- if (i < (initialDefaultCapacity * 4)) {
+ for (int i = 0; i < toVector.getValueCapacity(); i++) {
+ if (i < capacityAfterRealloc2) {
assertFalse("unexpected null value at index: " + i, toVector.isNull(i));
double value = toVector.get(i);
assertEquals("unexpected value at index: " + i, baseValue + (double)i, value, 0);
@@ -1103,7 +1125,7 @@ public class TestValueVector {
/* 4096 values with 10 byte per record */
vector.allocateNew(4096 * 10, 4096);
int valueCapacity = vector.getValueCapacity();
- assertEquals(4096, valueCapacity);
+ assertTrue(valueCapacity >= 4096);
/* populate the vector */
for (int i = 0; i < valueCapacity; i++) {
@@ -1125,7 +1147,10 @@ public class TestValueVector {
/* trigger first realloc */
vector.setSafe(valueCapacity, STR2, 0, STR2.length);
- assertEquals(valueCapacity * 2, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= 2 * valueCapacity);
+ while (vector.getByteCapacity() < 10 * vector.getValueCapacity()) {
+ vector.reallocDataBuffer();
+ }
/* populate the remaining vector */
for (int i = valueCapacity; i < vector.getValueCapacity(); i++) {
@@ -1148,7 +1173,10 @@ public class TestValueVector {
/* trigger second realloc */
vector.setSafe(valueCapacity + 10, STR2, 0, STR2.length);
- assertEquals(valueCapacity * 2, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= 2 * valueCapacity);
+ while (vector.getByteCapacity() < 10 * vector.getValueCapacity()) {
+ vector.reallocDataBuffer();
+ }
/* populate the remaining vector */
for (int i = valueCapacity; i < vector.getValueCapacity(); i++) {
@@ -1197,7 +1225,7 @@ public class TestValueVector {
/* 4096 values */
vector.allocateNew(4096);
int valueCapacity = vector.getValueCapacity();
- assertEquals(4096, valueCapacity);
+ assertTrue(valueCapacity >= 4096);
/* populate the vector */
int baseValue = 1000;
@@ -1218,7 +1246,7 @@ public class TestValueVector {
/* trigger first realloc */
vector.setSafe(valueCapacity, 10000000);
- assertEquals(valueCapacity * 2, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= valueCapacity * 2);
/* populate the remaining vector */
for (int i = valueCapacity; i < vector.getValueCapacity(); i++) {
@@ -1239,7 +1267,7 @@ public class TestValueVector {
/* trigger second realloc */
vector.setSafe(valueCapacity, 10000000);
- assertEquals(valueCapacity * 2, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= valueCapacity * 2);
/* populate the remaining vector */
for (int i = valueCapacity; i < vector.getValueCapacity(); i++) {
@@ -1288,7 +1316,8 @@ public class TestValueVector {
try (final Float4Vector vector = newVector(Float4Vector.class, EMPTY_SCHEMA_PATH, MinorType.FLOAT4, allocator)) {
vector.allocateNew(1024);
- assertEquals(1024, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= 1024);
+ int initialCapacity = vector.getValueCapacity();
// Put values in indexes that fall within the initial allocation
vector.setSafe(0, 100.1f);
@@ -1299,7 +1328,7 @@ public class TestValueVector {
vector.setSafe(2000, 105.5f);
// Check valueCapacity is more than initial allocation
- assertEquals(1024 * 2, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= 2 * initialCapacity);
assertEquals(100.1f, vector.get(0), 0);
assertEquals(102.3f, vector.get(100), 0);
@@ -1316,24 +1345,24 @@ public class TestValueVector {
@Test
public void testReAllocVariableWidthVector() {
try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
+ vector.setInitialCapacity(4095);
vector.allocateNew();
int initialCapacity = vector.getValueCapacity();
- assertEquals(4095, initialCapacity);
+ assertTrue(initialCapacity >= 4095);
/* Put values in indexes that fall within the initial allocation */
vector.setSafe(0, STR1, 0, STR1.length);
vector.setSafe(initialCapacity - 1, STR2, 0, STR2.length);
/* the above set calls should NOT have triggered a realloc */
- initialCapacity = vector.getValueCapacity();
- assertEquals(4095, initialCapacity);
+ assertEquals(initialCapacity, vector.getValueCapacity());
/* Now try to put values in space that falls beyond the initial allocation */
vector.setSafe(initialCapacity + 200, STR3, 0, STR3.length);
/* Check valueCapacity is more than initial allocation */
- assertEquals(((initialCapacity + 1) * 2) - 1, vector.getValueCapacity());
+ assertTrue(initialCapacity * 2 <= vector.getValueCapacity());
assertArrayEquals(STR1, vector.get(0));
assertArrayEquals(STR2, vector.get(initialCapacity - 1));
@@ -1348,20 +1377,20 @@ public class TestValueVector {
@Test
public void testFillEmptiesNotOverfill() {
try (final VarCharVector vector = newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
+ vector.setInitialCapacity(4095);
vector.allocateNew();
int initialCapacity = vector.getValueCapacity();
- assertEquals(4095, initialCapacity);
+ assertTrue(initialCapacity >= 4095);
vector.setSafe(4094, "hello".getBytes(), 0, 5);
/* the above set method should NOT have trigerred a realloc */
- initialCapacity = vector.getValueCapacity();
- assertEquals(4095, initialCapacity);
+ assertEquals(initialCapacity, vector.getValueCapacity());
- vector.setValueCount(4095);
- assertEquals(4096 * vector.OFFSET_WIDTH, vector.getFieldBuffers().get(1).capacity());
- initialCapacity = vector.getValueCapacity();
- assertEquals(4095, initialCapacity);
+ int bufSizeBefore = vector.getFieldBuffers().get(1).capacity();
+ vector.setValueCount(initialCapacity);
+ assertEquals(bufSizeBefore, vector.getFieldBuffers().get(1).capacity());
+ assertEquals(initialCapacity, vector.getValueCapacity());
}
}
@@ -1371,11 +1400,12 @@ public class TestValueVector {
final VarCharVector vector2 =
newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
+ vector.setInitialCapacity(4095);
vector.allocateNew();
int capacity = vector.getValueCapacity();
- assertEquals(4095, capacity);
+ assertTrue(capacity >= 4095);
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < capacity; i++) {
if (i % 3 == 0) {
continue;
}
@@ -1384,12 +1414,11 @@ public class TestValueVector {
}
/* NO reAlloc() should have happened in setSafe() */
- capacity = vector.getValueCapacity();
- assertEquals(4095, capacity);
+ assertEquals(capacity, vector.getValueCapacity());
- vector.setValueCount(4095);
+ vector.setValueCount(capacity);
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < capacity; i++) {
if (i % 3 == 0) {
assertNull(vector.getObject(i));
} else {
@@ -1397,11 +1426,12 @@ public class TestValueVector {
}
}
+ vector2.setInitialCapacity(4095);
vector2.allocateNew();
- capacity = vector2.getValueCapacity();
- assertEquals(4095, capacity);
+ int capacity2 = vector2.getValueCapacity();
+ assertEquals(capacity2, capacity);
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < capacity; i++) {
vector2.copyFromSafe(i, i, vector);
if (i % 3 == 0) {
assertNull(vector2.getObject(i));
@@ -1411,12 +1441,11 @@ public class TestValueVector {
}
/* NO reAlloc() should have happened in copyFrom */
- capacity = vector2.getValueCapacity();
- assertEquals(4095, capacity);
+ assertEquals(capacity, vector2.getValueCapacity());
- vector2.setValueCount(4095);
+ vector2.setValueCount(capacity);
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < capacity; i++) {
if (i % 3 == 0) {
assertNull(vector2.getObject(i));
} else {
@@ -1432,11 +1461,12 @@ public class TestValueVector {
final VarCharVector vector2 =
newVector(VarCharVector.class, EMPTY_SCHEMA_PATH, MinorType.VARCHAR, allocator)) {
+ vector.setInitialCapacity(4095);
vector.allocateNew();
int capacity = vector.getValueCapacity();
- assertEquals(4095, capacity);
+ assertTrue(capacity >= 4095);
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < capacity; i++) {
if (i % 3 == 0) {
continue;
}
@@ -1445,12 +1475,11 @@ public class TestValueVector {
}
/* NO reAlloc() should have happened in setSafe() */
- capacity = vector.getValueCapacity();
- assertEquals(4095, capacity);
+ assertEquals(capacity, vector.getValueCapacity());
- vector.setValueCount(4095);
+ vector.setValueCount(capacity);
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < capacity; i++) {
if (i % 3 == 0) {
assertNull(vector.getObject(i));
} else {
@@ -1463,10 +1492,11 @@ public class TestValueVector {
*/
vector2.allocateNew(1024 * 10, 1024);
- capacity = vector2.getValueCapacity();
- assertEquals(1024, capacity);
+ int capacity2 = vector2.getValueCapacity();
+ assertTrue(capacity2 >= 1024);
+ assertTrue(capacity2 <= capacity);
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < capacity; i++) {
vector2.copyFromSafe(i, i, vector);
if (i % 3 == 0) {
assertNull(vector2.getObject(i));
@@ -1476,12 +1506,11 @@ public class TestValueVector {
}
/* 2 reAllocs should have happened in copyFromSafe() */
- capacity = vector2.getValueCapacity();
- assertEquals(4096, capacity);
+ assertEquals(capacity, vector2.getValueCapacity());
- vector2.setValueCount(4095);
+ vector2.setValueCount(capacity);
- for (int i = 0; i < 4095; i++) {
+ for (int i = 0; i < capacity; i++) {
if (i % 3 == 0) {
assertNull(vector2.getObject(i));
} else {
@@ -1876,30 +1905,88 @@ public class TestValueVector {
try (final VarCharVector vector = new VarCharVector(EMPTY_SCHEMA_PATH, allocator)) {
/* use the default 8 data bytes on average per element */
- vector.setInitialCapacity(4096);
+ int defaultCapacity = BaseValueVector.INITIAL_VALUE_ALLOCATION - 1;
+ vector.setInitialCapacity(defaultCapacity);
vector.allocateNew();
- assertEquals(4096, vector.getValueCapacity());
- assertEquals(4096 * 8, vector.getDataBuffer().capacity());
+ assertEquals(defaultCapacity, vector.getValueCapacity());
+ assertEquals(BaseAllocator.nextPowerOfTwo(defaultCapacity * 8), vector.getDataBuffer().capacity());
- vector.setInitialCapacity(4096, 1);
+ vector.setInitialCapacity(defaultCapacity, 1);
vector.allocateNew();
- assertEquals(4096, vector.getValueCapacity());
- assertEquals(4096, vector.getDataBuffer().capacity());
+ assertEquals(defaultCapacity, vector.getValueCapacity());
+ assertEquals(BaseAllocator.nextPowerOfTwo(defaultCapacity), vector.getDataBuffer().capacity());
- vector.setInitialCapacity(4096, 0.1);
+ vector.setInitialCapacity(defaultCapacity, 0.1);
vector.allocateNew();
- assertEquals(4096, vector.getValueCapacity());
- assertEquals(512, vector.getDataBuffer().capacity());
+ assertEquals(defaultCapacity, vector.getValueCapacity());
+ assertEquals(BaseAllocator.nextPowerOfTwo((int)(defaultCapacity * 0.1)), vector.getDataBuffer().capacity());
- vector.setInitialCapacity(4096, 0.01);
+ vector.setInitialCapacity(defaultCapacity, 0.01);
vector.allocateNew();
- assertEquals(4096, vector.getValueCapacity());
- assertEquals(64, vector.getDataBuffer().capacity());
+ assertEquals(defaultCapacity, vector.getValueCapacity());
+ assertEquals(BaseAllocator.nextPowerOfTwo((int)(defaultCapacity * 0.01)), vector.getDataBuffer().capacity());
vector.setInitialCapacity(5, 0.01);
vector.allocateNew();
- assertEquals(7, vector.getValueCapacity());
+ assertEquals(5, vector.getValueCapacity());
assertEquals(2, vector.getDataBuffer().capacity());
}
}
+
+ @Test
+ public void testDefaultAllocNewAll() {
+ int defaultCapacity = BaseFixedWidthVector.INITIAL_VALUE_ALLOCATION;
+ int expectedSize;
+ long beforeSize;
+ try (BufferAllocator childAllocator = allocator.newChildAllocator("defaultAllocs", 0, Long.MAX_VALUE);
+ final IntVector intVector = new IntVector(EMPTY_SCHEMA_PATH, childAllocator);
+ final BigIntVector bigIntVector = new BigIntVector(EMPTY_SCHEMA_PATH, childAllocator);
+ final BitVector bitVector = new BitVector(EMPTY_SCHEMA_PATH, childAllocator);
+ final DecimalVector decimalVector = new DecimalVector(EMPTY_SCHEMA_PATH, childAllocator, 38, 6);
+ final VarCharVector varCharVector = new VarCharVector(EMPTY_SCHEMA_PATH, childAllocator)) {
+
+ // verify that the wastage is within bounds for IntVector.
+ beforeSize = childAllocator.getAllocatedMemory();
+ intVector.allocateNew();
+ assertTrue(intVector.getValueCapacity() >= defaultCapacity);
+ expectedSize = (defaultCapacity * IntVector.TYPE_WIDTH) +
+ BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity);
+ assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05);
+
+ // verify that the wastage is within bounds for BigIntVector.
+ beforeSize = childAllocator.getAllocatedMemory();
+ bigIntVector.allocateNew();
+ assertTrue(bigIntVector.getValueCapacity() >= defaultCapacity);
+ expectedSize = (defaultCapacity * bigIntVector.TYPE_WIDTH) +
+ BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity);
+ assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05);
+
+ // verify that the wastage is within bounds for DecimalVector.
+ beforeSize = childAllocator.getAllocatedMemory();
+ decimalVector.allocateNew();
+ assertTrue(decimalVector.getValueCapacity() >= defaultCapacity);
+ expectedSize = (defaultCapacity * decimalVector.TYPE_WIDTH) +
+ BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity);
+ assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05);
+
+ // verify that the wastage is within bounds for VarCharVector.
+ // var char vector have an offsets array that is 1 less than defaultCapacity
+ beforeSize = childAllocator.getAllocatedMemory();
+ varCharVector.allocateNew();
+ assertTrue(varCharVector.getValueCapacity() >= defaultCapacity - 1);
+ expectedSize = (defaultCapacity * VarCharVector.OFFSET_WIDTH) +
+ BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity) +
+ defaultCapacity * 8;
+ // wastage should be less than 5%.
+ assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05);
+
+ // verify that the wastage is within bounds for BitVector.
+ beforeSize = childAllocator.getAllocatedMemory();
+ bitVector.allocateNew();
+ assertTrue(bitVector.getValueCapacity() >= defaultCapacity);
+ expectedSize = BaseFixedWidthVector.getValidityBufferSizeFromCount(defaultCapacity) * 2;
+ assertTrue(childAllocator.getAllocatedMemory() - beforeSize <= expectedSize * 1.05);
+
+ }
+ }
}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
index 5474675..60747aa 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestVectorReAlloc.java
@@ -19,6 +19,7 @@ package org.apache.arrow.vector;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
import java.nio.charset.StandardCharsets;
@@ -54,20 +55,21 @@ public class TestVectorReAlloc {
vector.setInitialCapacity(512);
vector.allocateNew();
- assertEquals(512, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= 512);
+ int initialCapacity = vector.getValueCapacity();
try {
- vector.set(512, 0);
+ vector.set(initialCapacity, 0);
Assert.fail("Expected out of bounds exception");
} catch (Exception e) {
// ok
}
vector.reAlloc();
- assertEquals(1024, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= 2 * initialCapacity);
- vector.set(512, 100);
- assertEquals(100, vector.get(512));
+ vector.set(initialCapacity, 100);
+ assertEquals(100, vector.get(initialCapacity));
}
}
@@ -77,20 +79,21 @@ public class TestVectorReAlloc {
vector.setInitialCapacity(512);
vector.allocateNew();
- assertEquals(512, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= 512);
+ int initialCapacity = vector.getValueCapacity();
try {
- vector.set(512, "foo".getBytes(StandardCharsets.UTF_8));
+ vector.set(initialCapacity, "foo".getBytes(StandardCharsets.UTF_8));
Assert.fail("Expected out of bounds exception");
} catch (Exception e) {
// ok
}
vector.reAlloc();
- assertEquals(1024, vector.getValueCapacity());
+ assertTrue(vector.getValueCapacity() >= 2 * initialCapacity);
- vector.set(512, "foo".getBytes(StandardCharsets.UTF_8));
- assertEquals("foo", new String(vector.get(512), StandardCharsets.UTF_8));
+ vector.set(initialCapacity, "foo".getBytes(StandardCharsets.UTF_8));
+ assertEquals("foo", new String(vector.get(initialCapacity), StandardCharsets.UTF_8));
}
}
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index b7215ce..61c1b92 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -974,11 +974,16 @@ public class TestComplexWriter {
Float4Vector float4Vector = (Float4Vector)parent.getChild("float4Field");
Float8Vector float8Vector = (Float8Vector)parent.getChild("float8Field");
- assertEquals(initialCapacity, singleStructWriter.getValueCapacity());
- assertEquals(initialCapacity, intVector.getValueCapacity());
- assertEquals(initialCapacity, bigIntVector.getValueCapacity());
- assertEquals(initialCapacity, float4Vector.getValueCapacity());
- assertEquals(initialCapacity, float8Vector.getValueCapacity());
+ int capacity = singleStructWriter.getValueCapacity();
+ assertTrue(capacity >= initialCapacity && capacity < initialCapacity * 2);
+ capacity = intVector.getValueCapacity();
+ assertTrue(capacity >= initialCapacity && capacity < initialCapacity * 2);
+ capacity = bigIntVector.getValueCapacity();
+ assertTrue(capacity >= initialCapacity && capacity < initialCapacity * 2);
+ capacity = float4Vector.getValueCapacity();
+ assertTrue(capacity >= initialCapacity && capacity < initialCapacity * 2);
+ capacity = float8Vector.getValueCapacity();
+ assertTrue(capacity >= initialCapacity && capacity < initialCapacity * 2);
StructReader singleStructReader = new SingleStructReaderImpl(parent);