You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by uw...@apache.org on 2017/02/05 13:06:45 UTC
arrow git commit: ARROW-524: provide apis to access nested vectors
and buffers
Repository: arrow
Updated Branches:
refs/heads/master c45c3b3e1 -> 70c05be21
ARROW-524: provide apis to access nested vectors and buffers
Author: Julien Le Dem <ju...@dremio.com>
Closes #314 from julienledem/setRangeToOne and squashes the following commits:
0d526bd [Julien Le Dem] ARROW-524: provide apis to access nested vectors and buffers
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/70c05be2
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/70c05be2
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/70c05be2
Branch: refs/heads/master
Commit: 70c05be2130bdbb650a83bc46f7c4f8fc8a231df
Parents: c45c3b3
Author: Julien Le Dem <ju...@dremio.com>
Authored: Sun Feb 5 14:06:37 2017 +0100
Committer: Uwe L. Korn <uw...@xhochy.com>
Committed: Sun Feb 5 14:06:37 2017 +0100
----------------------------------------------------------------------
.../codegen/templates/NullableValueVectors.java | 21 +++--
.../java/org/apache/arrow/vector/BitVector.java | 88 +++++++++++++++++++-
.../org/apache/arrow/vector/NullableVector.java | 2 +
.../apache/arrow/vector/TestValueVector.java | 36 ++++++++
4 files changed, 137 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/70c05be2/java/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index ce63710..6b25fb3 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -132,6 +132,11 @@ public final class ${className} extends BaseDataValueVector implements <#if type
</#if>
@Override
+ public BitVector getValidityVector() {
+ return bits;
+ }
+
+ @Override
public List<BufferBacked> getFieldInnerVectors() {
return innerVectors;
}
@@ -426,7 +431,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
mutator.fillEmpties(thisIndex);
</#if>
values.copyFromSafe(fromIndex, thisIndex, from);
- bits.getMutator().setSafe(thisIndex, 1);
+ bits.getMutator().setSafeToOne(thisIndex);
}
public void copyFromSafe(int fromIndex, int thisIndex, ${className} from){
@@ -525,7 +530,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
@Override
public void setIndexDefined(int index){
- bits.getMutator().set(index, 1);
+ bits.getMutator().setToOne(index);
}
/**
@@ -543,7 +548,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
valuesMutator.set(i, emptyByteArray);
}
</#if>
- bitsMutator.set(index, 1);
+ bitsMutator.setToOne(index);
valuesMutator.set(index, value);
<#if type.major == "VarLen">lastSet = index;</#if>
}
@@ -574,7 +579,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
<#else>
fillEmpties(index);
- bits.getMutator().setSafe(index, 1);
+ bits.getMutator().setSafeToOne(index);
values.getMutator().setSafe(index, value, start, length);
setCount++;
<#if type.major == "VarLen">lastSet = index;</#if>
@@ -587,7 +592,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
<#else>
fillEmpties(index);
- bits.getMutator().setSafe(index, 1);
+ bits.getMutator().setSafeToOne(index);
values.getMutator().setSafe(index, value, start, length);
setCount++;
<#if type.major == "VarLen">lastSet = index;</#if>
@@ -626,7 +631,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
valuesMutator.set(i, emptyByteArray);
}
</#if>
- bits.getMutator().set(index, 1);
+ bits.getMutator().setToOne(index);
valuesMutator.set(index, holder);
<#if type.major == "VarLen">lastSet = index;</#if>
}
@@ -676,7 +681,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
<#if type.major == "VarLen">
fillEmpties(index);
</#if>
- bits.getMutator().setSafe(index, 1);
+ bits.getMutator().setSafeToOne(index);
values.getMutator().setSafe(index, value);
setCount++;
<#if type.major == "VarLen">lastSet = index;</#if>
@@ -687,7 +692,7 @@ public final class ${className} extends BaseDataValueVector implements <#if type
<#if type.major == "VarLen">
fillEmpties(index);
</#if>
- bits.getMutator().setSafe(index, 1);
+ bits.getMutator().setSafeToOne(index);
values.getMutator().setSafe(index, value);
setCount++;
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/70c05be2/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
index 9beabcb..d1e9abe 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BitVector.java
@@ -423,8 +423,8 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
* value to set (either 1 or 0)
*/
public final void set(int index, int value) {
- int byteIndex = index >> 3;
- int bitIndex = index & 7;
+ int byteIndex = byteIndex(index);
+ int bitIndex = bitIndex(index);
byte currentByte = data.getByte(byteIndex);
byte bitMask = (byte) (1L << bitIndex);
if (value != 0) {
@@ -432,10 +432,87 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
} else {
currentByte -= (bitMask & currentByte);
}
+ data.setByte(byteIndex, currentByte);
+ }
+ /**
+ * Set the bit at the given index to 1.
+ *
+ * @param index position of the bit to set
+ */
+ public final void setToOne(int index) {
+ int byteIndex = byteIndex(index);
+ int bitIndex = bitIndex(index);
+ byte currentByte = data.getByte(byteIndex);
+ byte bitMask = (byte) (1L << bitIndex);
+ currentByte |= bitMask;
data.setByte(byteIndex, currentByte);
}
+ /**
+ * set count bits to 1 in data starting at firstBitIndex
+ * @param data the buffer to set
+ * @param firstBitIndex the index of the first bit to set
+ * @param count the number of bits to set
+ */
+ public void setRangeToOne(int firstBitIndex, int count) {
+ int starByteIndex = byteIndex(firstBitIndex);
+ final int lastBitIndex = firstBitIndex + count;
+ final int endByteIndex = byteIndex(lastBitIndex);
+ final int startByteBitIndex = bitIndex(firstBitIndex);
+ final int endBytebitIndex = bitIndex(lastBitIndex);
+ if (count < 8 && starByteIndex == endByteIndex) {
+ // handles the case where we don't have a first and a last byte
+ byte bitMask = 0;
+ for (int i = startByteBitIndex; i < endBytebitIndex; ++i) {
+ bitMask |= (byte) (1L << i);
+ }
+ byte currentByte = data.getByte(starByteIndex);
+ currentByte |= bitMask;
+ data.setByte(starByteIndex, currentByte);
+ } else {
+ // fill in first byte (if it's not full)
+ if (startByteBitIndex != 0) {
+ byte currentByte = data.getByte(starByteIndex);
+ final byte bitMask = (byte) (0xFFL << startByteBitIndex);
+ currentByte |= bitMask;
+ data.setByte(starByteIndex, currentByte);
+ ++ starByteIndex;
+ }
+
+ // fill in one full byte at a time
+ for (int i = starByteIndex; i < endByteIndex; i++) {
+ data.setByte(i, 0xFF);
+ }
+
+ // fill in the last byte (if it's not full)
+ if (endBytebitIndex != 0) {
+ final int byteIndex = byteIndex(lastBitIndex - endBytebitIndex);
+ byte currentByte = data.getByte(byteIndex);
+ final byte bitMask = (byte) (0xFFL >>> ((8 - endBytebitIndex) & 7));
+ currentByte |= bitMask;
+ data.setByte(byteIndex, currentByte);
+ }
+
+ }
+ }
+
+ /**
+ * @param absoluteBitIndex the index of the bit in the buffer
+ * @return the index of the byte containing that bit
+ */
+ private int byteIndex(int absoluteBitIndex) {
+ return absoluteBitIndex >> 3;
+ }
+
+ /**
+ * @param absoluteBitIndex the index of the bit in the buffer
+ * @return the index of the bit inside the byte
+ */
+ private int bitIndex(int absoluteBitIndex) {
+ return absoluteBitIndex & 7;
+ }
+
public final void set(int index, BitHolder holder) {
set(index, holder.value);
}
@@ -451,6 +528,13 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe
set(index, value);
}
+ public void setSafeToOne(int index) {
+ while(index >= getValueCapacity()) {
+ reAlloc();
+ }
+ setToOne(index);
+ }
+
public void setSafe(int index, BitHolder holder) {
while(index >= getValueCapacity()) {
reAlloc();
http://git-wip-us.apache.org/repos/asf/arrow/blob/70c05be2/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
index 0212b3c..b49e916 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/NullableVector.java
@@ -19,5 +19,7 @@ package org.apache.arrow.vector;
public interface NullableVector extends ValueVector {
+ BitVector getValidityVector();
+
ValueVector getValuesVector();
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/70c05be2/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
index b33919b..774b59e 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/TestValueVector.java
@@ -30,6 +30,7 @@ import org.apache.arrow.vector.schema.TypeLayout;
import org.apache.arrow.vector.types.Types.MinorType;
import org.apache.arrow.vector.types.pojo.Field;
import org.junit.After;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -365,6 +366,41 @@ public class TestValueVector {
}
@Test
+ public void testBitVectorRangeSetAllOnes() {
+ validateRange(1000, 0, 1000);
+ validateRange(1000, 0, 1);
+ validateRange(1000, 1, 2);
+ validateRange(1000, 5, 6);
+ validateRange(1000, 5, 10);
+ validateRange(1000, 5, 150);
+ validateRange(1000, 5, 27);
+ for (int i = 0; i < 8; i++) {
+ for (int j = 0; j < 8; j++) {
+ validateRange(1000, 10 + i, 27 + j);
+ validateRange(1000, i, j);
+ }
+ }
+ }
+
+ private void validateRange(int length, int start, int count) {
+ String desc = "[" + start + ", " + (start + count) + ") ";
+ try (BitVector bitVector = new BitVector("bits", allocator)) {
+ bitVector.reset();
+ bitVector.allocateNew(length);
+ bitVector.getMutator().setRangeToOne(start, count);
+ for (int i = 0; i < start; i++) {
+ Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i));
+ }
+ for (int i = start; i < start + count; i++) {
+ Assert.assertEquals(desc + i, 1, bitVector.getAccessor().get(i));
+ }
+ for (int i = start + count; i < length; i++) {
+ Assert.assertEquals(desc + i, 0, bitVector.getAccessor().get(i));
+ }
+ }
+ }
+
+ @Test
public void testReAllocNullableFixedWidthVector() {
// Create a new value vector for 1024 integers
try (final NullableFloat4Vector vector = (NullableFloat4Vector) MinorType.FLOAT4.getNewVector(EMPTY_SCHEMA_PATH, allocator, null)) {