You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by li...@apache.org on 2024/05/16 01:09:04 UTC
(arrow) branch main updated: GH-40943: [Java] Implement RangeEqualsVisitor for StringView (#41636)
This is an automated email from the ASF dual-hosted git repository.
lidavidm pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 1c15c88b4b GH-40943: [Java] Implement RangeEqualsVisitor for StringView (#41636)
1c15c88b4b is described below
commit 1c15c88b4b62b19e7a226cc9e11946af1d9ac343
Author: Vibhatha Lakmal Abeykoon <vi...@users.noreply.github.com>
AuthorDate: Thu May 16 06:38:57 2024 +0530
GH-40943: [Java] Implement RangeEqualsVisitor for StringView (#41636)
### Rationale for this change
Adding `RangeEqualsVisitor` for StringView as discussed in https://github.com/apache/arrow/issues/40943.
### What changes are included in this PR?
Including `RangeEqualsVisitor` visitor method and test cases to validate it.
### Are these changes tested?
Yes
### Are there any user-facing changes?
No
* GitHub Issue: #40943
Authored-by: Vibhatha Abeykoon <vi...@gmail.com>
Signed-off-by: David Li <li...@gmail.com>
---
.../arrow/vector/BaseVariableWidthViewVector.java | 21 ++++--
.../arrow/vector/compare/RangeEqualsVisitor.java | 85 +++++++++++++++++++++-
.../vector/compare/TestRangeEqualsVisitor.java | 71 +++++++++++++++---
3 files changed, 161 insertions(+), 16 deletions(-)
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java
index 2f80775a48..ec700a0dc2 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/BaseVariableWidthViewVector.java
@@ -46,7 +46,7 @@ import org.apache.arrow.vector.util.TransferPair;
*/
public abstract class BaseVariableWidthViewVector extends BaseValueVector implements VariableWidthFieldVector {
// A single element of a view comprises 16 bytes
- protected static final int ELEMENT_SIZE = 16;
+ public static final int ELEMENT_SIZE = 16;
public static final int INITIAL_VIEW_VALUE_ALLOCATION = 4096;
private static final int INITIAL_BYTE_COUNT = INITIAL_VIEW_VALUE_ALLOCATION * ELEMENT_SIZE;
private static final int MAX_BUFFER_SIZE = (int) Math.min(MAX_ALLOCATION_SIZE, Integer.MAX_VALUE);
@@ -70,14 +70,14 @@ public abstract class BaseVariableWidthViewVector extends BaseValueVector implem
*
* */
// 12 byte unsigned int to track inline views
- protected static final int INLINE_SIZE = 12;
+ public static final int INLINE_SIZE = 12;
// The first 4 bytes of view are allocated for length
- protected static final int LENGTH_WIDTH = 4;
+ public static final int LENGTH_WIDTH = 4;
// The second 4 bytes of view are allocated for prefix width
- protected static final int PREFIX_WIDTH = 4;
+ public static final int PREFIX_WIDTH = 4;
// The third 4 bytes of view are allocated for buffer index
- protected static final int BUF_INDEX_WIDTH = 4;
- protected static final byte[] EMPTY_BYTE_ARRAY = new byte[]{};
+ public static final int BUF_INDEX_WIDTH = 4;
+ public static final byte[] EMPTY_BYTE_ARRAY = new byte[]{};
protected ArrowBuf validityBuffer;
// The view buffer is used to store the variable width view elements
protected ArrowBuf viewBuffer;
@@ -158,6 +158,15 @@ public abstract class BaseVariableWidthViewVector extends BaseValueVector implem
return viewBuffer;
}
+ /**
+ * Get the buffers that store the data for views in the vector.
+ *
+ * @return buffer
+ */
+ public List<ArrowBuf> getDataBuffers() {
+ return dataBuffers;
+ }
+
/**
* BaseVariableWidthViewVector doesn't support offset buffer.
*
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java
index 56220d270f..28da2a86a5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/compare/RangeEqualsVisitor.java
@@ -22,6 +22,7 @@ import static org.apache.arrow.memory.util.LargeMemoryUtil.checkedCastToInt;
import java.util.List;
import java.util.function.BiFunction;
+import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.memory.util.ByteFunctionHelpers;
import org.apache.arrow.util.Preconditions;
import org.apache.arrow.vector.BaseFixedWidthVector;
@@ -165,7 +166,10 @@ public class RangeEqualsVisitor implements VectorVisitor<Boolean, Range> {
@Override
public Boolean visit(BaseVariableWidthViewVector left, Range range) {
- throw new UnsupportedOperationException("View vectors are not supported.");
+ if (!validate(left)) {
+ return false;
+ }
+ return compareBaseVariableWidthViewVectors(range);
}
@Override
@@ -450,6 +454,85 @@ public class RangeEqualsVisitor implements VectorVisitor<Boolean, Range> {
return true;
}
+ protected boolean compareBaseVariableWidthViewVectors(Range range) {
+ BaseVariableWidthViewVector leftVector = (BaseVariableWidthViewVector) left;
+ BaseVariableWidthViewVector rightVector = (BaseVariableWidthViewVector) right;
+
+ final ArrowBuf leftViewBuffer = leftVector.getDataBuffer();
+ final ArrowBuf rightViewBuffer = rightVector.getDataBuffer();
+
+ final int elementSize = BaseVariableWidthViewVector.ELEMENT_SIZE;
+ final int lengthWidth = BaseVariableWidthViewVector.LENGTH_WIDTH;
+ final int prefixWidth = BaseVariableWidthViewVector.PREFIX_WIDTH;
+ final int bufIndexWidth = BaseVariableWidthViewVector.BUF_INDEX_WIDTH;
+
+ List<ArrowBuf> leftDataBuffers = leftVector.getDataBuffers();
+ List<ArrowBuf> rightDataBuffers = rightVector.getDataBuffers();
+
+ for (int i = 0; i < range.getLength(); i++) {
+ int leftIndex = range.getLeftStart() + i;
+ int rightIndex = range.getRightStart() + i;
+
+ boolean isNull = leftVector.isNull(leftIndex);
+ if (isNull != rightVector.isNull(rightIndex)) {
+ return false;
+ }
+
+ if (isNull) {
+ continue;
+ }
+
+ int startLeftByteOffset = leftIndex * elementSize;
+
+ int startRightByteOffset = rightIndex * elementSize;
+
+ int leftDataBufferValueLength = leftVector.getValueLength(leftIndex);
+ int rightDataBufferValueLength = rightVector.getValueLength(rightIndex);
+
+ if (leftDataBufferValueLength != rightDataBufferValueLength) {
+ return false;
+ }
+
+ if (leftDataBufferValueLength > BaseVariableWidthViewVector.INLINE_SIZE) {
+ // if the value is stored in the dataBuffers
+ int leftDataBufferIndex = leftViewBuffer.getInt(startLeftByteOffset + lengthWidth + prefixWidth);
+ int rightDataBufferIndex = rightViewBuffer.getInt(startRightByteOffset + lengthWidth + prefixWidth);
+
+ final int leftDataOffset =
+ leftViewBuffer.getInt(startLeftByteOffset + lengthWidth + prefixWidth + bufIndexWidth);
+ final int rightDataOffset =
+ rightViewBuffer.getInt(startRightByteOffset + lengthWidth + prefixWidth + bufIndexWidth);
+
+ ArrowBuf leftDataBuffer = leftDataBuffers.get(leftDataBufferIndex);
+ ArrowBuf rightDataBuffer = rightDataBuffers.get(rightDataBufferIndex);
+
+ // check equality in the considered string stored in the dataBuffers
+ int retDataBuf = ByteFunctionHelpers.equal(
+ leftDataBuffer, leftDataOffset, leftDataOffset + leftDataBufferValueLength,
+ rightDataBuffer, rightDataOffset, rightDataOffset + rightDataBufferValueLength);
+
+ if (retDataBuf == 0) {
+ return false;
+ }
+ } else {
+ // if the value is stored in the view
+ final int leftDataOffset = startLeftByteOffset + lengthWidth;
+ final int rightDataOffset = startRightByteOffset + lengthWidth;
+
+ // check equality in the considered string stored in the view
+ int retDataBuf = ByteFunctionHelpers.equal(
+ leftViewBuffer, leftDataOffset, leftDataOffset + leftDataBufferValueLength,
+ rightViewBuffer, rightDataOffset, rightDataOffset + rightDataBufferValueLength);
+
+ if (retDataBuf == 0) {
+ return false;
+ }
+ }
+
+ }
+ return true;
+ }
+
protected boolean compareListVectors(Range range) {
ListVector leftVector = (ListVector) left;
ListVector rightVector = (ListVector) right;
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java b/java/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java
index ab8c6c6348..c3e7ef8bf8 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/compare/TestRangeEqualsVisitor.java
@@ -18,8 +18,8 @@
package org.apache.arrow.vector.compare;
import static org.apache.arrow.vector.testing.ValueVectorDataPopulator.setVector;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertTrue;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.charset.Charset;
import java.util.Arrays;
@@ -33,6 +33,7 @@ import org.apache.arrow.vector.Float8Vector;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.LargeVarCharVector;
import org.apache.arrow.vector.VarCharVector;
+import org.apache.arrow.vector.ViewVarCharVector;
import org.apache.arrow.vector.ZeroVector;
import org.apache.arrow.vector.compare.util.ValueEpsilonEqualizers;
import org.apache.arrow.vector.complex.DenseUnionVector;
@@ -53,16 +54,16 @@ import org.apache.arrow.vector.types.Types;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
-import org.junit.After;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
public class TestRangeEqualsVisitor {
private BufferAllocator allocator;
- @Before
+ @BeforeEach
public void init() {
allocator = new RootAllocator(Long.MAX_VALUE);
}
@@ -71,8 +72,11 @@ public class TestRangeEqualsVisitor {
private static final byte[] STR1 = "AAAAA1".getBytes(utf8Charset);
private static final byte[] STR2 = "BBBBBBBBB2".getBytes(utf8Charset);
private static final byte[] STR3 = "CCCC3".getBytes(utf8Charset);
+ private static final byte[] STR4 = "12345678901234A".getBytes(utf8Charset);
+ private static final byte[] STR5 = "A2345678901234ABC".getBytes(utf8Charset);
+ private static final byte[] STR6 = "AB45678901234ABCD".getBytes(utf8Charset);
- @After
+ @AfterEach
public void terminate() throws Exception {
allocator.close();
}
@@ -132,6 +136,55 @@ public class TestRangeEqualsVisitor {
}
}
+ @Test
+ public void testBaseVariableViewVectorRangeEquals() {
+ try (final ViewVarCharVector vector1 = new ViewVarCharVector("varchar", allocator);
+ final ViewVarCharVector vector2 = new ViewVarCharVector("varchar", allocator)) {
+
+ setVector(vector1, STR1, STR2, STR4, STR3, STR2, STR5, STR1, STR6, STR1, STR2, STR4);
+ setVector(vector2, STR1, STR2, STR4, STR3, STR2, STR5, STR1, STR6, STR1, STR2, STR4);
+
+ RangeEqualsVisitor visitor = new RangeEqualsVisitor(vector1, vector2);
+ // inclusion of long string in the middle
+ assertTrue(visitor.rangeEquals(new Range(1, 1, 3)));
+ assertFalse(visitor.rangeEquals(new Range(0, 1, 4)));
+ // inclusion of long string at the start
+ assertTrue(visitor.rangeEquals(new Range(2, 2, 4)));
+ assertFalse(visitor.rangeEquals(new Range(2, 5, 4)));
+ // inclusion of long string at the end
+ assertTrue(visitor.rangeEquals(new Range(4, 4, 4)));
+ // unequal range
+ assertTrue(visitor.rangeEquals(new Range(8, 0, 3)));
+ assertFalse(visitor.rangeEquals(new Range(4, 5, 3)));
+
+ // checking the same ranges when nulls are set
+
+ vector1.setNull(1);
+ vector2.setNull(1);
+
+ vector1.setNull(3);
+ vector2.setNull(3);
+
+ vector1.setNull(5);
+ vector2.setNull(5);
+
+ vector1.setNull(9);
+ vector2.setNull(9);
+
+ // inclusion of long string in the middle
+ assertTrue(visitor.rangeEquals(new Range(1, 1, 3)));
+ assertFalse(visitor.rangeEquals(new Range(0, 1, 4)));
+ // inclusion of long string at the start
+ assertTrue(visitor.rangeEquals(new Range(2, 2, 4)));
+ assertFalse(visitor.rangeEquals(new Range(2, 5, 4)));
+ // inclusion of long string at the end
+ assertTrue(visitor.rangeEquals(new Range(4, 4, 4)));
+ // unequal range
+ assertTrue(visitor.rangeEquals(new Range(8, 0, 3)));
+ assertFalse(visitor.rangeEquals(new Range(4, 5, 3)));
+ }
+ }
+
@Test
public void testListVectorWithDifferentChild() {
try (final ListVector vector1 = ListVector.empty("list", allocator);
@@ -476,7 +529,7 @@ public class TestRangeEqualsVisitor {
}
}
- @Ignore
+ @Disabled
@Test
public void testEqualsWithOutTypeCheck() {
try (final IntVector intVector = new IntVector("int", allocator);