(datafusion-comet) branch main updated: perf: cache CometDecodedVector validityBufferAddress (#4435)

[mb...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

mbutrovich pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git


The following commit(s) were added to refs/heads/main by this push:
     new 0d5c592be4 perf: cache CometDecodedVector validityBufferAddress (#4435)
0d5c592be4 is described below

commit 0d5c592be47881b530011c41b076f0e1bdbd1af0
Author: Matt Butrovich <mb...@users.noreply.github.com>
AuthorDate: Tue May 26 16:41:04 2026 -0400

    perf: cache CometDecodedVector validityBufferAddress (#4435)
---
 .../apache/comet/vector/CometDecodedVector.java    | 10 +++++--
 .../apache/comet/vector/TestCometPlainVector.java  | 32 ++++++++++++++++++++++
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/spark/src/main/java/org/apache/comet/vector/CometDecodedVector.java b/spark/src/main/java/org/apache/comet/vector/CometDecodedVector.java
index 157184a0c2..7105dcc83e 100644
--- a/spark/src/main/java/org/apache/comet/vector/CometDecodedVector.java
+++ b/spark/src/main/java/org/apache/comet/vector/CometDecodedVector.java
@@ -19,6 +19,7 @@
 
 package org.apache.comet.vector;
 
+import org.apache.arrow.vector.NullVector;
 import org.apache.arrow.vector.ValueVector;
 import org.apache.arrow.vector.types.pojo.Field;
 import org.apache.spark.sql.comet.util.Utils;
@@ -44,7 +45,8 @@ public abstract class CometDecodedVector extends CometVector {
   private final int numValues;
   private int validityByteCacheIndex = -1;
   private byte validityByteCache;
-  protected boolean isUuid;
+  private final long validityBufferAddress;
+  protected final boolean isUuid;
 
   protected CometDecodedVector(ValueVector vector, Field valueField) {
     this(vector, valueField, false);
@@ -56,6 +58,11 @@ public abstract class CometDecodedVector extends CometVector {
     this.numNulls = valueVector.getNullCount();
     this.numValues = valueVector.getValueCount();
     this.hasNull = numNulls != 0;
+    // NullVector has no validity buffer (all values are logically null), so use a sentinel.
+    // Subclasses that wrap NullVector (e.g. CometPlainVector) short-circuit isNullAt before
+    // it reaches the base implementation, so the sentinel is never dereferenced.
+    this.validityBufferAddress =
+        (vector instanceof NullVector) ? -1L : valueVector.getValidityBuffer().memoryAddress();
     this.isUuid = isUuid;
   }
 
@@ -84,7 +91,6 @@ public abstract class CometDecodedVector extends CometVector {
 
     int byteIndex = rowId >> 3;
     if (byteIndex != validityByteCacheIndex) {
-      long validityBufferAddress = valueVector.getValidityBuffer().memoryAddress();
       validityByteCache = Platform.getByte(null, validityBufferAddress + byteIndex);
       validityByteCacheIndex = byteIndex;
     }
diff --git a/spark/src/test/java/org/apache/comet/vector/TestCometPlainVector.java b/spark/src/test/java/org/apache/comet/vector/TestCometPlainVector.java
index 79faa04c55..51bbd9387e 100644
--- a/spark/src/test/java/org/apache/comet/vector/TestCometPlainVector.java
+++ b/spark/src/test/java/org/apache/comet/vector/TestCometPlainVector.java
@@ -24,6 +24,7 @@ import java.nio.charset.StandardCharsets;
 import org.junit.Test;
 
 import org.apache.arrow.memory.RootAllocator;
+import org.apache.arrow.vector.IntVector;
 import org.apache.arrow.vector.VarBinaryVector;
 import org.apache.arrow.vector.VarCharVector;
 
@@ -74,4 +75,35 @@ public class TestCometPlainVector {
   private static byte[] bytes(String s) {
     return s.getBytes(StandardCharsets.UTF_8);
   }
+
+  @Test
+  public void testIsNullAtSequentialAcrossValidityBytes() {
+    // 20 rows spans three validity bytes; mixing nulls and non-nulls across byte boundaries
+    // forces a cache miss on every 8th row.
+    try (RootAllocator allocator = new RootAllocator(Integer.MAX_VALUE)) {
+      IntVector vector = new IntVector("ints", allocator);
+      vector.allocateNew(20);
+      for (int i = 0; i < 20; i++) {
+        if (i % 3 == 0) {
+          vector.setNull(i);
+        } else {
+          vector.setSafe(i, i * 10);
+        }
+      }
+      vector.setValueCount(20);
+
+      try (CometPlainVector cv = new CometPlainVector(vector, false)) {
+        for (int i = 0; i < 20; i++) {
+          boolean expectedNull = (i % 3 == 0);
+          assertEquals("row " + i, expectedNull, cv.isNullAt(i));
+          if (!expectedNull) {
+            assertEquals("row " + i, i * 10, cv.getInt(i));
+          }
+        }
+        for (int i = 19; i >= 0; i--) {
+          assertEquals("row " + i, (i % 3 == 0), cv.isNullAt(i));
+        }
+      }
+    }
+  }
 }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@datafusion.apache.org
For additional commands, e-mail: commits-help@datafusion.apache.org