You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by bb...@apache.org on 2024/01/16 14:04:06 UTC

(hbase) branch branch-2.5 updated: HBASE-28256 Enhance ByteBufferUtils.readVLong to read more bytes at a time (#5576)

This is an automated email from the ASF dual-hosted git repository.

bbeaudreault pushed a commit to branch branch-2.5
in repository https://gitbox.apache.org/repos/asf/hbase.git


The following commit(s) were added to refs/heads/branch-2.5 by this push:
     new 98cab249c75 HBASE-28256 Enhance ByteBufferUtils.readVLong to read more bytes at a time (#5576)
98cab249c75 is described below

commit 98cab249c758e84bb229b637e63ce1013c639871
Author: jbewing <jb...@live.com>
AuthorDate: Tue Jan 16 08:59:05 2024 -0500

    HBASE-28256 Enhance ByteBufferUtils.readVLong to read more bytes at a time (#5576)
    
    Signed-off-by: Bryan Beaudreault <bb...@apache.org>
    Signed-off-by: Duo Zhang <zh...@apache.org>
---
 .../apache/hadoop/hbase/util/ByteBufferUtils.java  | 83 ++++++++++++++++------
 .../hadoop/hbase/util/TestByteBufferUtils.java     | 18 +++++
 2 files changed, 78 insertions(+), 23 deletions(-)

diff --git a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
index 6746323e98c..9c10544fa5c 100644
--- a/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
+++ b/hbase-common/src/main/java/org/apache/hadoop/hbase/util/ByteBufferUtils.java
@@ -471,38 +471,75 @@ public final class ByteBufferUtils {
     }
   }
 
-  private interface ByteVisitor {
-    byte get();
-  }
-
-  private static long readVLong(ByteVisitor visitor) {
-    byte firstByte = visitor.get();
+  /**
+   * Similar to {@link WritableUtils#readVLong(java.io.DataInput)} but reads from a
+   * {@link ByteBuff}.
+   */
+  public static long readVLong(ByteBuff buf) {
+    byte firstByte = buf.get();
     int len = WritableUtils.decodeVIntSize(firstByte);
     if (len == 1) {
       return firstByte;
+    } else {
+      int remaining = len - 1;
+      long i = 0;
+      int offsetFromPos = 0;
+      if (remaining >= Bytes.SIZEOF_INT) {
+        // The int read has to be converted to unsigned long so the & op
+        i = (buf.getIntAfterPosition(offsetFromPos) & 0x00000000ffffffffL);
+        remaining -= Bytes.SIZEOF_INT;
+        offsetFromPos += Bytes.SIZEOF_INT;
+      }
+      if (remaining >= Bytes.SIZEOF_SHORT) {
+        short s = buf.getShortAfterPosition(offsetFromPos);
+        i = i << 16;
+        i = i | (s & 0xFFFF);
+        remaining -= Bytes.SIZEOF_SHORT;
+        offsetFromPos += Bytes.SIZEOF_SHORT;
+      }
+      for (int idx = 0; idx < remaining; idx++) {
+        byte b = buf.getByteAfterPosition(offsetFromPos + idx);
+        i = i << 8;
+        i = i | (b & 0xFF);
+      }
+      buf.skip(len - 1);
+      return WritableUtils.isNegativeVInt(firstByte) ? ~i : i;
     }
-    long i = 0;
-    for (int idx = 0; idx < len - 1; idx++) {
-      byte b = visitor.get();
-      i = i << 8;
-      i = i | (b & 0xFF);
-    }
-    return (WritableUtils.isNegativeVInt(firstByte) ? (i ^ -1L) : i);
   }
 
   /**
    * Similar to {@link WritableUtils#readVLong(DataInput)} but reads from a {@link ByteBuffer}.
    */
-  public static long readVLong(ByteBuffer in) {
-    return readVLong(in::get);
-  }
-
-  /**
-   * Similar to {@link WritableUtils#readVLong(java.io.DataInput)} but reads from a
-   * {@link ByteBuff}.
-   */
-  public static long readVLong(ByteBuff in) {
-    return readVLong(in::get);
+  public static long readVLong(ByteBuffer buf) {
+    byte firstByte = buf.get();
+    int len = WritableUtils.decodeVIntSize(firstByte);
+    if (len == 1) {
+      return firstByte;
+    } else {
+      int remaining = len - 1;
+      long i = 0;
+      int offsetFromPos = 0;
+      if (remaining >= Bytes.SIZEOF_INT) {
+        // The int read has to be converted to unsigned long so the & op
+        i = (buf.getInt(buf.position() + offsetFromPos) & 0x00000000ffffffffL);
+        remaining -= Bytes.SIZEOF_INT;
+        offsetFromPos += Bytes.SIZEOF_INT;
+      }
+      if (remaining >= Bytes.SIZEOF_SHORT) {
+        short s = buf.getShort(buf.position() + offsetFromPos);
+        i = i << 16;
+        i = i | (s & 0xFFFF);
+        remaining -= Bytes.SIZEOF_SHORT;
+        offsetFromPos += Bytes.SIZEOF_SHORT;
+      }
+      for (int idx = 0; idx < remaining; idx++) {
+        byte b = buf.get(buf.position() + offsetFromPos + idx);
+        i = i << 8;
+        i = i | (b & 0xFF);
+      }
+      buf.position(buf.position() + len - 1);
+      return WritableUtils.isNegativeVInt(firstByte) ? ~i : i;
+    }
   }
 
   /**
diff --git a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
index 7d38690da77..8f180d266c2 100644
--- a/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
+++ b/hbase-common/src/test/java/org/apache/hadoop/hbase/util/TestByteBufferUtils.java
@@ -46,6 +46,7 @@ import java.util.stream.Collectors;
 import org.apache.hadoop.hbase.HBaseClassTestRule;
 import org.apache.hadoop.hbase.HBaseCommonTestingUtility;
 import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.nio.ByteBuff;
 import org.apache.hadoop.hbase.testclassification.MediumTests;
 import org.apache.hadoop.hbase.testclassification.MiscTests;
 import org.apache.hadoop.hbase.unsafe.HBasePlatformDependent;
@@ -179,6 +180,23 @@ public class TestByteBufferUtils {
       ByteBufferUtils.writeVLong(b, l);
       b.flip();
       assertEquals(l, ByteBufferUtils.readVLong(b));
+      b.flip();
+      assertEquals(l, ByteBufferUtils.readVLong(ByteBuff.wrap(b)));
+    }
+  }
+
+  @Test
+  public void testReadWriteConsecutiveVLong() {
+    for (long l : testNumbers) {
+      ByteBuffer b = ByteBuffer.allocate(2 * MAX_VLONG_LENGTH);
+      ByteBufferUtils.writeVLong(b, l);
+      ByteBufferUtils.writeVLong(b, l - 4);
+      b.flip();
+      assertEquals(l, ByteBufferUtils.readVLong(b));
+      assertEquals(l - 4, ByteBufferUtils.readVLong(b));
+      b.flip();
+      assertEquals(l, ByteBufferUtils.readVLong(ByteBuff.wrap(b)));
+      assertEquals(l - 4, ByteBufferUtils.readVLong(ByteBuff.wrap(b)));
     }
   }