You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2021/09/20 13:37:40 UTC
[lucene] branch main updated: LUCENE-10113: Use VarHandles to
access int/long/short types in byte arrays (e.g. ByteArrayDataInput) (#308)
This is an automated email from the ASF dual-hosted git repository.
uschindler pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new c57d6e5 LUCENE-10113: Use VarHandles to access int/long/short types in byte arrays (e.g. ByteArrayDataInput) (#308)
c57d6e5 is described below
commit c57d6e5f8c8a8cae943f99ba7393550579666eb4
Author: Uwe Schindler <us...@apache.org>
AuthorDate: Mon Sep 20 15:37:33 2021 +0200
LUCENE-10113: Use VarHandles to access int/long/short types in byte arrays (e.g. ByteArrayDataInput) (#308)
Co-authored-by: Robert Muir <rm...@apache.org>
---
lucene/CHANGES.txt | 5 ++
.../lucene/analysis/minhash/MinHashFilter.java | 6 +-
.../lucene/analysis/payloads/PayloadHelper.java | 18 ++---
.../org/apache/lucene/index/ByteSliceReader.java | 7 +-
.../apache/lucene/store/ByteArrayDataInput.java | 40 ++++------
.../apache/lucene/store/ByteArrayDataOutput.java | 22 ++++++
.../src/java/org/apache/lucene/util/BitUtil.java | 92 +++++++++++++++++++++-
.../java/org/apache/lucene/util/NumericUtils.java | 30 +------
.../java/org/apache/lucene/util/StringHelper.java | 6 +-
.../apache/lucene/util/bkd/HeapPointReader.java | 6 +-
.../apache/lucene/util/bkd/HeapPointWriter.java | 6 +-
.../apache/lucene/util/bkd/OfflinePointReader.java | 6 +-
.../lucene/store/TestByteArrayDataInput.java | 32 ++++++++
.../java/org/apache/lucene/facet/FacetsConfig.java | 7 +-
.../facet/taxonomy/IntAssociationFacetField.java | 11 +--
.../TaxonomyFacetSumFloatAssociations.java | 15 +---
.../taxonomy/TaxonomyFacetSumIntAssociations.java | 13 +--
.../queries/payloads/PayloadMatcherFactory.java | 12 +--
.../lucene/replicator/nrt/SimpleTransLog.java | 12 +--
.../codecs/idversion/IDVersionPostingsFormat.java | 19 +----
.../lucene/sandbox/document/HalfFloatPoint.java | 6 +-
21 files changed, 213 insertions(+), 158 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 9dc16e4..b337e98 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -271,6 +271,11 @@ Improvements
The getPath API now throws an IAE instead of returning null if the ordinal is out of bounds.
(Gautam Worah, Mike McCandless)
+* LUCENE-10113: Use VarHandles to access int/long/short primitive types in byte arrays.
+ This improves readability and performance of encoding/decoding of primitives to index
+ file format in input/output classes like DataInput / DataOutput and codecs.
+ (Uwe Schindler, Robert Muir)
+
Bug fixes
* LUCENE-10070 Skip deleted docs when accumulating facet counts for all docs. (Ankur Goel, Greg Miller)
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/minhash/MinHashFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/minhash/MinHashFilter.java
index 8e9215d..cef5b6c 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/minhash/MinHashFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/minhash/MinHashFilter.java
@@ -28,6 +28,7 @@ import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.util.BitUtil;
/**
* Generate min hash tokens from an incoming stream of tokens. The incoming tokens would typically
@@ -97,10 +98,7 @@ public class MinHashFilter extends TokenFilter {
static byte[] getBytes(int i) {
byte[] answer = new byte[4];
- answer[3] = (byte) (i);
- answer[2] = (byte) (i >> 8);
- answer[1] = (byte) (i >> 16);
- answer[0] = (byte) (i >> 24);
+ BitUtil.VH_BE_INT.set(answer, 0, i);
return answer;
}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/payloads/PayloadHelper.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/payloads/PayloadHelper.java
index 6688544..cfcafb2 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/payloads/PayloadHelper.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/payloads/PayloadHelper.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.analysis.payloads;
+import org.apache.lucene.util.BitUtil;
+
/** Utility methods for encoding payloads. */
public class PayloadHelper {
@@ -24,7 +26,8 @@ public class PayloadHelper {
}
public static byte[] encodeFloat(float payload, byte[] data, int offset) {
- return encodeInt(Float.floatToIntBits(payload), data, offset);
+ BitUtil.VH_BE_FLOAT.set(data, offset, payload);
+ return data;
}
public static byte[] encodeInt(int payload) {
@@ -32,10 +35,7 @@ public class PayloadHelper {
}
public static byte[] encodeInt(int payload, byte[] data, int offset) {
- data[offset] = (byte) (payload >> 24);
- data[offset + 1] = (byte) (payload >> 16);
- data[offset + 2] = (byte) (payload >> 8);
- data[offset + 3] = (byte) payload;
+ BitUtil.VH_BE_INT.set(data, offset, payload);
return data;
}
@@ -58,14 +58,10 @@ public class PayloadHelper {
* @see #encodeFloat(float)
*/
public static final float decodeFloat(byte[] bytes, int offset) {
-
- return Float.intBitsToFloat(decodeInt(bytes, offset));
+ return (float) BitUtil.VH_BE_FLOAT.get(bytes, offset);
}
public static final int decodeInt(byte[] bytes, int offset) {
- return ((bytes[offset] & 0xFF) << 24)
- | ((bytes[offset + 1] & 0xFF) << 16)
- | ((bytes[offset + 2] & 0xFF) << 8)
- | (bytes[offset + 3] & 0xFF);
+ return (int) BitUtil.VH_BE_INT.get(bytes, offset);
}
}
diff --git a/lucene/core/src/java/org/apache/lucene/index/ByteSliceReader.java b/lucene/core/src/java/org/apache/lucene/index/ByteSliceReader.java
index b0b65de..857388a 100644
--- a/lucene/core/src/java/org/apache/lucene/index/ByteSliceReader.java
+++ b/lucene/core/src/java/org/apache/lucene/index/ByteSliceReader.java
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import java.io.IOException;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.ByteBlockPool;
/* IndexInput that knows how to read the byte slices written
@@ -94,11 +95,7 @@ final class ByteSliceReader extends DataInput {
public void nextSlice() {
// Skip to our next slice
- final int nextIndex =
- ((buffer[limit] & 0xff) << 24)
- + ((buffer[1 + limit] & 0xff) << 16)
- + ((buffer[2 + limit] & 0xff) << 8)
- + (buffer[3 + limit] & 0xff);
+ final int nextIndex = (int) BitUtil.VH_BE_INT.get(buffer, limit);
level = ByteBlockPool.NEXT_LEVEL_ARRAY[level];
final int newSize = ByteBlockPool.LEVEL_SIZE_ARRAY[level];
diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataInput.java b/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataInput.java
index 31c5e23..fa005a1 100644
--- a/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataInput.java
@@ -16,6 +16,7 @@
*/
package org.apache.lucene.store;
+import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
/**
@@ -81,38 +82,29 @@ public final class ByteArrayDataInput extends DataInput {
@Override
public short readShort() {
- final byte b1 = bytes[pos++];
- final byte b2 = bytes[pos++];
- return (short) ((b2 & 0xFF) << 8 | (b1 & 0xFF));
+ try {
+ return (short) BitUtil.VH_LE_SHORT.get(bytes, pos);
+ } finally {
+ pos += Short.BYTES;
+ }
}
@Override
public int readInt() {
- final byte b1 = bytes[pos++];
- final byte b2 = bytes[pos++];
- final byte b3 = bytes[pos++];
- final byte b4 = bytes[pos++];
- return (b4 & 0xFF) << 24 | (b3 & 0xFF) << 16 | (b2 & 0xFF) << 8 | (b1 & 0xFF);
+ try {
+ return (int) BitUtil.VH_LE_INT.get(bytes, pos);
+ } finally {
+ pos += Integer.BYTES;
+ }
}
@Override
public long readLong() {
- final byte b1 = bytes[pos++];
- final byte b2 = bytes[pos++];
- final byte b3 = bytes[pos++];
- final byte b4 = bytes[pos++];
- final byte b5 = bytes[pos++];
- final byte b6 = bytes[pos++];
- final byte b7 = bytes[pos++];
- final byte b8 = bytes[pos++];
- return (b8 & 0xFFL) << 56
- | (b7 & 0xFFL) << 48
- | (b6 & 0xFFL) << 40
- | (b5 & 0xFFL) << 32
- | (b4 & 0xFFL) << 24
- | (b3 & 0xFFL) << 16
- | (b2 & 0xFFL) << 8
- | (b1 & 0xFFL);
+ try {
+ return (long) BitUtil.VH_LE_LONG.get(bytes, pos);
+ } finally {
+ pos += Long.BYTES;
+ }
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataOutput.java b/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataOutput.java
index 358238e..33933ec 100644
--- a/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataOutput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataOutput.java
@@ -16,6 +16,7 @@
*/
package org.apache.lucene.store;
+import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
/**
@@ -68,4 +69,25 @@ public class ByteArrayDataOutput extends DataOutput {
System.arraycopy(b, offset, bytes, pos, length);
pos += length;
}
+
+ @Override
+ public void writeShort(short i) {
+ assert pos + Short.BYTES <= limit;
+ BitUtil.VH_LE_SHORT.set(bytes, pos, i);
+ pos += Short.BYTES;
+ }
+
+ @Override
+ public void writeInt(int i) {
+ assert pos + Integer.BYTES <= limit;
+ BitUtil.VH_LE_INT.set(bytes, pos, i);
+ pos += Integer.BYTES;
+ }
+
+ @Override
+ public void writeLong(long i) {
+ assert pos + Long.BYTES <= limit;
+ BitUtil.VH_LE_LONG.set(bytes, pos, i);
+ pos += Long.BYTES;
+ }
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/BitUtil.java b/lucene/core/src/java/org/apache/lucene/util/BitUtil.java
index d974c0e..9848117 100644
--- a/lucene/core/src/java/org/apache/lucene/util/BitUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/util/BitUtil.java
@@ -16,8 +16,12 @@
*/
package org.apache.lucene.util; // from org.apache.solr.util rev 555343
+import java.lang.invoke.MethodHandles;
+import java.lang.invoke.VarHandle;
+import java.nio.ByteOrder;
+
/**
- * A variety of high efficiency bit twiddling routines.
+ * A variety of high efficiency bit twiddling routines and encoders for primitives.
*
* @lucene.internal
*/
@@ -25,6 +29,92 @@ public final class BitUtil {
private BitUtil() {} // no instance
+ /**
+ * A {@link VarHandle} to read/write little endian {@code short} from/to a byte array. Shape:
+ * {@code short vh.get(byte[] arr, int ofs)} and {@code void vh.set(byte[] arr, int ofs, short
+ * val)}
+ */
+ public static final VarHandle VH_LE_SHORT =
+ MethodHandles.byteArrayViewVarHandle(short[].class, ByteOrder.LITTLE_ENDIAN);
+
+ /**
+ * A {@link VarHandle} to read/write little endian {@code int} from a byte array. Shape: {@code
+ * int vh.get(byte[] arr, int ofs)} and {@code void vh.set(byte[] arr, int ofs, int val)}
+ */
+ public static final VarHandle VH_LE_INT =
+ MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.LITTLE_ENDIAN);
+
+ /**
+ * A {@link VarHandle} to read/write little endian {@code long} from a byte array. Shape: {@code
+ * long vh.get(byte[] arr, int ofs)} and {@code void vh.set(byte[] arr, int ofs, long val)}
+ */
+ public static final VarHandle VH_LE_LONG =
+ MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.LITTLE_ENDIAN);
+
+ /**
+ * A {@link VarHandle} to read/write little endian {@code float} from a byte array. Shape: {@code
+ * float vh.get(byte[] arr, int ofs)} and {@code void vh.set(byte[] arr, int ofs, float val)}
+ */
+ public static final VarHandle VH_LE_FLOAT =
+ MethodHandles.byteArrayViewVarHandle(float[].class, ByteOrder.LITTLE_ENDIAN);
+
+ /**
+ * A {@link VarHandle} to read/write little endian {@code double} from a byte array. Shape: {@code
+ * double vh.get(byte[] arr, int ofs)} and {@code void vh.set(byte[] arr, int ofs, double val)}
+ */
+ public static final VarHandle VH_LE_DOUBLE =
+ MethodHandles.byteArrayViewVarHandle(double[].class, ByteOrder.LITTLE_ENDIAN);
+
+ /**
+ * A {@link VarHandle} to read/write big endian {@code short} from a byte array. Shape: {@code
+ * short vh.get(byte[] arr, int ofs)} and {@code void vh.set(byte[] arr, int ofs, short val)}
+ *
+ * @deprecated Better use little endian unless it is needed for backwards compatibility.
+ */
+ @Deprecated
+ public static final VarHandle VH_BE_SHORT =
+ MethodHandles.byteArrayViewVarHandle(short[].class, ByteOrder.BIG_ENDIAN);
+
+ /**
+ * A {@link VarHandle} to read/write big endian {@code int} from a byte array. Shape: {@code int
+ * vh.get(byte[] arr, int ofs)} and {@code void vh.set(byte[] arr, int ofs, int val)}
+ *
+ * @deprecated Better use little endian unless it is needed for backwards compatibility.
+ */
+ @Deprecated
+ public static final VarHandle VH_BE_INT =
+ MethodHandles.byteArrayViewVarHandle(int[].class, ByteOrder.BIG_ENDIAN);
+
+ /**
+ * A {@link VarHandle} to read/write big endian {@code long} from a byte array. Shape: {@code long
+ * vh.get(byte[] arr, int ofs)} and {@code void vh.set(byte[] arr, int ofs, long val)}
+ *
+ * @deprecated Better use little endian unless it is needed for backwards compatibility.
+ */
+ @Deprecated
+ public static final VarHandle VH_BE_LONG =
+ MethodHandles.byteArrayViewVarHandle(long[].class, ByteOrder.BIG_ENDIAN);
+
+ /**
+ * A {@link VarHandle} to read/write big endian {@code float} from a byte array. Shape: {@code
+ * float vh.get(byte[] arr, int ofs)} and {@code void vh.set(byte[] arr, int ofs, float val)}
+ *
+ * @deprecated Better use little endian unless it is needed for backwards compatibility.
+ */
+ @Deprecated
+ public static final VarHandle VH_BE_FLOAT =
+ MethodHandles.byteArrayViewVarHandle(float[].class, ByteOrder.BIG_ENDIAN);
+
+ /**
+ * A {@link VarHandle} to read/write big endian {@code double} from a byte array. Shape: {@code
+ * double vh.get(byte[] arr, int ofs)} and {@code void vh.set(byte[] arr, int ofs, double val)}
+ *
+ * @deprecated Better use little endian unless it is needed for backwards compatibility.
+ */
+ @Deprecated
+ public static final VarHandle VH_BE_DOUBLE =
+ MethodHandles.byteArrayViewVarHandle(double[].class, ByteOrder.BIG_ENDIAN);
+
// The pop methods used to rely on bit-manipulation tricks for speed but it
// turns out that it is faster to use the Long.bitCount method (which is an
// intrinsic since Java 6u18) in a naive loop, see LUCENE-2221
diff --git a/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java b/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java
index 1b20876..1d0a271 100644
--- a/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java
+++ b/lucene/core/src/java/org/apache/lucene/util/NumericUtils.java
@@ -141,10 +141,7 @@ public final class NumericUtils {
public static void intToSortableBytes(int value, byte[] result, int offset) {
// Flip the sign bit, so negative ints sort before positive ints correctly:
value ^= 0x80000000;
- result[offset] = (byte) (value >> 24);
- result[offset + 1] = (byte) (value >> 16);
- result[offset + 2] = (byte) (value >> 8);
- result[offset + 3] = (byte) value;
+ BitUtil.VH_BE_INT.set(result, offset, value);
}
/**
@@ -153,11 +150,7 @@ public final class NumericUtils {
* @see #intToSortableBytes(int, byte[], int)
*/
public static int sortableBytesToInt(byte[] encoded, int offset) {
- int x =
- ((encoded[offset] & 0xFF) << 24)
- | ((encoded[offset + 1] & 0xFF) << 16)
- | ((encoded[offset + 2] & 0xFF) << 8)
- | (encoded[offset + 3] & 0xFF);
+ int x = (int) BitUtil.VH_BE_INT.get(encoded, offset);
// Re-flip the sign bit to restore the original value:
return x ^ 0x80000000;
}
@@ -171,14 +164,7 @@ public final class NumericUtils {
public static void longToSortableBytes(long value, byte[] result, int offset) {
// Flip the sign bit so negative longs sort before positive longs:
value ^= 0x8000000000000000L;
- result[offset] = (byte) (value >> 56);
- result[offset + 1] = (byte) (value >> 48);
- result[offset + 2] = (byte) (value >> 40);
- result[offset + 3] = (byte) (value >> 32);
- result[offset + 4] = (byte) (value >> 24);
- result[offset + 5] = (byte) (value >> 16);
- result[offset + 6] = (byte) (value >> 8);
- result[offset + 7] = (byte) value;
+ BitUtil.VH_BE_LONG.set(result, offset, value);
}
/**
@@ -187,15 +173,7 @@ public final class NumericUtils {
* @see #longToSortableBytes(long, byte[], int)
*/
public static long sortableBytesToLong(byte[] encoded, int offset) {
- long v =
- ((encoded[offset] & 0xFFL) << 56)
- | ((encoded[offset + 1] & 0xFFL) << 48)
- | ((encoded[offset + 2] & 0xFFL) << 40)
- | ((encoded[offset + 3] & 0xFFL) << 32)
- | ((encoded[offset + 4] & 0xFFL) << 24)
- | ((encoded[offset + 5] & 0xFFL) << 16)
- | ((encoded[offset + 6] & 0xFFL) << 8)
- | (encoded[offset + 7] & 0xFFL);
+ long v = (long) BitUtil.VH_BE_LONG.get(encoded, offset);
// Flip the sign bit back
v ^= 0x8000000000000000L;
return v;
diff --git a/lucene/core/src/java/org/apache/lucene/util/StringHelper.java b/lucene/core/src/java/org/apache/lucene/util/StringHelper.java
index 48734f2..20a8cce 100644
--- a/lucene/core/src/java/org/apache/lucene/util/StringHelper.java
+++ b/lucene/core/src/java/org/apache/lucene/util/StringHelper.java
@@ -164,11 +164,7 @@ public abstract class StringHelper {
for (int i = offset; i < roundedEnd; i += 4) {
// little endian load order
- int k1 =
- (data[i] & 0xff)
- | ((data[i + 1] & 0xff) << 8)
- | ((data[i + 2] & 0xff) << 16)
- | (data[i + 3] << 24);
+ int k1 = (int) BitUtil.VH_LE_INT.get(data, i);
k1 *= c1;
k1 = Integer.rotateLeft(k1, 15);
k1 *= c2;
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
index c4dee5d..093b6dd 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointReader.java
@@ -16,6 +16,7 @@
*/
package org.apache.lucene.util.bkd;
+import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
/**
@@ -85,10 +86,7 @@ public final class HeapPointReader implements PointReader {
@Override
public int docID() {
int position = packedValueDocID.offset + packedValueLength;
- return ((packedValueDocID.bytes[position] & 0xFF) << 24)
- | ((packedValueDocID.bytes[++position] & 0xFF) << 16)
- | ((packedValueDocID.bytes[++position] & 0xFF) << 8)
- | (packedValueDocID.bytes[++position] & 0xFF);
+ return (int) BitUtil.VH_BE_INT.get(packedValueDocID.bytes, position);
}
@Override
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
index ebc4f6b..117ce6f 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/HeapPointWriter.java
@@ -17,6 +17,7 @@
package org.apache.lucene.util.bkd;
import java.util.Arrays;
+import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
/**
@@ -67,10 +68,7 @@ public final class HeapPointWriter implements PointWriter {
System.arraycopy(
packedValue, 0, block, nextWrite * config.bytesPerDoc, config.packedBytesLength);
int position = nextWrite * config.bytesPerDoc + config.packedBytesLength;
- block[position] = (byte) (docID >> 24);
- block[++position] = (byte) (docID >> 16);
- block[++position] = (byte) (docID >> 8);
- block[++position] = (byte) (docID >> 0);
+ BitUtil.VH_BE_INT.set(block, position, docID);
nextWrite++;
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java
index e115347..d3b7bae 100644
--- a/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java
+++ b/lucene/core/src/java/org/apache/lucene/util/bkd/OfflinePointReader.java
@@ -23,6 +23,7 @@ import org.apache.lucene.store.ChecksumIndexInput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
/**
@@ -180,10 +181,7 @@ public final class OfflinePointReader implements PointReader {
@Override
public int docID() {
int position = packedValueDocID.offset + packedValueLength;
- return ((packedValueDocID.bytes[position] & 0xFF) << 24)
- | ((packedValueDocID.bytes[++position] & 0xFF) << 16)
- | ((packedValueDocID.bytes[++position] & 0xFF) << 8)
- | (packedValueDocID.bytes[++position] & 0xFF);
+ return (int) BitUtil.VH_BE_INT.get(packedValueDocID.bytes, position);
}
@Override
diff --git a/lucene/core/src/test/org/apache/lucene/store/TestByteArrayDataInput.java b/lucene/core/src/test/org/apache/lucene/store/TestByteArrayDataInput.java
index 73b5e70..df0e8ea 100644
--- a/lucene/core/src/test/org/apache/lucene/store/TestByteArrayDataInput.java
+++ b/lucene/core/src/test/org/apache/lucene/store/TestByteArrayDataInput.java
@@ -16,6 +16,8 @@
*/
package org.apache.lucene.store;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
import org.apache.lucene.util.LuceneTestCase;
public class TestByteArrayDataInput extends LuceneTestCase {
@@ -24,9 +26,39 @@ public class TestByteArrayDataInput extends LuceneTestCase {
byte[] bytes = new byte[] {1, 65};
ByteArrayDataInput in = new ByteArrayDataInput(bytes);
assertEquals("A", in.readString());
+ assertTrue(in.eof());
bytes = new byte[] {1, 1, 65};
in.reset(bytes, 1, 2);
assertEquals("A", in.readString());
+ assertTrue(in.eof());
+ }
+
+ public void testDatatypes() throws Exception {
+ // write some primitives using ByteArrayDataOutput:
+ final byte[] bytes = new byte[32];
+ final ByteArrayDataOutput out = new ByteArrayDataOutput(bytes);
+ out.writeByte((byte) 43);
+ out.writeShort((short) 12345);
+ out.writeInt(1234567890);
+ out.writeLong(1234567890123456789L);
+ final int size = out.getPosition();
+ assertEquals(15, size);
+
+ // read the primitives using ByteBuffer to ensure encoding in byte array is LE:
+ final ByteBuffer buf = ByteBuffer.wrap(bytes, 0, size).order(ByteOrder.LITTLE_ENDIAN);
+ assertEquals(43, buf.get());
+ assertEquals(12345, buf.getShort());
+ assertEquals(1234567890, buf.getInt());
+ assertEquals(1234567890123456789L, buf.getLong());
+ assertEquals(0, buf.remaining());
+
+ // read the primitives using ByteArrayDataInput:
+ final ByteArrayDataInput in = new ByteArrayDataInput(bytes, 0, size);
+ assertEquals(43, in.readByte());
+ assertEquals(12345, in.readShort());
+ assertEquals(1234567890, in.readInt());
+ assertEquals(1234567890123456789L, in.readLong());
+ assertTrue(in.eof());
}
}
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java b/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java
index 02361b6..7558f61 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/FacetsConfig.java
@@ -39,6 +39,7 @@ import org.apache.lucene.facet.taxonomy.TaxonomyWriter;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.IndexableFieldType;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.IntsRefBuilder;
@@ -492,10 +493,8 @@ public class FacetsConfig {
bytes = ArrayUtil.grow(bytes, upto + 4);
}
// big-endian:
- bytes[upto++] = (byte) (ordinal >> 24);
- bytes[upto++] = (byte) (ordinal >> 16);
- bytes[upto++] = (byte) (ordinal >> 8);
- bytes[upto++] = (byte) ordinal;
+ BitUtil.VH_BE_INT.set(bytes, upto, ordinal);
+ upto += 4;
if (upto + field.assoc.length > bytes.length) {
bytes = ArrayUtil.grow(bytes, upto + field.assoc.length);
}
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/IntAssociationFacetField.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/IntAssociationFacetField.java
index 50086a9..cfeaf97 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/IntAssociationFacetField.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/IntAssociationFacetField.java
@@ -18,6 +18,7 @@ package org.apache.lucene.facet.taxonomy;
import java.util.Arrays;
import org.apache.lucene.document.Document;
+import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
/**
@@ -37,19 +38,13 @@ public class IntAssociationFacetField extends AssociationFacetField {
public static BytesRef intToBytesRef(int v) {
byte[] bytes = new byte[4];
// big-endian:
- bytes[0] = (byte) (v >> 24);
- bytes[1] = (byte) (v >> 16);
- bytes[2] = (byte) (v >> 8);
- bytes[3] = (byte) v;
+ BitUtil.VH_BE_INT.set(bytes, 0, v);
return new BytesRef(bytes);
}
/** Decodes a previously encoded {@code int}. */
public static int bytesRefToInt(BytesRef b) {
- return ((b.bytes[b.offset] & 0xFF) << 24)
- | ((b.bytes[b.offset + 1] & 0xFF) << 16)
- | ((b.bytes[b.offset + 2] & 0xFF) << 8)
- | (b.bytes[b.offset + 3] & 0xFF);
+ return (int) BitUtil.VH_BE_INT.get(b.bytes, b.offset);
}
@Override
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java
index a7d36ad..5773d2e 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumFloatAssociations.java
@@ -23,6 +23,7 @@ import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
/**
@@ -71,19 +72,11 @@ public class TaxonomyFacetSumFloatAssociations extends FloatTaxonomyFacets {
int end = bytesRef.offset + bytesRef.length;
int offset = bytesRef.offset;
while (offset < end) {
- int ord =
- ((bytes[offset] & 0xFF) << 24)
- | ((bytes[offset + 1] & 0xFF) << 16)
- | ((bytes[offset + 2] & 0xFF) << 8)
- | (bytes[offset + 3] & 0xFF);
+ int ord = (int) BitUtil.VH_BE_INT.get(bytes, offset);
offset += 4;
- int value =
- ((bytes[offset] & 0xFF) << 24)
- | ((bytes[offset + 1] & 0xFF) << 16)
- | ((bytes[offset + 2] & 0xFF) << 8)
- | (bytes[offset + 3] & 0xFF);
+ float value = (float) BitUtil.VH_BE_FLOAT.get(bytes, offset);
offset += 4;
- values[ord] += Float.intBitsToFloat(value);
+ values[ord] += value;
}
}
}
diff --git a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
index e418977..96c23a9 100644
--- a/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
+++ b/lucene/facet/src/java/org/apache/lucene/facet/taxonomy/TaxonomyFacetSumIntAssociations.java
@@ -23,6 +23,7 @@ import org.apache.lucene.facet.FacetsCollector.MatchingDocs;
import org.apache.lucene.facet.FacetsConfig;
import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
/**
@@ -71,17 +72,9 @@ public class TaxonomyFacetSumIntAssociations extends IntTaxonomyFacets {
int end = bytesRef.offset + bytesRef.length;
int offset = bytesRef.offset;
while (offset < end) {
- int ord =
- ((bytes[offset] & 0xFF) << 24)
- | ((bytes[offset + 1] & 0xFF) << 16)
- | ((bytes[offset + 2] & 0xFF) << 8)
- | (bytes[offset + 3] & 0xFF);
+ int ord = (int) BitUtil.VH_BE_INT.get(bytes, offset);
offset += 4;
- int value =
- ((bytes[offset] & 0xFF) << 24)
- | ((bytes[offset + 1] & 0xFF) << 16)
- | ((bytes[offset + 2] & 0xFF) << 8)
- | (bytes[offset + 3] & 0xFF);
+ int value = (int) BitUtil.VH_BE_INT.get(bytes, offset);
offset += 4;
increment(ord, value);
}
diff --git a/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadMatcherFactory.java b/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadMatcherFactory.java
index cc412da..575f42e 100755
--- a/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadMatcherFactory.java
+++ b/lucene/queries/src/java/org/apache/lucene/queries/payloads/PayloadMatcherFactory.java
@@ -21,6 +21,7 @@ import java.util.EnumMap;
import org.apache.lucene.queries.payloads.SpanPayloadCheckQuery.MatchOperation;
import org.apache.lucene.queries.payloads.SpanPayloadCheckQuery.PayloadType;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
/**
@@ -153,10 +154,7 @@ public class PayloadMatcherFactory {
}
private int decodeInt(byte[] bytes, int offset) {
- return ((bytes[offset] & 0xFF) << 24)
- | ((bytes[offset + 1] & 0xFF) << 16)
- | ((bytes[offset + 2] & 0xFF) << 8)
- | (bytes[offset + 3] & 0xFF);
+ return (int) BitUtil.VH_BE_INT.get(bytes, offset);
}
protected abstract boolean intCompare(int val, int threshold);
@@ -203,11 +201,7 @@ public class PayloadMatcherFactory {
}
private float decodeFloat(byte[] bytes, int offset) {
- return Float.intBitsToFloat(
- ((bytes[offset] & 0xFF) << 24)
- | ((bytes[offset + 1] & 0xFF) << 16)
- | ((bytes[offset + 2] & 0xFF) << 8)
- | (bytes[offset + 3] & 0xFF));
+ return (float) BitUtil.VH_BE_FLOAT.get(bytes, offset);
}
protected abstract boolean floatCompare(float val, float threshold);
diff --git a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleTransLog.java b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleTransLog.java
index 6509d9c..4d053b9 100644
--- a/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleTransLog.java
+++ b/lucene/replicator/src/test/org/apache/lucene/replicator/nrt/SimpleTransLog.java
@@ -32,6 +32,7 @@ import org.apache.lucene.index.CorruptIndexException;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteBuffersDataOutput;
import org.apache.lucene.store.DataInput;
+import org.apache.lucene.util.BitUtil;
/**
* This is a stupid yet functional transaction log: it never fsync's, never prunes, it's
@@ -91,10 +92,7 @@ class SimpleTransLog implements Closeable {
byte[] bytes = buffer.toArrayCopy();
buffer.reset();
- intBuffer[0] = (byte) (len >> 24);
- intBuffer[1] = (byte) (len >> 16);
- intBuffer[2] = (byte) (len >> 8);
- intBuffer[3] = (byte) len;
+ BitUtil.VH_BE_INT.set(intBuffer, 0, len);
intByteBuffer.limit(4);
intByteBuffer.position(0);
@@ -140,11 +138,7 @@ class SimpleTransLog implements Closeable {
intByteBuffer.limit(4);
readBytesFromChannel(pos, intByteBuffer);
pos += 4;
- int len =
- ((intBuffer[0] & 0xff) << 24)
- | (intBuffer[1] & 0xff) << 16
- | (intBuffer[2] & 0xff) << 8
- | (intBuffer[3] & 0xff);
+ int len = (int) BitUtil.VH_BE_INT.get(intBuffer, 0);
byte[] bytes = new byte[len];
readBytesFromChannel(pos, ByteBuffer.wrap(bytes));
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/IDVersionPostingsFormat.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/IDVersionPostingsFormat.java
index aea0442..d4c0866 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/IDVersionPostingsFormat.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/codecs/idversion/IDVersionPostingsFormat.java
@@ -26,6 +26,7 @@ import org.apache.lucene.codecs.lucene90.blocktree.Lucene90BlockTreeTermsWriter;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.search.LiveFieldValues;
+import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
@@ -108,14 +109,7 @@ public class IDVersionPostingsFormat extends PostingsFormat {
}
public static long bytesToLong(BytesRef bytes) {
- return ((bytes.bytes[bytes.offset] & 0xFFL) << 56)
- | ((bytes.bytes[bytes.offset + 1] & 0xFFL) << 48)
- | ((bytes.bytes[bytes.offset + 2] & 0xFFL) << 40)
- | ((bytes.bytes[bytes.offset + 3] & 0xFFL) << 32)
- | ((bytes.bytes[bytes.offset + 4] & 0xFFL) << 24)
- | ((bytes.bytes[bytes.offset + 5] & 0xFFL) << 16)
- | ((bytes.bytes[bytes.offset + 6] & 0xFFL) << 8)
- | (bytes.bytes[bytes.offset + 7] & 0xFFL);
+ return (long) BitUtil.VH_BE_LONG.get(bytes.bytes, bytes.offset);
}
public static void longToBytes(long v, BytesRef bytes) {
@@ -131,14 +125,7 @@ public class IDVersionPostingsFormat extends PostingsFormat {
}
bytes.offset = 0;
bytes.length = 8;
- bytes.bytes[0] = (byte) (v >> 56);
- bytes.bytes[1] = (byte) (v >> 48);
- bytes.bytes[2] = (byte) (v >> 40);
- bytes.bytes[3] = (byte) (v >> 32);
- bytes.bytes[4] = (byte) (v >> 24);
- bytes.bytes[5] = (byte) (v >> 16);
- bytes.bytes[6] = (byte) (v >> 8);
- bytes.bytes[7] = (byte) v;
+ BitUtil.VH_BE_LONG.set(bytes.bytes, 0, v);
assert bytesToLong(bytes) == v : bytesToLong(bytes) + " vs " + v + " bytes=" + bytes;
}
}
diff --git a/lucene/sandbox/src/java/org/apache/lucene/sandbox/document/HalfFloatPoint.java b/lucene/sandbox/src/java/org/apache/lucene/sandbox/document/HalfFloatPoint.java
index 9e031f5..4605a37 100644
--- a/lucene/sandbox/src/java/org/apache/lucene/sandbox/document/HalfFloatPoint.java
+++ b/lucene/sandbox/src/java/org/apache/lucene/sandbox/document/HalfFloatPoint.java
@@ -26,6 +26,7 @@ import org.apache.lucene.index.PointValues;
import org.apache.lucene.search.PointInSetQuery;
import org.apache.lucene.search.PointRangeQuery;
import org.apache.lucene.search.Query;
+import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
/**
@@ -190,12 +191,11 @@ public final class HalfFloatPoint extends Field {
static void shortToSortableBytes(short value, byte[] result, int offset) {
// Flip the sign bit, so negative shorts sort before positive shorts correctly:
value ^= 0x8000;
- result[offset] = (byte) (value >> 8);
- result[offset + 1] = (byte) value;
+ BitUtil.VH_BE_SHORT.set(result, offset, value);
}
static short sortableBytesToShort(byte[] encoded, int offset) {
- short x = (short) (((encoded[offset] & 0xFF) << 8) | (encoded[offset + 1] & 0xFF));
+ short x = (short) BitUtil.VH_BE_SHORT.get(encoded, offset);
// Re-flip the sign bit to restore the original value:
return (short) (x ^ 0x8000);
}