You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2024/10/09 07:43:42 UTC
(lucene) branch branch_10x updated: Move DataInput.readGroupVInts into GroupVIntUtil (#13830)
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch branch_10x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_10x by this push:
new 56c9d1cbebd Move DataInput.readGroupVInts into GroupVIntUtil (#13830)
56c9d1cbebd is described below
commit 56c9d1cbebd7ab9958cd758f7adba5fb37eb36f8
Author: Zhang Chao <80...@qq.com>
AuthorDate: Mon Sep 30 17:04:18 2024 +0800
Move DataInput.readGroupVInts into GroupVIntUtil (#13830)
---
.../backward_codecs/lucene99/PostingsUtil.java | 3 ++-
.../lucene/benchmark/jmh/GroupVIntBenchmark.java | 8 ++++----
.../lucene/codecs/lucene912/PostingsUtil.java | 3 ++-
.../apache/lucene/store/BufferedIndexInput.java | 2 +-
.../apache/lucene/store/ByteBuffersDataInput.java | 2 +-
.../apache/lucene/store/ByteBuffersIndexInput.java | 2 +-
.../java/org/apache/lucene/store/DataInput.java | 22 ++--------------------
.../java/org/apache/lucene/util/GroupVIntUtil.java | 21 ++++++++++++++++++++-
.../lucene/store/MemorySegmentIndexInput.java | 2 +-
.../tests/store/BaseChunkedDirectoryTestCase.java | 5 +++--
.../lucene/tests/store/BaseDirectoryTestCase.java | 7 ++++---
11 files changed, 41 insertions(+), 36 deletions(-)
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/PostingsUtil.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/PostingsUtil.java
index 7b95bada5bc..dce8c2b145d 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/PostingsUtil.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene99/PostingsUtil.java
@@ -19,6 +19,7 @@ package org.apache.lucene.backward_codecs.lucene99;
import java.io.IOException;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
+import org.apache.lucene.util.GroupVIntUtil;
/** Utility class to encode/decode postings block. */
final class PostingsUtil {
@@ -35,7 +36,7 @@ final class PostingsUtil {
boolean indexHasFreq,
boolean decodeFreq)
throws IOException {
- docIn.readGroupVInts(docBuffer, num);
+ GroupVIntUtil.readGroupVInts(docIn, docBuffer, num);
if (indexHasFreq && decodeFreq) {
for (int i = 0; i < num; ++i) {
freqBuffer[i] = docBuffer[i] & 0x01;
diff --git a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GroupVIntBenchmark.java b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GroupVIntBenchmark.java
index 0df0d7ecf50..48b95570694 100644
--- a/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GroupVIntBenchmark.java
+++ b/lucene/benchmark-jmh/src/java/org/apache/lucene/benchmark/jmh/GroupVIntBenchmark.java
@@ -186,7 +186,7 @@ public class GroupVIntBenchmark {
@Benchmark
public void benchMMapDirectoryInputs_readGroupVInt(Blackhole bh) throws IOException {
byteBufferGVIntIn.seek(0);
- byteBufferGVIntIn.readGroupVInts(values, size);
+ GroupVIntUtil.readGroupVInts(byteBufferGVIntIn, values, size);
bh.consume(values);
}
@@ -209,14 +209,14 @@ public class GroupVIntBenchmark {
@Benchmark
public void benchByteArrayDataInput_readGroupVInt(Blackhole bh) throws IOException {
byteArrayGVIntIn.rewind();
- byteArrayGVIntIn.readGroupVInts(values, size);
+ GroupVIntUtil.readGroupVInts(byteArrayGVIntIn, values, size);
bh.consume(values);
}
@Benchmark
public void benchNIOFSDirectoryInputs_readGroupVInt(Blackhole bh) throws IOException {
nioGVIntIn.seek(0);
- nioGVIntIn.readGroupVInts(values, size);
+ GroupVIntUtil.readGroupVInts(nioGVIntIn, values, size);
bh.consume(values);
}
@@ -230,7 +230,7 @@ public class GroupVIntBenchmark {
@Benchmark
public void benchByteBuffersIndexInput_readGroupVInt(Blackhole bh) throws IOException {
byteBuffersGVIntIn.seek(0);
- byteBuffersGVIntIn.readGroupVInts(values, size);
+ GroupVIntUtil.readGroupVInts(byteBuffersGVIntIn, values, size);
bh.consume(values);
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene912/PostingsUtil.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/PostingsUtil.java
index 4834dd73e22..1ae808d308f 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene912/PostingsUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene912/PostingsUtil.java
@@ -19,6 +19,7 @@ package org.apache.lucene.codecs.lucene912;
import java.io.IOException;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.IndexInput;
+import org.apache.lucene.util.GroupVIntUtil;
/** Utility class to encode/decode postings block. */
final class PostingsUtil {
@@ -35,7 +36,7 @@ final class PostingsUtil {
boolean indexHasFreq,
boolean decodeFreq)
throws IOException {
- docIn.readGroupVInts(docBuffer, num);
+ GroupVIntUtil.readGroupVInts(docIn, docBuffer, num);
if (indexHasFreq && decodeFreq) {
for (int i = 0; i < num; ++i) {
freqBuffer[i] = docBuffer[i] & 0x01;
diff --git a/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
index 13151692bc0..7f2aadf54a5 100644
--- a/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
@@ -151,7 +151,7 @@ public abstract class BufferedIndexInput extends IndexInput implements RandomAcc
}
@Override
- protected void readGroupVInt(long[] dst, int offset) throws IOException {
+ public void readGroupVInt(long[] dst, int offset) throws IOException {
final int len =
GroupVIntUtil.readGroupVInt(
this, buffer.remaining(), p -> buffer.getInt((int) p), buffer.position(), dst, offset);
diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java
index 4b722b61689..a09f78e5f3a 100644
--- a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java
@@ -204,7 +204,7 @@ public final class ByteBuffersDataInput extends DataInput
}
@Override
- protected void readGroupVInt(long[] dst, int offset) throws IOException {
+ public void readGroupVInt(long[] dst, int offset) throws IOException {
final ByteBuffer block = blocks[blockIndex(pos)];
final int blockOffset = blockOffset(pos);
// We MUST save the return value to local variable, could not use pos += readGroupVInt(...).
diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexInput.java
index c66d864d570..6aebb771b68 100644
--- a/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/ByteBuffersIndexInput.java
@@ -206,7 +206,7 @@ public final class ByteBuffersIndexInput extends IndexInput implements RandomAcc
}
@Override
- protected void readGroupVInt(long[] dst, int offset) throws IOException {
+ public void readGroupVInt(long[] dst, int offset) throws IOException {
ensureOpen();
in.readGroupVInt(dst, offset);
}
diff --git a/lucene/core/src/java/org/apache/lucene/store/DataInput.java b/lucene/core/src/java/org/apache/lucene/store/DataInput.java
index 427e81f2df2..70f9a96db9c 100644
--- a/lucene/core/src/java/org/apache/lucene/store/DataInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/DataInput.java
@@ -100,28 +100,10 @@ public abstract class DataInput implements Cloneable {
}
/**
- * Read all the group varints, including the tail vints. we need a long[] because this is what
- * postings are using, all longs are actually required to be integers.
- *
- * @param dst the array to read ints into.
- * @param limit the number of int values to read.
- * @lucene.experimental
- */
- public final void readGroupVInts(long[] dst, int limit) throws IOException {
- int i;
- for (i = 0; i <= limit - 4; i += 4) {
- readGroupVInt(dst, i);
- }
- for (; i < limit; ++i) {
- dst[i] = readVInt() & 0xFFFFFFFFL;
- }
- }
-
- /**
- * Override if you have a efficient implementation. In general this is when the input supports
+ * Override if you have an efficient implementation. In general this is when the input supports
* random access.
*/
- protected void readGroupVInt(long[] dst, int offset) throws IOException {
+ public void readGroupVInt(long[] dst, int offset) throws IOException {
GroupVIntUtil.readGroupVInt(this, dst, offset);
}
diff --git a/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.java b/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.java
index e1b5466342a..1c5033172db 100644
--- a/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.java
+++ b/lucene/core/src/java/org/apache/lucene/util/GroupVIntUtil.java
@@ -32,10 +32,29 @@ public final class GroupVIntUtil {
// we use long array instead of int array to make negative integer to be read as positive long.
private static final long[] MASKS = new long[] {0xFFL, 0xFFFFL, 0xFFFFFFL, 0xFFFFFFFFL};
+ /**
+ * Read all the group varints, including the tail vints. we need a long[] because this is what
+ * postings are using, all longs are actually required to be integers.
+ *
+ * @param dst the array to read ints into.
+ * @param limit the number of int values to read.
+ * @lucene.experimental
+ */
+ public static void readGroupVInts(DataInput in, long[] dst, int limit) throws IOException {
+ int i;
+ for (i = 0; i <= limit - 4; i += 4) {
+ in.readGroupVInt(dst, i);
+ }
+ for (; i < limit; ++i) {
+ dst[i] = in.readVInt() & 0xFFFFFFFFL;
+ }
+ }
+
/**
* Default implementation of read single group, for optimal performance, you should use {@link
- * DataInput#readGroupVInts(long[], int)} instead.
+ * GroupVIntUtil#readGroupVInts(DataInput, long[], int)} instead.
*
+ * @param in the input to use to read data.
* @param dst the array to read ints into.
* @param offset the offset in the array to start storing ints.
*/
diff --git a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
index c6ac3d23a12..8bb70ba009d 100644
--- a/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
+++ b/lucene/core/src/java21/org/apache/lucene/store/MemorySegmentIndexInput.java
@@ -419,7 +419,7 @@ abstract class MemorySegmentIndexInput extends IndexInput
}
@Override
- protected void readGroupVInt(long[] dst, int offset) throws IOException {
+ public void readGroupVInt(long[] dst, int offset) throws IOException {
try {
final int len =
GroupVIntUtil.readGroupVInt(
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseChunkedDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseChunkedDirectoryTestCase.java
index dd956c6c3fd..8de332eeec9 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseChunkedDirectoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseChunkedDirectoryTestCase.java
@@ -33,6 +33,7 @@ import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.index.RandomIndexWriter;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util.GroupVIntUtil;
/**
* Base class for Directories that "chunk" the input into blocks.
@@ -77,7 +78,7 @@ public abstract class BaseChunkedDirectoryTestCase extends BaseDirectoryTestCase
expectThrows(
AlreadyClosedException.class,
() -> {
- two.readGroupVInts(values, values.length);
+ GroupVIntUtil.readGroupVInts(two, values, values.length);
});
assertEquals(5, three.readVInt());
one.close();
@@ -105,7 +106,7 @@ public abstract class BaseChunkedDirectoryTestCase extends BaseDirectoryTestCase
expectThrows(
AlreadyClosedException.class,
() -> {
- one.readGroupVInts(values, values.length);
+ GroupVIntUtil.readGroupVInts(one, values, values.length);
});
assertEquals(2, two.readInt());
// reopen a new slice "another":
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
index 9cc271a9d61..41d72c509db 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/store/BaseDirectoryTestCase.java
@@ -59,6 +59,7 @@ import org.apache.lucene.tests.util.LuceneTestCase;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BitUtil;
+import org.apache.lucene.util.GroupVIntUtil;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.packed.PackedInts;
import org.junit.Assert;
@@ -1458,7 +1459,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
assertEquals(43, in.readByte());
assertEquals(12345, in.readShort());
assertEquals(1234567890, in.readInt());
- in.readGroupVInts(restored, 4);
+ GroupVIntUtil.readGroupVInts(in, restored, 4);
assertArrayEquals(values, restored);
assertEquals(1234567890123456789L, in.readLong());
in.close();
@@ -1485,7 +1486,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
out.writeGroupVInts(values, limit);
out.close();
try (IndexInput in = dir.openInput("test", IOContext.DEFAULT)) {
- in.readGroupVInts(restore, limit);
+ GroupVIntUtil.readGroupVInts(in, restore, limit);
for (int i = 0; i < limit; i++) {
assertEquals(values[i], restore[i]);
}
@@ -1533,7 +1534,7 @@ public abstract class BaseDirectoryTestCase extends LuceneTestCase {
IndexInput groupVIntIn = dir.openInput("group-varint", IOContext.DEFAULT);
IndexInput vIntIn = dir.openInput("vint", IOContext.DEFAULT);
for (int iter = 0; iter < iterations; iter++) {
- groupVIntIn.readGroupVInts(values, numValuesArray[iter]);
+ GroupVIntUtil.readGroupVInts(groupVIntIn, values, numValuesArray[iter]);
for (int j = 0; j < numValuesArray[iter]; j++) {
assertEquals(vIntIn.readVInt(), values[j]);
}