You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by gu...@apache.org on 2022/10/24 09:39:28 UTC
[lucene] branch main updated: LUCENE-10376: Roll up the loop in vint/vlong in DataInput (#602)
This is an automated email from the ASF dual-hosted git repository.
guofeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 8cfbc184976 LUCENE-10376: Roll up the loop in vint/vlong in DataInput (#602)
8cfbc184976 is described below
commit 8cfbc184976f7f169af17853beba0e4d27e842cb
Author: gf2121 <52...@users.noreply.github.com>
AuthorDate: Mon Oct 24 17:39:22 2022 +0800
LUCENE-10376: Roll up the loop in vint/vlong in DataInput (#602)
---
lucene/CHANGES.txt | 2 +
.../apache/lucene/store/BufferedIndexInput.java | 61 --------------------
.../apache/lucene/store/ByteArrayDataInput.java | 60 +++++---------------
.../java/org/apache/lucene/store/DataInput.java | 66 +---------------------
.../util/packed/BlockPackedReaderIterator.java | 36 +++---------
.../org/apache/lucene/index/TestIndexInput.java | 36 ------------
6 files changed, 27 insertions(+), 234 deletions(-)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 88eb798fb98..fef1f32fe0e 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -796,6 +796,8 @@ Bug Fixes
Other
---------------------
+* LUCENE-10376: Roll up the loop in VInt/VLong in DataInput. (Guo Feng)
+
* LUCENE-10273: Deprecate SpanishMinimalStemFilter in favor of SpanishPluralStemFilter. (Robert Muir)
* LUCENE-10284: Upgrade morfologik-stemming to 2.1.8. (Dawid Weiss)
diff --git a/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java b/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
index 974ad0c6874..8db6ba3031c 100644
--- a/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/BufferedIndexInput.java
@@ -159,67 +159,6 @@ public abstract class BufferedIndexInput extends IndexInput implements RandomAcc
}
}
- @Override
- public final int readVInt() throws IOException {
- if (5 <= buffer.remaining()) {
- byte b = buffer.get();
- if (b >= 0) return b;
- int i = b & 0x7F;
- b = buffer.get();
- i |= (b & 0x7F) << 7;
- if (b >= 0) return i;
- b = buffer.get();
- i |= (b & 0x7F) << 14;
- if (b >= 0) return i;
- b = buffer.get();
- i |= (b & 0x7F) << 21;
- if (b >= 0) return i;
- b = buffer.get();
- // Warning: the next ands use 0x0F / 0xF0 - beware copy/paste errors:
- i |= (b & 0x0F) << 28;
- if ((b & 0xF0) == 0) return i;
- throw new IOException("Invalid vInt detected (too many bits)");
- } else {
- return super.readVInt();
- }
- }
-
- @Override
- public final long readVLong() throws IOException {
- if (9 <= buffer.remaining()) {
- byte b = buffer.get();
- if (b >= 0) return b;
- long i = b & 0x7FL;
- b = buffer.get();
- i |= (b & 0x7FL) << 7;
- if (b >= 0) return i;
- b = buffer.get();
- i |= (b & 0x7FL) << 14;
- if (b >= 0) return i;
- b = buffer.get();
- i |= (b & 0x7FL) << 21;
- if (b >= 0) return i;
- b = buffer.get();
- i |= (b & 0x7FL) << 28;
- if (b >= 0) return i;
- b = buffer.get();
- i |= (b & 0x7FL) << 35;
- if (b >= 0) return i;
- b = buffer.get();
- i |= (b & 0x7FL) << 42;
- if (b >= 0) return i;
- b = buffer.get();
- i |= (b & 0x7FL) << 49;
- if (b >= 0) return i;
- b = buffer.get();
- i |= (b & 0x7FL) << 56;
- if (b >= 0) return i;
- throw new IOException("Invalid vLong detected (negative values disallowed)");
- } else {
- return super.readVLong();
- }
- }
-
@Override
public final byte readByte(long pos) throws IOException {
long index = pos - bufferStart;
diff --git a/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataInput.java b/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataInput.java
index fa005a160a9..afa2b635be6 100644
--- a/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataInput.java
@@ -16,6 +16,7 @@
*/
package org.apache.lucene.store;
+import java.io.IOException;
import org.apache.lucene.util.BitUtil;
import org.apache.lucene.util.BytesRef;
@@ -109,55 +110,24 @@ public final class ByteArrayDataInput extends DataInput {
@Override
public int readVInt() {
- byte b = bytes[pos++];
- if (b >= 0) return b;
- int i = b & 0x7F;
- b = bytes[pos++];
- i |= (b & 0x7F) << 7;
- if (b >= 0) return i;
- b = bytes[pos++];
- i |= (b & 0x7F) << 14;
- if (b >= 0) return i;
- b = bytes[pos++];
- i |= (b & 0x7F) << 21;
- if (b >= 0) return i;
- b = bytes[pos++];
- // Warning: the next ands use 0x0F / 0xF0 - beware copy/paste errors:
- i |= (b & 0x0F) << 28;
- if ((b & 0xF0) == 0) return i;
- throw new RuntimeException("Invalid vInt detected (too many bits)");
+ try {
+ return super.readVInt();
+ } catch (
+ @SuppressWarnings("unused")
+ IOException e) {
+ throw new AssertionError("ByteArrayDataInput#readByte should not throw IOException");
+ }
}
@Override
public long readVLong() {
- byte b = bytes[pos++];
- if (b >= 0) return b;
- long i = b & 0x7FL;
- b = bytes[pos++];
- i |= (b & 0x7FL) << 7;
- if (b >= 0) return i;
- b = bytes[pos++];
- i |= (b & 0x7FL) << 14;
- if (b >= 0) return i;
- b = bytes[pos++];
- i |= (b & 0x7FL) << 21;
- if (b >= 0) return i;
- b = bytes[pos++];
- i |= (b & 0x7FL) << 28;
- if (b >= 0) return i;
- b = bytes[pos++];
- i |= (b & 0x7FL) << 35;
- if (b >= 0) return i;
- b = bytes[pos++];
- i |= (b & 0x7FL) << 42;
- if (b >= 0) return i;
- b = bytes[pos++];
- i |= (b & 0x7FL) << 49;
- if (b >= 0) return i;
- b = bytes[pos++];
- i |= (b & 0x7FL) << 56;
- if (b >= 0) return i;
- throw new RuntimeException("Invalid vLong detected (negative values disallowed)");
+ try {
+ return super.readVLong();
+ } catch (
+ @SuppressWarnings("unused")
+ IOException e) {
+ throw new AssertionError("ByteArrayDataInput#readByte should not throw IOException");
+ }
}
// NOTE: AIOOBE not EOF if you read too much
diff --git a/lucene/core/src/java/org/apache/lucene/store/DataInput.java b/lucene/core/src/java/org/apache/lucene/store/DataInput.java
index cfb39e7593f..24aae0372d1 100644
--- a/lucene/core/src/java/org/apache/lucene/store/DataInput.java
+++ b/lucene/core/src/java/org/apache/lucene/store/DataInput.java
@@ -107,9 +107,6 @@ public abstract class DataInput implements Cloneable {
* @see DataOutput#writeVInt(int)
*/
public int readVInt() throws IOException {
- /* This is the original code of this method,
- * but a Hotspot bug (see LUCENE-2975) corrupts the for-loop if
- * readByte() is inlined. So the loop was unwinded!
byte b = readByte();
int i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
@@ -117,24 +114,6 @@ public abstract class DataInput implements Cloneable {
i |= (b & 0x7F) << shift;
}
return i;
- */
- byte b = readByte();
- if (b >= 0) return b;
- int i = b & 0x7F;
- b = readByte();
- i |= (b & 0x7F) << 7;
- if (b >= 0) return i;
- b = readByte();
- i |= (b & 0x7F) << 14;
- if (b >= 0) return i;
- b = readByte();
- i |= (b & 0x7F) << 21;
- if (b >= 0) return i;
- b = readByte();
- // Warning: the next ands use 0x0F / 0xF0 - beware copy/paste errors:
- i |= (b & 0x0F) << 28;
- if ((b & 0xF0) == 0) return i;
- throw new IOException("Invalid vInt detected (too many bits)");
}
/**
@@ -206,13 +185,6 @@ public abstract class DataInput implements Cloneable {
* @see DataOutput#writeVLong(long)
*/
public long readVLong() throws IOException {
- return readVLong(false);
- }
-
- private long readVLong(boolean allowNegative) throws IOException {
- /* This is the original code of this method,
- * but a Hotspot bug (see LUCENE-2975) corrupts the for-loop if
- * readByte() is inlined. So the loop was unwinded!
byte b = readByte();
long i = b & 0x7F;
for (int shift = 7; (b & 0x80) != 0; shift += 7) {
@@ -220,42 +192,6 @@ public abstract class DataInput implements Cloneable {
i |= (b & 0x7FL) << shift;
}
return i;
- */
- byte b = readByte();
- if (b >= 0) return b;
- long i = b & 0x7FL;
- b = readByte();
- i |= (b & 0x7FL) << 7;
- if (b >= 0) return i;
- b = readByte();
- i |= (b & 0x7FL) << 14;
- if (b >= 0) return i;
- b = readByte();
- i |= (b & 0x7FL) << 21;
- if (b >= 0) return i;
- b = readByte();
- i |= (b & 0x7FL) << 28;
- if (b >= 0) return i;
- b = readByte();
- i |= (b & 0x7FL) << 35;
- if (b >= 0) return i;
- b = readByte();
- i |= (b & 0x7FL) << 42;
- if (b >= 0) return i;
- b = readByte();
- i |= (b & 0x7FL) << 49;
- if (b >= 0) return i;
- b = readByte();
- i |= (b & 0x7FL) << 56;
- if (b >= 0) return i;
- if (allowNegative) {
- b = readByte();
- i |= (b & 0x7FL) << 63;
- if (b == 0 || b == 1) return i;
- throw new IOException("Invalid vLong detected (more than 64 bits)");
- } else {
- throw new IOException("Invalid vLong detected (negative values disallowed)");
- }
}
/**
@@ -265,7 +201,7 @@ public abstract class DataInput implements Cloneable {
* @see DataOutput#writeZLong(long)
*/
public long readZLong() throws IOException {
- return BitUtil.zigZagDecode(readVLong(true));
+ return BitUtil.zigZagDecode(readVLong());
}
/**
diff --git a/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReaderIterator.java b/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReaderIterator.java
index 904a2a27d5a..cbbf5d70713 100644
--- a/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReaderIterator.java
+++ b/lucene/core/src/java/org/apache/lucene/util/packed/BlockPackedReaderIterator.java
@@ -40,33 +40,15 @@ public final class BlockPackedReaderIterator {
// same as DataInput.readVLong but supports negative values
static long readVLong(DataInput in) throws IOException {
- byte b = in.readByte();
- if (b >= 0) return b;
- long i = b & 0x7FL;
- b = in.readByte();
- i |= (b & 0x7FL) << 7;
- if (b >= 0) return i;
- b = in.readByte();
- i |= (b & 0x7FL) << 14;
- if (b >= 0) return i;
- b = in.readByte();
- i |= (b & 0x7FL) << 21;
- if (b >= 0) return i;
- b = in.readByte();
- i |= (b & 0x7FL) << 28;
- if (b >= 0) return i;
- b = in.readByte();
- i |= (b & 0x7FL) << 35;
- if (b >= 0) return i;
- b = in.readByte();
- i |= (b & 0x7FL) << 42;
- if (b >= 0) return i;
- b = in.readByte();
- i |= (b & 0x7FL) << 49;
- if (b >= 0) return i;
- b = in.readByte();
- i |= (b & 0xFFL) << 56;
- return i;
+ long l = 0L;
+ for (int shift = 0; shift < 56; shift += 7) {
+ byte b = in.readByte();
+ l |= (b & 0x7FL) << shift;
+ if (b >= 0) {
+ return l;
+ }
+ }
+ return l | ((in.readByte() & 0xFFL) << 56);
}
DataInput in;
diff --git a/lucene/core/src/test/org/apache/lucene/index/TestIndexInput.java b/lucene/core/src/test/org/apache/lucene/index/TestIndexInput.java
index 3d52aa36fee..70ca5fe6d00 100644
--- a/lucene/core/src/test/org/apache/lucene/index/TestIndexInput.java
+++ b/lucene/core/src/test/org/apache/lucene/index/TestIndexInput.java
@@ -155,24 +155,6 @@ public class TestIndexInput extends LuceneTestCase {
0x00,
'n',
'e',
-
- // tests for Exceptions on invalid values
- (byte) 0xFF,
- (byte) 0xFF,
- (byte) 0xFF,
- (byte) 0xFF,
- (byte) 0x17,
- (byte) 0x01, // guard value
- (byte) 0xFF,
- (byte) 0xFF,
- (byte) 0xFF,
- (byte) 0xFF,
- (byte) 0xFF,
- (byte) 0xFF,
- (byte) 0xFF,
- (byte) 0xFF,
- (byte) 0xFF,
- (byte) 0x01, // guard value
};
static final int COUNT = RANDOM_MULTIPLIER * 65536;
@@ -234,24 +216,6 @@ public class TestIndexInput extends LuceneTestCase {
assertEquals("\u0000", is.readString());
assertEquals("Lu\u0000ce\u0000ne", is.readString());
-
- Exception expected =
- expectThrows(
- expectedEx,
- () -> {
- is.readVInt();
- });
- assertTrue(expected.getMessage().startsWith("Invalid vInt"));
- assertEquals(1, is.readVInt()); // guard value
-
- expected =
- expectThrows(
- expectedEx,
- () -> {
- is.readVLong();
- });
- assertTrue(expected.getMessage().startsWith("Invalid vLong"));
- assertEquals(1L, is.readVLong()); // guard value
}
private void checkRandomReads(DataInput is) throws IOException {