You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@solr.apache.org by kr...@apache.org on 2022/01/20 18:33:17 UTC
[solr] branch branch_9_0 updated: SOLR-15405: Make HDFS handle little endian LUCENE-9047
This is an automated email from the ASF dual-hosted git repository.
krisden pushed a commit to branch branch_9_0
in repository https://gitbox.apache.org/repos/asf/solr.git
The following commit(s) were added to refs/heads/branch_9_0 by this push:
new 87a700f SOLR-15405: Make HDFS handle little endian LUCENE-9047
87a700f is described below
commit 87a700fac6f86713681698b80397ce117d39876f
Author: Kevin Risden <kr...@apache.org>
AuthorDate: Thu Jan 13 19:52:33 2022 -0500
SOLR-15405: Make HDFS handle little endian LUCENE-9047
Uses Lucene `ByteArrayDataInput` and `ByeBuffersDataInput` as inspiration.
* https://github.com/apache/lucene/blob/main/lucene/core/src/java/org/apache/lucene/store/ByteArrayDataInput.java
* https://github.com/apache/lucene/blob/releases/lucene%2F9.0.0/lucene/core/src/java/org/apache/lucene/store/ByteBuffersDataInput.java#L212
`CustomBufferedIndexInput` needs to ensure that
it handles LUCNE-9047 changes for little
endianess. This replaces the existing methods
with little endian implementations.
---
solr/CHANGES.txt | 2 +
.../store/blockcache/CustomBufferedIndexInput.java | 89 +++++++++++++++-------
.../solr/cloud/hdfs/HDFSCollectionsAPITest.java | 2 -
.../apache/solr/index/hdfs/CheckHdfsIndexTest.java | 3 -
.../org/apache/solr/search/TestRecoveryHdfs.java | 2 -
.../org/apache/solr/update/TestHdfsUpdateLog.java | 2 -
6 files changed, 64 insertions(+), 36 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e850621..211b0c4 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -551,6 +551,8 @@ Bug Fixes
* SOLR-15324: Upgrade jaegertracing to 1.6.0 and libthrift to 0.14.1 (wcmrnd1, janhoy)
+* SOLR-15405: Make HDFS handle little endian LUCENE-9047 (Kevin Risden)
+
================== 8.11.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
diff --git a/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java b/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java
index c138c07..560be65 100644
--- a/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java
+++ b/solr/core/src/java/org/apache/solr/store/blockcache/CustomBufferedIndexInput.java
@@ -38,7 +38,7 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
private int bufferLength = 0; // end of valid bytes
private int bufferPosition = 0; // next byte to read
- private Store store;
+ private final Store store;
@Override
public byte readByte() throws IOException {
@@ -116,14 +116,26 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
}
}
}
-
+
+ @Override
+ public short readShort() throws IOException {
+ if (2 <= (bufferLength - bufferPosition)) {
+ final byte b1 = buffer[bufferPosition++];
+ final byte b2 = buffer[bufferPosition++];
+ return (short) ((b2 & 0xFF) << 8 | (b1 & 0xFF));
+ } else {
+ return super.readShort();
+ }
+ }
+
@Override
public int readInt() throws IOException {
if (4 <= (bufferLength - bufferPosition)) {
- return ((buffer[bufferPosition++] & 0xFF) << 24)
- | ((buffer[bufferPosition++] & 0xFF) << 16)
- | ((buffer[bufferPosition++] & 0xFF) << 8)
- | (buffer[bufferPosition++] & 0xFF);
+ final byte b1 = buffer[bufferPosition++];
+ final byte b2 = buffer[bufferPosition++];
+ final byte b3 = buffer[bufferPosition++];
+ final byte b4 = buffer[bufferPosition++];
+ return (b4 & 0xFF) << 24 | (b3 & 0xFF) << 16 | (b2 & 0xFF) << 8 | (b1 & 0xFF);
} else {
return super.readInt();
}
@@ -132,15 +144,7 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
@Override
public long readLong() throws IOException {
if (8 <= (bufferLength - bufferPosition)) {
- final int i1 = ((buffer[bufferPosition++] & 0xff) << 24)
- | ((buffer[bufferPosition++] & 0xff) << 16)
- | ((buffer[bufferPosition++] & 0xff) << 8)
- | (buffer[bufferPosition++] & 0xff);
- final int i2 = ((buffer[bufferPosition++] & 0xff) << 24)
- | ((buffer[bufferPosition++] & 0xff) << 16)
- | ((buffer[bufferPosition++] & 0xff) << 8)
- | (buffer[bufferPosition++] & 0xff);
- return (((long) i1) << 32) | (i2 & 0xFFFFFFFFL);
+ return (readInt() & 0xFFFFFFFFL) | (((long) readInt()) << 32);
} else {
return super.readLong();
}
@@ -150,12 +154,22 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
public int readVInt() throws IOException {
if (5 <= (bufferLength - bufferPosition)) {
byte b = buffer[bufferPosition++];
+ if (b >= 0) return b;
int i = b & 0x7F;
- for (int shift = 7; (b & 0x80) != 0; shift += 7) {
- b = buffer[bufferPosition++];
- i |= (b & 0x7F) << shift;
- }
- return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7F) << 7;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7F) << 14;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7F) << 21;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ // Warning: the next ands use 0x0F / 0xF0 - beware copy/paste errors:
+ i |= (b & 0x0F) << 28;
+ if ((b & 0xF0) == 0) return i;
+ throw new RuntimeException("Invalid vInt detected (too many bits)");
} else {
return super.readVInt();
}
@@ -164,13 +178,34 @@ public abstract class CustomBufferedIndexInput extends IndexInput {
@Override
public long readVLong() throws IOException {
if (9 <= bufferLength - bufferPosition) {
- byte b = buffer[bufferPosition++];
- long i = b & 0x7F;
- for (int shift = 7; (b & 0x80) != 0; shift += 7) {
- b = buffer[bufferPosition++];
- i |= (b & 0x7FL) << shift;
- }
- return i;
+ byte b =buffer[bufferPosition++];
+ if (b >= 0) return b;
+ long i = b & 0x7FL;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 7;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 14;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 21;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 28;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 35;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 42;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 49;
+ if (b >= 0) return i;
+ b = buffer[bufferPosition++];
+ i |= (b & 0x7FL) << 56;
+ if (b >= 0) return i;
+ throw new RuntimeException("Invalid vLong detected (negative values disallowed)");
} else {
return super.readVLong();
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
index b5a387e3..83bd930 100644
--- a/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/hdfs/HDFSCollectionsAPITest.java
@@ -20,7 +20,6 @@ package org.apache.solr.cloud.hdfs;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.client.solrj.SolrQuery;
@@ -39,7 +38,6 @@ import org.junit.BeforeClass;
QuickPatchThreadsFilter.class,
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
-@LuceneTestCase.AwaitsFix(bugUrl = "SOLR-15405")
@ThreadLeakLingering(linger = 10)
public class HDFSCollectionsAPITest extends SolrCloudTestCase {
diff --git a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
index 3b08bab..28e82ca 100644
--- a/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
+++ b/solr/core/src/test/org/apache/solr/index/hdfs/CheckHdfsIndexTest.java
@@ -24,7 +24,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.lucene.index.BaseTestCheckIndex;
import org.apache.lucene.store.Directory;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.SolrTestCaseJ4;
@@ -50,8 +49,6 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
@SolrTestCaseJ4.SuppressSSL
-@LuceneTestCase.AwaitsFix(bugUrl = "SOLR-15405")
-// commented out on: 24-Dec-2018 @LuceneTestCase.BadApple(bugUrl="https://issues.apache.org/jira/browse/SOLR-12028") // 12-Jun-2018
public class CheckHdfsIndexTest extends AbstractFullDistribZkTestBase {
private static MiniDFSCluster dfsCluster;
private static Path path;
diff --git a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
index 17a1d01..9e57717 100644
--- a/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
+++ b/solr/core/src/test/org/apache/solr/search/TestRecoveryHdfs.java
@@ -39,7 +39,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.SolrTestCaseJ4;
@@ -69,7 +68,6 @@ import static org.apache.solr.update.processor.DistributingUpdateProcessorFactor
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
// TODO: longer term this should be combined with TestRecovery somehow ??
-@LuceneTestCase.AwaitsFix(bugUrl = "SOLR-15405")
public class TestRecoveryHdfs extends SolrTestCaseJ4 {
// means that we've seen the leader and have version info (i.e. we are a non-leader replica)
private static final String FROM_LEADER = DistribPhase.FROMLEADER.toString();
diff --git a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
index 2e65db5..59f8fa9 100644
--- a/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
+++ b/solr/core/src/test/org/apache/solr/update/TestHdfsUpdateLog.java
@@ -23,7 +23,6 @@ import java.net.URISyntaxException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.QuickPatchThreadsFilter;
import org.apache.solr.SolrIgnoredThreadsFilter;
import org.apache.solr.SolrTestCaseJ4;
@@ -42,7 +41,6 @@ import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters;
QuickPatchThreadsFilter.class,
BadHdfsThreadsFilter.class // hdfs currently leaks thread(s)
})
-@LuceneTestCase.AwaitsFix(bugUrl = "SOLR-15405")
public class TestHdfsUpdateLog extends SolrTestCaseJ4 {
private static MiniDFSCluster dfsCluster;
private static String hdfsUri;