You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by gu...@apache.org on 2022/02/08 08:59:53 UTC
[lucene] branch main updated: LUCENE-10410: Add more tests for legacy decoding logic in DocIdsWriter (#654)
This is an automated email from the ASF dual-hosted git repository.
guofeng pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new 5250186 LUCENE-10410: Add more tests for legacy decoding logic in DocIdsWriter (#654)
5250186 is described below
commit 5250186bd15fdf27d59677943a00730e1b5aa47f
Author: gf2121 <52...@users.noreply.github.com>
AuthorDate: Tue Feb 8 16:59:32 2022 +0800
LUCENE-10410: Add more tests for legacy decoding logic in DocIdsWriter (#654)
---
.../apache/lucene/util/bkd/TestDocIdsWriter.java | 95 ++++++++++++++++++++--
1 file changed, 88 insertions(+), 7 deletions(-)
diff --git a/lucene/core/src/test/org/apache/lucene/util/bkd/TestDocIdsWriter.java b/lucene/core/src/test/org/apache/lucene/util/bkd/TestDocIdsWriter.java
index 7409c71..91d1129 100644
--- a/lucene/core/src/test/org/apache/lucene/util/bkd/TestDocIdsWriter.java
+++ b/lucene/core/src/test/org/apache/lucene/util/bkd/TestDocIdsWriter.java
@@ -22,6 +22,7 @@ import java.util.HashSet;
import java.util.Set;
import org.apache.lucene.index.PointValues.IntersectVisitor;
import org.apache.lucene.index.PointValues.Relation;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexInput;
@@ -32,6 +33,14 @@ import org.apache.lucene.tests.util.TestUtil;
public class TestDocIdsWriter extends LuceneTestCase {
public void testRandom() throws Exception {
+ innerTestRandom(false);
+ }
+
+ public void testLegacyRandom() throws Exception {
+ innerTestRandom(true);
+ }
+
+ private void innerTestRandom(boolean legacy) throws Exception {
int numIters = atLeast(100);
try (Directory dir = newDirectory()) {
for (int iter = 0; iter < numIters; ++iter) {
@@ -40,12 +49,20 @@ public class TestDocIdsWriter extends LuceneTestCase {
for (int i = 0; i < docIDs.length; ++i) {
docIDs[i] = TestUtil.nextInt(random(), 0, (1 << bpv) - 1);
}
- test(dir, docIDs);
+ test(dir, docIDs, legacy);
}
}
}
public void testSorted() throws Exception {
+ innerTestSorted(false);
+ }
+
+ public void testLegacySorted() throws Exception {
+ innerTestSorted(true);
+ }
+
+ private void innerTestSorted(boolean legacy) throws Exception {
int numIters = atLeast(100);
try (Directory dir = newDirectory()) {
for (int iter = 0; iter < numIters; ++iter) {
@@ -55,7 +72,7 @@ public class TestDocIdsWriter extends LuceneTestCase {
docIDs[i] = TestUtil.nextInt(random(), 0, (1 << bpv) - 1);
}
Arrays.sort(docIDs);
- test(dir, docIDs);
+ test(dir, docIDs, legacy);
}
}
}
@@ -70,7 +87,7 @@ public class TestDocIdsWriter extends LuceneTestCase {
for (int i = 0; i < docIDs.length; ++i) {
docIDs[i] = min + TestUtil.nextInt(random(), 0, (1 << bpv) - 1);
}
- test(dir, docIDs);
+ test(dir, docIDs, false);
}
}
}
@@ -86,7 +103,7 @@ public class TestDocIdsWriter extends LuceneTestCase {
set.add(small + random().nextInt(size * 16));
}
int[] docIDs = set.stream().mapToInt(t -> t).sorted().toArray();
- test(dir, docIDs);
+ test(dir, docIDs, false);
}
}
}
@@ -101,16 +118,20 @@ public class TestDocIdsWriter extends LuceneTestCase {
for (int i = 0; i < docIDs.length; i++) {
docIDs[i] = start + i;
}
- test(dir, docIDs);
+ test(dir, docIDs, false);
}
}
}
- private void test(Directory dir, int[] ints) throws Exception {
+ private void test(Directory dir, int[] ints, boolean legacy) throws Exception {
final long len;
DocIdsWriter docIdsWriter = new DocIdsWriter(ints.length);
try (IndexOutput out = dir.createOutput("tmp", IOContext.DEFAULT)) {
- docIdsWriter.writeDocIds(ints, 0, ints.length, out);
+ if (legacy) {
+ legacyWriteDocIds(ints, 0, ints.length, out);
+ } else {
+ docIdsWriter.writeDocIds(ints, 0, ints.length, out);
+ }
len = out.getFilePointer();
if (random().nextBoolean()) {
out.writeLong(0); // garbage
@@ -150,4 +171,64 @@ public class TestDocIdsWriter extends LuceneTestCase {
}
dir.deleteFile("tmp");
}
+
+ // This is a fork of legacy DocIdsWriter to test backward compatibility.
+ private static void legacyWriteDocIds(int[] docIds, int start, int count, DataOutput out)
+ throws IOException {
+ boolean sorted = true;
+ for (int i = 1; i < count; ++i) {
+ if (docIds[start + i - 1] > docIds[start + i]) {
+ sorted = false;
+ break;
+ }
+ }
+ if (sorted) {
+ out.writeByte((byte) 0);
+ int previous = 0;
+ for (int i = 0; i < count; ++i) {
+ int doc = docIds[start + i];
+ out.writeVInt(doc - previous);
+ previous = doc;
+ }
+ } else {
+ long max = 0;
+ for (int i = 0; i < count; ++i) {
+ max |= Integer.toUnsignedLong(docIds[start + i]);
+ }
+ if (max <= 0xffffff) {
+ out.writeByte((byte) 24);
+ // write them the same way we are reading them.
+ int i;
+ for (i = 0; i < count - 7; i += 8) {
+ int doc1 = docIds[start + i];
+ int doc2 = docIds[start + i + 1];
+ int doc3 = docIds[start + i + 2];
+ int doc4 = docIds[start + i + 3];
+ int doc5 = docIds[start + i + 4];
+ int doc6 = docIds[start + i + 5];
+ int doc7 = docIds[start + i + 6];
+ int doc8 = docIds[start + i + 7];
+ long l1 = (doc1 & 0xffffffL) << 40 | (doc2 & 0xffffffL) << 16 | ((doc3 >>> 8) & 0xffffL);
+ long l2 =
+ (doc3 & 0xffL) << 56
+ | (doc4 & 0xffffffL) << 32
+ | (doc5 & 0xffffffL) << 8
+ | ((doc6 >> 16) & 0xffL);
+ long l3 = (doc6 & 0xffffL) << 48 | (doc7 & 0xffffffL) << 24 | (doc8 & 0xffffffL);
+ out.writeLong(l1);
+ out.writeLong(l2);
+ out.writeLong(l3);
+ }
+ for (; i < count; ++i) {
+ out.writeShort((short) (docIds[start + i] >>> 8));
+ out.writeByte((byte) docIds[start + i]);
+ }
+ } else {
+ out.writeByte((byte) 32);
+ for (int i = 0; i < count; ++i) {
+ out.writeInt(docIds[start + i]);
+ }
+ }
+ }
+ }
}