You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by so...@apache.org on 2025/11/25 13:59:31 UTC
(lucene) branch main updated: Speed up slow LegacyBaseDocValuesFormatTestCase tests (fixes #1544) (#15448)
This is an automated email from the ASF dual-hosted git repository.
sokolov pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/main by this push:
new a7270d59c58 Speed up slow LegacyBaseDocValuesFormatTestCase tests (fixes #1544) (#15448)
a7270d59c58 is described below
commit a7270d59c586e6845a0e4810bf0fc3a073c5e497
Author: Michael Sokolov <so...@falutin.net>
AuthorDate: Tue Nov 25 08:59:26 2025 -0500
Speed up slow LegacyBaseDocValuesFormatTestCase tests (fixes #1544) (#15448)
---
.../BaseLucene80DocValuesFormatTestCase.java | 35 ++++++++++++++++------
.../lucene90/TestLucene90DocValuesFormat.java | 4 +--
.../index/LegacyBaseDocValuesFormatTestCase.java | 14 +++++++--
3 files changed, 40 insertions(+), 13 deletions(-)
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/BaseLucene80DocValuesFormatTestCase.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/BaseLucene80DocValuesFormatTestCase.java
index 376b72d79a0..312623f8a1b 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/BaseLucene80DocValuesFormatTestCase.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene80/BaseLucene80DocValuesFormatTestCase.java
@@ -66,6 +66,7 @@ import org.apache.lucene.tests.analysis.MockAnalyzer;
import org.apache.lucene.tests.codecs.asserting.AssertingCodec;
import org.apache.lucene.tests.index.LegacyBaseDocValuesFormatTestCase;
import org.apache.lucene.tests.index.RandomIndexWriter;
+import org.apache.lucene.tests.store.BaseDirectoryWrapper;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefBuilder;
@@ -657,11 +658,11 @@ public abstract class BaseLucene80DocValuesFormatTestCase
@Nightly
public void testNumericFieldJumpTables() throws Exception {
// IndexedDISI block skipping only activated if target >= current+2, so we need at least 5
- // blocks to
- // trigger consecutive block skips
+ // blocks to trigger consecutive block skips
final int maxDoc = atLeast(5 * 65536);
-
- Directory dir = newDirectory();
+ long start = System.nanoTime();
+ BaseDirectoryWrapper dir = newDirectory();
+ dir.setCheckIndexOnClose(false);
IndexWriter iw = createFastIndexWriter(dir, maxDoc);
Field idField = newStringField("id", "", Field.Store.NO);
@@ -685,11 +686,20 @@ public abstract class BaseLucene80DocValuesFormatTestCase
iw.forceMerge(1, true); // Single segment to force large enough structures
iw.commit();
iw.close();
-
- assertDVIterate(dir);
+ if (VERBOSE) {
+ System.out.println("index created " + (System.nanoTime() - start) / 1e9 + " s");
+ }
+ start = System.nanoTime();
+ assertDVIterate(dir, 16);
+ if (VERBOSE) {
+ System.out.println("assertDVIterate " + (System.nanoTime() - start) / 1e9 + " s");
+ }
+ start = System.nanoTime();
assertDVAdvance(
dir, rarely() ? 1 : 7); // 1 is heavy (~20 s), so we do it rarely. 7 is a lot faster (8 s)
-
+ if (VERBOSE) {
+ System.out.println("assertDVAdvance " + (System.nanoTime() - start) / 1e9 + " s");
+ }
dir.close();
}
@@ -823,7 +833,7 @@ public abstract class BaseLucene80DocValuesFormatTestCase
writer.close();
// compare
- assertDVIterate(dir);
+ assertDVIterate(dir, 1);
assertDVAdvance(
dir, 1); // Tests all jump-lengths from 1 to maxDoc (quite slow ~= 1 minute for 200K docs)
@@ -838,7 +848,7 @@ public abstract class BaseLucene80DocValuesFormatTestCase
LeafReader r = context.reader();
StoredFields storedFields = r.storedFields();
- for (int jump = jumpStep; jump < r.maxDoc(); jump += jumpStep) {
+ for (int jump = jumpStep; jump < r.maxDoc(); jump += jumpStep * random().nextInt(1, 10)) {
// Create a new instance each time to ensure jumps from the beginning
NumericDocValues docValues = DocValues.getNumeric(r, "dv");
for (int docID = 0; docID < r.maxDoc(); docID += jump) {
@@ -861,6 +871,13 @@ public abstract class BaseLucene80DocValuesFormatTestCase
Long.parseLong(storedValue),
docValues.longValue());
}
+ if (random().nextInt(10) == 3) {
+ // sometimes skip more
+ int skip = (r.maxDoc() - docID) / 3;
+ if (skip > 0) {
+ docID += random().nextInt(skip);
+ }
+ }
}
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
index 489ee868139..ea51b30b3da 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90DocValuesFormat.java
@@ -601,7 +601,7 @@ public class TestLucene90DocValuesFormat extends BaseCompressingDocValuesFormatT
iw.commit();
iw.close();
- assertDVIterate(dir);
+ assertDVIterate(dir, 4);
assertDVAdvance(
dir, rarely() ? 1 : 7); // 1 is heavy (~20 s), so we do it rarely. 7 is a lot faster (8 s)
@@ -740,7 +740,7 @@ public class TestLucene90DocValuesFormat extends BaseCompressingDocValuesFormatT
writer.close();
// compare
- assertDVIterate(dir);
+ assertDVIterate(dir, 4);
assertDVAdvance(
dir, 1); // Tests all jump-lengths from 1 to maxDoc (quite slow ~= 1 minute for 200K docs)
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/LegacyBaseDocValuesFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/LegacyBaseDocValuesFormatTestCase.java
index 78e36dcd1fc..2c15fa0f558 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/LegacyBaseDocValuesFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/LegacyBaseDocValuesFormatTestCase.java
@@ -1386,12 +1386,13 @@ public abstract class LegacyBaseDocValuesFormatTestCase extends BaseIndexFileFor
writer.close();
// compare
- assertDVIterate(dir);
+ int skip = minDocs < 1000 ? 0 : minDocs < 10000 ? 1 : 2;
+ assertDVIterate(dir, skip);
dir.close();
}
// Asserts equality of stored value vs. DocValue by iterating DocValues one at a time
- protected void assertDVIterate(Directory dir) throws IOException {
+ protected void assertDVIterate(Directory dir, int skip) throws IOException {
DirectoryReader ir = maybeWrapWithMergingReader(DirectoryReader.open(dir));
TestUtil.checkReader(ir);
for (LeafReaderContext context : ir.leaves()) {
@@ -1408,6 +1409,15 @@ public abstract class LegacyBaseDocValuesFormatTestCase extends BaseIndexFileFor
assertEquals(Long.parseLong(storedValue), docValues.longValue());
docValues.nextDoc();
}
+ if (random().nextInt(10) == 3) {
+ // Sometimes skip ahead. We used to exhaustively check every doc, but it is too
+ // time-consuming
+ int skipAmount = skip * (r.maxDoc() - i) / 10;
+ if (skipAmount > 0) {
+ i += random().nextInt(skipAmount);
+ docValues.advance(i + 1);
+ }
+ }
}
assertEquals(DocIdSetIterator.NO_MORE_DOCS, docValues.docID());
}