You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2024/01/19 15:30:33 UTC
(lucene) branch branch_9x updated: Run BWC indices generation code together with unittest (#13023)
This is an automated email from the ASF dual-hosted git repository.
simonw pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
The following commit(s) were added to refs/heads/branch_9x by this push:
new c8980471e12 Run BWC indices generation code together with unittest (#13023)
c8980471e12 is described below
commit c8980471e12efdb72bbdb2aac21ca5c5541e8b46
Author: Simon Willnauer <si...@apache.org>
AuthorDate: Fri Jan 19 15:47:01 2024 +0100
Run BWC indices generation code together with unittest (#13023)
This change runs the current BWC indices generation code together
with the unittest to catch issues with the generated indices earliy.
Each generation method runs a sanity check on the generated indices.
---
.../backward_index/TestBackwardsCompatibility.java | 129 ++++++++++++---------
1 file changed, 73 insertions(+), 56 deletions(-)
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
index 6835eb2cfed..ec57b945815 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_index/TestBackwardsCompatibility.java
@@ -176,11 +176,19 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
private static final float[] KNN_VECTOR = {0.2f, -0.1f, 0.1f};
public void testCreateCFS() throws IOException {
- createIndex("index.cfs", true, false);
+ Path indexDir = getIndexDir().resolve("index.cfs");
+ Files.deleteIfExists(indexDir);
+ try (Directory dir = newFSDirectory(indexDir)) {
+ createIndex(dir, true, false);
+ }
}
public void testCreateNoCFS() throws IOException {
- createIndex("index.nocfs", false, false);
+ Path indexDir = getIndexDir().resolve("index.nocfs");
+ Files.deleteIfExists(indexDir);
+ try (Directory dir = newFSDirectory(indexDir)) {
+ createIndex(dir, false, false);
+ }
}
// These are only needed for the special upgrade test to verify
@@ -189,11 +197,26 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
// "old" segment format, version is unimportant:
public void testCreateSingleSegmentCFS() throws IOException {
- createIndex("index.singlesegment-cfs", true, true);
+ Path indexDir = getIndexDir().resolve("index.singlesegment-cfs");
+ Files.deleteIfExists(indexDir);
+ try (Directory dir = newFSDirectory(indexDir)) {
+ createIndex(dir, true, true);
+ }
}
public void testCreateSingleSegmentNoCFS() throws IOException {
- createIndex("index.singlesegment-nocfs", false, true);
+ Path indexDir = getIndexDir().resolve("index.singlesegment-nocfs");
+ Files.deleteIfExists(indexDir);
+ try (Directory dir = newFSDirectory(indexDir)) {
+ createIndex(dir, false, true);
+ }
+ }
+
+ public void testCreateIndexInternal() throws IOException {
+ try (Directory dir = newDirectory()) {
+ createIndex(dir, random().nextBoolean(), false);
+ searchIndex(dir, Version.LATEST.toString(), Version.MIN_SUPPORTED_MAJOR, Version.LATEST);
+ }
}
private Path getIndexDir() {
@@ -226,24 +249,23 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
analyzer.setMaxTokenLength(TestUtil.nextInt(random(), 1, IndexWriter.MAX_TERM_LENGTH));
IndexWriterConfig conf =
- new IndexWriterConfig(analyzer).setMergePolicy(mp).setUseCompoundFile(false);
+ new IndexWriterConfig(analyzer)
+ .setMergePolicy(mp)
+ .setCodec(TestUtil.getDefaultCodec())
+ .setUseCompoundFile(false);
IndexWriter writer = new IndexWriter(dir, conf);
LineFileDocs docs = new LineFileDocs(new Random(0));
- Field docIdDV = null;
- Field titleDVField = null;
for (int i = 0; i < 50; i++) {
- Document doc = docs.nextDoc();
- if (docIdDV == null) {
- docIdDV = new NumericDocValuesField("docid_intDV", 0);
- doc.add(docIdDV);
- }
- docIdDV.setLongValue(doc.getField("docid_int").numericValue().longValue());
- if (titleDVField == null) {
- titleDVField = new SortedDocValuesField("titleDV", new BytesRef());
- doc.add(titleDVField);
- }
- titleDVField.setBytesValue(new BytesRef(doc.getField("title").stringValue()));
+ Document doc = TestUtil.cloneDocument(docs.nextDoc());
+ doc.add(
+ new NumericDocValuesField(
+ "docid_intDV", doc.getField("docid_int").numericValue().longValue()));
+ doc.add(
+ new SortedDocValuesField("titleDV", new BytesRef(doc.getField("title").stringValue())));
writer.addDocument(doc);
+ if (i % 10 == 0) { // commit every 10 documents
+ writer.commit();
+ }
}
docs.close();
writer.close();
@@ -281,20 +303,16 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
IndexWriterConfig conf = new IndexWriterConfig(analyzer);
conf.setMergePolicy(mp);
conf.setUseCompoundFile(false);
+ conf.setCodec(TestUtil.getDefaultCodec());
conf.setIndexSort(new Sort(new SortField("dateDV", SortField.Type.LONG, true)));
IndexWriter writer = new IndexWriter(dir, conf);
LineFileDocs docs = new LineFileDocs(new Random(0));
SimpleDateFormat parser = new SimpleDateFormat("yyyy-MM-dd", Locale.ROOT);
parser.setTimeZone(TimeZone.getTimeZone("UTC"));
ParsePosition position = new ParsePosition(0);
- Field docIdDV = null;
- Field dateDVField = null;
- Field titleDVField = null;
-
for (int i = 0; i < 50; i++) {
- Document doc = docs.nextDoc();
+ Document doc = TestUtil.cloneDocument(docs.nextDoc());
String dateString = doc.get("date");
-
position.setIndex(0);
Date date = parser.parse(dateString, position);
if (position.getErrorIndex() != -1) {
@@ -303,23 +321,13 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
if (position.getIndex() != dateString.length()) {
throw new AssertionError("failed to parse \"" + dateString + "\" as date");
}
- if (dateDVField == null) {
- dateDVField = new NumericDocValuesField("dateDV", 0l);
- doc.add(dateDVField);
- }
- dateDVField.setLongValue(date.getTime());
- if (docIdDV == null) {
- docIdDV = new NumericDocValuesField("docid_intDV", 0);
- doc.add(docIdDV);
- }
-
- docIdDV.setLongValue(doc.getField("docid_int").numericValue().longValue());
- if (titleDVField == null) {
- titleDVField = new SortedDocValuesField("titleDV", new BytesRef());
- doc.add(titleDVField);
- }
- titleDVField.setBytesValue(new BytesRef(doc.getField("title").stringValue()));
- if (i == 250) {
+ doc.add(
+ new NumericDocValuesField(
+ "docid_intDV", doc.getField("docid_int").numericValue().longValue()));
+ doc.add(
+ new SortedDocValuesField("titleDV", new BytesRef(doc.getField("title").stringValue())));
+ doc.add(new NumericDocValuesField("dateDV", date.getTime()));
+ if (i % 10 == 0) { // commit every 10 documents
writer.commit();
}
writer.addDocument(doc);
@@ -345,13 +353,26 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
// Creates an index with DocValues updates
- public void testCreateIndexWithDocValuesUpdates() throws Exception {
+ public void testCreateIndexWithDocValuesUpdates() throws IOException {
Path indexDir = getIndexDir().resolve("dvupdates");
Files.deleteIfExists(indexDir);
- Directory dir = newFSDirectory(indexDir);
+ try (Directory dir = newFSDirectory(indexDir)) {
+ createIndexWithDocValuesUpdates(dir);
+ searchDocValuesUpdatesIndex(dir);
+ }
+ }
+ public void testCreateIndexWithDocValuesUpdatesInternal() throws IOException {
+ try (Directory dir = newDirectory()) {
+ createIndexWithDocValuesUpdates(dir);
+ searchDocValuesUpdatesIndex(dir);
+ }
+ }
+
+ private void createIndexWithDocValuesUpdates(Directory dir) throws IOException {
IndexWriterConfig conf =
new IndexWriterConfig(new MockAnalyzer(random()))
+ .setCodec(TestUtil.getDefaultCodec())
.setUseCompoundFile(false)
.setMergePolicy(NoMergePolicy.INSTANCE);
IndexWriter writer = new IndexWriter(dir, conf);
@@ -385,10 +406,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
updateBinary(writer, "21", "bdv1", "bdv1_c", 100L);
writer.commit();
updateNumeric(writer, "22", "ndv1", "ndv1_c", 200L); // update the field again
- writer.commit();
-
writer.close();
- dir.close();
}
public void testCreateEmptyIndex() throws Exception {
@@ -1211,6 +1229,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
final Bits liveDocs = MultiBits.getLiveDocs(reader);
assertNotNull(liveDocs);
+
StoredFields storedFields = reader.storedFields();
TermVectors termVectors = reader.termVectors();
@@ -1575,10 +1594,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
reader.close();
}
- public void createIndex(String dirName, boolean doCFS, boolean fullyMerged) throws IOException {
- Path indexDir = getIndexDir().resolve(dirName);
- Files.deleteIfExists(indexDir);
- Directory dir = newFSDirectory(indexDir);
+ public void createIndex(Directory dir, boolean doCFS, boolean fullyMerged) throws IOException {
LogByteSizeMergePolicy mp = new LogByteSizeMergePolicy();
mp.setNoCFSRatio(doCFS ? 1.0 : 0.0);
mp.setMaxCFSSegmentSizeMB(Double.POSITIVE_INFINITY);
@@ -1620,8 +1636,6 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
writer.deleteDocuments(searchTerm);
writer.close();
}
-
- dir.close();
}
private void addDoc(IndexWriter writer, int id) throws IOException {
@@ -1813,7 +1827,7 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
}
}
- public void verifyUsesDefaultCodec(Directory dir, String name) throws Exception {
+ public void verifyUsesDefaultCodec(Directory dir, String name) throws IOException {
DirectoryReader r = DirectoryReader.open(dir);
for (LeafReaderContext context : r.leaves()) {
SegmentReader air = (SegmentReader) context.reader();
@@ -2056,9 +2070,13 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
public void testDocValuesUpdates() throws Exception {
Path oldIndexDir = createTempDir("dvupdates");
TestUtil.unzip(getDataInputStream(dvUpdatesIndex), oldIndexDir);
- Directory dir = newFSDirectory(oldIndexDir);
- verifyUsesDefaultCodec(dir, dvUpdatesIndex);
+ try (Directory dir = newFSDirectory(oldIndexDir)) {
+ searchDocValuesUpdatesIndex(dir);
+ }
+ }
+ private void searchDocValuesUpdatesIndex(Directory dir) throws IOException {
+ verifyUsesDefaultCodec(dir, dvUpdatesIndex);
verifyDocValues(dir);
// update fields and verify index
@@ -2078,7 +2096,6 @@ public class TestBackwardsCompatibility extends LuceneTestCase {
verifyDocValues(dir);
writer.close();
- dir.close();
}
public void testDeletes() throws Exception {