You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2023/12/12 13:15:05 UTC

(lucene) branch branch_9x updated: Check `Terms#intersect` in CheckIndex. (#12925)

This is an automated email from the ASF dual-hosted git repository.

jpountz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git


The following commit(s) were added to refs/heads/branch_9x by this push:
     new cb438575237 Check `Terms#intersect` in CheckIndex. (#12925)
cb438575237 is described below

commit cb438575237a220b63b88f6d712b3b900947f6e2
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Tue Dec 12 14:08:44 2023 +0100

    Check `Terms#intersect` in CheckIndex. (#12925)
    
    This commit adds coverage to `Terms#intersect` to `CheckIndex` and indexes
    `LineFileDocs` in `BasePostingsFormatTestCase` to get some coverage with
    real-world data.
    
    With this change, `TestLucene90PostingsFormat` now exhibits #12895.
---
 .../java/org/apache/lucene/index/CheckIndex.java   | 32 ++++++++++++++++++++++
 .../tests/index/BasePostingsFormatTestCase.java    | 27 ++++++++++++++++++
 2 files changed, 59 insertions(+)

diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
index 3e90c432ebb..9eed2526396 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
@@ -83,6 +83,11 @@ import org.apache.lucene.util.NamedThreadFactory;
 import org.apache.lucene.util.StringHelper;
 import org.apache.lucene.util.SuppressForbidden;
 import org.apache.lucene.util.Version;
+import org.apache.lucene.util.automaton.Automata;
+import org.apache.lucene.util.automaton.Automaton;
+import org.apache.lucene.util.automaton.ByteRunAutomaton;
+import org.apache.lucene.util.automaton.CompiledAutomaton;
+import org.apache.lucene.util.automaton.Operations;
 
 /**
  * Basic tool and API to check the health of an index and write a new segments file that removes
@@ -2315,6 +2320,33 @@ public final class CheckIndex implements Closeable {
             }
           }
         }
+
+        // Test Terms#intersect
+        TermsEnum allTerms = terms.iterator();
+        // An automaton that should match a good number of terms
+        Automaton a =
+            Operations.concatenate(
+                Arrays.asList(
+                    Automata.makeAnyBinary(),
+                    Automata.makeCharRange('a', 'e'),
+                    Automata.makeAnyBinary()));
+        a = Operations.determinize(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+        CompiledAutomaton ca = new CompiledAutomaton(a);
+        ByteRunAutomaton runAutomaton = new ByteRunAutomaton(a);
+        TermsEnum filteredTerms = terms.intersect(ca, null);
+        for (BytesRef term = allTerms.next(); term != null; term = allTerms.next()) {
+          if (runAutomaton.run(term.bytes, term.offset, term.length)) {
+            BytesRef filteredTerm = filteredTerms.next();
+            if (Objects.equals(term, filteredTerm) == false) {
+              throw new CheckIndexException(
+                  "Expected next filtered term: " + term + ", but got " + filteredTerm);
+            }
+          }
+        }
+        BytesRef filteredTerm = filteredTerms.next();
+        if (filteredTerm != null) {
+          throw new CheckIndexException("Expected exhausted TermsEnum, but got " + filteredTerm);
+        }
       }
     }
 
diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java
index d98fa17b974..181be045fd1 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/index/BasePostingsFormatTestCase.java
@@ -38,6 +38,7 @@ import org.apache.lucene.codecs.NormsProducer;
 import org.apache.lucene.codecs.PostingsFormat;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
+import org.apache.lucene.document.Field.Store;
 import org.apache.lucene.document.FieldType;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.document.TextField;
@@ -47,6 +48,7 @@ import org.apache.lucene.index.IndexOptions;
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.MultiTerms;
 import org.apache.lucene.index.PostingsEnum;
@@ -1610,4 +1612,29 @@ public abstract class BasePostingsFormatTestCase extends BaseIndexFileFormatTest
       }
     }
   }
+
+  /** Test realistic data, which is often better at uncovering real bugs. */
+  @Nightly // this test takes a few seconds
+  public void testLineFileDocs() throws IOException {
+    // Use a FS dir and a non-randomized IWC to not slow down indexing
+    try (Directory dir = newFSDirectory(createTempDir())) {
+      try (LineFileDocs docs = new LineFileDocs(random());
+          IndexWriter w = new IndexWriter(dir, new IndexWriterConfig())) {
+        final int numDocs = atLeast(10_000);
+        for (int i = 0; i < numDocs; ++i) {
+          // Only keep the body field, and don't index term vectors on it, we only care about
+          // postings
+          Document doc = docs.nextDoc();
+          IndexableField body = doc.getField("body");
+          assertNotNull(body);
+          assertNotNull(body.stringValue());
+          assertNotEquals(IndexOptions.NONE, body.fieldType().indexOptions());
+          body = new TextField("body", body.stringValue(), Store.NO);
+          w.addDocument(Collections.singletonList(body));
+        }
+        w.forceMerge(1);
+      }
+      TestUtil.checkIndex(dir);
+    }
+  }
 }