You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by jp...@apache.org on 2023/12/19 16:33:53 UTC
(lucene) 01/03: Beef up `Terms#intersect` checks in `CheckIndex`. (#12926)
This is an automated email from the ASF dual-hosted git repository.
jpountz pushed a commit to branch branch_9x
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit ce4e78316fc4e41ecf2c0514b040afd56330a9cb
Author: Adrien Grand <jp...@gmail.com>
AuthorDate: Tue Dec 19 11:17:38 2023 +0100
Beef up `Terms#intersect` checks in `CheckIndex`. (#12926)
Now also testing what happens with a non-null `startTerm`. This found bugs in
`DirectPostingsFormat`.
---
.../lucene/codecs/memory/DirectPostingsFormat.java | 18 ++++--
.../java/org/apache/lucene/index/CheckIndex.java | 72 ++++++++++++++++------
2 files changed, 65 insertions(+), 25 deletions(-)
diff --git a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
index dfd0291019c..7b0e53d35b5 100644
--- a/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
+++ b/lucene/codecs/src/java/org/apache/lucene/codecs/memory/DirectPostingsFormat.java
@@ -991,7 +991,11 @@ public final class DirectPostingsFormat extends PostingsFormat {
while (label > states[i].transitionMax) {
states[i].transitionUpto++;
- assert states[i].transitionUpto < states[i].transitionCount;
+ if (states[i].transitionUpto >= states[i].transitionCount) {
+ // All transitions compare less than the required label
+ break;
+ }
+
compiledAutomaton.automaton.getNextTransition(states[i].transition);
states[i].transitionMin = states[i].transition.min;
states[i].transitionMax = states[i].transition.max;
@@ -1116,12 +1120,14 @@ public final class DirectPostingsFormat extends PostingsFormat {
}
}
- final int termOffset = termOffsets[termOrd];
- final int termLen = termOffsets[1 + termOrd] - termOffset;
+ if (termOrd >= 0) {
+ final int termOffset = termOffsets[termOrd];
+ final int termLen = termOffsets[1 + termOrd] - termOffset;
- if (termOrd >= 0 && !startTerm.equals(new BytesRef(termBytes, termOffset, termLen))) {
- stateUpto -= skipUpto;
- termOrd--;
+ if (!startTerm.equals(new BytesRef(termBytes, termOffset, termLen))) {
+ stateUpto -= skipUpto;
+ termOrd--;
+ }
}
// if (DEBUG) {
// System.out.println(" loop end; return termOrd=" + termOrd + " stateUpto=" +
diff --git a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
index 9eed2526396..5d5772fbc33 100644
--- a/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
+++ b/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
@@ -2322,31 +2322,26 @@ public final class CheckIndex implements Closeable {
}
// Test Terms#intersect
- TermsEnum allTerms = terms.iterator();
// An automaton that should match a good number of terms
- Automaton a =
+ Automaton automaton =
Operations.concatenate(
Arrays.asList(
Automata.makeAnyBinary(),
Automata.makeCharRange('a', 'e'),
Automata.makeAnyBinary()));
- a = Operations.determinize(a, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
- CompiledAutomaton ca = new CompiledAutomaton(a);
- ByteRunAutomaton runAutomaton = new ByteRunAutomaton(a);
- TermsEnum filteredTerms = terms.intersect(ca, null);
- for (BytesRef term = allTerms.next(); term != null; term = allTerms.next()) {
- if (runAutomaton.run(term.bytes, term.offset, term.length)) {
- BytesRef filteredTerm = filteredTerms.next();
- if (Objects.equals(term, filteredTerm) == false) {
- throw new CheckIndexException(
- "Expected next filtered term: " + term + ", but got " + filteredTerm);
- }
- }
- }
- BytesRef filteredTerm = filteredTerms.next();
- if (filteredTerm != null) {
- throw new CheckIndexException("Expected exhausted TermsEnum, but got " + filteredTerm);
- }
+ BytesRef startTerm = null;
+ checkTermsIntersect(terms, automaton, startTerm);
+
+ startTerm = new BytesRef();
+ checkTermsIntersect(terms, automaton, startTerm);
+
+ automaton = Automata.makeAnyBinary();
+ startTerm = new BytesRef(new byte[] {'l'});
+ checkTermsIntersect(terms, automaton, startTerm);
+
+ // a term that likely compares greater than every other term in the dictionary
+ startTerm = new BytesRef(new byte[] {(byte) 0xFF, (byte) 0xFF, (byte) 0xFF, (byte) 0xFF});
+ checkTermsIntersect(terms, automaton, startTerm);
}
}
@@ -2387,6 +2382,45 @@ public final class CheckIndex implements Closeable {
return status;
}
+ private static void checkTermsIntersect(Terms terms, Automaton automaton, BytesRef startTerm)
+ throws IOException {
+ TermsEnum allTerms = terms.iterator();
+ automaton = Operations.determinize(automaton, Operations.DEFAULT_DETERMINIZE_WORK_LIMIT);
+ CompiledAutomaton compiledAutomaton = new CompiledAutomaton(automaton);
+ ByteRunAutomaton runAutomaton = new ByteRunAutomaton(automaton);
+ TermsEnum filteredTerms = terms.intersect(compiledAutomaton, startTerm);
+ BytesRef term;
+ if (startTerm != null) {
+ switch (allTerms.seekCeil(startTerm)) {
+ case FOUND:
+ term = allTerms.next();
+ break;
+ case NOT_FOUND:
+ term = allTerms.term();
+ break;
+ case END:
+ default:
+ term = null;
+ break;
+ }
+ } else {
+ term = allTerms.next();
+ }
+ for (; term != null; term = allTerms.next()) {
+ if (runAutomaton.run(term.bytes, term.offset, term.length)) {
+ BytesRef filteredTerm = filteredTerms.next();
+ if (Objects.equals(term, filteredTerm) == false) {
+ throw new CheckIndexException(
+ "Expected next filtered term: " + term + ", but got " + filteredTerm);
+ }
+ }
+ }
+ BytesRef filteredTerm = filteredTerms.next();
+ if (filteredTerm != null) {
+ throw new CheckIndexException("Expected exhausted TermsEnum, but got " + filteredTerm);
+ }
+ }
+
/**
* For use in tests only.
*