You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/07/28 13:28:27 UTC
svn commit: r1366643 [11/19] - in /lucene/dev/branches/lucene3312: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/copyright/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/ dev-tools/maven/
dev-tools/maven/lucene/benchmark/ dev-tools/mave...
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Sat Jul 28 11:27:51 2012
@@ -842,7 +842,7 @@ public class IndexWriter implements Clos
if (hitOOM) {
rollbackInternal();
} else {
- closeInternal(waitForMerges, !hitOOM);
+ closeInternal(waitForMerges, true);
}
}
}
@@ -870,7 +870,7 @@ public class IndexWriter implements Clos
}
private void closeInternal(boolean waitForMerges, boolean doFlush) throws IOException {
-
+ boolean interrupted = false;
try {
if (pendingCommit != null) {
@@ -883,26 +883,57 @@ public class IndexWriter implements Clos
docWriter.close();
- // Only allow a new merge to be triggered if we are
- // going to wait for merges:
- if (doFlush) {
- flush(waitForMerges, true);
- } else {
- docWriter.abort(); // already closed
- }
-
- if (waitForMerges)
- // Give merge scheduler last chance to run, in case
- // any pending merges are waiting:
- mergeScheduler.merge(this);
-
- mergePolicy.close();
-
- synchronized(this) {
- finishMerges(waitForMerges);
- stopMerges = true;
+ try {
+ // Only allow a new merge to be triggered if we are
+ // going to wait for merges:
+ if (doFlush) {
+ flush(waitForMerges, true);
+ } else {
+ docWriter.abort(); // already closed
+ }
+
+ } finally {
+ try {
+ // clean up merge scheduler in all cases, although flushing may have failed:
+ interrupted = Thread.interrupted();
+
+ if (waitForMerges) {
+ try {
+ // Give merge scheduler last chance to run, in case
+ // any pending merges are waiting:
+ mergeScheduler.merge(this);
+ } catch (ThreadInterruptedException tie) {
+ // ignore any interruption, does not matter
+ interrupted = true;
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "interrupted while waiting for final merges");
+ }
+ }
+ }
+
+ synchronized(this) {
+ for (;;) {
+ try {
+ finishMerges(waitForMerges && !interrupted);
+ break;
+ } catch (ThreadInterruptedException tie) {
+ // by setting the interrupted status, the
+ // next call to finishMerges will pass false,
+ // so it will not wait
+ interrupted = true;
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "interrupted while waiting for merges to finish");
+ }
+ }
+ }
+ stopMerges = true;
+ }
+
+ } finally {
+ // shutdown policy, scheduler and all threads (this call is not interruptible):
+ IOUtils.closeWhileHandlingException(mergePolicy, mergeScheduler);
+ }
}
- mergeScheduler.close();
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "now call final commit()");
@@ -943,6 +974,8 @@ public class IndexWriter implements Clos
}
}
}
+ // finally, restore interrupt status:
+ if (interrupted) Thread.currentThread().interrupt();
}
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java Sat Jul 28 11:27:51 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.Closeable;
import java.io.IOException;
/** <p>Expert: {@link IndexWriter} uses an instance
@@ -26,7 +27,7 @@ import java.io.IOException;
*
* @lucene.experimental
*/
-public abstract class MergeScheduler {
+public abstract class MergeScheduler implements Closeable {
/** Run the merges provided by {@link IndexWriter#getNextMerge()}. */
public abstract void merge(IndexWriter writer) throws IOException;
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiDocsAndPositionsEnum.java Sat Jul 28 11:27:51 2012
@@ -20,6 +20,7 @@ package org.apache.lucene.index;
import org.apache.lucene.util.BytesRef;
import java.io.IOException;
+import java.util.Arrays;
/**
* Exposes flex API, merged from flex API of sub-segments.
@@ -55,6 +56,7 @@ public final class MultiDocsAndPositions
this.subs[i].slice = subs[i].slice;
}
upto = -1;
+ doc = -1;
current = null;
return this;
}
@@ -69,6 +71,7 @@ public final class MultiDocsAndPositions
@Override
public int freq() throws IOException {
+ assert current != null;
return current.freq();
}
@@ -148,6 +151,16 @@ public final class MultiDocsAndPositions
public final static class EnumWithSlice {
public DocsAndPositionsEnum docsAndPositionsEnum;
public ReaderSlice slice;
+
+ @Override
+ public String toString() {
+ return slice.toString()+":"+docsAndPositionsEnum;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "MultiDocsAndPositionsEnum(" + Arrays.toString(getSubs()) + ")";
}
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiDocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiDocsEnum.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiDocsEnum.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/MultiDocsEnum.java Sat Jul 28 11:27:51 2012
@@ -52,6 +52,7 @@ public final class MultiDocsEnum extends
this.subs[i].slice = subs[i].slice;
}
upto = -1;
+ doc = -1;
current = null;
return this;
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/index/TieredMergePolicy.java Sat Jul 28 11:27:51 2012
@@ -168,7 +168,7 @@ public class TieredMergePolicy extends M
/** @see #setFloorSegmentMB */
public double getFloorSegmentMB() {
- return floorSegmentBytes/1024*1024.;
+ return floorSegmentBytes/(1024*1024.);
}
/** When forceMergeDeletes is called, we only merge away a
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java Sat Jul 28 11:27:51 2012
@@ -358,49 +358,17 @@ public class BooleanQuery extends Query
final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()];
for (int i = 0; i < docsAndFreqs.length; i++) {
final TermWeight weight = (TermWeight) weights.get(i);
- final TermsEnum termsEnum = weight.getTermsEnum(context);
- if (termsEnum == null) {
+ final Scorer scorer = weight.scorer(context, true, false, acceptDocs);
+ if (scorer == null) {
return null;
}
- final ExactSimScorer docScorer = weight.createDocScorer(context);
- final DocsEnum docsAndFreqsEnum = termsEnum.docs(acceptDocs, null, true);
- if (docsAndFreqsEnum == null) {
- // TODO: we could carry over TermState from the
- // terms we already seek'd to, to save re-seeking
- // to make the match-only scorer, but it's likely
- // rare that BQ mixes terms from omitTf and
- // non-omitTF fields:
-
- // At least one sub cannot provide freqs; abort
- // and fallback to full match-only scorer:
- return createMatchOnlyConjunctionTermScorer(context, acceptDocs);
- }
-
- docsAndFreqs[i] = new DocsAndFreqs(docsAndFreqsEnum,
- docsAndFreqsEnum,
- termsEnum.docFreq(), docScorer);
- }
- return new ConjunctionTermScorer(this, disableCoord ? 1.0f : coord(
- docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs);
- }
-
- private Scorer createMatchOnlyConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs)
- throws IOException {
-
- final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()];
- for (int i = 0; i < docsAndFreqs.length; i++) {
- final TermWeight weight = (TermWeight) weights.get(i);
- final TermsEnum termsEnum = weight.getTermsEnum(context);
- if (termsEnum == null) {
- return null;
+ if (scorer instanceof TermScorer) {
+ docsAndFreqs[i] = new DocsAndFreqs((TermScorer) scorer);
+ } else {
+ docsAndFreqs[i] = new DocsAndFreqs((MatchOnlyTermScorer) scorer);
}
- final ExactSimScorer docScorer = weight.createDocScorer(context);
- docsAndFreqs[i] = new DocsAndFreqs(null,
- termsEnum.docs(acceptDocs, null, false),
- termsEnum.docFreq(), docScorer);
}
-
- return new MatchOnlyConjunctionTermScorer(this, disableCoord ? 1.0f : coord(
+ return new ConjunctionTermScorer(this, disableCoord ? 1.0f : coord(
docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs);
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanScorer.java Sat Jul 28 11:27:51 2012
@@ -318,6 +318,11 @@ final class BooleanScorer extends Scorer
}
@Override
+ public float freq() throws IOException {
+ return current.coord;
+ }
+
+ @Override
public void score(Collector collector) throws IOException {
score(collector, Integer.MAX_VALUE, -1);
}
@@ -338,7 +343,8 @@ final class BooleanScorer extends Scorer
public Collection<ChildScorer> getChildren() {
List<ChildScorer> children = new ArrayList<ChildScorer>();
for (SubScorer sub = scorers; sub != null; sub = sub.next) {
- children.add(new ChildScorer(sub.scorer, sub.prohibited ? Occur.MUST_NOT.toString() : Occur.SHOULD.toString()));
+ // TODO: fix this if BQ ever sends us required clauses
+ children.add(new ChildScorer(sub.scorer, sub.prohibited ? "MUST_NOT" : "SHOULD"));
}
return children;
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/BooleanScorer2.java Sat Jul 28 11:27:51 2012
@@ -131,6 +131,11 @@ class BooleanScorer2 extends Scorer {
}
@Override
+ public float freq() throws IOException {
+ return 1;
+ }
+
+ @Override
public int docID() {
return scorer.docID();
}
@@ -310,8 +315,8 @@ class BooleanScorer2 extends Scorer {
}
@Override
- public float freq() {
- return coordinator.nrMatchers;
+ public float freq() throws IOException {
+ return countingSumScorer.freq();
}
@Override
@@ -323,13 +328,13 @@ class BooleanScorer2 extends Scorer {
public Collection<ChildScorer> getChildren() {
ArrayList<ChildScorer> children = new ArrayList<ChildScorer>();
for (Scorer s : optionalScorers) {
- children.add(new ChildScorer(s, Occur.SHOULD.toString()));
+ children.add(new ChildScorer(s, "SHOULD"));
}
for (Scorer s : prohibitedScorers) {
- children.add(new ChildScorer(s, Occur.MUST_NOT.toString()));
+ children.add(new ChildScorer(s, "MUST_NOT"));
}
for (Scorer s : requiredScorers) {
- children.add(new ChildScorer(s, Occur.MUST.toString()));
+ children.add(new ChildScorer(s, "MUST"));
}
return children;
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java Sat Jul 28 11:27:51 2012
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
import org.apache.lucene.util.ArrayUtil;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.Comparator;
@@ -136,4 +137,18 @@ class ConjunctionScorer extends Scorer {
}
return sum * coord;
}
+
+ @Override
+ public float freq() throws IOException {
+ return scorers.length;
+ }
+
+ @Override
+ public Collection<ChildScorer> getChildren() {
+ ArrayList<ChildScorer> children = new ArrayList<ChildScorer>(scorers.length);
+ for (Scorer scorer : scorers) {
+ children.add(new ChildScorer(scorer, "MUST"));
+ }
+ return children;
+ }
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java Sat Jul 28 11:27:51 2012
@@ -18,10 +18,11 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.Comparator;
import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.search.similarities.Similarity.ExactSimScorer;
import org.apache.lucene.util.ArrayUtil;
/** Scorer for conjunctions, sets of terms, all of which are required. */
@@ -91,23 +92,43 @@ class ConjunctionTermScorer extends Scor
public float score() throws IOException {
float sum = 0.0f;
for (DocsAndFreqs docs : docsAndFreqs) {
- sum += docs.docScorer.score(lastDoc, docs.docs.freq());
+ sum += docs.scorer.score();
}
return sum * coord;
}
+
+ @Override
+ public float freq() {
+ return docsAndFreqs.length;
+ }
+
+ @Override
+ public Collection<ChildScorer> getChildren() {
+ ArrayList<ChildScorer> children = new ArrayList<ChildScorer>(docsAndFreqs.length);
+ for (DocsAndFreqs docs : docsAndFreqs) {
+ children.add(new ChildScorer(docs.scorer, "MUST"));
+ }
+ return children;
+ }
static final class DocsAndFreqs {
- final DocsEnum docsAndFreqs;
final DocsEnum docs;
final int docFreq;
- final ExactSimScorer docScorer;
+ final Scorer scorer;
int doc = -1;
- DocsAndFreqs(DocsEnum docsAndFreqs, DocsEnum docs, int docFreq, ExactSimScorer docScorer) {
- this.docsAndFreqs = docsAndFreqs;
+ DocsAndFreqs(TermScorer termScorer) {
+ this(termScorer, termScorer.getDocsEnum(), termScorer.getDocFreq());
+ }
+
+ DocsAndFreqs(MatchOnlyTermScorer termScorer) {
+ this(termScorer, termScorer.getDocsEnum(), termScorer.getDocFreq());
+ }
+
+ DocsAndFreqs(Scorer scorer, DocsEnum docs, int docFreq) {
this.docs = docs;
this.docFreq = docFreq;
- this.docScorer = docScorer;
+ this.scorer = scorer;
}
}
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ConstantScoreQuery.java Sat Jul 28 11:27:51 2012
@@ -194,6 +194,11 @@ public class ConstantScoreQuery extends
}
@Override
+ public float freq() throws IOException {
+ return 1;
+ }
+
+ @Override
public int advance(int target) throws IOException {
return docIdSetIterator.advance(target);
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxQuery.java Sat Jul 28 11:27:51 2012
@@ -158,7 +158,7 @@ public class DisjunctionMaxQuery extends
for (Weight w : weights) {
// we will advance() subscorers
Scorer subScorer = w.scorer(context, true, false, acceptDocs);
- if (subScorer != null && subScorer.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+ if (subScorer != null) {
scorers[idx++] = subScorer;
}
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/DisjunctionMaxScorer.java Sat Jul 28 11:27:51 2012
@@ -17,9 +17,6 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
/**
* The Scorer for DisjunctionMaxQuery. The union of all documents generated by the the subquery scorers
@@ -27,11 +24,7 @@ import java.util.Collections;
* by the subquery scorers that generate that document, plus tieBreakerMultiplier times the sum of the scores
* for the other subqueries that generate the document.
*/
-class DisjunctionMaxScorer extends Scorer {
-
- /* The scorers for subqueries that have remaining docs, kept as a min heap by number of next doc. */
- private final Scorer[] subScorers;
- private int numScorers;
+class DisjunctionMaxScorer extends DisjunctionScorer {
/* Multiplier applied to non-maximum-scoring subqueries for a document as they are summed into the result. */
private final float tieBreakerMultiplier;
private int doc = -1;
@@ -56,15 +49,8 @@ class DisjunctionMaxScorer extends Score
*/
public DisjunctionMaxScorer(Weight weight, float tieBreakerMultiplier,
Scorer[] subScorers, int numScorers) {
- super(weight);
+ super(weight, subScorers, numScorers);
this.tieBreakerMultiplier = tieBreakerMultiplier;
- // The passed subScorers array includes only scorers which have documents
- // (DisjunctionMaxQuery takes care of that), and their nextDoc() was already
- // called.
- this.subScorers = subScorers;
- this.numScorers = numScorers;
-
- heapify();
}
@Override
@@ -114,6 +100,24 @@ class DisjunctionMaxScorer extends Score
}
@Override
+ public float freq() throws IOException {
+ int doc = subScorers[0].docID();
+ int size = numScorers;
+ return 1 + freq(1, size, doc) + freq(2, size, doc);
+ }
+
+ // Recursively iterate all subScorers that generated last doc computing sum and max
+ private int freq(int root, int size, int doc) throws IOException {
+ int freq = 0;
+ if (root < size && subScorers[root].docID() == doc) {
+ freq++;
+ freq += freq((root<<1)+1, size, doc);
+ freq += freq((root<<1)+2, size, doc);
+ }
+ return freq;
+ }
+
+ @Override
public int advance(int target) throws IOException {
if (numScorers == 0) return doc = NO_MORE_DOCS;
while (subScorers[0].docID() < target) {
@@ -128,70 +132,4 @@ class DisjunctionMaxScorer extends Score
}
return doc = subScorers[0].docID();
}
-
- // Organize subScorers into a min heap with scorers generating the earliest document on top.
- private void heapify() {
- for (int i = (numScorers >> 1) - 1; i >= 0; i--) {
- heapAdjust(i);
- }
- }
-
- /* The subtree of subScorers at root is a min heap except possibly for its root element.
- * Bubble the root down as required to make the subtree a heap.
- */
- private void heapAdjust(int root) {
- Scorer scorer = subScorers[root];
- int doc = scorer.docID();
- int i = root;
- while (i <= (numScorers >> 1) - 1) {
- int lchild = (i << 1) + 1;
- Scorer lscorer = subScorers[lchild];
- int ldoc = lscorer.docID();
- int rdoc = Integer.MAX_VALUE, rchild = (i << 1) + 2;
- Scorer rscorer = null;
- if (rchild < numScorers) {
- rscorer = subScorers[rchild];
- rdoc = rscorer.docID();
- }
- if (ldoc < doc) {
- if (rdoc < ldoc) {
- subScorers[i] = rscorer;
- subScorers[rchild] = scorer;
- i = rchild;
- } else {
- subScorers[i] = lscorer;
- subScorers[lchild] = scorer;
- i = lchild;
- }
- } else if (rdoc < doc) {
- subScorers[i] = rscorer;
- subScorers[rchild] = scorer;
- i = rchild;
- } else {
- return;
- }
- }
- }
-
- // Remove the root Scorer from subScorers and re-establish it as a heap
- private void heapRemoveRoot() {
- if (numScorers == 1) {
- subScorers[0] = null;
- numScorers = 0;
- } else {
- subScorers[0] = subScorers[numScorers - 1];
- subScorers[numScorers - 1] = null;
- --numScorers;
- heapAdjust(0);
- }
- }
-
- @Override
- public Collection<ChildScorer> getChildren() {
- final ChildScorer[] children = new ChildScorer[numScorers];
- for (int i = 0; i< numScorers; i++) {
- children[i] = new ChildScorer(subScorers[i], BooleanClause.Occur.SHOULD.toString());
- }
- return Collections.unmodifiableCollection(Arrays.asList(children));
- }
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/DisjunctionSumScorer.java Sat Jul 28 11:27:51 2012
@@ -20,42 +20,20 @@ package org.apache.lucene.search;
import java.util.List;
import java.io.IOException;
-import org.apache.lucene.util.ScorerDocQueue;
-
/** A Scorer for OR like queries, counterpart of <code>ConjunctionScorer</code>.
* This Scorer implements {@link Scorer#advance(int)} and uses advance() on the given Scorers.
*/
-class DisjunctionSumScorer extends Scorer {
- /** The number of subscorers. */
- private final int nrScorers;
-
- /** The subscorers. */
- protected final List<Scorer> subScorers;
-
+class DisjunctionSumScorer extends DisjunctionScorer {
/** The minimum number of scorers that should match. */
private final int minimumNrMatchers;
- /** The scorerDocQueue contains all subscorers ordered by their current doc(),
- * with the minimum at the top.
- * <br>The scorerDocQueue is initialized the first time nextDoc() or advance() is called.
- * <br>An exhausted scorer is immediately removed from the scorerDocQueue.
- * <br>If less than the minimumNrMatchers scorers
- * remain in the scorerDocQueue nextDoc() and advance() return false.
- * <p>
- * After each to call to nextDoc() or advance()
- * <code>currentSumScore</code> is the total score of the current matching doc,
- * <code>nrMatchers</code> is the number of matching scorers,
- * and all scorers are after the matching doc, or are exhausted.
- */
- private final ScorerDocQueue scorerDocQueue;
-
/** The document number of the current match. */
- private int currentDoc = -1;
+ private int doc = -1;
/** The number of subscorers that provide the current match. */
protected int nrMatchers = -1;
- private double currentScore = Float.NaN;
+ private double score = Float.NaN;
/** Construct a <code>DisjunctionScorer</code>.
* @param weight The weight to be used.
@@ -69,21 +47,16 @@ class DisjunctionSumScorer extends Score
* it more efficient to use <code>ConjunctionScorer</code>.
*/
public DisjunctionSumScorer(Weight weight, List<Scorer> subScorers, int minimumNrMatchers) throws IOException {
- super(weight);
-
- nrScorers = subScorers.size();
+ super(weight, subScorers.toArray(new Scorer[subScorers.size()]), subScorers.size());
if (minimumNrMatchers <= 0) {
throw new IllegalArgumentException("Minimum nr of matchers must be positive");
}
- if (nrScorers <= 1) {
+ if (numScorers <= 1) {
throw new IllegalArgumentException("There must be at least 2 subScorers");
}
this.minimumNrMatchers = minimumNrMatchers;
- this.subScorers = subScorers;
-
- scorerDocQueue = initScorerDocQueue();
}
/** Construct a <code>DisjunctionScorer</code>, using one as the minimum number
@@ -93,119 +66,66 @@ class DisjunctionSumScorer extends Score
this(weight, subScorers, 1);
}
- /** Called the first time nextDoc() or advance() is called to
- * initialize <code>scorerDocQueue</code>.
- * @return
- */
- private ScorerDocQueue initScorerDocQueue() throws IOException {
- final ScorerDocQueue docQueue = new ScorerDocQueue(nrScorers);
- for (final Scorer se : subScorers) {
- if (se.nextDoc() != NO_MORE_DOCS) {
- docQueue.insert(se);
- }
- }
- return docQueue;
- }
-
- /** Scores and collects all matching documents.
- * @param collector The collector to which all matching documents are passed through.
- */
- @Override
- public void score(Collector collector) throws IOException {
- collector.setScorer(this);
- while (nextDoc() != NO_MORE_DOCS) {
- collector.collect(currentDoc);
- }
- }
-
- /** Expert: Collects matching documents in a range. Hook for optimization.
- * Note that {@link #nextDoc()} must be called once before this method is called
- * for the first time.
- * @param collector The collector to which all matching documents are passed through.
- * @param max Do not score documents past this.
- * @return true if more matching documents may remain.
- */
- @Override
- public boolean score(Collector collector, int max, int firstDocID) throws IOException {
- // firstDocID is ignored since nextDoc() sets 'currentDoc'
- collector.setScorer(this);
- while (currentDoc < max) {
- collector.collect(currentDoc);
- if (nextDoc() == NO_MORE_DOCS) {
- return false;
- }
- }
- return true;
- }
-
@Override
public int nextDoc() throws IOException {
-
- if (scorerDocQueue.size() < minimumNrMatchers || !advanceAfterCurrent()) {
- currentDoc = NO_MORE_DOCS;
- }
- return currentDoc;
- }
-
- /** Advance all subscorers after the current document determined by the
- * top of the <code>scorerDocQueue</code>.
- * Repeat until at least the minimum number of subscorers match on the same
- * document and all subscorers are after that document or are exhausted.
- * <br>On entry the <code>scorerDocQueue</code> has at least <code>minimumNrMatchers</code>
- * available. At least the scorer with the minimum document number will be advanced.
- * @return true iff there is a match.
- * <br>In case there is a match, </code>currentDoc</code>, </code>currentSumScore</code>,
- * and </code>nrMatchers</code> describe the match.
- *
- * TODO: Investigate whether it is possible to use advance() when
- * the minimum number of matchers is bigger than one, ie. try and use the
- * character of ConjunctionScorer for the minimum number of matchers.
- * Also delay calling score() on the sub scorers until the minimum number of
- * matchers is reached.
- * <br>For this, a Scorer array with minimumNrMatchers elements might
- * hold Scorers at currentDoc that are temporarily popped from scorerQueue.
- */
- protected boolean advanceAfterCurrent() throws IOException {
- do { // repeat until minimum nr of matchers
- currentDoc = scorerDocQueue.topDoc();
- currentScore = scorerDocQueue.topScore();
- nrMatchers = 1;
- do { // Until all subscorers are after currentDoc
- if (!scorerDocQueue.topNextAndAdjustElsePop()) {
- if (scorerDocQueue.size() == 0) {
- break; // nothing more to advance, check for last match.
+ while(true) {
+ while (subScorers[0].docID() == doc) {
+ if (subScorers[0].nextDoc() != NO_MORE_DOCS) {
+ heapAdjust(0);
+ } else {
+ heapRemoveRoot();
+ if (numScorers < minimumNrMatchers) {
+ return doc = NO_MORE_DOCS;
}
}
- if (scorerDocQueue.topDoc() != currentDoc) {
- break; // All remaining subscorers are after currentDoc.
- }
- currentScore += scorerDocQueue.topScore();
- nrMatchers++;
- } while (true);
-
+ }
+ afterNext();
if (nrMatchers >= minimumNrMatchers) {
- return true;
- } else if (scorerDocQueue.size() < minimumNrMatchers) {
- return false;
+ break;
}
- } while (true);
+ }
+
+ return doc;
+ }
+
+ private void afterNext() throws IOException {
+ final Scorer sub = subScorers[0];
+ doc = sub.docID();
+ score = sub.score();
+ nrMatchers = 1;
+ countMatches(1);
+ countMatches(2);
+ }
+
+ // TODO: this currently scores, but so did the previous impl
+ // TODO: remove recursion.
+ // TODO: if we separate scoring, out of here, modify this
+ // and afterNext() to terminate when nrMatchers == minimumNrMatchers
+ // then also change freq() to just always compute it from scratch
+ private void countMatches(int root) throws IOException {
+ if (root < numScorers && subScorers[root].docID() == doc) {
+ nrMatchers++;
+ score += subScorers[root].score();
+ countMatches((root<<1)+1);
+ countMatches((root<<1)+2);
+ }
}
/** Returns the score of the current document matching the query.
* Initially invalid, until {@link #nextDoc()} is called the first time.
*/
@Override
- public float score() throws IOException { return (float)currentScore; }
+ public float score() throws IOException {
+ return (float)score;
+ }
@Override
public int docID() {
- return currentDoc;
+ return doc;
}
-
- /** Returns the number of subscorers matching the current document.
- * Initially invalid, until {@link #nextDoc()} is called the first time.
- */
- public int nrMatchers() {
+
+ @Override
+ public float freq() throws IOException {
return nrMatchers;
}
@@ -221,20 +141,24 @@ class DisjunctionSumScorer extends Score
*/
@Override
public int advance(int target) throws IOException {
- if (scorerDocQueue.size() < minimumNrMatchers) {
- return currentDoc = NO_MORE_DOCS;
- }
- if (target <= currentDoc) {
- return currentDoc;
- }
- do {
- if (scorerDocQueue.topDoc() >= target) {
- return advanceAfterCurrent() ? currentDoc : (currentDoc = NO_MORE_DOCS);
- } else if (!scorerDocQueue.topSkipToAndAdjustElsePop(target)) {
- if (scorerDocQueue.size() < minimumNrMatchers) {
- return currentDoc = NO_MORE_DOCS;
+ if (numScorers == 0) return doc = NO_MORE_DOCS;
+ while (subScorers[0].docID() < target) {
+ if (subScorers[0].advance(target) != NO_MORE_DOCS) {
+ heapAdjust(0);
+ } else {
+ heapRemoveRoot();
+ if (numScorers == 0) {
+ return doc = NO_MORE_DOCS;
}
}
- } while (true);
+ }
+
+ afterNext();
+
+ if (nrMatchers >= minimumNrMatchers) {
+ return doc;
+ } else {
+ return nextDoc();
+ }
}
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/FilteredQuery.java Sat Jul 28 11:27:51 2012
@@ -24,6 +24,8 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ToStringUtils;
import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
import java.util.Set;
@@ -221,6 +223,14 @@ public class FilteredQuery extends Query
public float score() throws IOException {
return scorer.score();
}
+
+ @Override
+ public float freq() throws IOException { return scorer.freq(); }
+
+ @Override
+ public Collection<ChildScorer> getChildren() {
+ return Collections.singleton(new ChildScorer(scorer, "FILTERED"));
+ }
};
}
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/MatchAllDocsQuery.java Sat Jul 28 11:27:51 2012
@@ -68,6 +68,11 @@ public class MatchAllDocsQuery extends Q
}
@Override
+ public float freq() {
+ return 1;
+ }
+
+ @Override
public int advance(int target) throws IOException {
doc = target-1;
return nextDoc();
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java Sat Jul 28 11:27:51 2012
@@ -30,6 +30,7 @@ import org.apache.lucene.search.similari
final class MatchOnlyTermScorer extends Scorer {
private final DocsEnum docsEnum;
private final Similarity.ExactSimScorer docScorer;
+ private final int docFreq;
/**
* Construct a <code>TermScorer</code>.
@@ -41,11 +42,14 @@ final class MatchOnlyTermScorer extends
* @param docScorer
* The </code>Similarity.ExactSimScorer</code> implementation
* to be used for score computations.
+ * @param docFreq
+ * per-segment docFreq of this term
*/
- MatchOnlyTermScorer(Weight weight, DocsEnum td, Similarity.ExactSimScorer docScorer) {
+ MatchOnlyTermScorer(Weight weight, DocsEnum td, Similarity.ExactSimScorer docScorer, int docFreq) {
super(weight);
this.docScorer = docScorer;
this.docsEnum = td;
+ this.docFreq = docFreq;
}
@Override
@@ -91,4 +95,18 @@ final class MatchOnlyTermScorer extends
/** Returns a string representation of this <code>TermScorer</code>. */
@Override
public String toString() { return "scorer(" + weight + ")"; }
+
+ // TODO: benchmark if the specialized conjunction really benefits
+ // from these, or if instead its from sorting by docFreq, or both
+
+ DocsEnum getDocsEnum() {
+ return docsEnum;
+ }
+
+ // TODO: generalize something like this for scorers?
+ // even this is just an estimation...
+
+ int getDocFreq() {
+ return docFreq;
+ }
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/NumericRangeQuery.java Sat Jul 28 11:27:51 2012
@@ -173,35 +173,6 @@ public final class NumericRangeQuery<T e
this.max = max;
this.minInclusive = minInclusive;
this.maxInclusive = maxInclusive;
-
- // For bigger precisionSteps this query likely
- // hits too many terms, so set to CONSTANT_SCORE_FILTER right off
- // (especially as the FilteredTermsEnum is costly if wasted only for AUTO tests because it
- // creates new enums from IndexReader for each sub-range)
- switch (dataType) {
- case LONG:
- case DOUBLE:
- setRewriteMethod( (precisionStep > 6) ?
- CONSTANT_SCORE_FILTER_REWRITE :
- CONSTANT_SCORE_AUTO_REWRITE_DEFAULT
- );
- break;
- case INT:
- case FLOAT:
- setRewriteMethod( (precisionStep > 8) ?
- CONSTANT_SCORE_FILTER_REWRITE :
- CONSTANT_SCORE_AUTO_REWRITE_DEFAULT
- );
- break;
- default:
- // should never happen
- throw new IllegalArgumentException("Invalid numeric NumericType");
- }
-
- // shortcut if upper bound == lower bound
- if (min != null && min.equals(max)) {
- setRewriteMethod(CONSTANT_SCORE_BOOLEAN_QUERY_REWRITE);
- }
}
/**
@@ -319,9 +290,10 @@ public final class NumericRangeQuery<T e
@Override @SuppressWarnings("unchecked")
protected TermsEnum getTermsEnum(final Terms terms, AttributeSource atts) throws IOException {
// very strange: java.lang.Number itsself is not Comparable, but all subclasses used here are
- return (min != null && max != null && ((Comparable<T>) min).compareTo(max) > 0) ?
- TermsEnum.EMPTY :
- new NumericRangeTermsEnum(terms.iterator(null));
+ if (min != null && max != null && ((Comparable<T>) min).compareTo(max) > 0) {
+ return TermsEnum.EMPTY;
+ }
+ return new NumericRangeTermsEnum(terms.iterator(null));
}
/** Returns <code>true</code> if the lower endpoint is inclusive */
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ReqExclScorer.java Sat Jul 28 11:27:51 2012
@@ -18,7 +18,8 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
-
+import java.util.Collection;
+import java.util.Collections;
/** A Scorer for queries with a required subscorer
* and an excluding (prohibited) sub DocIdSetIterator.
@@ -104,6 +105,16 @@ class ReqExclScorer extends Scorer {
}
@Override
+ public float freq() throws IOException {
+ return reqScorer.freq();
+ }
+
+ @Override
+ public Collection<ChildScorer> getChildren() {
+ return Collections.singleton(new ChildScorer(reqScorer, "FILTERED"));
+ }
+
+ @Override
public int advance(int target) throws IOException {
if (reqScorer == null) {
return doc = NO_MORE_DOCS;
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java Sat Jul 28 11:27:51 2012
@@ -17,6 +17,8 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
/** A Scorer for queries with a required part and an optional part.
* Delays skipTo() on the optional part until a score() is needed.
@@ -39,6 +41,8 @@ class ReqOptSumScorer extends Scorer {
Scorer optScorer)
{
super(reqScorer.weight);
+ assert reqScorer != null;
+ assert optScorer != null;
this.reqScorer = reqScorer;
this.optScorer = optScorer;
}
@@ -80,5 +84,19 @@ class ReqOptSumScorer extends Scorer {
return optScorerDoc == curDoc ? reqScore + optScorer.score() : reqScore;
}
+ @Override
+ public float freq() throws IOException {
+ // we might have deferred advance()
+ score();
+ return (optScorer != null && optScorer.docID() == reqScorer.docID()) ? 2 : 1;
+ }
+
+ @Override
+ public Collection<ChildScorer> getChildren() {
+ ArrayList<ChildScorer> children = new ArrayList<ChildScorer>(2);
+ children.add(new ChildScorer(reqScorer, "MUST"));
+ children.add(new ChildScorer(optScorer, "SHOULD"));
+ return children;
+ }
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/ScoreCachingWrappingScorer.java Sat Jul 28 11:27:51 2012
@@ -18,6 +18,8 @@ package org.apache.lucene.search;
*/
import java.io.IOException;
+import java.util.Collection;
+import java.util.Collections;
/**
* A {@link Scorer} which wraps another scorer and caches the score of the
@@ -59,6 +61,11 @@ public class ScoreCachingWrappingScorer
}
@Override
+ public float freq() throws IOException {
+ return scorer.freq();
+ }
+
+ @Override
public int docID() {
return scorer.docID();
}
@@ -77,5 +84,9 @@ public class ScoreCachingWrappingScorer
public int advance(int target) throws IOException {
return scorer.advance(target);
}
-
+
+ @Override
+ public Collection<ChildScorer> getChildren() {
+ return Collections.singleton(new ChildScorer(scorer, "CACHED"));
+ }
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/Scorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/Scorer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/Scorer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/Scorer.java Sat Jul 28 11:27:51 2012
@@ -98,9 +98,7 @@ public abstract class Scorer extends Doc
* "sloppy" the match was.
*
* @lucene.experimental */
- public float freq() throws IOException {
- throw new UnsupportedOperationException(this + " does not implement freq()");
- }
+ public abstract float freq() throws IOException;
/** returns parent Weight
* @lucene.experimental
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/TermQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/TermQuery.java Sat Jul 28 11:27:51 2012
@@ -85,27 +85,20 @@ public class TermQuery extends Query {
}
DocsEnum docs = termsEnum.docs(acceptDocs, null, true);
if (docs != null) {
- return new TermScorer(this, docs, createDocScorer(context));
+ return new TermScorer(this, docs, similarity.exactSimScorer(stats, context), termsEnum.docFreq());
} else {
// Index does not store freq info
docs = termsEnum.docs(acceptDocs, null, false);
assert docs != null;
- return new MatchOnlyTermScorer(this, docs, createDocScorer(context));
+ return new MatchOnlyTermScorer(this, docs, similarity.exactSimScorer(stats, context), termsEnum.docFreq());
}
}
/**
- * Creates an {@link ExactSimScorer} for this {@link TermWeight}*/
- ExactSimScorer createDocScorer(AtomicReaderContext context)
- throws IOException {
- return similarity.exactSimScorer(stats, context);
- }
-
- /**
* Returns a {@link TermsEnum} positioned at this weights Term or null if
* the term does not exist in the given context
*/
- TermsEnum getTermsEnum(AtomicReaderContext context) throws IOException {
+ private TermsEnum getTermsEnum(AtomicReaderContext context) throws IOException {
final TermState state = termStates.get(context.ord);
if (state == null) { // term is not present in that reader
assert termNotInReader(context.reader(), term.field(), term.bytes()) : "no termstate found but term exists in reader term=" + term;
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/TermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/TermScorer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/TermScorer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/TermScorer.java Sat Jul 28 11:27:51 2012
@@ -27,6 +27,7 @@ import org.apache.lucene.search.similari
final class TermScorer extends Scorer {
private final DocsEnum docsEnum;
private final Similarity.ExactSimScorer docScorer;
+ private final int docFreq;
/**
* Construct a <code>TermScorer</code>.
@@ -38,11 +39,14 @@ final class TermScorer extends Scorer {
* @param docScorer
* The </code>Similarity.ExactSimScorer</code> implementation
* to be used for score computations.
+ * @param docFreq
+ * per-segment docFreq of this term
*/
- TermScorer(Weight weight, DocsEnum td, Similarity.ExactSimScorer docScorer) {
+ TermScorer(Weight weight, DocsEnum td, Similarity.ExactSimScorer docScorer, int docFreq) {
super(weight);
this.docScorer = docScorer;
this.docsEnum = td;
+ this.docFreq = docFreq;
}
@Override
@@ -89,4 +93,17 @@ final class TermScorer extends Scorer {
@Override
public String toString() { return "scorer(" + weight + ")"; }
+ // TODO: benchmark if the specialized conjunction really benefits
+ // from this, or if instead its from sorting by docFreq, or both
+
+ DocsEnum getDocsEnum() {
+ return docsEnum;
+ }
+
+ // TODO: generalize something like this for scorers?
+ // even this is just an estimation...
+
+ int getDocFreq() {
+ return docFreq;
+ }
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java Sat Jul 28 11:27:51 2012
@@ -1,6 +1,5 @@
package org.apache.lucene.search.payloads;
-import org.apache.lucene.search.Explanation;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -36,14 +35,6 @@ public class AveragePayloadFunction exte
public float docScore(int docId, String field, int numPayloadsSeen, float payloadScore) {
return numPayloadsSeen > 0 ? (payloadScore / numPayloadsSeen) : 1;
}
- @Override
- public Explanation explain(int doc, int numPayloadsSeen, float payloadScore) {
- Explanation payloadBoost = new Explanation();
- float avgPayloadScore = (numPayloadsSeen > 0 ? (payloadScore / numPayloadsSeen) : 1);
- payloadBoost.setValue(avgPayloadScore);
- payloadBoost.setDescription("AveragePayloadFunction(...)");
- return payloadBoost;
- }
@Override
public int hashCode() {
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java Sat Jul 28 11:27:51 2012
@@ -1,6 +1,5 @@
package org.apache.lucene.search.payloads;
-import org.apache.lucene.search.Explanation;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -41,14 +40,6 @@ public class MaxPayloadFunction extends
}
@Override
- public Explanation explain(int doc, int numPayloadsSeen, float payloadScore) {
- Explanation expl = new Explanation();
- float maxPayloadScore = (numPayloadsSeen > 0 ? payloadScore : 1);
- expl.setValue(maxPayloadScore);
- expl.setDescription("MaxPayloadFunction(...)");
- return expl;
- }
- @Override
public int hashCode() {
final int prime = 31;
int result = 1;
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java Sat Jul 28 11:27:51 2012
@@ -1,6 +1,5 @@
package org.apache.lucene.search.payloads;
-import org.apache.lucene.search.Explanation;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -39,14 +38,6 @@ public class MinPayloadFunction extends
}
@Override
- public Explanation explain(int doc, int numPayloadsSeen, float payloadScore) {
- Explanation expl = new Explanation();
- float minPayloadScore = (numPayloadsSeen > 0 ? payloadScore : 1);
- expl.setValue(minPayloadScore);
- expl.setDescription("MinPayloadFunction(...)");
- return expl;
- }
- @Override
public int hashCode() {
final int prime = 31;
int result = 1;
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java Sat Jul 28 11:27:51 2012
@@ -55,10 +55,10 @@ public abstract class PayloadFunction {
*/
public abstract float docScore(int docId, String field, int numPayloadsSeen, float payloadScore);
- public Explanation explain(int docId, int numPayloadsSeen, float payloadScore){
+ public Explanation explain(int docId, String field, int numPayloadsSeen, float payloadScore){
Explanation result = new Explanation();
- result.setDescription("Unimpl Payload Function Explain");
- result.setValue(1);
+ result.setDescription(getClass().getSimpleName() + ".docScore()");
+ result.setValue(docScore(docId, field, numPayloadsSeen, payloadScore));
return result;
};
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java Sat Jul 28 11:27:51 2012
@@ -167,8 +167,9 @@ public class PayloadNearQuery extends Sp
Explanation scoreExplanation = docScorer.explain(doc, new Explanation(freq, "phraseFreq=" + freq));
expl.addDetail(scoreExplanation);
expl.setValue(scoreExplanation.getValue());
+ String field = ((SpanQuery)getQuery()).getField();
// now the payloads part
- Explanation payloadExpl = function.explain(doc, scorer.payloadsSeen, scorer.payloadScore);
+ Explanation payloadExpl = function.explain(doc, field, scorer.payloadsSeen, scorer.payloadScore);
// combined
ComplexExplanation result = new ComplexExplanation();
result.addDetail(expl);
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java Sat Jul 28 11:27:51 2012
@@ -28,6 +28,7 @@ import org.apache.lucene.search.ComplexE
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.Similarity.SloppySimScorer;
+import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.TermSpans;
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.search.spans.SpanWeight;
@@ -190,7 +191,8 @@ public class PayloadTermQuery extends Sp
// whether to load the payload or not
// GSI: I suppose we could toString the payload, but I don't think that
// would be a good idea
- Explanation payloadExpl = new Explanation(scorer.getPayloadScore(), "scorePayload(...)");
+ String field = ((SpanQuery)getQuery()).getField();
+ Explanation payloadExpl = function.explain(doc, field, scorer.payloadsSeen, scorer.payloadScore);
payloadExpl.setValue(scorer.getPayloadScore());
// combined
ComplexExplanation result = new ComplexExplanation();
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/Constants.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/Constants.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/Constants.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/Constants.java Sat Jul 28 11:27:51 2012
@@ -117,9 +117,14 @@ public final class Constants {
Package pkg = LucenePackage.get();
String v = (pkg == null) ? null : pkg.getImplementationVersion();
if (v == null) {
- v = LUCENE_MAIN_VERSION + "-SNAPSHOT";
- } else if (!v.startsWith(LUCENE_MAIN_VERSION)) {
- v = LUCENE_MAIN_VERSION + "-SNAPSHOT " + v;
+ String parts[] = LUCENE_MAIN_VERSION.split("\\.");
+ if (parts.length == 4) {
+ // alpha/beta
+ assert parts[2].equals("0");
+ v = parts[0] + "." + parts[1] + "-SNAPSHOT";
+ } else {
+ v = LUCENE_MAIN_VERSION + "-SNAPSHOT";
+ }
}
LUCENE_VERSION = ident(v);
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/IOUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/IOUtils.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/IOUtils.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/IOUtils.java Sat Jul 28 11:27:51 2012
@@ -65,7 +65,7 @@ public final class IOUtils {
* } catch (ExpectedException e) {
* priorE = e;
* } finally {
- * closeSafely(priorE, resource1, resource2, resource3);
+ * closeWhileHandlingException(priorE, resource1, resource2, resource3);
* }
* </pre>
* </p>
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/NamedSPILoader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/NamedSPILoader.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/NamedSPILoader.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/NamedSPILoader.java Sat Jul 28 11:27:51 2012
@@ -22,73 +22,80 @@ import java.util.Iterator;
import java.util.Map;
import java.util.LinkedHashMap;
import java.util.Set;
-import java.util.ServiceLoader;
+import java.util.ServiceConfigurationError;
/**
* Helper class for loading named SPIs from classpath (e.g. Codec, PostingsFormat).
* @lucene.internal
*/
-// TODO: would be nice to have case insensitive lookups.
public final class NamedSPILoader<S extends NamedSPILoader.NamedSPI> implements Iterable<S> {
- private final Map<String,S> services;
-
- /** This field is a hack for LuceneTestCase to get access
- * to the modifiable map (to work around bugs in IBM J9) */
- @SuppressWarnings("unused")
- @Deprecated
- // Hackidy-Häck-Hack for bugs in IBM J9 ServiceLoader
- private final Map<String,S> modifiableServices;
-
+ private volatile Map<String,S> services = Collections.emptyMap();
private final Class<S> clazz;
public NamedSPILoader(Class<S> clazz) {
+ this(clazz, Thread.currentThread().getContextClassLoader());
+ }
+
+ public NamedSPILoader(Class<S> clazz, ClassLoader classloader) {
this.clazz = clazz;
- final ServiceLoader<S> loader = ServiceLoader.load(clazz);
- final LinkedHashMap<String,S> services = new LinkedHashMap<String,S>();
- for (final S service : loader) {
- final String name = service.getName();
- // only add the first one for each name, later services will be ignored
- // this allows to place services before others in classpath to make
- // them used instead of others
- if (!services.containsKey(name)) {
- assert checkServiceName(name);
- services.put(name, service);
+ reload(classloader);
+ }
+
+ /**
+ * Reloads the internal SPI list from the given {@link ClassLoader}.
+ * Changes to the service list are visible after the method ends, all
+ * iterators ({@link #iterator()},...) stay consistent.
+ *
+ * <p><b>NOTE:</b> Only new service providers are added, existing ones are
+ * never removed or replaced.
+ *
+ * <p><em>This method is expensive and should only be called for discovery
+ * of new service providers on the given classpath/classloader!</em>
+ */
+ public void reload(ClassLoader classloader) {
+ final LinkedHashMap<String,S> services = new LinkedHashMap<String,S>(this.services);
+ final SPIClassIterator<S> loader = SPIClassIterator.get(clazz, classloader);
+ while (loader.hasNext()) {
+ final Class<? extends S> c = loader.next();
+ try {
+ final S service = c.newInstance();
+ final String name = service.getName();
+ // only add the first one for each name, later services will be ignored
+ // this allows to place services before others in classpath to make
+ // them used instead of others
+ if (!services.containsKey(name)) {
+ checkServiceName(name);
+ services.put(name, service);
+ }
+ } catch (Exception e) {
+ throw new ServiceConfigurationError("Cannot instantiate SPI class: " + c.getName(), e);
}
}
- this.modifiableServices = services; // hack, remove when IBM J9 is fixed!
this.services = Collections.unmodifiableMap(services);
}
/**
* Validates that a service name meets the requirements of {@link NamedSPI}
*/
- public static boolean checkServiceName(String name) {
+ public static void checkServiceName(String name) {
// based on harmony charset.java
if (name.length() >= 128) {
throw new IllegalArgumentException("Illegal service name: '" + name + "' is too long (must be < 128 chars).");
}
- for (int i = 0; i < name.length(); i++) {
+ for (int i = 0, len = name.length(); i < len; i++) {
char c = name.charAt(i);
- if (!isLetter(c) && !isDigit(c)) {
+ if (!isLetterOrDigit(c)) {
throw new IllegalArgumentException("Illegal service name: '" + name + "' must be simple ascii alphanumeric.");
}
}
- return true;
}
- /*
- * Checks whether a character is a letter (ascii) which are defined in the spec.
- */
- private static boolean isLetter(char c) {
- return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z');
- }
-
- /*
- * Checks whether a character is a digit (ascii) which are defined in the spec.
+ /**
+ * Checks whether a character is a letter or digit (ascii) which are defined in the spec.
*/
- private static boolean isDigit(char c) {
- return ('0' <= c && c <= '9');
+ private static boolean isLetterOrDigit(char c) {
+ return ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') || ('0' <= c && c <= '9');
}
public S lookup(String name) {
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/fst/FST.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/fst/FST.java Sat Jul 28 11:27:51 2012
@@ -669,6 +669,8 @@ public final class FST<T> {
// 2nd pass just "expands" all arcs to take up a fixed
// byte size
final int sizeNeeded = fixedArrayStart + nodeIn.numArcs * maxBytesPerArc;
+ assert ((long) fixedArrayStart) + ((long) nodeIn.numArcs) * maxBytesPerArc < Integer.MAX_VALUE: "FST too large (> 2.1 GB)";
+
bytes = ArrayUtil.grow(bytes, sizeNeeded);
// TODO: we could make this a vInt instead
bytes[fixedArrayStart-4] = (byte) (maxBytesPerArc >> 24);
@@ -685,7 +687,7 @@ public final class FST<T> {
destPos -= maxBytesPerArc;
srcPos -= bytesPerArc[arcIdx];
if (srcPos != destPos) {
- assert destPos > srcPos;
+ assert destPos > srcPos: "destPos=" + destPos + " srcPos=" + srcPos + " arcIdx=" + arcIdx + " maxBytesPerArc=" + maxBytesPerArc + " bytesPerArc[arcIdx]=" + bytesPerArc[arcIdx] + " nodeIn.numArcs=" + nodeIn.numArcs;
System.arraycopy(bytes, srcPos, bytes, destPos, bytesPerArc[arcIdx]);
}
}
@@ -1194,6 +1196,7 @@ public final class FST<T> {
public void writeByte(byte b) {
assert posWrite <= bytes.length;
if (bytes.length == posWrite) {
+ assert bytes.length < Integer.MAX_VALUE: "FST too large (> 2.1 GB)";
bytes = ArrayUtil.grow(bytes);
}
assert posWrite < bytes.length: "posWrite=" + posWrite + " bytes.length=" + bytes.length;
@@ -1203,6 +1206,7 @@ public final class FST<T> {
public void setPosWrite(int posWrite) {
this.posWrite = posWrite;
if (bytes.length < posWrite) {
+ assert bytes.length < Integer.MAX_VALUE: "FST too large (> 2.1 GB)";
bytes = ArrayUtil.grow(bytes, posWrite);
}
}
@@ -1210,6 +1214,7 @@ public final class FST<T> {
@Override
public void writeBytes(byte[] b, int offset, int length) {
final int size = posWrite + length;
+ assert bytes.length < Integer.MAX_VALUE: "FST too large (> 2.1 GB)";
bytes = ArrayUtil.grow(bytes, size);
System.arraycopy(b, offset, bytes, posWrite, length);
posWrite += length;
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/fst/NodeHash.java Sat Jul 28 11:27:51 2012
@@ -155,6 +155,11 @@ final class NodeHash<T> {
private void rehash() throws IOException {
final int[] oldTable = table;
+
+ if (oldTable.length >= Integer.MAX_VALUE/2) {
+ throw new IllegalStateException("FST too large (> 2.1 GB)");
+ }
+
table = new int[2*table.length];
mask = table.length-1;
for(int idx=0;idx<oldTable.length;idx++) {
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/fst/PositiveIntOutputs.java Sat Jul 28 11:27:51 2012
@@ -42,6 +42,15 @@ public final class PositiveIntOutputs ex
this.doShare = doShare;
}
+ /** Returns the instance of PositiveIntOutputs. */
+ public static PositiveIntOutputs getSingleton() {
+ return getSingleton(true);
+ }
+
+ /** Expert: pass doShare=false to disable output sharing.
+ * In some cases this may result in a smaller FST,
+ * however it will also break methods like {@link
+ * Util#getByOutput} and {@link Util#shortestPaths}. */
public static PositiveIntOutputs getSingleton(boolean doShare) {
return doShare ? singletonShare : singletonNoShare;
}
Modified: lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/java/org/apache/lucene/util/packed/PackedInts.java Sat Jul 28 11:27:51 2012
@@ -585,7 +585,7 @@ public class PackedInts {
* @lucene.internal
*/
public static Reader getReader(DataInput in) throws IOException {
- final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START);
+ final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
final int bitsPerValue = in.readVInt();
assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
final int valueCount = in.readVInt();
@@ -625,7 +625,7 @@ public class PackedInts {
* @lucene.internal
*/
public static ReaderIterator getReaderIterator(DataInput in, int mem) throws IOException {
- final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START);
+ final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
final int bitsPerValue = in.readVInt();
assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
final int valueCount = in.readVInt();
@@ -677,7 +677,7 @@ public class PackedInts {
* @lucene.internal
*/
public static Reader getDirectReader(IndexInput in) throws IOException {
- final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START);
+ final int version = CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT);
final int bitsPerValue = in.readVInt();
assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
final int valueCount = in.readVInt();
Modified: lucene/dev/branches/lucene3312/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/resources/META-INF/services/org.apache.lucene.codecs.PostingsFormat Sat Jul 28 11:27:51 2012
@@ -17,3 +17,4 @@ org.apache.lucene.codecs.lucene40.Lucene
org.apache.lucene.codecs.pulsing.Pulsing40PostingsFormat
org.apache.lucene.codecs.simpletext.SimpleTextPostingsFormat
org.apache.lucene.codecs.memory.MemoryPostingsFormat
+org.apache.lucene.codecs.memory.DirectPostingsFormat
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/TestExternalCodecs.java Sat Jul 28 11:27:51 2012
@@ -59,7 +59,7 @@ public class TestExternalCodecs extends
System.out.println("TEST: NUM_DOCS=" + NUM_DOCS);
}
- MockDirectoryWrapper dir = newDirectory();
+ BaseDirectoryWrapper dir = newDirectory();
dir.setCheckIndexOnClose(false); // we use a custom codec provider
IndexWriter w = new IndexWriter(
dir,
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/TestMergeSchedulerExternal.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/TestMergeSchedulerExternal.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/TestMergeSchedulerExternal.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/TestMergeSchedulerExternal.java Sat Jul 28 11:27:51 2012
@@ -85,7 +85,7 @@ public class TestMergeSchedulerExternal
}
public void testSubclassConcurrentMergeScheduler() throws IOException {
- MockDirectoryWrapper dir = newDirectory();
+ MockDirectoryWrapper dir = newMockDirectory();
dir.failOn(new FailOnlyOnMerge());
Document doc = new Document();
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/analysis/TestMockAnalyzer.java Sat Jul 28 11:27:51 2012
@@ -123,7 +123,7 @@ public class TestMockAnalyzer extends Ba
for (int i = 0; i < num; i++) {
String s = _TestUtil.randomHtmlishString(random(), 20);
StringReader reader = new StringReader(s);
- MockCharFilter charfilter = new MockCharFilter(CharReader.get(reader), 2);
+ MockCharFilter charfilter = new MockCharFilter(reader, 2);
MockAnalyzer analyzer = new MockAnalyzer(random());
TokenStream ts = analyzer.tokenStream("bogus", charfilter);
ts.reset();
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/analysis/TestMockCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/analysis/TestMockCharFilter.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/analysis/TestMockCharFilter.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/analysis/TestMockCharFilter.java Sat Jul 28 11:27:51 2012
@@ -33,7 +33,7 @@ public class TestMockCharFilter extends
@Override
protected Reader initReader(String fieldName, Reader reader) {
- return new MockCharFilter(CharReader.get(reader), 7);
+ return new MockCharFilter(reader, 7);
}
};
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/codecs/pulsing/Test10KPulsings.java Sat Jul 28 11:27:51 2012
@@ -37,6 +37,7 @@ import org.apache.lucene.index.MultiFiel
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -54,7 +55,7 @@ public class Test10KPulsings extends Luc
Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(1));
File f = _TestUtil.getTempDir("10kpulsed");
- MockDirectoryWrapper dir = newFSDirectory(f);
+ BaseDirectoryWrapper dir = newFSDirectory(f);
dir.setCheckIndexOnClose(false); // we do this ourselves explicitly
RandomIndexWriter iw = new RandomIndexWriter(random(), dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(cp));
@@ -105,7 +106,7 @@ public class Test10KPulsings extends Luc
Codec cp = _TestUtil.alwaysPostingsFormat(new Pulsing40PostingsFormat(freqCutoff));
File f = _TestUtil.getTempDir("10knotpulsed");
- MockDirectoryWrapper dir = newFSDirectory(f);
+ BaseDirectoryWrapper dir = newFSDirectory(f);
dir.setCheckIndexOnClose(false); // we do this ourselves explicitly
RandomIndexWriter iw = new RandomIndexWriter(random(), dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(cp));
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/codecs/pulsing/TestPulsingReuse.java Sat Jul 28 11:27:51 2012
@@ -33,6 +33,7 @@ import org.apache.lucene.index.DocsAndPo
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
@@ -84,7 +85,7 @@ public class TestPulsingReuse extends Lu
public void testNestedPulsing() throws Exception {
// we always run this test with pulsing codec.
Codec cp = _TestUtil.alwaysPostingsFormat(new NestedPulsingPostingsFormat());
- MockDirectoryWrapper dir = newDirectory();
+ BaseDirectoryWrapper dir = newDirectory();
dir.setCheckIndexOnClose(false); // will do this ourselves, custom codec
RandomIndexWriter iw = new RandomIndexWriter(random(), dir,
newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random())).setCodec(cp));
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/Test2BPostings.java Sat Jul 28 11:27:51 2012
@@ -25,6 +25,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -34,13 +35,15 @@ import org.apache.lucene.util.LuceneTest
* Test indexes ~82M docs with 26 terms each, so you get > Integer.MAX_VALUE terms/docs pairs
* @lucene.experimental
*/
-@SuppressCodecs({ "SimpleText", "Memory" })
+@SuppressCodecs({ "SimpleText", "Memory", "Direct" })
public class Test2BPostings extends LuceneTestCase {
@Nightly
public void test() throws Exception {
- MockDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BPostings"));
- dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER);
+ BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BPostings"));
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
+ }
dir.setCheckIndexOnClose(false); // don't double-checkindex
IndexWriter w = new IndexWriter(dir,
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/Test2BTerms.java Sat Jul 28 11:27:51 2012
@@ -41,7 +41,7 @@ import java.util.Random;
//
// java -server -Xmx8g -d64 -cp .:lib/junit-4.10.jar:./build/classes/test:./build/classes/test-framework:./build/classes/java -Dlucene.version=4.0-dev -Dtests.directory=MMapDirectory -DtempDir=build -ea org.junit.runner.JUnitCore org.apache.lucene.index.Test2BTerms
//
-@SuppressCodecs({ "SimpleText", "Memory" })
+@SuppressCodecs({ "SimpleText", "Memory", "Direct" })
public class Test2BTerms extends LuceneTestCase {
private final static int TOKEN_LEN = 10;
@@ -146,9 +146,11 @@ public class Test2BTerms extends LuceneT
List<BytesRef> savedTerms = null;
- MockDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BTerms"));
+ BaseDirectoryWrapper dir = newFSDirectory(_TestUtil.getTempDir("2BTerms"));
//MockDirectoryWrapper dir = newFSDirectory(new File("/p/lucene/indices/2bindex"));
- dir.setThrottling(MockDirectoryWrapper.Throttling.NEVER);
+ if (dir instanceof MockDirectoryWrapper) {
+ ((MockDirectoryWrapper)dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
+ }
dir.setCheckIndexOnClose(false); // don't double-checkindex
if (true) {
Modified: lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/lucene3312/lucene/core/src/test/org/apache/lucene/index/TestAddIndexes.java Sat Jul 28 11:27:51 2012
@@ -52,6 +52,7 @@ import org.apache.lucene.index.IndexWrit
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.store.BaseDirectoryWrapper;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
@@ -1169,7 +1170,7 @@ public class TestAddIndexes extends Luce
* simple test that ensures we getting expected exceptions
*/
public void testAddIndexMissingCodec() throws IOException {
- MockDirectoryWrapper toAdd = newDirectory();
+ BaseDirectoryWrapper toAdd = newDirectory();
// Disable checkIndex, else we get an exception because
// of the unregistered codec:
toAdd.setCheckIndexOnClose(false);