You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/11/04 16:27:20 UTC
svn commit: r1197598 - in /lucene/dev/branches/lucene2621: ./
dev-tools/idea/lucene/contrib/ lucene/ lucene/contrib/
lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/
lucene/src/java/org/apache/lucene/index/ lucene/src/java/org/a...
Author: rmuir
Date: Fri Nov 4 15:27:17 2011
New Revision: 1197598
URL: http://svn.apache.org/viewvc?rev=1197598&view=rev
Log:
LUCENE-3490: merge trunk (1197231:1197591)
Added:
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/CollectionStatistics.java
- copied unchanged from r1197591, lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/CollectionStatistics.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/TermStatistics.java
- copied unchanged from r1197591, lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/TermStatistics.java
Removed:
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/similarities/SpoofIndexSearcher.java
Modified:
lucene/dev/branches/lucene2621/ (props changed)
lucene/dev/branches/lucene2621/dev-tools/idea/lucene/contrib/ (props changed)
lucene/dev/branches/lucene2621/lucene/ (props changed)
lucene/dev/branches/lucene2621/lucene/CHANGES.txt
lucene/dev/branches/lucene2621/lucene/MIGRATE.txt
lucene/dev/branches/lucene2621/lucene/contrib/CHANGES.txt
lucene/dev/branches/lucene2621/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/regex/TestSpanRegexQuery.java (props changed)
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/TermInfosReaderIndex.java (props changed)
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/IndexSearcher.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/TermQuery.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java
lucene/dev/branches/lucene2621/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSearchAfter.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSearchWithThreads.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSimilarity.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
lucene/dev/branches/lucene2621/modules/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java
lucene/dev/branches/lucene2621/modules/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java
lucene/dev/branches/lucene2621/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
lucene/dev/branches/lucene2621/modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/DocFreqValueSource.java
lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java
lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxDocValueSource.java
lucene/dev/branches/lucene2621/modules/queryparser/src/test/org/apache/lucene/queryparser/xml/builders/TestNumericRangeFilterBuilder.java (props changed)
lucene/dev/branches/lucene2621/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java
lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java
lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java
lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java
lucene/dev/branches/lucene2621/solr/ (props changed)
lucene/dev/branches/lucene2621/solr/CHANGES.txt (props changed)
lucene/dev/branches/lucene2621/solr/LICENSE.txt (props changed)
lucene/dev/branches/lucene2621/solr/NOTICE.txt (props changed)
lucene/dev/branches/lucene2621/solr/README.txt (props changed)
lucene/dev/branches/lucene2621/solr/build.xml (props changed)
lucene/dev/branches/lucene2621/solr/client/ (props changed)
lucene/dev/branches/lucene2621/solr/common-build.xml (props changed)
lucene/dev/branches/lucene2621/solr/contrib/ (props changed)
lucene/dev/branches/lucene2621/solr/contrib/clustering/src/test-files/ (props changed)
lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler-extras/src/java/ (props changed)
lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler/src/java/ (props changed)
lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler/src/test-files/ (props changed)
lucene/dev/branches/lucene2621/solr/contrib/dataimporthandler/src/test/org/ (props changed)
lucene/dev/branches/lucene2621/solr/contrib/uima/src/java/ (props changed)
lucene/dev/branches/lucene2621/solr/contrib/uima/src/test-files/ (props changed)
lucene/dev/branches/lucene2621/solr/core/ (props changed)
lucene/dev/branches/lucene2621/solr/core/src/java/ (props changed)
lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/LuceneQueryOptimizer.java
lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java
lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java
lucene/dev/branches/lucene2621/solr/core/src/test/ (props changed)
lucene/dev/branches/lucene2621/solr/dev-tools/ (props changed)
lucene/dev/branches/lucene2621/solr/example/ (props changed)
lucene/dev/branches/lucene2621/solr/lib/ (props changed)
lucene/dev/branches/lucene2621/solr/scripts/ (props changed)
lucene/dev/branches/lucene2621/solr/site/ (props changed)
lucene/dev/branches/lucene2621/solr/site-src/ (props changed)
lucene/dev/branches/lucene2621/solr/solrj/ (props changed)
lucene/dev/branches/lucene2621/solr/solrj/src/java/ (props changed)
lucene/dev/branches/lucene2621/solr/solrj/src/test/org/apache/solr/client/ (props changed)
lucene/dev/branches/lucene2621/solr/solrj/src/test/org/apache/solr/client/solrj/ (props changed)
lucene/dev/branches/lucene2621/solr/solrj/src/test/org/apache/solr/common/ (props changed)
lucene/dev/branches/lucene2621/solr/test-framework/ (props changed)
lucene/dev/branches/lucene2621/solr/testlogging.properties (props changed)
lucene/dev/branches/lucene2621/solr/webapp/ (props changed)
Modified: lucene/dev/branches/lucene2621/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/CHANGES.txt?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/lucene2621/lucene/CHANGES.txt Fri Nov 4 15:27:17 2011
@@ -194,6 +194,10 @@ Changes in backwards compatibility polic
to the Bits interface. This method should never do I/O for performance reasons.
(Mike McCandless, Uwe Schindler, Robert Muir, Chris Male, Yonik Seeley,
Jason Rutherglen, Paul Elschot)
+
+* LUCENE-3559: The methods "docFreq" and "maxDoc" on IndexSearcher were removed,
+ as these are no longer used by the scoring system. See MIGRATE.txt for more
+ details. (Robert Muir)
Changes in Runtime Behavior
Modified: lucene/dev/branches/lucene2621/lucene/MIGRATE.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/MIGRATE.txt?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/MIGRATE.txt (original)
+++ lucene/dev/branches/lucene2621/lucene/MIGRATE.txt Fri Nov 4 15:27:17 2011
@@ -395,6 +395,16 @@ LUCENE-1458, LUCENE-2111: Flexible Index
The scorePayload method now takes a BytesRef. It is never null.
+* LUCENE-3559: The methods "docFreq" and "maxDoc" on IndexSearcher were removed,
+ as these are no longer used by the scoring system.
+
+ If you were using these casually in your code for reasons unrelated to scoring,
+ call them on the IndexSearcher's reader instead: getIndexReader().
+
+ If you were subclassing IndexSearcher and overriding these methods to alter
+ scoring, override IndexSearcher's termStatistics() and collectionStatistics()
+ methods instead.
+
* LUCENE-3283: Lucene's core o.a.l.queryParser QueryParsers have been consolidated into module/queryparser,
where other QueryParsers from the codebase will also be placed. The following classes were moved:
- o.a.l.queryParser.CharStream -> o.a.l.queryparser.classic.CharStream
Modified: lucene/dev/branches/lucene2621/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/contrib/CHANGES.txt?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/branches/lucene2621/lucene/contrib/CHANGES.txt Fri Nov 4 15:27:17 2011
@@ -88,6 +88,14 @@ Changes in backwards compatibility polic
* LUCENE-3508: Changed some method signatures in decompounding TokenFilters
to make them no longer use the Token class. (Uwe Schindler)
+
+ * LUCENE-3557: The various SpellChecker.indexDictionary methods were removed,
+ and consolidated to one:
+
+ indexDictionary(Dictionary dict, IndexWriterConfig config, boolean optimize)
+
+ Previously, there was no way to specify an IndexWriterConfig, and some
+ of these methods would sneakily pass 'true' to optimize. (Robert Muir)
New Features
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/index/IndexableFieldType.java Fri Nov 4 15:27:17 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.index;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+/** @lucene.experimental */
public interface IndexableFieldType {
/** True if this field should be indexed (inverted) */
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/IndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/IndexSearcher.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/IndexSearcher.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/IndexSearcher.java Fri Nov 4 15:27:17 2011
@@ -35,14 +35,18 @@ import org.apache.lucene.index.CorruptIn
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.StoredFieldVisitor;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
import org.apache.lucene.search.similarities.DefaultSimilarityProvider;
import org.apache.lucene.search.similarities.SimilarityProvider;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NIOFSDirectory; // javadoc
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ReaderUtil;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ThreadInterruptedException;
/** Implements search over a single IndexReader.
@@ -211,36 +215,6 @@ public class IndexSearcher implements Cl
return reader;
}
- /** Expert: Returns one greater than the largest possible document number.
- *
- * @see org.apache.lucene.index.IndexReader#maxDoc()
- */
- public int maxDoc() {
- return reader.maxDoc();
- }
-
- /** Returns total docFreq for this term. */
- public int docFreq(final Term term) throws IOException {
- if (executor == null) {
- return reader.docFreq(term);
- } else {
- final ExecutionHelper<Integer> runner = new ExecutionHelper<Integer>(executor);
- for(int i = 0; i < leafContexts.length; i++) {
- final IndexReader leaf = leafContexts[i].reader;
- runner.submit(new Callable<Integer>() {
- public Integer call() throws IOException {
- return Integer.valueOf(leaf.docFreq(term));
- }
- });
- }
- int docFreq = 0;
- for (Integer num : runner) {
- docFreq += num.intValue();
- }
- return docFreq;
- }
- }
-
/* Sugar for <code>.getIndexReader().document(docID)</code> */
public Document doc(int docID) throws CorruptIndexException, IOException {
return reader.document(docID);
@@ -860,4 +834,34 @@ public class IndexSearcher implements Cl
public String toString() {
return "IndexSearcher(" + reader + "; executor=" + executor + ")";
}
+
+ /**
+ * Returns {@link TermStatistics} for a term
+ * @lucene.experimental
+ */
+ public TermStatistics termStatistics(Term term, TermContext context) throws IOException {
+ return new TermStatistics(term.bytes(), context.docFreq(), context.totalTermFreq());
+ };
+
+ /**
+ * Returns {@link CollectionStatistics} for a field
+ * @lucene.experimental
+ */
+ public CollectionStatistics collectionStatistics(String field) throws IOException {
+ final int docCount;
+ final long sumTotalTermFreq;
+ final long sumDocFreq;
+
+ Terms terms = MultiFields.getTerms(reader, field);
+ if (terms == null) {
+ docCount = 0;
+ sumTotalTermFreq = 0;
+ sumDocFreq = 0;
+ } else {
+ docCount = terms.getDocCount();
+ sumTotalTermFreq = terms.getSumTotalTermFreq();
+ sumDocFreq = terms.getSumDocFreq();
+ }
+ return new CollectionStatistics(field, reader.maxDoc(), docCount, sumTotalTermFreq, sumDocFreq);
+ }
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java Fri Nov 4 15:27:17 2011
@@ -141,13 +141,15 @@ public class MultiPhraseQuery extends Qu
final ReaderContext context = searcher.getTopReaderContext();
// compute idf
- ArrayList<TermContext> allTerms = new ArrayList<TermContext>();
+ ArrayList<TermStatistics> allTermStats = new ArrayList<TermStatistics>();
for(final Term[] terms: termArrays) {
for (Term term: terms) {
- allTerms.add(TermContext.build(context, term, true));
+ TermContext termContext = TermContext.build(context, term, true);
+ allTermStats.add(searcher.termStatistics(term, termContext));
}
}
- stats = similarity.computeStats(searcher, field, getBoost(), allTerms.toArray(new TermContext[allTerms.size()]));
+ stats = similarity.computeStats(searcher.collectionStatistics(field),
+ getBoost(), allTermStats.toArray(new TermStatistics[allTermStats.size()]));
}
@Override
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/PhraseQuery.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/PhraseQuery.java Fri Nov 4 15:27:17 2011
@@ -190,9 +190,13 @@ public class PhraseQuery extends Query {
this.similarity = searcher.getSimilarityProvider().get(field);
final ReaderContext context = searcher.getTopReaderContext();
states = new TermContext[terms.size()];
- for (int i = 0; i < terms.size(); i++)
- states[i] = TermContext.build(context, terms.get(i), true);
- stats = similarity.computeStats(searcher, field, getBoost(), states);
+ TermStatistics termStats[] = new TermStatistics[terms.size()];
+ for (int i = 0; i < terms.size(); i++) {
+ final Term term = terms.get(i);
+ states[i] = TermContext.build(context, term, true);
+ termStats[i] = searcher.termStatistics(term, states[i]);
+ }
+ stats = similarity.computeStats(searcher.collectionStatistics(field), getBoost(), termStats);
}
@Override
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/TermQuery.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/TermQuery.java Fri Nov 4 15:27:17 2011
@@ -54,7 +54,10 @@ public class TermQuery extends Query {
assert termStates != null : "TermContext must not be null";
this.termStates = termStates;
this.similarity = searcher.getSimilarityProvider().get(term.field());
- this.stats = similarity.computeStats(searcher, term.field(), getBoost(), termStates);
+ this.stats = similarity.computeStats(
+ searcher.collectionStatistics(term.field()),
+ getBoost(),
+ searcher.termStatistics(term, termStates));
}
@Override
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java Fri Nov 4 15:27:17 2011
@@ -20,14 +20,12 @@ package org.apache.lucene.search.similar
import java.io.IOException;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.index.Terms;
+import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SmallFloat;
-import org.apache.lucene.util.TermContext;
/**
* BM25 Similarity. Introduced in Stephen E. Robertson, Steve Walker,
@@ -75,15 +73,13 @@ public class BM25Similarity extends Simi
/** The default implementation computes the average as <code>sumTotalTermFreq / maxDoc</code>,
* or returns <code>1</code> if the index does not store sumTotalTermFreq (Lucene 3.x indexes
* or any field that omits frequency information). */
- protected float avgFieldLength(IndexSearcher searcher, String field) throws IOException {
- Terms terms = MultiFields.getTerms(searcher.getIndexReader(), field);
- if (terms == null) {
- // field does not exist;
- return 1f;
- }
- long sumTotalTermFreq = terms.getSumTotalTermFreq();
- long maxdoc = searcher.maxDoc();
- return sumTotalTermFreq == -1 ? 1f : (float) (sumTotalTermFreq / (double) maxdoc);
+ protected float avgFieldLength(CollectionStatistics collectionStats) {
+ final long sumTotalTermFreq = collectionStats.sumTotalTermFreq();
+ if (sumTotalTermFreq <= 0) {
+ return 1f; // field does not exist, or stat is unsupported
+ } else {
+ return (float) (sumTotalTermFreq / (double) collectionStats.maxDoc());
+ }
}
/** The default implementation encodes <code>boost / sqrt(length)</code>
@@ -131,19 +127,19 @@ public class BM25Similarity extends Simi
return encodeNormValue(state.getBoost(), numTerms);
}
- public Explanation idfExplain(TermContext stats, final IndexSearcher searcher) throws IOException {
- final int df = stats.docFreq();
- final int max = searcher.maxDoc();
+ public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) {
+ final int df = termStats.docFreq();
+ final int max = collectionStats.maxDoc();
final float idf = idf(df, max);
return new Explanation(idf, "idf(docFreq=" + df + ", maxDocs=" + max + ")");
}
- public Explanation idfExplain(final TermContext stats[], IndexSearcher searcher) throws IOException {
- final int max = searcher.maxDoc();
+ public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats[]) {
+ final int max = collectionStats.maxDoc();
float idf = 0.0f;
final Explanation exp = new Explanation();
exp.setDescription("idf(), sum of:");
- for (final TermContext stat : stats ) {
+ for (final TermStatistics stat : termStats ) {
final int df = stat.docFreq();
final float termIdf = idf(df, max);
exp.addDetail(new Explanation(termIdf, "idf(docFreq=" + df + ", maxDocs=" + max + ")"));
@@ -154,10 +150,10 @@ public class BM25Similarity extends Simi
}
@Override
- public final Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termStats) throws IOException {
- Explanation idf = termStats.length == 1 ? idfExplain(termStats[0], searcher) : idfExplain(termStats, searcher);
+ public final Stats computeStats(CollectionStatistics collectionStats, float queryBoost, TermStatistics... termStats) {
+ Explanation idf = termStats.length == 1 ? idfExplain(collectionStats, termStats[0]) : idfExplain(collectionStats, termStats);
- float avgdl = avgFieldLength(searcher, fieldName);
+ float avgdl = avgFieldLength(collectionStats);
// compute freq-independent part of bm25 equation across all norm values
float cache[] = new float[256];
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/LMSimilarity.java Fri Nov 4 15:27:17 2011
@@ -17,11 +17,9 @@ package org.apache.lucene.search.similar
* limitations under the License.
*/
-import java.io.IOException;
-
+import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.util.TermContext;
+import org.apache.lucene.search.TermStatistics;
/**
* Abstract superclass for language modeling Similarities. The following inner
@@ -62,8 +60,8 @@ public abstract class LMSimilarity exten
* usual statistics.
*/
@Override
- protected void fillBasicStats(BasicStats stats, IndexSearcher searcher, String fieldName, TermContext termContext) throws IOException {
- super.fillBasicStats(stats, searcher, fieldName, termContext);
+ protected void fillBasicStats(BasicStats stats, CollectionStatistics collectionStats, TermStatistics termStats) {
+ super.fillBasicStats(stats, collectionStats, termStats);
LMStats lmStats = (LMStats) stats;
lmStats.setCollectionProbability(collectionModel.computeProbability(stats));
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/MultiSimilarity.java Fri Nov 4 15:27:17 2011
@@ -21,10 +21,10 @@ import java.io.IOException;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.TermContext;
/**
* Implements the CombSUM method for combining evidence from multiple
@@ -45,10 +45,10 @@ public class MultiSimilarity extends Sim
}
@Override
- public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException {
+ public Stats computeStats(CollectionStatistics collectionStats, float queryBoost, TermStatistics... termStats) {
Stats subStats[] = new Stats[sims.length];
for (int i = 0; i < subStats.length; i++) {
- subStats[i] = sims[i].computeStats(searcher, fieldName, queryBoost, termContexts);
+ subStats[i] = sims[i].computeStats(collectionStats, queryBoost, termStats);
}
return new MultiStats(subStats);
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/Similarity.java Fri Nov 4 15:27:17 2011
@@ -26,11 +26,13 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Terms; // javadoc
import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.spans.SpanQuery; // javadoc
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SmallFloat; // javadoc
@@ -81,10 +83,10 @@ import org.apache.lucene.util.TermContex
* <a name="querytime"/>
* At query-time, Queries interact with the Similarity via these steps:
* <ol>
- * <li>The {@link #computeStats(IndexSearcher, String, float, TermContext...)} method is called a single time,
+ * <li>The {@link #computeStats(CollectionStatistics, float, TermStatistics...)} method is called a single time,
* allowing the implementation to compute any statistics (such as IDF, average document length, etc)
- * across <i>the entire collection</i>. The {@link TermContext}s passed in are already positioned
- * to the terms involved with the raw statistics involved, so a Similarity can freely use any combination
+ * across <i>the entire collection</i>. The {@link TermStatistics} passed in already contain
+ * the raw statistics involved, so a Similarity can freely use any combination
* of term statistics without causing any additional I/O. Lucene makes no assumption about what is
* stored in the returned {@link Similarity.Stats} object.
* <li>The query normalization process occurs a single time: {@link Similarity.Stats#getValueForNormalization()}
@@ -128,7 +130,7 @@ public abstract class Similarity {
/**
* Compute any collection-level stats (e.g. IDF, average document length, etc) needed for scoring a query.
*/
- public abstract Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException;
+ public abstract Stats computeStats(CollectionStatistics collectionStats, float queryBoost, TermStatistics... termStats);
/**
* returns a new {@link Similarity.ExactDocScorer}.
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/SimilarityBase.java Fri Nov 4 15:27:17 2011
@@ -20,15 +20,12 @@ package org.apache.lucene.search.similar
import java.io.IOException;
import org.apache.lucene.index.FieldInvertState;
-import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
-import org.apache.lucene.index.MultiFields;
-import org.apache.lucene.index.Terms;
+import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
-import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.SmallFloat;
-import org.apache.lucene.util.TermContext;
/**
* A subclass of {@code Similarity} that provides a simplified API for its
@@ -71,12 +68,11 @@ public abstract class SimilarityBase ext
}
@Override
- public final Stats computeStats(IndexSearcher searcher, String fieldName,
- float queryBoost, TermContext... termContexts) throws IOException {
- BasicStats stats[] = new BasicStats[termContexts.length];
- for (int i = 0; i < termContexts.length; i++) {
+ public final Stats computeStats(CollectionStatistics collectionStats, float queryBoost, TermStatistics... termStats) {
+ BasicStats stats[] = new BasicStats[termStats.length];
+ for (int i = 0; i < termStats.length; i++) {
stats[i] = newStats(queryBoost);
- fillBasicStats(stats[i], searcher, fieldName, termContexts[i]);
+ fillBasicStats(stats[i], collectionStats, termStats[i]);
}
return stats.length == 1 ? stats[0] : new MultiSimilarity.MultiStats(stats);
}
@@ -88,13 +84,11 @@ public abstract class SimilarityBase ext
/** Fills all member fields defined in {@code BasicStats} in {@code stats}.
* Subclasses can override this method to fill additional stats. */
- protected void fillBasicStats(BasicStats stats, IndexSearcher searcher,
- String fieldName, TermContext termContext) throws IOException {
- IndexReader reader = searcher.getIndexReader();
- int numberOfDocuments = reader.maxDoc();
+ protected void fillBasicStats(BasicStats stats, CollectionStatistics collectionStats, TermStatistics termStats) {
+ int numberOfDocuments = collectionStats.maxDoc();
- int docFreq = termContext.docFreq();
- long totalTermFreq = termContext.totalTermFreq();
+ int docFreq = termStats.docFreq();
+ long totalTermFreq = termStats.totalTermFreq();
// codec does not supply totalTermFreq: substitute docFreq
if (totalTermFreq == -1) {
@@ -103,25 +97,19 @@ public abstract class SimilarityBase ext
final long numberOfFieldTokens;
final float avgFieldLength;
-
- Terms terms = MultiFields.getTerms(searcher.getIndexReader(), fieldName);
- if (terms == null) {
- // field does not exist;
- numberOfFieldTokens = 0;
- avgFieldLength = 1;
- } else {
- long sumTotalTermFreq = terms.getSumTotalTermFreq();
+ long sumTotalTermFreq = collectionStats.sumTotalTermFreq();
+
+ if (sumTotalTermFreq <= 0) {
+ // field does not exist;
// We have to provide something if codec doesnt supply these measures,
// or if someone omitted frequencies for the field... negative values cause
// NaN/Inf for some scorers.
- if (sumTotalTermFreq == -1) {
- numberOfFieldTokens = docFreq;
- avgFieldLength = 1;
- } else {
- numberOfFieldTokens = sumTotalTermFreq;
- avgFieldLength = (float)numberOfFieldTokens / numberOfDocuments;
- }
+ numberOfFieldTokens = docFreq;
+ avgFieldLength = 1;
+ } else {
+ numberOfFieldTokens = sumTotalTermFreq;
+ avgFieldLength = (float)numberOfFieldTokens / numberOfDocuments;
}
// TODO: add sumDocFreq for field (numberOfFieldPostings)
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/similarities/TFIDFSimilarity.java Fri Nov 4 15:27:17 2011
@@ -22,9 +22,11 @@ import java.io.IOException;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.PhraseQuery;
+import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.SmallFloat;
@@ -569,21 +571,21 @@ public abstract class TFIDFSimilarity ex
* idf(docFreq, searcher.maxDoc());
* </pre>
*
- * Note that {@link IndexSearcher#maxDoc()} is used instead of
+ * Note that {@link CollectionStatistics#maxDoc()} is used instead of
* {@link org.apache.lucene.index.IndexReader#numDocs() IndexReader#numDocs()} because also
- * {@link IndexSearcher#docFreq(Term)} is used, and when the latter
- * is inaccurate, so is {@link IndexSearcher#maxDoc()}, and in the same direction.
- * In addition, {@link IndexSearcher#maxDoc()} is more efficient to compute
+ * {@link TermStatistics#docFreq()} is used, and when the latter
+ * is inaccurate, so is {@link CollectionStatistics#maxDoc()}, and in the same direction.
+ * In addition, {@link CollectionStatistics#maxDoc()} is more efficient to compute
*
- * @param stats statistics of the term in question
- * @param searcher the document collection being searched
+ * @param collectionStats collection-level statistics
+ * @param termStats term-level statistics for the term
* @return an Explain object that includes both an idf score factor
and an explanation for the term.
* @throws IOException
*/
- public Explanation idfExplain(TermContext stats, final IndexSearcher searcher) throws IOException {
- final int df = stats.docFreq();
- final int max = searcher.maxDoc();
+ public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats) {
+ final int df = termStats.docFreq();
+ final int max = collectionStats.maxDoc();
final float idf = idf(df, max);
return new Explanation(idf, "idf(docFreq=" + df + ", maxDocs=" + max + ")");
}
@@ -595,19 +597,19 @@ public abstract class TFIDFSimilarity ex
* The default implementation sums the idf factor for
* each term in the phrase.
*
- * @param stats statistics of the terms in the phrase
- * @param searcher the document collection being searched
+ * @param collectionStats collection-level statistics
+ * @param termStats term-level statistics for the terms in the phrase
* @return an Explain object that includes both an idf
* score factor for the phrase and an explanation
* for each term.
* @throws IOException
*/
- public Explanation idfExplain(final TermContext stats[], IndexSearcher searcher) throws IOException {
- final int max = searcher.maxDoc();
+ public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats[]) {
+ final int max = collectionStats.maxDoc();
float idf = 0.0f;
final Explanation exp = new Explanation();
exp.setDescription("idf(), sum of:");
- for (final TermContext stat : stats ) {
+ for (final TermStatistics stat : termStats ) {
final int df = stat.docFreq();
final float termIdf = idf(df, max);
exp.addDetail(new Explanation(termIdf, "idf(docFreq=" + df + ", maxDocs=" + max + ")"));
@@ -693,11 +695,10 @@ public abstract class TFIDFSimilarity ex
public abstract float scorePayload(int doc, int start, int end, BytesRef payload);
@Override
- public final Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost,
- TermContext... termContexts) throws IOException {
- final Explanation idf = termContexts.length == 1
- ? idfExplain(termContexts[0], searcher)
- : idfExplain(termContexts, searcher);
+ public final Stats computeStats(CollectionStatistics collectionStats, float queryBoost, TermStatistics... termStats) {
+ final Explanation idf = termStats.length == 1
+ ? idfExplain(collectionStats, termStats[0])
+ : idfExplain(collectionStats, termStats);
return new IDFStats(idf, queryBoost);
}
Modified: lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java Fri Nov 4 15:27:17 2011
@@ -48,10 +48,17 @@ public class SpanWeight extends Weight {
query.extractTerms(terms);
final ReaderContext context = searcher.getTopReaderContext();
final TermContext states[] = new TermContext[terms.size()];
+ final TermStatistics termStats[] = new TermStatistics[terms.size()];
int i = 0;
- for (Term term : terms)
- states[i++] = TermContext.build(context, term, true);
- stats = similarity.computeStats(searcher, query.getField(), query.getBoost(), states);
+ for (Term term : terms) {
+ states[i] = TermContext.build(context, term, true);
+ termStats[i] = searcher.termStatistics(term, states[i]);
+ i++;
+ }
+ stats = similarity.computeStats(
+ searcher.collectionStatistics(query.getField()),
+ query.getBoost(),
+ termStats);
}
@Override
Modified: lucene/dev/branches/lucene2621/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test-framework/java/org/apache/lucene/search/CheckHits.java Fri Nov 4 15:27:17 2011
@@ -61,7 +61,7 @@ public class CheckHits {
ignore.add(Integer.valueOf(results[i]));
}
- int maxDoc = searcher.maxDoc();
+ int maxDoc = searcher.getIndexReader().maxDoc();
for (int doc = 0; doc < maxDoc; doc++) {
if (ignore.contains(Integer.valueOf(doc))) continue;
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestDeletionPolicy.java Fri Nov 4 15:27:17 2011
@@ -717,7 +717,7 @@ public class TestDeletionPolicy extends
expectedCount -= 17;
}
}
- assertEquals("maxDoc=" + searcher.maxDoc() + " numDocs=" + searcher.getIndexReader().numDocs(), expectedCount, hits.length);
+ assertEquals("maxDoc=" + searcher.getIndexReader().maxDoc() + " numDocs=" + searcher.getIndexReader().numDocs(), expectedCount, hits.length);
searcher.close();
reader.close();
if (i == N) {
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestOmitTf.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestOmitTf.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestOmitTf.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/index/TestOmitTf.java Fri Nov 4 15:27:17 2011
@@ -21,7 +21,6 @@ import java.io.IOException;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TermContext;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -50,7 +49,7 @@ public class TestOmitTf extends LuceneTe
@Override public float tf(float freq) { return freq; }
@Override public float sloppyFreq(int distance) { return 2.0f; }
@Override public float idf(int docFreq, int numDocs) { return 1.0f; }
- @Override public Explanation idfExplain(TermContext[] terms, IndexSearcher searcher) throws IOException {
+ @Override public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics[] termStats) {
return new Explanation(1.0f, "Inexplicable");
}
@Override public float scorePayload(int doc, int start, int end, BytesRef payload) { return 1.0f; }
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/JustCompileSearch.java Fri Nov 4 15:27:17 2011
@@ -22,12 +22,8 @@ import java.io.IOException;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.SimilarityProvider;
-import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
-import org.apache.lucene.search.similarities.Similarity.SloppyDocScorer;
-import org.apache.lucene.search.similarities.Similarity.Stats;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.TermContext;
import org.apache.lucene.index.FieldInvertState;
import org.apache.lucene.util.PriorityQueue;
@@ -250,7 +246,7 @@ final class JustCompileSearch {
static final class JustCompileSimilarity extends Similarity {
@Override
- public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException {
+ public Stats computeStats(CollectionStatistics collectionStats, float queryBoost, TermStatistics... termStats) {
throw new UnsupportedOperationException(UNSUPPORTED_MSG);
}
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestDocValuesScoring.java Fri Nov 4 15:27:17 2011
@@ -161,8 +161,8 @@ public class TestDocValuesScoring extend
}
@Override
- public Stats computeStats(IndexSearcher searcher, String fieldName, float queryBoost, TermContext... termContexts) throws IOException {
- return sim.computeStats(searcher, fieldName, queryBoost, termContexts);
+ public Stats computeStats(CollectionStatistics collectionStats, float queryBoost, TermStatistics... termStats) {
+ return sim.computeStats(collectionStats, queryBoost, termStats);
}
@Override
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java Fri Nov 4 15:27:17 2011
@@ -316,8 +316,7 @@ public class TestMultiPhraseQuery extend
return new DefaultSimilarity() {
@Override
- public Explanation idfExplain(TermContext stats[],
- IndexSearcher searcher) throws IOException {
+ public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats[]) {
return new Explanation(10f, "just a test");
}
};
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSearchAfter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSearchAfter.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSearchAfter.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSearchAfter.java Fri Nov 4 15:27:17 2011
@@ -73,8 +73,9 @@ public class TestSearchAfter extends Luc
}
void assertQuery(Query query, Filter filter) throws Exception {
- TopDocs all = searcher.search(query, filter, searcher.maxDoc());
- int pageSize = _TestUtil.nextInt(random, 1, searcher.maxDoc()*2);
+ int maxDoc = searcher.getIndexReader().maxDoc();
+ TopDocs all = searcher.search(query, filter, maxDoc);
+ int pageSize = _TestUtil.nextInt(random, 1, maxDoc*2);
int pageStart = 0;
ScoreDoc lastBottom = null;
while (pageStart < all.totalHits) {
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSearchWithThreads.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSearchWithThreads.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSearchWithThreads.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSearchWithThreads.java Fri Nov 4 15:27:17 2011
@@ -27,8 +27,10 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec;
import org.apache.lucene.util.LuceneTestCase;
+@UseNoMemoryExpensiveCodec
public class TestSearchWithThreads extends LuceneTestCase {
final int NUM_DOCS = atLeast(10000);
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSimilarity.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSimilarity.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSimilarity.java Fri Nov 4 15:27:17 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.search;
*/
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TermContext;
import java.io.IOException;
@@ -50,7 +49,7 @@ public class TestSimilarity extends Luce
@Override public float tf(float freq) { return freq; }
@Override public float sloppyFreq(int distance) { return 2.0f; }
@Override public float idf(int docFreq, int numDocs) { return 1.0f; }
- @Override public Explanation idfExplain(TermContext[] stats, IndexSearcher searcher) throws IOException {
+ @Override public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics[] stats) {
return new Explanation(1.0f, "Inexplicable");
}
};
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSubScorerFreqs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSubScorerFreqs.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSubScorerFreqs.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/TestSubScorerFreqs.java Fri Nov 4 15:27:17 2011
@@ -134,7 +134,7 @@ public class TestSubScorerFreqs extends
CountingCollector c = new CountingCollector(TopScoreDocCollector.create(10,
true));
s.search(q, null, c);
- final int maxDocs = s.maxDoc();
+ final int maxDocs = s.getIndexReader().maxDoc();
assertEquals(maxDocs, c.docCounts.size());
for (int i = 0; i < maxDocs; i++) {
Map<Query, Float> doc0 = c.docCounts.get(i);
@@ -171,7 +171,7 @@ public class TestSubScorerFreqs extends
CountingCollector c = new CountingCollector(TopScoreDocCollector.create(
10, true), occur);
s.search(query, null, c);
- final int maxDocs = s.maxDoc();
+ final int maxDocs = s.getIndexReader().maxDoc();
assertEquals(maxDocs, c.docCounts.size());
boolean includeOptional = occur.contains(Occur.SHOULD.toString());
for (int i = 0; i < maxDocs; i++) {
@@ -201,7 +201,7 @@ public class TestSubScorerFreqs extends
CountingCollector c = new CountingCollector(TopScoreDocCollector.create(10,
true));
s.search(q, null, c);
- final int maxDocs = s.maxDoc();
+ final int maxDocs = s.getIndexReader().maxDoc();
assertEquals(maxDocs, c.docCounts.size());
for (int i = 0; i < maxDocs; i++) {
Map<Query, Float> doc0 = c.docCounts.get(i);
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/payloads/TestPayloadNearQuery.java Fri Nov 4 15:27:17 2011
@@ -27,10 +27,12 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.Payload;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.QueryUtils;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.Similarity;
@@ -42,7 +44,6 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.English;
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.TermContext;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -346,7 +347,7 @@ public class TestPayloadNearQuery extend
// idf used for phrase queries
@Override
- public Explanation idfExplain(TermContext states[], IndexSearcher searcher) throws IOException {
+ public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics[] termStats) {
return new Explanation(1.0f, "Inexplicable");
}
};
Modified: lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java (original)
+++ lucene/dev/branches/lucene2621/lucene/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java Fri Nov 4 15:27:17 2011
@@ -30,12 +30,15 @@ import org.apache.lucene.index.OrdTermSt
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.codecs.Codec;
+import org.apache.lucene.search.CollectionStatistics;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.TermStatistics;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TermContext;
import org.junit.Ignore;
@@ -172,7 +175,14 @@ public class TestSimilarityBase extends
stats.setTotalTermFreq(TOTAL_TERM_FREQ);
return stats;
}
-
+
+ private CollectionStatistics toCollectionStats(BasicStats stats) {
+ return new CollectionStatistics("spoof", stats.getNumberOfDocuments(), -1, stats.getNumberOfFieldTokens(), -1);
+ }
+
+ private TermStatistics toTermStats(BasicStats stats) {
+ return new TermStatistics(new BytesRef("spoofyText"), stats.getDocFreq(), stats.getTotalTermFreq());
+ }
/**
* The generic test core called by all unit test methods. It calls the
* {@link SimilarityBase#score(BasicStats, float, int)} method of all
@@ -180,17 +190,11 @@ public class TestSimilarityBase extends
* is a finite positive real number.
*/
private void unitTestCore(BasicStats stats, float freq, int docLen)
- throws IOException {
- // We have to fake everything, because computeStats() can be overridden and
- // there is no way to inject false data after fillBasicStats().
- SpoofIndexSearcher searcher = new SpoofIndexSearcher(stats);
- TermContext tc = new TermContext(
- searcher.getIndexReader().getTopReaderContext(),
- new OrdTermState(), 0, stats.getDocFreq(), stats.getTotalTermFreq());
-
+ throws IOException {
for (SimilarityBase sim : sims) {
- BasicStats realStats = (BasicStats) sim.computeStats(new SpoofIndexSearcher(stats),
- "spoof", stats.getTotalBoost(), tc);
+ BasicStats realStats = (BasicStats) sim.computeStats(toCollectionStats(stats),
+ stats.getTotalBoost(),
+ toTermStats(stats));
float score = sim.score(realStats, freq, docLen);
float explScore = sim.explain(
realStats, 1, new Explanation(freq, "freq"), docLen).getValue();
@@ -520,16 +524,10 @@ public class TestSimilarityBase extends
*/
private void correctnessTestCore(SimilarityBase sim, float gold)
throws IOException {
- // We have to fake everything, because computeStats() can be overridden and
- // there is no way to inject false data after fillBasicStats().
BasicStats stats = createStats();
- SpoofIndexSearcher searcher = new SpoofIndexSearcher(stats);
- TermContext tc = new TermContext(
- searcher.getIndexReader().getTopReaderContext(),
- new OrdTermState(), 0, stats.getDocFreq(), stats.getTotalTermFreq());
-
- BasicStats realStats = (BasicStats) sim.computeStats(
- searcher, "spoof", stats.getTotalBoost(), tc);
+ BasicStats realStats = (BasicStats) sim.computeStats(toCollectionStats(stats),
+ stats.getTotalBoost(),
+ toTermStats(stats));
float score = sim.score(realStats, FREQ, DOC_LEN);
assertEquals(
sim.toString() + " score not correct.", gold, score, FLOAT_EPSILON);
Modified: lucene/dev/branches/lucene2621/modules/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java (original)
+++ lucene/dev/branches/lucene2621/modules/facet/src/test/org/apache/lucene/facet/search/TestTopKInEachNodeResultHandler.java Fri Nov 4 15:27:17 2011
@@ -109,7 +109,7 @@ public class TestTopKInEachNodeResultHan
// Get all of the documents and run the query, then do different
// facet counts and compare to control
Query q = new TermQuery(new Term("content", "alpha"));
- ScoredDocIdCollector scoredDoc = ScoredDocIdCollector.create(is.maxDoc(), true);
+ ScoredDocIdCollector scoredDoc = ScoredDocIdCollector.create(ir.maxDoc(), true);
// Collector collector = new MultiCollector(scoredDoc);
is.search(q, scoredDoc);
Modified: lucene/dev/branches/lucene2621/modules/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java (original)
+++ lucene/dev/branches/lucene2621/modules/facet/src/test/org/apache/lucene/facet/search/sampling/BaseSampleTestTopK.java Fri Nov 4 15:27:17 2011
@@ -61,7 +61,7 @@ public abstract class BaseSampleTestTopK
// Get all of the documents and run the query, then do different
// facet counts and compare to control
Query q = new TermQuery(new Term(CONTENT_FIELD, BETA)); // 90% of the docs
- ScoredDocIdCollector docCollector = ScoredDocIdCollector.create(searcher.maxDoc(), false);
+ ScoredDocIdCollector docCollector = ScoredDocIdCollector.create(indexReader.maxDoc(), false);
FacetSearchParams expectedSearchParams = searchParamsWithRequests(K, partitionSize);
FacetsCollector fc = new FacetsCollector(expectedSearchParams, indexReader, taxoReader);
Modified: lucene/dev/branches/lucene2621/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java (original)
+++ lucene/dev/branches/lucene2621/modules/grouping/src/test/org/apache/lucene/search/grouping/AllGroupHeadsCollectorTest.java Fri Nov 4 15:27:17 2011
@@ -106,9 +106,10 @@ public class AllGroupHeadsCollectorTest
doc.add(newField("id", "7", StringField.TYPE_STORED));
w.addDocument(doc);
- IndexSearcher indexSearcher = new IndexSearcher(w.getReader());
+ IndexReader reader = w.getReader();
+ IndexSearcher indexSearcher = new IndexSearcher(reader);
w.close();
- int maxDoc = indexSearcher.maxDoc();
+ int maxDoc = reader.maxDoc();
Sort sortWithinGroup = new Sort(new SortField("id", SortField.Type.INT, true));
AbstractAllGroupHeadsCollector c1 = createRandomCollector(groupField, sortWithinGroup);
Modified: lucene/dev/branches/lucene2621/modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java (original)
+++ lucene/dev/branches/lucene2621/modules/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java Fri Nov 4 15:27:17 2011
@@ -950,7 +950,7 @@ public class TestGrouping extends Lucene
}
if (searchIter == 14) {
- for(int docIDX=0;docIDX<s.maxDoc();docIDX++) {
+ for(int docIDX=0;docIDX<s.getIndexReader().maxDoc();docIDX++) {
System.out.println("ID=" + docIDToID[docIDX] + " explain=" + s.explain(query, docIDX));
}
}
Modified: lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/DocFreqValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/DocFreqValueSource.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/DocFreqValueSource.java (original)
+++ lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/DocFreqValueSource.java Fri Nov 4 15:27:17 2011
@@ -148,7 +148,7 @@ public class DocFreqValueSource extends
@Override
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
IndexSearcher searcher = (IndexSearcher)context.get("searcher");
- int docfreq = searcher.docFreq(new Term(indexedField, indexedBytes));
+ int docfreq = searcher.getIndexReader().docFreq(new Term(indexedField, indexedBytes));
return new ConstIntDocValues(docfreq, this);
}
Modified: lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java (original)
+++ lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java Fri Nov 4 15:27:17 2011
@@ -46,8 +46,8 @@ public class IDFValueSource extends DocF
if (!(sim instanceof TFIDFSimilarity)) {
throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
}
- int docfreq = searcher.docFreq(new Term(indexedField, indexedBytes));
- float idf = ((TFIDFSimilarity)sim).idf(docfreq, searcher.maxDoc());
+ int docfreq = searcher.getIndexReader().docFreq(new Term(indexedField, indexedBytes));
+ float idf = ((TFIDFSimilarity)sim).idf(docfreq, searcher.getIndexReader().maxDoc());
return new ConstDoubleDocValues(idf, this);
}
}
Modified: lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxDocValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxDocValueSource.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxDocValueSource.java (original)
+++ lucene/dev/branches/lucene2621/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/MaxDocValueSource.java Fri Nov 4 15:27:17 2011
@@ -42,7 +42,7 @@ public class MaxDocValueSource extends V
@Override
public DocValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
IndexSearcher searcher = (IndexSearcher)context.get("searcher");
- return new ConstIntDocValues(searcher.maxDoc(), this);
+ return new ConstIntDocValues(searcher.getIndexReader().maxDoc(), this);
}
@Override
Modified: lucene/dev/branches/lucene2621/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java (original)
+++ lucene/dev/branches/lucene2621/modules/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java Fri Nov 4 15:27:17 2011
@@ -31,7 +31,6 @@ import org.apache.lucene.document.String
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.Terms;
@@ -470,7 +469,9 @@ public class SpellChecker implements jav
// obtainSearcher calls ensureOpen
final IndexSearcher indexSearcher = obtainSearcher();
try{
- return indexSearcher.docFreq(new Term(F_WORD, word)) > 0;
+ // TODO: we should use ReaderUtil+seekExact, we dont care about the docFreq
+ // this is just an existence check
+ return indexSearcher.getIndexReader().docFreq(new Term(F_WORD, word)) > 0;
} finally {
releaseSearcher(indexSearcher);
}
@@ -479,19 +480,11 @@ public class SpellChecker implements jav
/**
* Indexes the data from the given {@link Dictionary}.
* @param dict Dictionary to index
- * @param mergeFactor mergeFactor to use when indexing
- * @param ramMB the max amount or memory in MB to use
+ * @param config {@link IndexWriterConfig} to use
* @param optimize whether or not the spellcheck index should be optimized
* @throws AlreadyClosedException if the Spellchecker is already closed
* @throws IOException
*/
- public final void indexDictionary(Dictionary dict, int mergeFactor, int ramMB, boolean optimize) throws IOException {
- IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null)
- .setRAMBufferSizeMB(ramMB);
- ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(mergeFactor);
- indexDictionary(dict, config, optimize);
- }
-
public final void indexDictionary(Dictionary dict, IndexWriterConfig config, boolean optimize) throws IOException {
synchronized (modifyCurrentIndexLock) {
ensureOpen();
@@ -500,8 +493,9 @@ public class SpellChecker implements jav
IndexSearcher indexSearcher = obtainSearcher();
final List<TermsEnum> termsEnums = new ArrayList<TermsEnum>();
- if (searcher.maxDoc() > 0) {
- new ReaderUtil.Gather(searcher.getIndexReader()) {
+ final IndexReader reader = searcher.getIndexReader();
+ if (reader.maxDoc() > 0) {
+ new ReaderUtil.Gather(reader) {
@Override
protected void add(int base, IndexReader r) throws IOException {
Terms terms = r.terms(F_WORD);
@@ -546,32 +540,15 @@ public class SpellChecker implements jav
if (optimize)
writer.optimize();
writer.close();
+ // TODO: this isn't that great, maybe in the future SpellChecker should take
+ // IWC in its ctor / keep its writer open?
+
// also re-open the spell index to see our own changes when the next suggestion
// is fetched:
swapSearcher(dir);
}
}
- /**
- * Indexes the data from the given {@link Dictionary}.
- * @param dict the dictionary to index
- * @param mergeFactor mergeFactor to use when indexing
- * @param ramMB the max amount or memory in MB to use
- * @throws IOException
- */
- public final void indexDictionary(Dictionary dict, int mergeFactor, int ramMB) throws IOException {
- indexDictionary(dict, mergeFactor, ramMB, true);
- }
-
- /**
- * Indexes the data from the given {@link Dictionary}.
- * @param dict the dictionary to index
- * @throws IOException
- */
- public final void indexDictionary(Dictionary dict) throws IOException {
- indexDictionary(dict, 300, (int)IndexWriterConfig.DEFAULT_RAM_BUFFER_SIZE_MB);
- }
-
private static int getMin(int l) {
if (l > 5) {
return 3;
Modified: lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java (original)
+++ lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java Fri Nov 4 15:27:17 2011
@@ -195,7 +195,7 @@ public class TestLuceneDictionary extend
Directory dir = newDirectory();
SpellChecker sc = new SpellChecker(dir);
indexReader = IndexReader.open(store, true);
- sc.indexDictionary(new LuceneDictionary(indexReader, "contents"));
+ sc.indexDictionary(new LuceneDictionary(indexReader, "contents"), newIndexWriterConfig(TEST_VERSION_CURRENT, null), false);
String[] suggestions = sc.suggestSimilar("Tam", 1);
assertEquals(1, suggestions.length);
assertEquals("Tom", suggestions[0]);
Modified: lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java (original)
+++ lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestPlainTextDictionary.java Fri Nov 4 15:27:17 2011
@@ -35,7 +35,7 @@ public class TestPlainTextDictionary ext
PlainTextDictionary ptd = new PlainTextDictionary(new StringReader(input));
Directory ramDir = newDirectory();
SpellChecker spellChecker = new SpellChecker(ramDir);
- spellChecker.indexDictionary(ptd);
+ spellChecker.indexDictionary(ptd, newIndexWriterConfig(TEST_VERSION_CURRENT, null), false);
String[] similar = spellChecker.suggestSimilar("treeword", 2);
assertEquals(2, similar.length);
assertEquals(similar[0], "threeword");
Modified: lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java (original)
+++ lucene/dev/branches/lucene2621/modules/suggest/src/test/org/apache/lucene/search/spell/TestSpellChecker.java Fri Nov 4 15:27:17 2011
@@ -331,7 +331,7 @@ public class TestSpellChecker extends Lu
private void addwords(IndexReader r, SpellChecker sc, String field) throws IOException {
long time = System.currentTimeMillis();
- sc.indexDictionary(new LuceneDictionary(r, field));
+ sc.indexDictionary(new LuceneDictionary(r, field), newIndexWriterConfig(TEST_VERSION_CURRENT, null), false);
time = System.currentTimeMillis() - time;
//System.out.println("time to build " + field + ": " + time);
}
@@ -379,7 +379,7 @@ public class TestSpellChecker extends Lu
}
try {
- spellChecker.indexDictionary(new LuceneDictionary(r, field));
+ spellChecker.indexDictionary(new LuceneDictionary(r, field), newIndexWriterConfig(TEST_VERSION_CURRENT, null), false);
fail("spellchecker was already closed");
} catch (AlreadyClosedException e) {
// expected
Modified: lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/LuceneQueryOptimizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/LuceneQueryOptimizer.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/LuceneQueryOptimizer.java (original)
+++ lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/LuceneQueryOptimizer.java Fri Nov 4 15:27:17 2011
@@ -55,7 +55,7 @@ class LuceneQueryOptimizer {
}
public TopDocs optimize(BooleanQuery original,
- IndexSearcher searcher,
+ SolrIndexSearcher searcher,
int numHits,
Query[] queryOut,
Filter[] filterOut
Modified: lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java (original)
+++ lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java Fri Nov 4 15:27:17 2011
@@ -191,6 +191,13 @@ public class SolrIndexSearcher extends I
return core;
}
+ public final int maxDoc() {
+ return reader.maxDoc();
+ }
+
+ public final int docFreq(Term term) throws IOException {
+ return reader.docFreq(term);
+ }
/** Register sub-objects such as caches
*/
Modified: lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java (original)
+++ lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/spelling/FileBasedSpellChecker.java Fri Nov 4 15:27:17 2011
@@ -29,7 +29,6 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.search.spell.HighFrequencyDictionary;
import org.apache.lucene.search.spell.PlainTextDictionary;
import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.Version;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
import org.apache.solr.schema.FieldType;
@@ -64,13 +63,9 @@ public class FileBasedSpellChecker exten
loadExternalFileDictionary(core);
spellChecker.clearIndex();
// TODO: you should be able to specify the IWC params?
- IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null);
// TODO: if we enable this, codec gets angry since field won't exist in the schema
// config.setCodec(core.getCodec());
- ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(300);
- // TODO: does Solr really want to continue passing 'optimize=true' to the spellchecker here?
- // (its been doing this behind the scenes all along, but its wasteful.
- spellChecker.indexDictionary(dictionary, config, true);
+ spellChecker.indexDictionary(dictionary, new IndexWriterConfig(core.getSolrConfig().luceneMatchVersion, null), false);
} catch (IOException e) {
throw new RuntimeException(e);
}
Modified: lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java?rev=1197598&r1=1197597&r2=1197598&view=diff
==============================================================================
--- lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java (original)
+++ lucene/dev/branches/lucene2621/solr/core/src/java/org/apache/solr/spelling/IndexBasedSpellChecker.java Fri Nov 4 15:27:17 2011
@@ -18,10 +18,8 @@ package org.apache.solr.spelling;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.search.spell.HighFrequencyDictionary;
-import org.apache.lucene.util.Version;
import org.apache.solr.common.util.NamedList;
import org.apache.solr.core.SolrCore;
@@ -93,15 +91,9 @@ public class IndexBasedSpellChecker exte
// old terms I think they might hang around.
spellChecker.clearIndex();
// TODO: you should be able to specify the IWC params?
- IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_CURRENT, null);
-
// TODO: if we enable this, codec gets angry since field won't exist in the schema
// config.setCodec(core.getCodec());
- ((TieredMergePolicy)config.getMergePolicy()).setMaxMergeAtOnce(300);
- // TODO: does Solr really want to continue passing 'optimize=true' to the spellchecker here?
- // (its been doing this behind the scenes all along, but its wasteful.
- spellChecker.indexDictionary(dictionary, config, true);
-
+ spellChecker.indexDictionary(dictionary, new IndexWriterConfig(core.getSolrConfig().luceneMatchVersion, null), false);
} catch (IOException e) {
throw new RuntimeException(e);
}