You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2011/01/20 20:52:08 UTC
svn commit: r1061480 [3/5] - in /lucene/dev/branches/bulkpostings: ./
dev-tools/idea/.idea/copyright/ lucene/ lucene/contrib/
lucene/contrib/demo/src/java/org/apache/lucene/demo/
lucene/contrib/highlighter/src/java/org/apache/lucene/search/highlight/ l...
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/BooleanScorer2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/BooleanScorer2.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/BooleanScorer2.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/BooleanScorer2.java Thu Jan 20 19:52:03 2011
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.search.BooleanClause.Occur;
+import org.apache.lucene.search.BooleanQuery.BooleanWeight;
/* See the description in BooleanScorer.java, comparing
* BooleanScorer & BooleanScorer2 */
@@ -42,10 +43,10 @@ class BooleanScorer2 extends Scorer {
int maxCoord = 0; // to be increased for each non prohibited scorer
int nrMatchers; // to be increased by score() of match counting scorers.
- void init(Similarity sim, boolean disableCoord) { // use after all scorers have been added.
+ void init(boolean disableCoord) { // use after all scorers have been added.
coordFactors = new float[optionalScorers.size() + requiredScorers.size() + 1];
for (int i = 0; i < coordFactors.length; i++) {
- coordFactors[i] = disableCoord ? 1.0f : sim.coord(i, maxCoord);
+ coordFactors[i] = disableCoord ? 1.0f : ((BooleanWeight)weight).coord(i, maxCoord);
}
}
}
@@ -80,7 +81,7 @@ class BooleanScorer2 extends Scorer {
* @param optional
* the list of optional scorers.
*/
- public BooleanScorer2(Weight weight, boolean disableCoord, Similarity similarity, int minNrShouldMatch,
+ public BooleanScorer2(BooleanWeight weight, boolean disableCoord, int minNrShouldMatch,
List<Scorer> required, List<Scorer> prohibited, List<Scorer> optional, int maxCoord) throws IOException {
super(null, weight); // Similarity not used
if (minNrShouldMatch < 0) {
@@ -94,8 +95,8 @@ class BooleanScorer2 extends Scorer {
requiredScorers = required;
prohibitedScorers = prohibited;
- coordinator.init(similarity, disableCoord);
- countingSumScorer = makeCountingSumScorer(disableCoord, similarity);
+ coordinator.init(disableCoord);
+ countingSumScorer = makeCountingSumScorer(disableCoord);
}
/** Count a scorer as a single match. */
@@ -163,11 +164,10 @@ class BooleanScorer2 extends Scorer {
}
private Scorer countingConjunctionSumScorer(boolean disableCoord,
- Similarity similarity,
List<Scorer> requiredScorers) throws IOException {
// each scorer from the list counted as a single matcher
final int requiredNrMatchers = requiredScorers.size();
- return new ConjunctionScorer(disableCoord ? 1.0f : similarity.coord(requiredScorers.size(), requiredScorers.size()), requiredScorers) {
+ return new ConjunctionScorer(disableCoord ? 1.0f : ((BooleanWeight)weight).coord(requiredScorers.size(), requiredScorers.size()), requiredScorers) {
private int lastScoredDoc = -1;
// Save the score of lastScoredDoc, so that we don't compute it more than
// once in score().
@@ -191,9 +191,8 @@ class BooleanScorer2 extends Scorer {
}
private Scorer dualConjunctionSumScorer(boolean disableCoord,
- Similarity similarity,
Scorer req1, Scorer req2) throws IOException { // non counting.
- return new ConjunctionScorer(disableCoord ? 1.0f : similarity.coord(2, 2), req1, req2);
+ return new ConjunctionScorer(disableCoord ? 1.0f : ((BooleanWeight)weight).coord(2, 2), req1, req2);
// All scorers match, so defaultSimilarity always has 1 as
// the coordination factor.
// Therefore the sum of the scores of two scorers
@@ -203,14 +202,13 @@ class BooleanScorer2 extends Scorer {
/** Returns the scorer to be used for match counting and score summing.
* Uses requiredScorers, optionalScorers and prohibitedScorers.
*/
- private Scorer makeCountingSumScorer(boolean disableCoord,
- Similarity similarity) throws IOException { // each scorer counted as a single matcher
+ private Scorer makeCountingSumScorer(boolean disableCoord) throws IOException { // each scorer counted as a single matcher
return (requiredScorers.size() == 0)
- ? makeCountingSumScorerNoReq(disableCoord, similarity)
- : makeCountingSumScorerSomeReq(disableCoord, similarity);
+ ? makeCountingSumScorerNoReq(disableCoord)
+ : makeCountingSumScorerSomeReq(disableCoord);
}
- private Scorer makeCountingSumScorerNoReq(boolean disableCoord, Similarity similarity) throws IOException { // No required scorers
+ private Scorer makeCountingSumScorerNoReq(boolean disableCoord) throws IOException { // No required scorers
// minNrShouldMatch optional scorers are required, but at least 1
int nrOptRequired = (minNrShouldMatch < 1) ? 1 : minNrShouldMatch;
Scorer requiredCountingSumScorer;
@@ -219,26 +217,25 @@ class BooleanScorer2 extends Scorer {
else if (optionalScorers.size() == 1)
requiredCountingSumScorer = new SingleMatchScorer(optionalScorers.get(0));
else {
- requiredCountingSumScorer = countingConjunctionSumScorer(disableCoord, similarity, optionalScorers);
+ requiredCountingSumScorer = countingConjunctionSumScorer(disableCoord, optionalScorers);
}
return addProhibitedScorers(requiredCountingSumScorer);
}
- private Scorer makeCountingSumScorerSomeReq(boolean disableCoord, Similarity similarity) throws IOException { // At least one required scorer.
+ private Scorer makeCountingSumScorerSomeReq(boolean disableCoord) throws IOException { // At least one required scorer.
if (optionalScorers.size() == minNrShouldMatch) { // all optional scorers also required.
ArrayList<Scorer> allReq = new ArrayList<Scorer>(requiredScorers);
allReq.addAll(optionalScorers);
- return addProhibitedScorers(countingConjunctionSumScorer(disableCoord, similarity, allReq));
+ return addProhibitedScorers(countingConjunctionSumScorer(disableCoord, allReq));
} else { // optionalScorers.size() > minNrShouldMatch, and at least one required scorer
Scorer requiredCountingSumScorer =
requiredScorers.size() == 1
? new SingleMatchScorer(requiredScorers.get(0))
- : countingConjunctionSumScorer(disableCoord, similarity, requiredScorers);
+ : countingConjunctionSumScorer(disableCoord, requiredScorers);
if (minNrShouldMatch > 0) { // use a required disjunction scorer over the optional scorers
return addProhibitedScorers(
dualConjunctionSumScorer( // non counting
disableCoord,
- similarity,
requiredCountingSumScorer,
countingDisjunctionSumScorer(
optionalScorers,
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/BoostAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/BoostAttributeImpl.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/BoostAttributeImpl.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/BoostAttributeImpl.java Thu Jan 20 19:52:03 2011
@@ -37,20 +37,6 @@ public final class BoostAttributeImpl ex
public void clear() {
boost = 1.0f;
}
-
- @Override
- public boolean equals(Object other) {
- if (this == other)
- return true;
- if (other instanceof BoostAttributeImpl)
- return ((BoostAttributeImpl) other).boost == boost;
- return false;
- }
-
- @Override
- public int hashCode() {
- return Float.floatToIntBits(boost);
- }
@Override
public void copyTo(AttributeImpl target) {
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/ConstantScoreQuery.java Thu Jan 20 19:52:03 2011
@@ -102,7 +102,7 @@ public class ConstantScoreQuery extends
private float queryWeight;
public ConstantWeight(IndexSearcher searcher) throws IOException {
- this.similarity = getSimilarity(searcher);
+ this.similarity = searcher.getSimilarity();
this.innerWeight = (query == null) ? null : query.createWeight(searcher);
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FieldComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FieldComparator.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FieldComparator.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FieldComparator.java Thu Jan 20 19:52:03 2011
@@ -82,7 +82,7 @@ import org.apache.lucene.util.packed.Pac
* priority queue. The {@link FieldValueHitQueue}
* calls this method when a new hit is competitive.
*
- * <li> {@link #setNextReader(AtomicReaderContext)} Invoked
+ * <li> {@link #setNextReader(IndexReader.AtomicReaderContext)} Invoked
* when the search is switching to the next segment.
* You may need to update internal state of the
* comparator, for example retrieving new values from
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FieldValueHitQueue.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FieldValueHitQueue.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FieldValueHitQueue.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FieldValueHitQueue.java Thu Jan 20 19:52:03 2011
@@ -57,9 +57,6 @@ public abstract class FieldValueHitQueue
public OneComparatorFieldValueHitQueue(SortField[] fields, int size)
throws IOException {
super(fields);
- if (fields.length == 0) {
- throw new IllegalArgumentException("Sort must contain at least one field");
- }
SortField field = fields[0];
setComparator(0,field.getComparator(size, 0));
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Filter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Filter.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Filter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Filter.java Thu Jan 20 19:52:03 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.search;
import java.io.IOException;
+import org.apache.lucene.index.IndexReader; // javadocs
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.util.DocIdBitSet;
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FilteredQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FilteredQuery.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FilteredQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FilteredQuery.java Thu Jan 20 19:52:03 2011
@@ -62,7 +62,7 @@ extends Query {
@Override
public Weight createWeight(final IndexSearcher searcher) throws IOException {
final Weight weight = query.createWeight (searcher);
- final Similarity similarity = query.getSimilarity(searcher);
+ final Similarity similarity = searcher.getSimilarity();
return new Weight() {
private float value;
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FilteredTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FilteredTermsEnum.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FilteredTermsEnum.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FilteredTermsEnum.java Thu Jan 20 19:52:03 2011
@@ -127,6 +127,11 @@ public abstract class FilteredTermsEnum
return tenum.docFreq();
}
+ @Override
+ public long totalTermFreq() {
+ return tenum.totalTermFreq();
+ }
+
/** This enum does not support seeking!
* @throws UnsupportedOperationException
*/
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java Thu Jan 20 19:52:03 2011
@@ -244,6 +244,11 @@ public final class FuzzyTermsEnum extend
public int docFreq() {
return actualEnum.docFreq();
}
+
+ @Override
+ public long totalTermFreq() {
+ return actualEnum.totalTermFreq();
+ }
@Override
public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException {
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MaxNonCompetitiveBoostAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MaxNonCompetitiveBoostAttributeImpl.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MaxNonCompetitiveBoostAttributeImpl.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MaxNonCompetitiveBoostAttributeImpl.java Thu Jan 20 19:52:03 2011
@@ -48,25 +48,6 @@ public final class MaxNonCompetitiveBoos
maxNonCompetitiveBoost = Float.NEGATIVE_INFINITY;
competitiveTerm = null;
}
-
- @Override
- public boolean equals(Object other) {
- if (this == other)
- return true;
- if (other instanceof MaxNonCompetitiveBoostAttributeImpl) {
- final MaxNonCompetitiveBoostAttributeImpl o = (MaxNonCompetitiveBoostAttributeImpl) other;
- return (o.maxNonCompetitiveBoost == maxNonCompetitiveBoost)
- && (o.competitiveTerm == null ? competitiveTerm == null : o.competitiveTerm.equals(competitiveTerm));
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- int hash = Float.floatToIntBits(maxNonCompetitiveBoost);
- if (competitiveTerm != null) hash = 31 * hash + competitiveTerm.hashCode();
- return hash;
- }
@Override
public void copyTo(AttributeImpl target) {
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java Thu Jan 20 19:52:03 2011
@@ -137,7 +137,7 @@ public class MultiPhraseQuery extends Qu
public MultiPhraseWeight(IndexSearcher searcher)
throws IOException {
- this.similarity = getSimilarity(searcher);
+ this.similarity = searcher.getSimilarity();
// compute idf
final int maxDoc = searcher.maxDoc();
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/PhraseQuery.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/PhraseQuery.java Thu Jan 20 19:52:03 2011
@@ -146,7 +146,7 @@ public class PhraseQuery extends Query {
public PhraseWeight(IndexSearcher searcher)
throws IOException {
- this.similarity = getSimilarity(searcher);
+ this.similarity = searcher.getSimilarity();
idfExp = similarity.idfExplain(terms, searcher);
idf = idfExp.getIdf();
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Query.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Query.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Query.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Query.java Thu Jan 20 19:52:03 2011
@@ -98,7 +98,7 @@ public abstract class Query implements j
Query query = searcher.rewrite(this);
Weight weight = query.createWeight(searcher);
float sum = weight.sumOfSquaredWeights();
- float norm = getSimilarity(searcher).queryNorm(sum);
+ float norm = searcher.getSimilarity().queryNorm(sum);
if (Float.isInfinite(norm) || Float.isNaN(norm))
norm = 1.0f;
weight.normalize(norm);
@@ -124,15 +124,6 @@ public abstract class Query implements j
// needs to be implemented by query subclasses
throw new UnsupportedOperationException();
}
-
-
- /** Expert: Returns the Similarity implementation to be used for this query.
- * Subclasses may override this method to specify their own Similarity
- * implementation, perhaps one that delegates through that of the Searcher.
- * By default the Searcher's Similarity implementation is returned.*/
- public Similarity getSimilarity(IndexSearcher searcher) {
- return searcher.getSimilarity();
- }
/** Returns a clone of this query. */
@Override
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/TermQuery.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/TermQuery.java Thu Jan 20 19:52:03 2011
@@ -58,7 +58,7 @@ public class TermQuery extends Query {
throws IOException {
assert termStates != null : "PerReaderTermState must not be null";
this.termStates = termStates;
- this.similarity = getSimilarity(searcher);
+ this.similarity = searcher.getSimilarity();
if (docFreq != -1) {
idfExp = similarity.idfExplain(term, searcher, docFreq);
} else {
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Weight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Weight.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Weight.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/Weight.java Thu Jan 20 19:52:03 2011
@@ -34,7 +34,7 @@ import org.apache.lucene.index.IndexRead
* {@link IndexReader} dependent state should reside in the {@link Scorer}.
* <p>
* Since {@link Weight} creates {@link Scorer} instances for a given
- * {@link AtomicReaderContext} ({@link #scorer(AtomicReaderContext, boolean, boolean)})
+ * {@link AtomicReaderContext} ({@link #scorer(IndexReader.AtomicReaderContext, ScorerContext)})
* callers must maintain the relationship between the searcher's top-level
* {@link ReaderContext} and the context used to create a {@link Scorer}.
* <p>
@@ -49,7 +49,7 @@ import org.apache.lucene.index.IndexRead
* <li>The query normalization factor is passed to {@link #normalize(float)}. At
* this point the weighting is complete.
* <li>A <code>Scorer</code> is constructed by
- * {@link #scorer(AtomicReaderContext, ScorerContext)}.
+ * {@link #scorer(IndexReader.AtomicReaderContext, ScorerContext)}.
* </ol>
*
* @since 2.9
@@ -89,7 +89,7 @@ public abstract class Weight implements
*
* @param context
* the {@link AtomicReaderContext} for which to return the {@link Scorer}.
- * @param scorercontext the {@link ScorerContext} holding the scores context variables
+ * @param scorerContext the {@link ScorerContext} holding the scores context variables
* @return a {@link Scorer} which scores documents in/out-of order.
* @throws IOException
*/
@@ -102,7 +102,7 @@ public abstract class Weight implements
* Returns true iff this implementation scores docs only out of order. This
* method is used in conjunction with {@link Collector}'s
* {@link Collector#acceptsDocsOutOfOrder() acceptsDocsOutOfOrder} and
- * {@link #scorer(AtomicReaderContext, boolean, boolean)} to
+ * {@link #scorer(IndexReader.AtomicReaderContext, ScorerContext)} to
* create a matching {@link Scorer} instance for a given {@link Collector}, or
* vice versa.
* <p>
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/cache/DocTermsIndexCreator.java Thu Jan 20 19:52:03 2011
@@ -28,7 +28,6 @@ import org.apache.lucene.index.OrdTermSt
import org.apache.lucene.index.TermState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.codecs.PrefixCodedTermState;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.FieldCache.DocTermsIndex;
import org.apache.lucene.util.ArrayUtil;
@@ -322,6 +321,11 @@ public class DocTermsIndexCreator extend
}
@Override
+ public long totalTermFreq() {
+ return -1;
+ }
+
+ @Override
public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException {
throw new UnsupportedOperationException();
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/CustomScoreQuery.java Thu Jan 20 19:52:03 2011
@@ -189,7 +189,7 @@ public class CustomScoreQuery extends Qu
boolean qStrict;
public CustomWeight(IndexSearcher searcher) throws IOException {
- this.similarity = getSimilarity(searcher);
+ this.similarity = searcher.getSimilarity();
this.subQueryWeight = subQuery.weight(searcher);
this.valSrcWeights = new Weight[valSrcQueries.length];
for(int i = 0; i < valSrcQueries.length; i++) {
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ValueSource.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ValueSource.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ValueSource.java Thu Jan 20 19:52:03 2011
@@ -19,7 +19,6 @@ package org.apache.lucene.search.functio
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.CompositeReaderContext;
-import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.search.function.DocValues;
@@ -52,7 +51,7 @@ public abstract class ValueSource implem
* Return the DocValues used by the function query.
* @deprecated (4.0) This method is temporary, to ease the migration to segment-based
* searching. Please change your code to not pass {@link CompositeReaderContext} to these
- * APIs. Use {@link #getValues(AtomicReaderContext)} instead
+ * APIs. Use {@link #getValues(IndexReader.AtomicReaderContext)} instead
*/
@Deprecated
public DocValues getValues(ReaderContext context) throws IOException {
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/function/ValueSourceQuery.java Thu Jan 20 19:52:03 2011
@@ -69,7 +69,7 @@ public class ValueSourceQuery extends Qu
float queryWeight;
public ValueSourceWeight(IndexSearcher searcher) {
- this.similarity = getSimilarity(searcher);
+ this.similarity = searcher.getSimilarity();
}
/*(non-Javadoc) @see org.apache.lucene.search.Weight#getQuery() */
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java Thu Jan 20 19:52:03 2011
@@ -107,11 +107,6 @@ public class FieldMaskingSpanQuery exten
}
@Override
- public Similarity getSimilarity(IndexSearcher searcher) {
- return maskedQuery.getSimilarity(searcher);
- }
-
- @Override
public Query rewrite(IndexReader reader) throws IOException {
FieldMaskingSpanQuery clone = null;
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java Thu Jan 20 19:52:03 2011
@@ -42,7 +42,7 @@ public class SpanWeight extends Weight {
public SpanWeight(SpanQuery query, IndexSearcher searcher)
throws IOException {
- this.similarity = query.getSimilarity(searcher);
+ this.similarity = searcher.getSimilarity();
this.query = query;
terms=new HashSet<Term>();
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/AttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/AttributeImpl.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/AttributeImpl.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/AttributeImpl.java Thu Jan 20 19:52:03 2011
@@ -20,6 +20,8 @@ package org.apache.lucene.util;
import java.io.Serializable;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
+import java.lang.ref.WeakReference;
+import java.util.LinkedList;
/**
* Base class for Attributes that can be added to a
@@ -37,72 +39,80 @@ public abstract class AttributeImpl impl
public abstract void clear();
/**
- * The default implementation of this method accesses all declared
- * fields of this object and prints the values in the following syntax:
+ * This method returns the current attribute values as a string in the following format
+ * by calling the {@link #reflectWith(AttributeReflector)} method:
*
+ * <ul>
+ * <li><em>iff {@code prependAttClass=true}:</em> {@code "AttributeClass#key=value,AttributeClass#key=value"}
+ * <li><em>iff {@code prependAttClass=false}:</em> {@code "key=value,key=value"}
+ * </ul>
+ *
+ * @see #reflectWith(AttributeReflector)
+ */
+ public final String reflectAsString(final boolean prependAttClass) {
+ final StringBuilder buffer = new StringBuilder();
+ reflectWith(new AttributeReflector() {
+ public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
+ if (buffer.length() > 0) {
+ buffer.append(',');
+ }
+ if (prependAttClass) {
+ buffer.append(attClass.getName()).append('#');
+ }
+ buffer.append(key).append('=').append((value == null) ? "null" : value);
+ }
+ });
+ return buffer.toString();
+ }
+
+ /**
+ * This method is for introspection of attributes, it should simply
+ * add the key/values this attribute holds to the given {@link AttributeReflector}.
+ *
+ * <p>The default implementation calls {@link AttributeReflector#reflect} for all
+ * non-static fields from the implementing class, using the field name as key
+ * and the field value as value. The Attribute class is also determined by reflection.
+ * Please note that the default implementation can only handle single-Attribute
+ * implementations.
+ *
+ * <p>Custom implementations look like this (e.g. for a combined attribute implementation):
* <pre>
- * public String toString() {
- * return "start=" + startOffset + ",end=" + endOffset;
+ * public void reflectWith(AttributeReflector reflector) {
+ * reflector.reflect(CharTermAttribute.class, "term", term());
+ * reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", getPositionIncrement());
* }
* </pre>
- *
- * This method may be overridden by subclasses.
- */
- @Override
- public String toString() {
- StringBuilder buffer = new StringBuilder();
- Class<?> clazz = this.getClass();
- Field[] fields = clazz.getDeclaredFields();
+ *
+ * <p>If you implement this method, make sure that for each invocation, the same set of {@link Attribute}
+ * interfaces and keys are passed to {@link AttributeReflector#reflect} in the same order, but possibly
+ * different values. So don't automatically exclude e.g. {@code null} properties!
+ *
+ * @see #reflectAsString(boolean)
+ */
+ public void reflectWith(AttributeReflector reflector) {
+ final Class<? extends AttributeImpl> clazz = this.getClass();
+ final LinkedList<WeakReference<Class<? extends Attribute>>> interfaces = AttributeSource.getAttributeInterfaces(clazz);
+ if (interfaces.size() != 1) {
+ throw new UnsupportedOperationException(clazz.getName() +
+ " implements more than one Attribute interface, the default reflectWith() implementation cannot handle this.");
+ }
+ final Class<? extends Attribute> interf = interfaces.getFirst().get();
+ final Field[] fields = clazz.getDeclaredFields();
try {
for (int i = 0; i < fields.length; i++) {
- Field f = fields[i];
+ final Field f = fields[i];
if (Modifier.isStatic(f.getModifiers())) continue;
f.setAccessible(true);
- Object value = f.get(this);
- if (buffer.length()>0) {
- buffer.append(',');
- }
- if (value == null) {
- buffer.append(f.getName() + "=null");
- } else {
- buffer.append(f.getName() + "=" + value);
- }
+ reflector.reflect(interf, f.getName(), f.get(this));
}
} catch (IllegalAccessException e) {
// this should never happen, because we're just accessing fields
// from 'this'
throw new RuntimeException(e);
}
-
- return buffer.toString();
}
/**
- * Subclasses must implement this method and should compute
- * a hashCode similar to this:
- * <pre>
- * public int hashCode() {
- * int code = startOffset;
- * code = code * 31 + endOffset;
- * return code;
- * }
- * </pre>
- *
- * see also {@link #equals(Object)}
- */
- @Override
- public abstract int hashCode();
-
- /**
- * All values used for computation of {@link #hashCode()}
- * should be checked here for equality.
- *
- * see also {@link Object#equals(Object)}
- */
- @Override
- public abstract boolean equals(Object other);
-
- /**
* Copies the values from this Attribute into the passed-in
* target attribute. The target implementation must support all the
* Attributes this implementation supports.
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/AttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/AttributeSource.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/AttributeSource.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/AttributeSource.java Thu Jan 20 19:52:03 2011
@@ -180,20 +180,9 @@ public class AttributeSource {
private static final WeakHashMap<Class<? extends AttributeImpl>,LinkedList<WeakReference<Class<? extends Attribute>>>> knownImplClasses =
new WeakHashMap<Class<? extends AttributeImpl>,LinkedList<WeakReference<Class<? extends Attribute>>>>();
- /** <b>Expert:</b> Adds a custom AttributeImpl instance with one or more Attribute interfaces.
- * <p><font color="red"><b>Please note:</b> It is not guaranteed, that <code>att</code> is added to
- * the <code>AttributeSource</code>, because the provided attributes may already exist.
- * You should always retrieve the wanted attributes using {@link #getAttribute} after adding
- * with this method and cast to your class.
- * The recommended way to use custom implementations is using an {@link AttributeFactory}.
- * </font></p>
- */
- public void addAttributeImpl(final AttributeImpl att) {
- final Class<? extends AttributeImpl> clazz = att.getClass();
- if (attributeImpls.containsKey(clazz)) return;
- LinkedList<WeakReference<Class<? extends Attribute>>> foundInterfaces;
+ static LinkedList<WeakReference<Class<? extends Attribute>>> getAttributeInterfaces(final Class<? extends AttributeImpl> clazz) {
synchronized(knownImplClasses) {
- foundInterfaces = knownImplClasses.get(clazz);
+ LinkedList<WeakReference<Class<? extends Attribute>>> foundInterfaces = knownImplClasses.get(clazz);
if (foundInterfaces == null) {
// we have a strong reference to the class instance holding all interfaces in the list (parameter "att"),
// so all WeakReferences are never evicted by GC
@@ -210,7 +199,23 @@ public class AttributeSource {
actClazz = actClazz.getSuperclass();
} while (actClazz != null);
}
+ return foundInterfaces;
}
+ }
+
+ /** <b>Expert:</b> Adds a custom AttributeImpl instance with one or more Attribute interfaces.
+ * <p><font color="red"><b>Please note:</b> It is not guaranteed, that <code>att</code> is added to
+ * the <code>AttributeSource</code>, because the provided attributes may already exist.
+ * You should always retrieve the wanted attributes using {@link #getAttribute} after adding
+ * with this method and cast to your class.
+ * The recommended way to use custom implementations is using an {@link AttributeFactory}.
+ * </font></p>
+ */
+ public final void addAttributeImpl(final AttributeImpl att) {
+ final Class<? extends AttributeImpl> clazz = att.getClass();
+ if (attributeImpls.containsKey(clazz)) return;
+ final LinkedList<WeakReference<Class<? extends Attribute>>> foundInterfaces =
+ getAttributeInterfaces(clazz);
// add all interfaces of this AttributeImpl to the maps
for (WeakReference<Class<? extends Attribute>> curInterfaceRef : foundInterfaces) {
@@ -233,7 +238,7 @@ public class AttributeSource {
* already in this AttributeSource and returns it. Otherwise a
* new instance is created, added to this AttributeSource and returned.
*/
- public <A extends Attribute> A addAttribute(Class<A> attClass) {
+ public final <A extends Attribute> A addAttribute(Class<A> attClass) {
AttributeImpl attImpl = attributes.get(attClass);
if (attImpl == null) {
if (!(attClass.isInterface() && Attribute.class.isAssignableFrom(attClass))) {
@@ -248,7 +253,7 @@ public class AttributeSource {
}
/** Returns true, iff this AttributeSource has any attributes */
- public boolean hasAttributes() {
+ public final boolean hasAttributes() {
return !this.attributes.isEmpty();
}
@@ -256,7 +261,7 @@ public class AttributeSource {
* The caller must pass in a Class<? extends Attribute> value.
* Returns true, iff this AttributeSource contains the passed-in Attribute.
*/
- public boolean hasAttribute(Class<? extends Attribute> attClass) {
+ public final boolean hasAttribute(Class<? extends Attribute> attClass) {
return this.attributes.containsKey(attClass);
}
@@ -271,7 +276,7 @@ public class AttributeSource {
* available. If you want to only use the attribute, if it is available (to optimize
* consuming), use {@link #hasAttribute}.
*/
- public <A extends Attribute> A getAttribute(Class<A> attClass) {
+ public final <A extends Attribute> A getAttribute(Class<A> attClass) {
AttributeImpl attImpl = attributes.get(attClass);
if (attImpl == null) {
throw new IllegalArgumentException("This AttributeSource does not have the attribute '" + attClass.getName() + "'.");
@@ -319,7 +324,7 @@ public class AttributeSource {
* Resets all Attributes in this AttributeSource by calling
* {@link AttributeImpl#clear()} on each Attribute implementation.
*/
- public void clearAttributes() {
+ public final void clearAttributes() {
if (hasAttributes()) {
if (currentState == null) {
computeCurrentState();
@@ -334,7 +339,7 @@ public class AttributeSource {
* Captures the state of all Attributes. The return value can be passed to
* {@link #restoreState} to restore the state of this or another AttributeSource.
*/
- public State captureState() {
+ public final State captureState() {
if (!hasAttributes()) {
return null;
}
@@ -360,7 +365,7 @@ public class AttributeSource {
* reset its value to the default, in which case the caller should first
* call {@link TokenStream#clearAttributes()} on the targetStream.
*/
- public void restoreState(State state) {
+ public final void restoreState(State state) {
if (state == null) return;
do {
@@ -431,21 +436,53 @@ public class AttributeSource {
return false;
}
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder().append('(');
+ /**
+ * This method returns the current attribute values as a string in the following format
+ * by calling the {@link #reflectWith(AttributeReflector)} method:
+ *
+ * <ul>
+ * <li><em>iff {@code prependAttClass=true}:</em> {@code "AttributeClass#key=value,AttributeClass#key=value"}
+ * <li><em>iff {@code prependAttClass=false}:</em> {@code "key=value,key=value"}
+ * </ul>
+ *
+ * @see #reflectWith(AttributeReflector)
+ */
+ public final String reflectAsString(final boolean prependAttClass) {
+ final StringBuilder buffer = new StringBuilder();
+ reflectWith(new AttributeReflector() {
+ public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
+ if (buffer.length() > 0) {
+ buffer.append(',');
+ }
+ if (prependAttClass) {
+ buffer.append(attClass.getName()).append('#');
+ }
+ buffer.append(key).append('=').append((value == null) ? "null" : value);
+ }
+ });
+ return buffer.toString();
+ }
+
+ /**
+ * This method is for introspection of attributes, it should simply
+ * add the key/values this AttributeSource holds to the given {@link AttributeReflector}.
+ *
+ * <p>This method iterates over all Attribute implementations and calls the
+ * corresponding {@link AttributeImpl#reflectWith} method.</p>
+ *
+ * @see AttributeImpl#reflectWith
+ */
+ public final void reflectWith(AttributeReflector reflector) {
if (hasAttributes()) {
if (currentState == null) {
computeCurrentState();
}
for (State state = currentState; state != null; state = state.next) {
- if (state != currentState) sb.append(',');
- sb.append(state.attribute.toString());
+ state.attribute.reflectWith(reflector);
}
}
- return sb.append(')').toString();
}
-
+
/**
* Performs a clone of all {@link AttributeImpl} instances returned in a new
* {@code AttributeSource} instance. This method can be used to e.g. create another TokenStream
@@ -453,7 +490,7 @@ public class AttributeSource {
* You can also use it as a (non-performant) replacement for {@link #captureState}, if you need to look
* into / modify the captured state.
*/
- public AttributeSource cloneAttributes() {
+ public final AttributeSource cloneAttributes() {
final AttributeSource clone = new AttributeSource(this.factory);
if (hasAttributes()) {
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/PerReaderTermState.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/PerReaderTermState.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/PerReaderTermState.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/PerReaderTermState.java Thu Jan 20 19:52:03 2011
@@ -138,9 +138,9 @@ public final class PerReaderTermState {
/**
* Returns the accumulated document frequency of all {@link TermState}
- * instances passed to {@link #register(TermState, int)}.
+ * instances passed to {@link #register(TermState, int, int)}.
* @return the accumulated document frequency of all {@link TermState}
- * instances passed to {@link #register(TermState, int)}.
+ * instances passed to {@link #register(TermState, int, int)}.
*/
public int docFreq() {
return docFreq;
Modified: lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/SorterTemplate.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/SorterTemplate.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/SorterTemplate.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/java/org/apache/lucene/util/SorterTemplate.java Thu Jan 20 19:52:03 2011
@@ -36,14 +36,14 @@ public abstract class SorterTemplate {
protected abstract void swap(int i, int j);
/** Compares slots {@code i} and {@code j} of you data.
- * Should be implemented like <code><em>valueOf(j)</em>.compareTo(<em>valueOf(i)</em>)</code> */
+ * Should be implemented like <code><em>valueOf(i)</em>.compareTo(<em>valueOf(j)</em>)</code> */
protected abstract int compare(int i, int j);
/** Implement this method, that stores the value of slot {@code i} as pivot value */
protected abstract void setPivot(int i);
/** Implements the compare function for the previously stored pivot value.
- * Should be implemented like <code>pivot.compareTo(<em>valueOf(i)</em>)</code> */
+ * Should be implemented like <code>pivot.compareTo(<em>valueOf(j)</em>)</code> */
protected abstract int comparePivot(int j);
/** Sorts via stable in-place InsertionSort algorithm
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestExternalCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestExternalCodecs.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestExternalCodecs.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/TestExternalCodecs.java Thu Jan 20 19:52:03 2011
@@ -102,6 +102,8 @@ public class TestExternalCodecs extends
static class RAMField extends Terms {
final String field;
final SortedMap<String,RAMTerm> termToDocs = new TreeMap<String,RAMTerm>();
+ long sumTotalTermFreq;
+
RAMField(String field) {
this.field = field;
}
@@ -112,6 +114,11 @@ public class TestExternalCodecs extends
}
@Override
+ public long getSumTotalTermFreq() {
+ return sumTotalTermFreq;
+ }
+
+ @Override
public TermsEnum iterator() {
return new RAMTermsEnum(RAMOnlyCodec.RAMField.this);
}
@@ -124,6 +131,7 @@ public class TestExternalCodecs extends
static class RAMTerm {
final String term;
+ long totalTermFreq;
final List<RAMDoc> docs = new ArrayList<RAMDoc>();
public RAMTerm(String term) {
this.term = term;
@@ -189,14 +197,16 @@ public class TestExternalCodecs extends
}
@Override
- public void finishTerm(BytesRef text, int numDocs) {
- assert numDocs > 0;
- assert numDocs == current.docs.size();
+ public void finishTerm(BytesRef text, TermStats stats) {
+ assert stats.docFreq > 0;
+ assert stats.docFreq == current.docs.size();
+ current.totalTermFreq = stats.totalTermFreq;
field.termToDocs.put(current.term, current);
}
@Override
- public void finish() {
+ public void finish(long sumTotalTermFreq) {
+ field.sumTotalTermFreq = sumTotalTermFreq;
}
}
@@ -331,6 +341,10 @@ public class TestExternalCodecs extends
}
@Override
+ public long totalTermFreq() {
+ return ramField.termToDocs.get(current).totalTermFreq;
+ }
+
public DocsEnum docs(Bits skipDocs, DocsEnum reuse) {
return new RAMDocsEnum(ramField.termToDocs.get(current), skipDocs);
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/analysis/TestNumericTokenStream.java Thu Jan 20 19:52:03 2011
@@ -38,13 +38,13 @@ public class TestNumericTokenStream exte
final BytesRef bytes = new BytesRef();
stream.reset();
assertEquals(64, numericAtt.getValueSize());
- assertEquals(lvalue, numericAtt.getRawValue());
for (int shift=0; shift<64; shift+=NumericUtils.PRECISION_STEP_DEFAULT) {
assertTrue("New token is available", stream.incrementToken());
assertEquals("Shift value wrong", shift, numericAtt.getShift());
final int hash = bytesAtt.toBytesRef(bytes);
assertEquals("Hash incorrect", bytes.hashCode(), hash);
assertEquals("Term is incorrectly encoded", lvalue & ~((1L << shift) - 1L), NumericUtils.prefixCodedToLong(bytes));
+ assertEquals("Term raw value is incorrectly encoded", lvalue & ~((1L << shift) - 1L), numericAtt.getRawValue());
assertEquals("Type incorrect", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
}
assertFalse("More tokens available", stream.incrementToken());
@@ -61,13 +61,13 @@ public class TestNumericTokenStream exte
final BytesRef bytes = new BytesRef();
stream.reset();
assertEquals(32, numericAtt.getValueSize());
- assertEquals(ivalue, numericAtt.getRawValue());
for (int shift=0; shift<32; shift+=NumericUtils.PRECISION_STEP_DEFAULT) {
assertTrue("New token is available", stream.incrementToken());
assertEquals("Shift value wrong", shift, numericAtt.getShift());
final int hash = bytesAtt.toBytesRef(bytes);
assertEquals("Hash incorrect", bytes.hashCode(), hash);
assertEquals("Term is incorrectly encoded", ivalue & ~((1 << shift) - 1), NumericUtils.prefixCodedToInt(bytes));
+ assertEquals("Term raw value is incorrectly encoded", ((long) ivalue) & ~((1L << shift) - 1L), numericAtt.getRawValue());
assertEquals("Type incorrect", (shift == 0) ? NumericTokenStream.TOKEN_TYPE_FULL_PREC : NumericTokenStream.TOKEN_TYPE_LOWER_PREC, typeAtt.type());
}
assertFalse("More tokens available", stream.incrementToken());
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/analysis/TestToken.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/analysis/TestToken.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/analysis/TestToken.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/analysis/TestToken.java Thu Jan 20 19:52:03 2011
@@ -22,8 +22,11 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util._TestUtil;
import java.io.StringReader;
+import java.util.HashMap;
public class TestToken extends LuceneTestCase {
@@ -241,6 +244,22 @@ public class TestToken extends LuceneTes
ts.addAttribute(TypeAttribute.class) instanceof Token);
}
+ public void testAttributeReflection() throws Exception {
+ Token t = new Token("foobar", 6, 22, 8);
+ _TestUtil.assertAttributeReflection(t,
+ new HashMap<String,Object>() {{
+ put(CharTermAttribute.class.getName() + "#term", "foobar");
+ put(TermToBytesRefAttribute.class.getName() + "#bytes", new BytesRef("foobar"));
+ put(OffsetAttribute.class.getName() + "#startOffset", 6);
+ put(OffsetAttribute.class.getName() + "#endOffset", 22);
+ put(PositionIncrementAttribute.class.getName() + "#positionIncrement", 1);
+ put(PayloadAttribute.class.getName() + "#payload", null);
+ put(TypeAttribute.class.getName() + "#type", TypeAttribute.DEFAULT_TYPE);
+ put(FlagsAttribute.class.getName() + "#flags", 8);
+ }});
+ }
+
+
public static <T extends AttributeImpl> T assertCloneIsEqual(T att) {
@SuppressWarnings("unchecked")
T clone = (T) att.clone();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java Thu Jan 20 19:52:03 2011
@@ -19,7 +19,10 @@ package org.apache.lucene.analysis.token
import org.apache.lucene.analysis.TestToken;
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.BytesRef;
+import org.apache.lucene.util._TestUtil;
import java.nio.CharBuffer;
+import java.util.HashMap;
import java.util.Formatter;
import java.util.Locale;
import java.util.regex.Pattern;
@@ -126,6 +129,15 @@ public class TestCharTermAttributeImpl e
assertNotSame(buf, copy.buffer());
}
+ public void testAttributeReflection() throws Exception {
+ CharTermAttributeImpl t = new CharTermAttributeImpl();
+ t.append("foobar");
+ _TestUtil.assertAttributeReflection(t, new HashMap<String,Object>() {{
+ put(CharTermAttribute.class.getName() + "#term", "foobar");
+ put(TermToBytesRefAttribute.class.getName() + "#bytes", new BytesRef("foobar"));
+ }});
+ }
+
public void testCharSequenceInterface() {
final String s = "0123456789";
final CharTermAttributeImpl t = new CharTermAttributeImpl();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestCodecs.java Thu Jan 20 19:52:03 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.index.codecs.Fi
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.PostingsConsumer;
import org.apache.lucene.index.codecs.TermsConsumer;
+import org.apache.lucene.index.codecs.TermStats;
import org.apache.lucene.index.codecs.mocksep.MockSepCodec;
import org.apache.lucene.index.codecs.preflex.PreFlexCodec;
import org.apache.lucene.search.DocIdSetIterator;
@@ -97,9 +98,11 @@ public class TestCodecs extends LuceneTe
public void write(final FieldsConsumer consumer) throws Throwable {
Arrays.sort(terms);
final TermsConsumer termsConsumer = consumer.addField(fieldInfo);
- for (final TermData term : terms)
- term.write(termsConsumer);
- termsConsumer.finish();
+ long sumTotalTermCount = 0;
+ for (final TermData term : terms) {
+ sumTotalTermCount += term.write(termsConsumer);
+ }
+ termsConsumer.finish(sumTotalTermCount);
}
}
@@ -131,8 +134,9 @@ public class TestCodecs extends LuceneTe
return text.compareTo(((TermData) o).text);
}
- public void write(final TermsConsumer termsConsumer) throws Throwable {
+ public long write(final TermsConsumer termsConsumer) throws Throwable {
final PostingsConsumer postingsConsumer = termsConsumer.startTerm(text);
+ long totTF = 0;
for(int i=0;i<docs.length;i++) {
final int termDocFreq;
if (field.omitTF) {
@@ -142,6 +146,7 @@ public class TestCodecs extends LuceneTe
}
postingsConsumer.startDoc(docs[i], termDocFreq);
if (!field.omitTF) {
+ totTF += positions[i].length;
for(int j=0;j<positions[i].length;j++) {
final PositionData pos = positions[i][j];
postingsConsumer.addPosition(pos.pos, pos.payload);
@@ -149,7 +154,8 @@ public class TestCodecs extends LuceneTe
postingsConsumer.finishDoc();
}
}
- termsConsumer.finishTerm(text, docs.length);
+ termsConsumer.finishTerm(text, new TermStats(docs.length, totTF));
+ return totTF;
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Thu Jan 20 19:52:03 2011
@@ -29,7 +29,7 @@ import java.util.Map;
import java.util.HashMap;
import java.util.Set;
import java.util.SortedSet;
-
+import org.junit.Assume;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
@@ -1865,4 +1865,27 @@ public class TestIndexReader extends Luc
assertTrue(IndexReader.indexExists(dir));
dir.close();
}
+
+ // Make sure totalTermFreq works correctly in the terms
+ // dict cache
+ public void testTotalTermFreqCached() throws Exception {
+ Directory dir = newDirectory();
+ IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer()));
+ Document d = new Document();
+ d.add(newField("f", "a a b", Field.Index.ANALYZED));
+ writer.addDocument(d);
+ IndexReader r = writer.getReader();
+ writer.close();
+ Terms terms = MultiFields.getTerms(r, "f");
+ try {
+ // Make sure codec impls totalTermFreq (eg PreFlex doesn't)
+ Assume.assumeTrue(terms.totalTermFreq(new BytesRef("b")) != -1);
+ assertEquals(1, terms.totalTermFreq(new BytesRef("b")));
+ assertEquals(2, terms.totalTermFreq(new BytesRef("a")));
+ assertEquals(1, terms.totalTermFreq(new BytesRef("b")));
+ } finally {
+ r.close();
+ dir.close();
+ }
+ }
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Thu Jan 20 19:52:03 2011
@@ -785,7 +785,7 @@ public class TestIndexWriter extends Luc
public void testHighFreqTerm() throws IOException {
MockDirectoryWrapper dir = newDirectory();
IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxFieldLength(100000000).setRAMBufferSizeMB(0.01));
+ TEST_VERSION_CURRENT, new MockAnalyzer()).setRAMBufferSizeMB(0.01));
// Massive doc that has 128 K a's
StringBuilder b = new StringBuilder(1024*1024);
for(int i=0;i<4096;i++) {
@@ -1237,30 +1237,7 @@ public class TestIndexWriter extends Luc
writer.close();
dir.close();
}
-
- // LUCENE-1084: test user-specified field length
- public void testUserSpecifiedMaxFieldLength() throws IOException {
- Directory dir = newDirectory();
-
- IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
- TEST_VERSION_CURRENT, new MockAnalyzer()).setMaxFieldLength(100000));
-
- Document doc = new Document();
- StringBuilder b = new StringBuilder();
- for(int i=0;i<10000;i++)
- b.append(" a");
- b.append(" x");
- doc.add(newField("field", b.toString(), Field.Store.NO, Field.Index.ANALYZED));
- writer.addDocument(doc);
- writer.close();
-
- IndexReader reader = IndexReader.open(dir, true);
- Term t = new Term("field", "x");
- assertEquals(1, reader.docFreq(t));
- reader.close();
- dir.close();
- }
-
+
// LUCENE-325: test expungeDeletes, when 2 singular merges
// are required
public void testExpungeDeletes() throws IOException {
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterConfig.java Thu Jan 20 19:52:03 2011
@@ -17,7 +17,6 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import java.io.IOException;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
@@ -26,7 +25,6 @@ import java.util.Set;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.index.DocumentsWriter.IndexingChain;
-import org.apache.lucene.index.IndexWriter.IndexReaderWarmer;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.search.DefaultSimilarity;
import org.apache.lucene.search.Similarity;
@@ -49,22 +47,12 @@ public class TestIndexWriterConfig exten
}
- private static final class MyWarmer extends IndexReaderWarmer {
- // Does not implement anything - used only for type checking on IndexWriterConfig.
-
- @Override
- public void warm(IndexReader reader) throws IOException {
- }
-
- }
-
@Test
public void testDefaults() throws Exception {
IndexWriterConfig conf = new IndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer());
assertEquals(MockAnalyzer.class, conf.getAnalyzer().getClass());
assertNull(conf.getIndexCommit());
assertEquals(KeepOnlyLastCommitDeletionPolicy.class, conf.getIndexDeletionPolicy().getClass());
- assertEquals(IndexWriterConfig.UNLIMITED_FIELD_LENGTH, conf.getMaxFieldLength());
assertEquals(ConcurrentMergeScheduler.class, conf.getMergeScheduler().getClass());
assertEquals(OpenMode.CREATE_OR_APPEND, conf.getOpenMode());
assertTrue(Similarity.getDefault() == conf.getSimilarity());
@@ -129,7 +117,6 @@ public class TestIndexWriterConfig exten
// Tests that the values of the constants does not change
assertEquals(1000, IndexWriterConfig.WRITE_LOCK_TIMEOUT);
assertEquals(32, IndexWriterConfig.DEFAULT_TERM_INDEX_INTERVAL);
- assertEquals(Integer.MAX_VALUE, IndexWriterConfig.UNLIMITED_FIELD_LENGTH);
assertEquals(-1, IndexWriterConfig.DISABLE_AUTO_FLUSH);
assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, IndexWriterConfig.DEFAULT_MAX_BUFFERED_DELETE_TERMS);
assertEquals(IndexWriterConfig.DISABLE_AUTO_FLUSH, IndexWriterConfig.DEFAULT_MAX_BUFFERED_DOCS);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestIndexWriterOnJRECrash.java Thu Jan 20 19:52:03 2011
@@ -50,7 +50,7 @@ public class TestIndexWriterOnJRECrash e
public void testNRTThreads() throws Exception {
String vendor = Constants.JAVA_VENDOR;
assumeTrue(vendor + " JRE not supported.",
- vendor.startsWith("Sun") || vendor.startsWith("IBM") || vendor.startsWith("Apple"));
+ vendor.startsWith("Sun") || vendor.startsWith("Apple"));
// if we are not the fork
if (System.getProperty("tests.crashmode") == null) {
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/TestSizeBoundedOptimize.java Thu Jan 20 19:52:03 2011
@@ -63,7 +63,7 @@ public class TestSizeBoundedOptimize ext
conf = newWriterConfig();
LogByteSizeMergePolicy lmp = new LogByteSizeMergePolicy();
- lmp.setMaxMergeMB((min + 1) / (1 << 20));
+ lmp.setMaxMergeMBForOptimize((min + 1) / (1 << 20));
conf.setMergePolicy(lmp);
writer = new IndexWriter(dir, conf);
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java Thu Jan 20 19:52:03 2011
@@ -39,6 +39,7 @@ import org.apache.lucene.index.codecs.Te
import org.apache.lucene.index.codecs.TermsIndexWriterBase;
import org.apache.lucene.index.codecs.VariableGapTermsIndexReader;
import org.apache.lucene.index.codecs.VariableGapTermsIndexWriter;
+import org.apache.lucene.index.codecs.TermStats;
import org.apache.lucene.index.codecs.mockintblock.MockFixedIntBlockCodec;
import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec;
import org.apache.lucene.index.codecs.mocksep.MockSingleIntFactory;
@@ -66,7 +67,7 @@ public class MockRandomCodec extends Cod
public MockRandomCodec(Random random) {
name = "MockRandom";
- this.seedRandom = random;
+ this.seedRandom = new Random(random.nextLong());
}
@Override
@@ -148,7 +149,7 @@ public class MockRandomCodec extends Cod
final Random rand = new Random(seed2);
@Override
- public boolean isIndexTerm(BytesRef term, int docFreq) {
+ public boolean isIndexTerm(BytesRef term, TermStats stats) {
return random.nextInt(gap) == 17;
}
};
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/index/codecs/preflexrw/PreFlexFieldsWriter.java Thu Jan 20 19:52:03 2011
@@ -21,6 +21,7 @@ import org.apache.lucene.util.BytesRef;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.TermsConsumer;
import org.apache.lucene.index.codecs.PostingsConsumer;
+import org.apache.lucene.index.codecs.TermStats;
import org.apache.lucene.index.codecs.standard.DefaultSkipListWriter;
import org.apache.lucene.index.codecs.preflex.PreFlexCodec;
import org.apache.lucene.index.CorruptIndexException;
@@ -184,10 +185,10 @@ class PreFlexFieldsWriter extends Fields
}
@Override
- public void finishTerm(BytesRef text, int numDocs) throws IOException {
- if (numDocs > 0) {
+ public void finishTerm(BytesRef text, TermStats stats) throws IOException {
+ if (stats.docFreq > 0) {
long skipPointer = skipListWriter.writeSkip(freqOut);
- termInfo.docFreq = numDocs;
+ termInfo.docFreq = stats.docFreq;
termInfo.skipOffset = (int) (skipPointer - termInfo.freqPointer);
//System.out.println(" w finish term=" + text.utf8ToString() + " fnum=" + fieldInfo.number);
termsOut.add(fieldInfo.number,
@@ -197,7 +198,7 @@ class PreFlexFieldsWriter extends Fields
}
@Override
- public void finish() throws IOException {
+ public void finish(long sumTotalTermCount) throws IOException {
}
@Override
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/TestBooleanScorer.java Thu Jan 20 19:52:03 2011
@@ -25,6 +25,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.BooleanQuery.BooleanWeight;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -68,6 +69,13 @@ public class TestBooleanScorer extends L
// 'more' variable to work properly, and this test ensures that if the logic
// changes, we have a test to back it up.
+ Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random, directory);
+ writer.commit();
+ IndexReader ir = writer.getReader();
+ writer.close();
+ IndexSearcher searcher = new IndexSearcher(ir);
+
Similarity sim = Similarity.getDefault();
Scorer[] scorers = new Scorer[] {new Scorer(sim) {
private int doc = -1;
@@ -83,10 +91,15 @@ public class TestBooleanScorer extends L
}
}};
- BooleanScorer bs = new BooleanScorer(null, false, sim, 1, Arrays.asList(scorers), null, scorers.length);
+ BooleanWeight weight = (BooleanWeight) new BooleanQuery().createWeight(searcher);
+ BooleanScorer bs = new BooleanScorer(weight, false, 1, Arrays.asList(scorers), null, scorers.length);
assertEquals("should have received 3000", 3000, bs.nextDoc());
assertEquals("should have received NO_MORE_DOCS", DocIdSetIterator.NO_MORE_DOCS, bs.nextDoc());
+ searcher.close();
+ ir.close();
+ directory.close();
+
}
}
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/search/spans/TestSpans.java Thu Jan 20 19:52:03 2011
@@ -410,20 +410,21 @@ public class TestSpans extends LuceneTes
}
};
- SpanNearQuery snq = new SpanNearQuery(
+ final Similarity oldSim = searcher.getSimilarity();
+ Scorer spanScorer;
+ try {
+ searcher.setSimilarity(sim);
+ SpanNearQuery snq = new SpanNearQuery(
new SpanQuery[] {
makeSpanTermQuery("t1"),
makeSpanTermQuery("t2") },
slop,
- ordered) {
- @Override
- public Similarity getSimilarity(IndexSearcher s) {
- return sim;
- }
- };
-
- Scorer spanScorer = snq.weight(searcher).scorer(new AtomicReaderContext(new SlowMultiReaderWrapper(searcher.getIndexReader())), ScorerContext.def());
+ ordered);
+ spanScorer = snq.weight(searcher).scorer(new AtomicReaderContext(new SlowMultiReaderWrapper(searcher.getIndexReader())), ScorerContext.def());
+ } finally {
+ searcher.setSimilarity(oldSim);
+ }
assertTrue("first doc", spanScorer.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
assertEquals("first doc number", spanScorer.docID(), 11);
float score = spanScorer.score();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java Thu Jan 20 19:52:03 2011
@@ -109,34 +109,6 @@ public class TestAttributeSource extends
assertEquals("TypeAttribute of original and clone must be equal", typeAtt2, typeAtt);
}
- public void testToStringAndMultiAttributeImplementations() {
- AttributeSource src = new AttributeSource();
- CharTermAttribute termAtt = src.addAttribute(CharTermAttribute.class);
- TypeAttribute typeAtt = src.addAttribute(TypeAttribute.class);
- termAtt.append("TestTerm");
- typeAtt.setType("TestType");
- assertEquals("Attributes should appear in original order", "("+termAtt.toString()+","+typeAtt.toString()+")", src.toString());
- Iterator<AttributeImpl> it = src.getAttributeImplsIterator();
- assertTrue("Iterator should have 2 attributes left", it.hasNext());
- assertSame("First AttributeImpl from iterator should be termAtt", termAtt, it.next());
- assertTrue("Iterator should have 1 attributes left", it.hasNext());
- assertSame("Second AttributeImpl from iterator should be typeAtt", typeAtt, it.next());
- assertFalse("Iterator should have 0 attributes left", it.hasNext());
-
- src = new AttributeSource();
- src.addAttributeImpl(new Token());
- // this should not add a new attribute as Token implements CharTermAttribute, too
- termAtt = src.addAttribute(CharTermAttribute.class);
- assertTrue("CharTermAttribute should be implemented by Token", termAtt instanceof Token);
- // get the Token attribute and check, that it is the only one
- it = src.getAttributeImplsIterator();
- Token tok = (Token) it.next();
- assertFalse("There should be only one attribute implementation instance", it.hasNext());
-
- termAtt.setEmpty().append("TestTerm");
- assertEquals("Token should only printed once", "("+tok.toString()+")", src.toString());
- }
-
public void testDefaultAttributeFactory() throws Exception {
AttributeSource src = new AttributeSource();
Modified: lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/_TestUtil.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/bulkpostings/lucene/src/test/org/apache/lucene/util/_TestUtil.java Thu Jan 20 19:52:03 2011
@@ -22,6 +22,10 @@ import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.util.Random;
+import java.util.Map;
+import java.util.HashMap;
+
+import org.junit.Assert;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.ConcurrentMergeScheduler;
@@ -238,4 +242,17 @@ public class _TestUtil {
((ConcurrentMergeScheduler) ms).setMaxMergeCount(3);
}
}
+
+ /** Checks some basic behaviour of an AttributeImpl
+ * @param reflectedValues contains a map with "AttributeClass#key" as values
+ */
+ public static <T> void assertAttributeReflection(final AttributeImpl att, Map<String,T> reflectedValues) {
+ final Map<String,Object> map = new HashMap<String,Object>();
+ att.reflectWith(new AttributeReflector() {
+ public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
+ map.put(attClass.getName() + '#' + key, value);
+ }
+ });
+ Assert.assertEquals("Reflection does not produce same map", reflectedValues, map);
+ }
}
Modified: lucene/dev/branches/bulkpostings/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/dev/branches/bulkpostings/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Thu Jan 20 19:52:03 2011
@@ -34,14 +34,6 @@ import org.apache.lucene.util.Version;
* Unicode Text Segmentation algorithm, as specified in
* <a href="http://unicode.org/reports/tr29/">Unicode Standard Annex #29</a>.
* <p/>
- * <b>WARNING</b>: Because JFlex does not support Unicode supplementary
- * characters (characters above the Basic Multilingual Plane, which contains
- * those up to and including U+FFFF), this scanner will not recognize them
- * properly. If you need to be able to process text containing supplementary
- * characters, consider using the ICU4J-backed implementation in modules/analysis/icu
- * (org.apache.lucene.analysis.icu.segmentation.ICUTokenizer)
- * instead of this class, since the ICU4J-backed implementation does not have
- * this limitation.
* <p>Many applications have specific tokenizer needs. If this tokenizer does
* not suit your application, please consider copying this source code
* directory to your project and maintaining your own grammar-based tokenizer.
Modified: lucene/dev/branches/bulkpostings/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java (original)
+++ lucene/dev/branches/bulkpostings/modules/analysis/common/src/java/org/apache/lucene/analysis/synonym/SynonymFilter.java Thu Jan 20 19:52:03 2011
@@ -46,6 +46,9 @@ public final class SynonymFilter extends
public SynonymFilter(TokenStream in, SynonymMap map) {
super(in);
+ if (map == null)
+ throw new IllegalArgumentException("map is required");
+
this.map = map;
// just ensuring these attributes exist...
addAttribute(CharTermAttribute.class);
Modified: lucene/dev/branches/bulkpostings/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java (original)
+++ lucene/dev/branches/bulkpostings/modules/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestLimitTokenCountAnalyzer.java Thu Jan 20 19:52:03 2011
@@ -22,8 +22,16 @@ import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.analysis.core.WhitespaceAnalyzer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.document.Document;
+import org.apache.lucene.document.Field;
+import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.IndexWriterConfig;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.store.Directory;
public class TestLimitTokenCountAnalyzer extends BaseTokenStreamTestCase {
@@ -39,4 +47,26 @@ public class TestLimitTokenCountAnalyzer
assertTokenStreamContents(a.reusableTokenStream("dummy", new StringReader("1 2 3 4 5")), new String[] { "1", "2" }, new int[] { 0, 2 }, new int[] { 1, 3 }, 3);
}
+ public void testLimitTokenCountIndexWriter() throws IOException {
+ Directory dir = newDirectory();
+
+ IndexWriter writer = new IndexWriter(dir, new IndexWriterConfig(
+ TEST_VERSION_CURRENT, new LimitTokenCountAnalyzer(new MockAnalyzer(), 100000)));
+
+ Document doc = new Document();
+ StringBuilder b = new StringBuilder();
+ for(int i=0;i<10000;i++)
+ b.append(" a");
+ b.append(" x");
+ doc.add(newField("field", b.toString(), Field.Store.NO, Field.Index.ANALYZED));
+ writer.addDocument(doc);
+ writer.close();
+
+ IndexReader reader = IndexReader.open(dir, true);
+ Term t = new Term("field", "x");
+ assertEquals(1, reader.docFreq(t));
+ reader.close();
+ dir.close();
+ }
+
}
\ No newline at end of file
Modified: lucene/dev/branches/bulkpostings/modules/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/modules/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/modules/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java (original)
+++ lucene/dev/branches/bulkpostings/modules/analysis/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java Thu Jan 20 19:52:03 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.analysis.icu.t
import java.io.Serializable;
import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeReflector;
import com.ibm.icu.lang.UScript;
@@ -77,7 +78,7 @@ public class ScriptAttributeImpl extends
}
@Override
- public String toString() {
- return "script=" + getName();
+ public void reflectWith(AttributeReflector reflector) {
+ reflector.reflect(ScriptAttribute.class, "script", getName());
}
}
Modified: lucene/dev/branches/bulkpostings/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/bulkpostings/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java?rev=1061480&r1=1061479&r2=1061480&view=diff
==============================================================================
--- lucene/dev/branches/bulkpostings/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java (original)
+++ lucene/dev/branches/bulkpostings/modules/benchmark/src/java/org/apache/lucene/benchmark/byTask/tasks/CreateIndexTask.java Thu Jan 20 19:52:03 2011
@@ -46,8 +46,7 @@ import java.io.PrintStream;
* Create an index. <br>
* Other side effects: index writer object in perfRunData is set. <br>
* Relevant properties: <code>merge.factor (default 10),
- * max.buffered (default no flush), max.field.length (default
- * 10,000 tokens), max.field.length, compound (default true), ram.flush.mb [default 0],
+ * max.buffered (default no flush), compound (default true), ram.flush.mb [default 0],
* merge.policy (default org.apache.lucene.index.LogByteSizeMergePolicy),
* merge.scheduler (default
* org.apache.lucene.index.ConcurrentMergeScheduler),
@@ -153,7 +152,6 @@ public class CreateIndexTask extends Per
logMergePolicy.setMergeFactor(config.get("merge.factor",OpenIndexTask.DEFAULT_MERGE_PFACTOR));
}
}
- iwConf.setMaxFieldLength(config.get("max.field.length",OpenIndexTask.DEFAULT_MAX_FIELD_LENGTH));
final double ramBuffer = config.get("ram.flush.mb",OpenIndexTask.DEFAULT_RAM_FLUSH_MB);
final int maxBuffered = config.get("max.buffered",OpenIndexTask.DEFAULT_MAX_BUFFERED);
if (maxBuffered == IndexWriterConfig.DISABLE_AUTO_FLUSH) {