You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2011/10/10 21:58:32 UTC
svn commit: r1181188 [5/8] - in /lucene/dev/branches/solrcloud: ./
dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/lucene/contrib/
dev-tools/idea/modules/grouping/ dev-tools/idea/solr/
dev-tools/idea/solr/contrib/analysis-extras/ dev-tools/idea...
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/TermQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/TermQuery.java Mon Oct 10 19:58:24 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.Term;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ReaderUtil;
@@ -73,14 +74,15 @@ public class TermQuery extends Query {
}
@Override
- public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
+ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+ boolean topScorer, Bits acceptDocs) throws IOException {
assert termStates.topReaderContext == ReaderUtil.getTopLevelContext(context) : "The top-reader used to create Weight (" + termStates.topReaderContext + ") is not the same as the current reader's top-reader (" + ReaderUtil.getTopLevelContext(context);
final TermsEnum termsEnum = getTermsEnum(context);
if (termsEnum == null) {
return null;
}
// TODO should we reuse the DocsEnum here?
- final DocsEnum docs = termsEnum.docs(context.reader.getLiveDocs(), null);
+ final DocsEnum docs = termsEnum.docs(acceptDocs, null);
assert docs != null;
return new TermScorer(this, docs, createDocScorer(context));
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/Weight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/Weight.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/Weight.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/Weight.java Mon Oct 10 19:58:24 2011
@@ -23,6 +23,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
import org.apache.lucene.search.similarities.SimilarityProvider;
+import org.apache.lucene.util.Bits;
/**
* Expert: Calculate query weights and build query scorers.
@@ -34,7 +35,8 @@ import org.apache.lucene.search.similari
* {@link IndexReader} dependent state should reside in the {@link Scorer}.
* <p>
* Since {@link Weight} creates {@link Scorer} instances for a given
- * {@link AtomicReaderContext} ({@link #scorer(IndexReader.AtomicReaderContext, ScorerContext)})
+ * {@link AtomicReaderContext} ({@link #scorer(IndexReader.AtomicReaderContext,
+ * boolean, boolean, Bits)})
* callers must maintain the relationship between the searcher's top-level
* {@link ReaderContext} and the context used to create a {@link Scorer}.
* <p>
@@ -49,7 +51,7 @@ import org.apache.lucene.search.similari
* <li>The query normalization factor is passed to {@link #normalize(float, float)}. At
* this point the weighting is complete.
* <li>A <code>Scorer</code> is constructed by
- * {@link #scorer(IndexReader.AtomicReaderContext, ScorerContext)}.
+ * {@link #scorer(IndexReader.AtomicReaderContext, boolean, boolean, Bits)}.
* </ol>
*
* @since 2.9
@@ -89,17 +91,33 @@ public abstract class Weight {
*
* @param context
* the {@link AtomicReaderContext} for which to return the {@link Scorer}.
- * @param scorerContext the {@link ScorerContext} holding the scores context variables
+ * @param scoreDocsInOrder
+ * specifies whether in-order scoring of documents is required. Note
+ * that if set to false (i.e., out-of-order scoring is required),
+ * this method can return whatever scoring mode it supports, as every
+ * in-order scorer is also an out-of-order one. However, an
+ * out-of-order scorer may not support {@link Scorer#nextDoc()}
+ * and/or {@link Scorer#advance(int)}, therefore it is recommended to
+ * request an in-order scorer if use of these methods is required.
+ * @param topScorer
+ * if true, {@link Scorer#score(Collector)} will be called; if false,
+ * {@link Scorer#nextDoc()} and/or {@link Scorer#advance(int)} will
+ * be called.
+ * @param acceptDocs
+ * Bits that represent the allowable docs to match (typically deleted docs
+ * but possibly filtering other documents)
+ *
* @return a {@link Scorer} which scores documents in/out-of order.
* @throws IOException
*/
- public abstract Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException;
+ public abstract Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+ boolean topScorer, Bits acceptDocs) throws IOException;
/**
* Returns true iff this implementation scores docs only out of order. This
* method is used in conjunction with {@link Collector}'s
* {@link Collector#acceptsDocsOutOfOrder() acceptsDocsOutOfOrder} and
- * {@link #scorer(IndexReader.AtomicReaderContext, ScorerContext)} to
+ * {@link #scorer(IndexReader.AtomicReaderContext, boolean, boolean, Bits)} to
* create a matching {@link Scorer} instance for a given {@link Collector}, or
* vice versa.
* <p>
@@ -107,83 +125,4 @@ public abstract class Weight {
* the <code>Scorer</code> scores documents in-order.
*/
public boolean scoresDocsOutOfOrder() { return false; }
-
- /**
- * A struct like class encapsulating a scorer's context variables.
- * ScorerContex is a strictly immutable struct that follows a
- * <tt>create on modification</tt> pattern. If a context variable changes
- * through one of the modifiers like {@link #topScorer(boolean)} a new
- * {@link ScorerContext} instance is creates. If the modifier call doesn't
- * change the instance the method call has no effect and the same instance is
- * returned from the modifier.
- *
- * @lucene.experimental
- */
- public static final class ScorerContext {
-
- /**
- * Specifies whether in-order scoring of documents is required. Note that if
- * set to false (i.e., out-of-order scoring is required), this method can
- * return whatever scoring mode it supports, as every in-order scorer is
- * also an out-of-order one. However, an out-of-order scorer may not support
- * {@link Scorer#nextDoc()} and/or {@link Scorer#advance(int)}, therefore it
- * is recommended to request an in-order scorer if use of these methods is
- * required.
- */
- public final boolean scoreDocsInOrder;
-
- /**
- * if <code>true</code>, {@link Scorer#score(Collector)} will be called; if
- * false, {@link Scorer#nextDoc()} and/or {@link Scorer#advance(int)} will
- * be called instead.
- */
- public final boolean topScorer;
-
-
- private static final ScorerContext DEFAULT_CONTEXT = new ScorerContext(true, false);
-
- /**
- * Returns a default {@link ScorerContext} template initialized with:
- * <ul>
- * <li>{@link #scoreDocsInOrder} = <code>true</code></li>
- * <li>{@link #topScorer} = <code>false</code></li>
- * </ul>
- */
- public static ScorerContext def() {
- return DEFAULT_CONTEXT;
- }
-
- private ScorerContext(boolean scoreDocsInOrder, boolean topScorer) {
- this.scoreDocsInOrder = scoreDocsInOrder;
- this.topScorer = topScorer;
- }
-
- /**
- * Creates and returns a copy of this context with the given value for
- * {@link #scoreDocsInOrder} and returns a new instance of
- * {@link ScorerContext} iff the given value differs from the
- * {@link #scoreDocsInOrder}. Otherwise, this method has no effect and
- * returns this instance.
- */
- public ScorerContext scoreDocsInOrder(boolean scoreDocsInOrder) {
- if (this.scoreDocsInOrder == scoreDocsInOrder) {
- return this;
- }
- return new ScorerContext(scoreDocsInOrder, topScorer);
- }
-
- /**
- * Creates and returns a copy of this context with the given value for
- * {@link #topScorer} and returns a new instance of
- * {@link ScorerContext} iff the given value differs from the
- * {@link #topScorer}. Otherwise, this method has no effect and
- * returns this instance.
- */
- public ScorerContext topScorer(boolean topScorer) {
- if (this.topScorer == topScorer) {
- return this;
- }
- return new ScorerContext(scoreDocsInOrder, topScorer);
- }
- }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java Mon Oct 10 19:58:24 2011
@@ -33,6 +33,7 @@ import org.apache.lucene.search.spans.Sp
import org.apache.lucene.search.spans.SpanScorer;
import org.apache.lucene.search.spans.SpanWeight;
import org.apache.lucene.search.spans.Spans;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.ToStringUtils;
@@ -147,14 +148,15 @@ public class PayloadNearQuery extends Sp
}
@Override
- public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- return new PayloadNearSpanScorer(query.getSpans(context), this,
+ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+ boolean topScorer, Bits acceptDocs) throws IOException {
+ return new PayloadNearSpanScorer(query.getSpans(context, acceptDocs), this,
similarity, similarity.sloppyDocScorer(stats, query.getField(), context));
}
@Override
public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
- PayloadNearSpanScorer scorer = (PayloadNearSpanScorer) scorer(context, ScorerContext.def());
+ PayloadNearSpanScorer scorer = (PayloadNearSpanScorer) scorer(context, true, false, context.reader.getLiveDocs());
if (scorer != null) {
int newDoc = scorer.advance(doc);
if (newDoc == doc) {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/payloads/PayloadSpanUtil.java Mon Oct 10 19:58:24 2011
@@ -176,7 +176,7 @@ public class PayloadSpanUtil {
throws IOException {
final AtomicReaderContext[] leaves = ReaderUtil.leaves(context);
for (AtomicReaderContext atomicReaderContext : leaves) {
- final Spans spans = query.getSpans(atomicReaderContext);
+ final Spans spans = query.getSpans(atomicReaderContext, atomicReaderContext.reader.getLiveDocs());
while (spans.next() == true) {
if (spans.isPayloadAvailable()) {
Collection<byte[]> payload = spans.getPayload();
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java Mon Oct 10 19:58:24 2011
@@ -25,7 +25,6 @@ import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.ComplexExplanation;
-import org.apache.lucene.search.Weight.ScorerContext;
import org.apache.lucene.search.payloads.PayloadNearQuery.PayloadNearSpanScorer;
import org.apache.lucene.search.similarities.DefaultSimilarity;
import org.apache.lucene.search.similarities.Similarity;
@@ -34,6 +33,7 @@ import org.apache.lucene.search.spans.Te
import org.apache.lucene.search.spans.SpanTermQuery;
import org.apache.lucene.search.spans.SpanWeight;
import org.apache.lucene.search.spans.SpanScorer;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import java.io.IOException;
@@ -79,8 +79,9 @@ public class PayloadTermQuery extends Sp
}
@Override
- public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- return new PayloadTermSpanScorer((TermSpans) query.getSpans(context),
+ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+ boolean topScorer, Bits acceptDocs) throws IOException {
+ return new PayloadTermSpanScorer((TermSpans) query.getSpans(context, acceptDocs),
this, similarity.sloppyDocScorer(stats, query.getField(), context));
}
@@ -174,7 +175,7 @@ public class PayloadTermQuery extends Sp
@Override
public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
- PayloadTermSpanScorer scorer = (PayloadTermSpanScorer) scorer(context, ScorerContext.def());
+ PayloadTermSpanScorer scorer = (PayloadTermSpanScorer) scorer(context, true, false, context.reader.getLiveDocs());
if (scorer != null) {
int newDoc = scorer.advance(doc);
if (newDoc == doc) {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/similarities/BM25Similarity.java Mon Oct 10 19:58:24 2011
@@ -277,6 +277,8 @@ public class BM25Similarity extends Simi
private final float avgdl;
/** query's inner boost */
private final float queryBoost;
+ /** query's outer boost (only for explain) */
+ private float topLevelBoost;
/** weight (idf * boost) */
private float weight;
/** precomputed norm[256] with k1 * ((1 - b) + b * dl / avgdl) */
@@ -299,6 +301,7 @@ public class BM25Similarity extends Simi
@Override
public void normalize(float queryNorm, float topLevelBoost) {
// we don't normalize with queryNorm at all, we just capture the top-level boost
+ this.topLevelBoost = topLevelBoost;
this.weight = idf.getValue() * queryBoost * topLevelBoost;
}
}
@@ -307,8 +310,8 @@ public class BM25Similarity extends Simi
Explanation result = new Explanation();
result.setDescription("score(doc="+doc+",freq="+freq+"), product of:");
- Explanation boostExpl = new Explanation(stats.queryBoost, "boost");
- if (stats.queryBoost != 1.0f)
+ Explanation boostExpl = new Explanation(stats.queryBoost * stats.topLevelBoost, "boost");
+ if (boostExpl.getValue() != 1.0f)
result.addDetail(boostExpl);
result.addDetail(stats.idf);
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/FieldMaskingSpanQuery.java Mon Oct 10 19:58:24 2011
@@ -26,6 +26,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Weight;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ToStringUtils;
/**
@@ -92,8 +93,8 @@ public class FieldMaskingSpanQuery exten
// ...this is done to be more consistent with things like SpanFirstQuery
@Override
- public Spans getSpans(AtomicReaderContext context) throws IOException {
- return maskedQuery.getSpans(context);
+ public Spans getSpans(AtomicReaderContext context, Bits acceptDocs) throws IOException {
+ return maskedQuery.getSpans(context, acceptDocs);
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/NearSpansOrdered.java Mon Oct 10 19:58:24 2011
@@ -19,6 +19,7 @@ package org.apache.lucene.search.spans;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.util.ArrayUtil;
+import org.apache.lucene.util.Bits;
import java.io.IOException;
import java.util.ArrayList;
@@ -77,11 +78,11 @@ public class NearSpansOrdered extends Sp
private SpanNearQuery query;
private boolean collectPayloads = true;
- public NearSpansOrdered(SpanNearQuery spanNearQuery, AtomicReaderContext context) throws IOException {
- this(spanNearQuery, context, true);
+ public NearSpansOrdered(SpanNearQuery spanNearQuery, AtomicReaderContext context, Bits acceptDocs) throws IOException {
+ this(spanNearQuery, context, acceptDocs, true);
}
- public NearSpansOrdered(SpanNearQuery spanNearQuery, AtomicReaderContext context, boolean collectPayloads)
+ public NearSpansOrdered(SpanNearQuery spanNearQuery, AtomicReaderContext context, Bits acceptDocs, boolean collectPayloads)
throws IOException {
if (spanNearQuery.getClauses().length < 2) {
throw new IllegalArgumentException("Less than 2 clauses: "
@@ -94,7 +95,7 @@ public class NearSpansOrdered extends Sp
matchPayload = new LinkedList<byte[]>();
subSpansByDoc = new Spans[clauses.length];
for (int i = 0; i < clauses.length; i++) {
- subSpans[i] = clauses[i].getSpans(context);
+ subSpans[i] = clauses[i].getSpans(context, acceptDocs);
subSpansByDoc[i] = subSpans[i]; // used in toSameDoc()
}
query = spanNearQuery; // kept for toString() only.
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/NearSpansUnordered.java Mon Oct 10 19:58:24 2011
@@ -18,6 +18,7 @@ package org.apache.lucene.search.spans;
*/
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.PriorityQueue;
import java.io.IOException;
@@ -131,7 +132,7 @@ public class NearSpansUnordered extends
}
- public NearSpansUnordered(SpanNearQuery query, AtomicReaderContext context)
+ public NearSpansUnordered(SpanNearQuery query, AtomicReaderContext context, Bits acceptDocs)
throws IOException {
this.query = query;
this.slop = query.getSlop();
@@ -141,7 +142,7 @@ public class NearSpansUnordered extends
subSpans = new Spans[clauses.length];
for (int i = 0; i < clauses.length; i++) {
SpansCell cell =
- new SpansCell(clauses[i].getSpans(context), i);
+ new SpansCell(clauses[i].getSpans(context, acceptDocs), i);
ordered.add(cell);
subSpans[i] = cell.spans;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanMultiTermQueryWrapper.java Mon Oct 10 19:58:24 2011
@@ -27,6 +27,7 @@ import org.apache.lucene.search.Query;
import org.apache.lucene.search.TopTermsRewrite;
import org.apache.lucene.search.ScoringRewrite;
import org.apache.lucene.search.BooleanClause.Occur; // javadocs only
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.TermContext;
/**
@@ -89,7 +90,7 @@ public class SpanMultiTermQueryWrapper<Q
}
@Override
- public Spans getSpans(AtomicReaderContext context) throws IOException {
+ public Spans getSpans(AtomicReaderContext context, Bits acceptDocs) throws IOException {
throw new UnsupportedOperationException("Query should have been rewritten");
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanNearQuery.java Mon Oct 10 19:58:24 2011
@@ -30,6 +30,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ToStringUtils;
/** Matches spans which are near one another. One can specify <i>slop</i>, the
@@ -117,16 +118,16 @@ public class SpanNearQuery extends SpanQ
}
@Override
- public Spans getSpans(final AtomicReaderContext context) throws IOException {
+ public Spans getSpans(final AtomicReaderContext context, Bits acceptDocs) throws IOException {
if (clauses.size() == 0) // optimize 0-clause case
- return new SpanOrQuery(getClauses()).getSpans(context);
+ return new SpanOrQuery(getClauses()).getSpans(context, acceptDocs);
if (clauses.size() == 1) // optimize 1-clause case
- return clauses.get(0).getSpans(context);
+ return clauses.get(0).getSpans(context, acceptDocs);
return inOrder
- ? (Spans) new NearSpansOrdered(this, context, collectPayloads)
- : (Spans) new NearSpansUnordered(this, context);
+ ? (Spans) new NearSpansOrdered(this, context, acceptDocs, collectPayloads)
+ : (Spans) new NearSpansUnordered(this, context, acceptDocs);
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanNotQuery.java Mon Oct 10 19:58:24 2011
@@ -21,6 +21,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ToStringUtils;
import java.io.IOException;
@@ -75,12 +76,12 @@ public class SpanNotQuery extends SpanQu
}
@Override
- public Spans getSpans(final AtomicReaderContext context) throws IOException {
+ public Spans getSpans(final AtomicReaderContext context, final Bits acceptDocs) throws IOException {
return new Spans() {
- private Spans includeSpans = include.getSpans(context);
+ private Spans includeSpans = include.getSpans(context, acceptDocs);
private boolean moreInclude = true;
- private Spans excludeSpans = exclude.getSpans(context);
+ private Spans excludeSpans = exclude.getSpans(context, acceptDocs);
private boolean moreExclude = excludeSpans.next();
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanOrQuery.java Mon Oct 10 19:58:24 2011
@@ -28,6 +28,7 @@ import java.util.Set;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.PriorityQueue;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.search.Query;
@@ -163,9 +164,9 @@ public class SpanOrQuery extends SpanQue
}
@Override
- public Spans getSpans(final AtomicReaderContext context) throws IOException {
+ public Spans getSpans(final AtomicReaderContext context, final Bits acceptDocs) throws IOException {
if (clauses.size() == 1) // optimize 1-clause case
- return (clauses.get(0)).getSpans(context);
+ return (clauses.get(0)).getSpans(context, acceptDocs);
return new Spans() {
private SpanQueue queue = null;
@@ -174,7 +175,7 @@ public class SpanOrQuery extends SpanQue
queue = new SpanQueue(clauses.size());
Iterator<SpanQuery> i = clauses.iterator();
while (i.hasNext()) {
- Spans spans = i.next().getSpans(context);
+ Spans spans = i.next().getSpans(context, acceptDocs);
if ( ((target == -1) && spans.next())
|| ((target != -1) && spans.skipTo(target))) {
queue.add(spans);
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanPositionCheckQuery.java Mon Oct 10 19:58:24 2011
@@ -21,6 +21,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.Query;
+import org.apache.lucene.util.Bits;
import java.io.IOException;
import java.util.ArrayList;
@@ -81,8 +82,8 @@ public abstract class SpanPositionCheckQ
protected abstract AcceptStatus acceptPosition(Spans spans) throws IOException;
@Override
- public Spans getSpans(final AtomicReaderContext context) throws IOException {
- return new PositionCheckSpan(context);
+ public Spans getSpans(final AtomicReaderContext context, Bits acceptDocs) throws IOException {
+ return new PositionCheckSpan(context, acceptDocs);
}
@@ -106,8 +107,8 @@ public abstract class SpanPositionCheckQ
protected class PositionCheckSpan extends Spans {
private Spans spans;
- public PositionCheckSpan(AtomicReaderContext context) throws IOException {
- spans = match.getSpans(context);
+ public PositionCheckSpan(AtomicReaderContext context, Bits acceptDocs) throws IOException {
+ spans = match.getSpans(context, acceptDocs);
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanQuery.java Mon Oct 10 19:58:24 2011
@@ -23,12 +23,13 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.search.Query;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Weight;
+import org.apache.lucene.util.Bits;
/** Base class for span-based queries. */
public abstract class SpanQuery extends Query {
/** Expert: Returns the matches for this query in an index. Used internally
* to search for spans. */
- public abstract Spans getSpans(AtomicReaderContext context) throws IOException;
+ public abstract Spans getSpans(AtomicReaderContext context, Bits acceptDocs) throws IOException;
/** Returns the name of the field matched by this query.*/
public abstract String getField();
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanTermQuery.java Mon Oct 10 19:58:24 2011
@@ -21,6 +21,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.ToStringUtils;
import java.io.IOException;
@@ -81,9 +82,9 @@ public class SpanTermQuery extends SpanQ
}
@Override
- public Spans getSpans(final AtomicReaderContext context) throws IOException {
+ public Spans getSpans(final AtomicReaderContext context, Bits acceptDocs) throws IOException {
final IndexReader reader = context.reader;
- final DocsAndPositionsEnum postings = reader.termPositionsEnum(reader.getLiveDocs(),
+ final DocsAndPositionsEnum postings = reader.termPositionsEnum(acceptDocs,
term.field(),
term.bytes());
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/spans/SpanWeight.java Mon Oct 10 19:58:24 2011
@@ -23,6 +23,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.search.*;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.Similarity.SloppyDocScorer;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.TermContext;
import java.io.IOException;
@@ -67,13 +68,14 @@ public class SpanWeight extends Weight {
}
@Override
- public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- return new SpanScorer(query.getSpans(context), this, similarity.sloppyDocScorer(stats, query.getField(), context));
+ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+ boolean topScorer, Bits acceptDocs) throws IOException {
+ return new SpanScorer(query.getSpans(context, acceptDocs), this, similarity.sloppyDocScorer(stats, query.getField(), context));
}
@Override
public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
- Scorer scorer = scorer(context, ScorerContext.def());
+ Scorer scorer = scorer(context, true, false, context.reader.getLiveDocs());
if (scorer != null) {
int newDoc = scorer.advance(doc);
if (newDoc == doc) {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct16.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct16.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct16.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct16.java Mon Oct 10 19:58:24 2011
@@ -28,7 +28,7 @@ import java.util.Arrays;
* @lucene.internal
*/
-public class Direct16 extends PackedInts.ReaderImpl
+class Direct16 extends PackedInts.ReaderImpl
implements PackedInts.Mutable {
private short[] values;
private static final int BITS_PER_VALUE = 16;
@@ -68,10 +68,6 @@ public class Direct16 extends PackedInts
this.values = values;
}
- public short[] getArray() {
- return values;
- }
-
public long get(final int index) {
return 0xFFFFL & values[index];
}
@@ -88,4 +84,14 @@ public class Direct16 extends PackedInts
public void clear() {
Arrays.fill(values, (short)0);
}
+
+ @Override
+ public Object getArray() {
+ return values;
+ }
+
+ @Override
+ public boolean hasArray() {
+ return true;
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct32.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct32.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct32.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct32.java Mon Oct 10 19:58:24 2011
@@ -28,7 +28,7 @@ import java.util.Arrays;
* @lucene.internal
*/
-public class Direct32 extends PackedInts.ReaderImpl
+class Direct32 extends PackedInts.ReaderImpl
implements PackedInts.Mutable {
private int[] values;
private static final int BITS_PER_VALUE = 32;
@@ -64,10 +64,6 @@ public class Direct32 extends PackedInts
this.values = values;
}
- public int[] getArray() {
- return values;
- }
-
public long get(final int index) {
return 0xFFFFFFFFL & values[index];
}
@@ -84,4 +80,14 @@ public class Direct32 extends PackedInts
public void clear() {
Arrays.fill(values, 0);
}
+
+ @Override
+ public int[] getArray() {
+ return values;
+ }
+
+ @Override
+ public boolean hasArray() {
+ return true;
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct64.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct64.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct64.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct64.java Mon Oct 10 19:58:24 2011
@@ -28,7 +28,7 @@ import java.util.Arrays;
* @lucene.internal
*/
-public class Direct64 extends PackedInts.ReaderImpl
+class Direct64 extends PackedInts.ReaderImpl
implements PackedInts.Mutable {
private long[] values;
private static final int BITS_PER_VALUE = 64;
@@ -48,10 +48,6 @@ public class Direct64 extends PackedInts
this.values = values;
}
- public long[] getArray() {
- return values;
- }
-
/**
* Creates an array backed by the given values.
* </p><p>
@@ -80,4 +76,14 @@ public class Direct64 extends PackedInts
public void clear() {
Arrays.fill(values, 0L);
}
+
+ @Override
+ public long[] getArray() {
+ return values;
+ }
+
+ @Override
+ public boolean hasArray() {
+ return true;
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct8.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct8.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct8.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/Direct8.java Mon Oct 10 19:58:24 2011
@@ -28,7 +28,7 @@ import java.util.Arrays;
* @lucene.internal
*/
-public class Direct8 extends PackedInts.ReaderImpl
+class Direct8 extends PackedInts.ReaderImpl
implements PackedInts.Mutable {
private byte[] values;
private static final int BITS_PER_VALUE = 8;
@@ -69,10 +69,6 @@ public class Direct8 extends PackedInts.
this.values = values;
}
- public byte[] getArray() {
- return values;
- }
-
public long get(final int index) {
return 0xFFL & values[index];
}
@@ -88,4 +84,14 @@ public class Direct8 extends PackedInts.
public void clear() {
Arrays.fill(values, (byte)0);
}
+
+ @Override
+ public Object getArray() {
+ return values;
+ }
+
+ @Override
+ public boolean hasArray() {
+ return true;
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/GrowableWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/GrowableWriter.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/GrowableWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/GrowableWriter.java Mon Oct 10 19:58:24 2011
@@ -60,6 +60,16 @@ public class GrowableWriter implements P
return current;
}
+ @Override
+ public Object getArray() {
+ return current.getArray();
+ }
+
+ @Override
+ public boolean hasArray() {
+ return current.hasArray();
+ }
+
public void set(int index, long value) {
if (value >= currentMaxValue) {
int bpv = getBitsPerValue();
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/PackedInts.java Mon Oct 10 19:58:24 2011
@@ -65,6 +65,25 @@ public class PackedInts {
* @return the number of values.
*/
int size();
+
+ /**
+ * Expert: if the bit-width of this reader matches one of
+ * java's native types, returns the underlying array
+ * (ie, byte[], short[], int[], long[]); else, returns
+ * null. Note that when accessing the array you must
+ * upgrade the type (bitwise AND with all ones), to
+ * interpret the full value as unsigned. Ie,
+ * bytes[idx]&0xFF, shorts[idx]&0xFFFF, etc.
+ */
+ Object getArray();
+
+ /**
+ * Returns true if this implementation is backed by a
+ * native java array.
+ *
+ * @see #getArray
+ */
+ boolean hasArray();
}
/**
@@ -85,6 +104,14 @@ public class PackedInts {
long advance(int ord) throws IOException;
}
+ public static interface RandomAccessReaderIterator extends ReaderIterator {
+ /**
+ * @param index the position of the wanted value.
+ * @return the value at the stated index.
+ */
+ long get(int index) throws IOException;
+ }
+
/**
* A packed integer array that can be modified.
* @lucene.internal
@@ -129,6 +156,14 @@ public class PackedInts {
public long getMaxValue() { // Convenience method
return maxValue(bitsPerValue);
}
+
+ public Object getArray() {
+ return null;
+ }
+
+ public boolean hasArray() {
+ return false;
+ }
}
/** A write-once Writer.
@@ -195,6 +230,17 @@ public class PackedInts {
* @lucene.internal
*/
public static ReaderIterator getReaderIterator(IndexInput in) throws IOException {
+ return getRandomAccessReaderIterator(in);
+ }
+
+ /**
+ * Retrieve PackedInts as a {@link RandomAccessReaderIterator}
+ * @param in positioned at the beginning of a stored packed int structure.
+ * @return an iterator to access the values
+ * @throws IOException if the structure could not be retrieved.
+ * @lucene.internal
+ */
+ public static RandomAccessReaderIterator getRandomAccessReaderIterator(IndexInput in) throws IOException {
CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START);
final int bitsPerValue = in.readVInt();
assert bitsPerValue > 0 && bitsPerValue <= 64: "bitsPerValue=" + bitsPerValue;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/util/packed/PackedReaderIterator.java Mon Oct 10 19:58:24 2011
@@ -21,13 +21,15 @@ import org.apache.lucene.store.IndexInpu
import java.io.IOException;
-final class PackedReaderIterator implements PackedInts.ReaderIterator {
+final class PackedReaderIterator implements PackedInts.RandomAccessReaderIterator {
private long pending;
private int pendingBitsLeft;
private final IndexInput in;
private final int bitsPerValue;
private final int valueCount;
private int position = -1;
+ private long currentValue;
+ private final long startPointer;
// masks[n-1] masks for bottom n bits
private final long[] masks;
@@ -39,6 +41,7 @@ final class PackedReaderIterator impleme
this.bitsPerValue = bitsPerValue;
this.in = in;
+ startPointer = in.getFilePointer();
masks = new long[bitsPerValue];
long v = 1;
@@ -76,7 +79,7 @@ final class PackedReaderIterator impleme
}
++position;
- return result;
+ return currentValue = result;
}
public void close() throws IOException {
@@ -106,6 +109,26 @@ final class PackedReaderIterator impleme
pendingBitsLeft = 64 - (int)(skip % 64);
}
position = ord-1;
- return next();
+ return currentValue = next();
+ }
+
+
+ @Override
+ public long get(int index) throws IOException {
+ assert index < valueCount : "ord must be less than valueCount";
+ if (index < position) {
+ pendingBitsLeft = 0;
+ final long bitsToSkip = (((long) bitsPerValue) * (long) index);
+ final long skip = bitsToSkip - pendingBitsLeft;
+ final long closestByte = (skip >> 6) << 3;
+ in.seek(startPointer + closestByte);
+ pending = in.readLong();
+ pendingBitsLeft = 64 - (int) (skip % 64);
+ position = index - 1;
+ return currentValue = next();
+ } else if (index == position) {
+ return currentValue;
+ }
+ return advance(index);
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/RandomIndexWriter.java Mon Oct 10 19:58:24 2011
@@ -176,9 +176,10 @@ public class RandomIndexWriter implement
IndexDocValuesField docValuesField = new IndexDocValuesField(name);
switch (type) {
case BYTES_FIXED_DEREF:
- case BYTES_FIXED_SORTED:
case BYTES_FIXED_STRAIGHT:
- final String randomUnicodeString = _TestUtil.randomUnicodeString(random, fixedBytesLength);
+ case BYTES_FIXED_SORTED:
+ //make sure we use a valid unicode string with a fixed size byte length
+ final String randomUnicodeString = _TestUtil.randomFixedByteLengthUnicodeString(random, fixedBytesLength);
BytesRef fixedRef = new BytesRef(randomUnicodeString);
if (fixedRef.length > fixedBytesLength) {
fixedRef = new BytesRef(fixedRef.bytes, 0, fixedBytesLength);
@@ -189,8 +190,8 @@ public class RandomIndexWriter implement
docValuesField.setBytes(fixedRef, type);
break;
case BYTES_VAR_DEREF:
- case BYTES_VAR_SORTED:
case BYTES_VAR_STRAIGHT:
+ case BYTES_VAR_SORTED:
BytesRef ref = new BytesRef(_TestUtil.randomUnicodeString(random, 200));
docValuesField.setBytes(ref, type);
break;
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockFixedIntBlockCodec.java Mon Oct 10 19:58:24 2011
@@ -30,16 +30,16 @@ import org.apache.lucene.index.codecs.Fi
import org.apache.lucene.index.codecs.sep.IntStreamFactory;
import org.apache.lucene.index.codecs.sep.IntIndexInput;
import org.apache.lucene.index.codecs.sep.IntIndexOutput;
+import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer;
+import org.apache.lucene.index.codecs.sep.SepDocValuesProducer;
import org.apache.lucene.index.codecs.sep.SepPostingsReader;
import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
import org.apache.lucene.index.codecs.standard.StandardCodec;
import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexInput;
import org.apache.lucene.index.codecs.intblock.FixedIntBlockIndexOutput;
-import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.PostingsWriterBase;
import org.apache.lucene.index.codecs.PostingsReaderBase;
@@ -207,7 +207,7 @@ public class MockFixedIntBlockCodec exte
SepPostingsReader.files(segmentInfo, codecId, files);
BlockTermsReader.files(dir, segmentInfo, codecId, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
- DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
+ SepDocValuesConsumer.files(dir, segmentInfo, codecId, files);
}
@Override
@@ -215,16 +215,16 @@ public class MockFixedIntBlockCodec exte
SepPostingsWriter.getExtensions(extensions);
BlockTermsReader.getExtensions(extensions);
FixedGapTermsIndexReader.getIndexExtensions(extensions);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+ SepDocValuesConsumer.getExtensions(extensions);
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+ return new SepDocValuesConsumer(state);
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+ return new SepDocValuesProducer(state);
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mockintblock/MockVariableIntBlockCodec.java Mon Oct 10 19:58:24 2011
@@ -30,16 +30,16 @@ import org.apache.lucene.index.codecs.Fi
import org.apache.lucene.index.codecs.sep.IntStreamFactory;
import org.apache.lucene.index.codecs.sep.IntIndexInput;
import org.apache.lucene.index.codecs.sep.IntIndexOutput;
+import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer;
+import org.apache.lucene.index.codecs.sep.SepDocValuesProducer;
import org.apache.lucene.index.codecs.sep.SepPostingsReader;
import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
import org.apache.lucene.index.codecs.standard.StandardCodec;
import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexInput;
import org.apache.lucene.index.codecs.intblock.VariableIntBlockIndexOutput;
-import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.PostingsWriterBase;
import org.apache.lucene.index.codecs.PostingsReaderBase;
@@ -230,7 +230,7 @@ public class MockVariableIntBlockCodec e
SepPostingsReader.files(segmentInfo, codecId, files);
BlockTermsReader.files(dir, segmentInfo, codecId, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
- DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
+ SepDocValuesConsumer.files(dir, segmentInfo, codecId, files);
}
@Override
@@ -238,16 +238,16 @@ public class MockVariableIntBlockCodec e
SepPostingsWriter.getExtensions(extensions);
BlockTermsReader.getExtensions(extensions);
FixedGapTermsIndexReader.getIndexExtensions(extensions);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+ SepDocValuesConsumer.getExtensions(extensions);
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+ return new SepDocValuesConsumer(state);
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+ return new SepDocValuesProducer(state);
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mockrandom/MockRandomCodec.java Mon Oct 10 19:58:24 2011
@@ -58,6 +58,8 @@ import org.apache.lucene.index.codecs.pu
import org.apache.lucene.index.codecs.sep.IntIndexInput;
import org.apache.lucene.index.codecs.sep.IntIndexOutput;
import org.apache.lucene.index.codecs.sep.IntStreamFactory;
+import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer;
+import org.apache.lucene.index.codecs.sep.SepDocValuesProducer;
import org.apache.lucene.index.codecs.sep.SepPostingsReader;
import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
import org.apache.lucene.index.codecs.standard.StandardPostingsReader;
@@ -75,17 +77,13 @@ import org.apache.lucene.util._TestUtil;
*/
public class MockRandomCodec extends Codec {
-
+ private final boolean useSepDocValues;
private final Random seedRandom;
private final String SEED_EXT = "sd";
public MockRandomCodec(Random random) {
- this(random, "MockRandom", true);
-
- }
-
- protected MockRandomCodec(Random random, String name, boolean docValuesUseCompoundFile) {
- super(name, docValuesUseCompoundFile);
+ super("MockRandom");
+ this.useSepDocValues = random.nextBoolean();
this.seedRandom = new Random(random.nextLong());
}
@@ -428,7 +426,11 @@ public class MockRandomCodec extends Cod
BlockTreeTermsReader.files(dir, segmentInfo, codecId, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
VariableGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
- DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
+ if (useSepDocValues) {
+ SepDocValuesConsumer.files(dir, segmentInfo, codecId, files);
+ } else {
+ DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files);
+ }
// hackish!
Iterator<String> it = files.iterator();
while(it.hasNext()) {
@@ -447,7 +449,11 @@ public class MockRandomCodec extends Cod
BlockTreeTermsReader.getExtensions(extensions);
FixedGapTermsIndexReader.getIndexExtensions(extensions);
VariableGapTermsIndexReader.getIndexExtensions(extensions);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+ if (useSepDocValues) {
+ SepDocValuesConsumer.getExtensions(extensions);
+ } else {
+ DefaultDocValuesConsumer.getExtensions(extensions);
+ }
extensions.add(SEED_EXT);
//System.out.println("MockRandom.getExtensions return " + extensions);
}
@@ -455,11 +461,19 @@ public class MockRandomCodec extends Cod
// can we make this more evil?
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+ if (useSepDocValues) {
+ return new SepDocValuesConsumer(state);
+ } else {
+ return new DefaultDocValuesConsumer(state);
+ }
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+ if (useSepDocValues) {
+ return new SepDocValuesProducer(state);
+ } else {
+ return new DefaultDocValuesProducer(state);
+ }
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/index/codecs/mocksep/MockSepCodec.java Mon Oct 10 19:58:24 2011
@@ -25,13 +25,11 @@ import org.apache.lucene.index.SegmentIn
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.codecs.Codec;
-import org.apache.lucene.index.codecs.DefaultDocValuesProducer;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.FixedGapTermsIndexReader;
import org.apache.lucene.index.codecs.FixedGapTermsIndexWriter;
import org.apache.lucene.index.codecs.PerDocConsumer;
-import org.apache.lucene.index.codecs.DefaultDocValuesConsumer;
import org.apache.lucene.index.codecs.PerDocValues;
import org.apache.lucene.index.codecs.PostingsReaderBase;
import org.apache.lucene.index.codecs.PostingsWriterBase;
@@ -40,6 +38,8 @@ import org.apache.lucene.index.codecs.Bl
import org.apache.lucene.index.codecs.TermsIndexReaderBase;
import org.apache.lucene.index.codecs.TermsIndexWriterBase;
import org.apache.lucene.index.codecs.standard.StandardCodec;
+import org.apache.lucene.index.codecs.sep.SepDocValuesConsumer;
+import org.apache.lucene.index.codecs.sep.SepDocValuesProducer;
import org.apache.lucene.index.codecs.sep.SepPostingsWriter;
import org.apache.lucene.index.codecs.sep.SepPostingsReader;
import org.apache.lucene.store.Directory;
@@ -139,28 +139,28 @@ public class MockSepCodec extends Codec
SepPostingsReader.files(segmentInfo, codecId, files);
BlockTermsReader.files(dir, segmentInfo, codecId, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, codecId, files);
- DefaultDocValuesConsumer.files(dir, segmentInfo, codecId, files, getDocValuesUseCFS());
+ SepDocValuesConsumer.files(dir, segmentInfo, codecId, files);
}
@Override
public void getExtensions(Set<String> extensions) {
getSepExtensions(extensions);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
}
public static void getSepExtensions(Set<String> extensions) {
SepPostingsWriter.getExtensions(extensions);
BlockTermsReader.getExtensions(extensions);
FixedGapTermsIndexReader.getIndexExtensions(extensions);
+ SepDocValuesConsumer.getExtensions(extensions);
}
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+ return new SepDocValuesConsumer(state);
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+ return new SepDocValuesProducer(state);
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/search/AssertingIndexSearcher.java Mon Oct 10 19:58:24 2011
@@ -23,6 +23,7 @@ import java.io.IOException;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
+import org.apache.lucene.util.Bits;
/**
* Helper class that adds some extra checks to ensure correct
@@ -67,8 +68,9 @@ public class AssertingIndexSearcher exte
}
@Override
- public Scorer scorer(AtomicReaderContext context, ScorerContext scorerContext) throws IOException {
- return w.scorer(context, scorerContext);
+ public Scorer scorer(AtomicReaderContext context, boolean scoreDocsInOrder,
+ boolean topScorer, Bits acceptDocs) throws IOException {
+ return w.scorer(context, scoreDocsInOrder, topScorer, acceptDocs);
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/search/QueryUtils.java Mon Oct 10 19:58:24 2011
@@ -29,10 +29,10 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.MultiReader;
-import org.apache.lucene.search.Weight.ScorerContext;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.MockDirectoryWrapper;
import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ReaderUtil;
import org.apache.lucene.util._TestUtil;
@@ -256,7 +256,8 @@ public class QueryUtils {
try {
if (scorer == null) {
Weight w = s.createNormalizedWeight(q);
- scorer = w.scorer(readerContextArray[leafPtr], ScorerContext.def());
+ AtomicReaderContext context = readerContextArray[leafPtr];
+ scorer = w.scorer(context, true, false, context.reader.getLiveDocs());
}
int op = order[(opidx[0]++) % order.length];
@@ -301,7 +302,8 @@ public class QueryUtils {
final IndexReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader);
Weight w = indexSearcher.createNormalizedWeight(q);
- Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def());
+ AtomicReaderContext ctx = (AtomicReaderContext)indexSearcher.getTopReaderContext();
+ Scorer scorer = w.scorer(ctx, true, false, ctx.reader.getLiveDocs());
if (scorer != null) {
boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more);
@@ -327,7 +329,8 @@ public class QueryUtils {
final IndexReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader, false);
Weight w = indexSearcher.createNormalizedWeight(q);
- Scorer scorer = w.scorer((AtomicReaderContext)previousReader.getTopReaderContext(), ScorerContext.def());
+ AtomicReaderContext ctx = (AtomicReaderContext)previousReader.getTopReaderContext();
+ Scorer scorer = w.scorer(ctx, true, false, ctx.reader.getLiveDocs());
if (scorer != null) {
boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more);
@@ -347,6 +350,7 @@ public class QueryUtils {
s.search(q,new Collector() {
private Scorer scorer;
private int leafPtr;
+ private Bits liveDocs;
@Override
public void setScorer(Scorer scorer) throws IOException {
this.scorer = scorer;
@@ -358,7 +362,7 @@ public class QueryUtils {
long startMS = System.currentTimeMillis();
for (int i=lastDoc[0]+1; i<=doc; i++) {
Weight w = s.createNormalizedWeight(q);
- Scorer scorer = w.scorer(context[leafPtr], ScorerContext.def());
+ Scorer scorer = w.scorer(context[leafPtr], true, false, liveDocs);
Assert.assertTrue("query collected "+doc+" but skipTo("+i+") says no more docs!",scorer.advance(i) != DocIdSetIterator.NO_MORE_DOCS);
Assert.assertEquals("query collected "+doc+" but skipTo("+i+") got to "+scorer.docID(),doc,scorer.docID());
float skipToScore = scorer.score();
@@ -385,7 +389,7 @@ public class QueryUtils {
final IndexReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader);
Weight w = indexSearcher.createNormalizedWeight(q);
- Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def());
+ Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), true, false, previousReader.getLiveDocs());
if (scorer != null) {
boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more);
@@ -396,6 +400,7 @@ public class QueryUtils {
lastReader[0] = context.reader;
lastDoc[0] = -1;
+ liveDocs = context.reader.getLiveDocs();
}
@Override
public boolean acceptsDocsOutOfOrder() {
@@ -409,7 +414,7 @@ public class QueryUtils {
final IndexReader previousReader = lastReader[0];
IndexSearcher indexSearcher = LuceneTestCase.newSearcher(previousReader);
Weight w = indexSearcher.createNormalizedWeight(q);
- Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), ScorerContext.def());
+ Scorer scorer = w.scorer((AtomicReaderContext)indexSearcher.getTopReaderContext(), true, false, previousReader.getLiveDocs());
if (scorer != null) {
boolean more = scorer.advance(lastDoc[0] + 1) != DocIdSetIterator.NO_MORE_DOCS;
Assert.assertFalse("query's last doc was "+ lastDoc[0] +" but skipTo("+(lastDoc[0]+1)+") got to "+scorer.docID(),more);
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/org/apache/lucene/util/LuceneTestCase.java Mon Oct 10 19:58:24 2011
@@ -45,7 +45,6 @@ import org.apache.lucene.index.codecs.mo
import org.apache.lucene.index.codecs.mockintblock.MockVariableIntBlockCodec;
import org.apache.lucene.index.codecs.mocksep.MockSepCodec;
import org.apache.lucene.index.codecs.mockrandom.MockRandomCodec;
-import org.apache.lucene.index.codecs.mockrandom.MockRandomDocValuesCodec;
import org.apache.lucene.index.codecs.preflex.PreFlexCodec;
import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec;
import org.apache.lucene.index.codecs.pulsing.PulsingCodec;
@@ -283,8 +282,6 @@ public abstract class LuceneTestCase ext
// baseBlockSize cannot be over 127:
swapCodec(new MockVariableIntBlockCodec(codecHasParam && "MockVariableIntBlock".equals(codec) ? codecParam : _TestUtil.nextInt(random, 1, 127)), cp);
swapCodec(new MockRandomCodec(random), cp);
- // give docvalues non-cfs testcoverage
- swapCodec(new MockRandomDocValuesCodec(random), cp);
return cp.lookup(codec);
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocTermOrds.java Mon Oct 10 19:58:24 2011
@@ -203,13 +203,13 @@ public class TestDocTermOrds extends Luc
StandardPostingsReader.files(dir, segmentInfo, id, files);
BlockTermsReader.files(dir, segmentInfo, id, files);
FixedGapTermsIndexReader.files(dir, segmentInfo, id, files);
- DefaultDocValuesConsumer.files(dir, segmentInfo, id, files, getDocValuesUseCFS());
+ DefaultDocValuesConsumer.files(dir, segmentInfo, id, files);
}
@Override
public void getExtensions(Set<String> extensions) {
getStandardExtensions(extensions);
- DefaultDocValuesConsumer.getDocValuesExtensions(extensions, getDocValuesUseCFS());
+ DefaultDocValuesConsumer.getExtensions(extensions);
}
public static void getStandardExtensions(Set<String> extensions) {
@@ -221,12 +221,12 @@ public class TestDocTermOrds extends Luc
@Override
public PerDocConsumer docsConsumer(PerDocWriteState state) throws IOException {
- return new DefaultDocValuesConsumer(state, getDocValuesSortComparator(), getDocValuesUseCFS());
+ return new DefaultDocValuesConsumer(state);
}
@Override
public PerDocValues docsProducer(SegmentReadState state) throws IOException {
- return new DefaultDocValuesProducer(state.segmentInfo, state.dir, state.fieldInfos, state.codecId, getDocValuesUseCFS(), getDocValuesSortComparator(), state.context);
+ return new DefaultDocValuesProducer(state);
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java?rev=1181188&r1=1181187&r2=1181188&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/values/TestDocValues.java Mon Oct 10 19:58:24 2011
@@ -25,14 +25,12 @@ import org.apache.lucene.index.values.In
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Counter;
-import org.apache.lucene.util.FloatsRef;
-import org.apache.lucene.util.LongsRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util._TestUtil;
public class TestDocValues extends LuceneTestCase {
-
+ private static final Comparator<BytesRef> COMP = BytesRef.getUTF8SortedAsUnicodeComparator();
// TODO -- for sorted test, do our own Sort of the
// values and verify it's identical
@@ -45,23 +43,20 @@ public class TestDocValues extends Lucen
runTestBytes(Bytes.Mode.DEREF, true);
runTestBytes(Bytes.Mode.DEREF, false);
}
-
+
public void testBytesSorted() throws IOException {
runTestBytes(Bytes.Mode.SORTED, true);
runTestBytes(Bytes.Mode.SORTED, false);
}
-
+
public void runTestBytes(final Bytes.Mode mode, final boolean fixedSize)
throws IOException {
final BytesRef bytesRef = new BytesRef();
- final Comparator<BytesRef> comp = mode == Bytes.Mode.SORTED ? BytesRef
- .getUTF8SortedAsUnicodeComparator() : null;
-
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Bytes.getWriter(dir, "test", mode, comp, fixedSize, trackBytes, newIOContext(random));
+ Writer w = Bytes.getWriter(dir, "test", mode, fixedSize, COMP, trackBytes, newIOContext(random));
int maxDoc = 220;
final String[] values = new String[maxDoc];
final int fixedLength = 1 + atLeast(50);
@@ -81,24 +76,7 @@ public class TestDocValues extends Lucen
w.finish(maxDoc);
assertEquals(0, trackBytes.get());
- IndexDocValues r = Bytes.getValues(dir, "test", mode, fixedSize, maxDoc, comp, newIOContext(random));
- for (int iter = 0; iter < 2; iter++) {
- ValuesEnum bytesEnum = getEnum(r);
- assertNotNull("enum is null", bytesEnum);
- BytesRef ref = bytesEnum.bytes();
-
- for (int i = 0; i < 2; i++) {
- final int idx = 2 * i;
- assertEquals("doc: " + idx, idx, bytesEnum.advance(idx));
- String utf8String = ref.utf8ToString();
- assertEquals("doc: " + idx + " lenLeft: " + values[idx].length()
- + " lenRight: " + utf8String.length(), values[idx], utf8String);
- }
- assertEquals(ValuesEnum.NO_MORE_DOCS, bytesEnum.advance(maxDoc));
- assertEquals(ValuesEnum.NO_MORE_DOCS, bytesEnum.advance(maxDoc + 1));
-
- bytesEnum.close();
- }
+ IndexDocValues r = Bytes.getValues(dir, "test", mode, fixedSize, maxDoc, COMP, newIOContext(random));
// Verify we can load source twice:
for (int iter = 0; iter < 2; iter++) {
@@ -106,7 +84,7 @@ public class TestDocValues extends Lucen
IndexDocValues.SortedSource ss;
if (mode == Bytes.Mode.SORTED) {
// default is unicode so we can simply pass null here
- s = ss = getSortedSource(r, random.nextBoolean() ? comp : null);
+ s = ss = getSortedSource(r);
} else {
s = getSource(r);
ss = null;
@@ -121,7 +99,7 @@ public class TestDocValues extends Lucen
assertEquals("doc " + idx, values[idx], ss.getByOrd(ss.ord(idx),
bytesRef).utf8ToString());
int ord = ss
- .getByValue(new BytesRef(values[idx]));
+ .getByValue(new BytesRef(values[idx]), new BytesRef());
assertTrue(ord >= 0);
assertEquals(ss.ord(idx), ord);
}
@@ -129,10 +107,10 @@ public class TestDocValues extends Lucen
// Lookup random strings:
if (mode == Bytes.Mode.SORTED) {
- final int numValues = ss.getValueCount();
+ final int valueCount = ss.getValueCount();
for (int i = 0; i < 1000; i++) {
BytesRef bytesValue = new BytesRef(_TestUtil.randomFixedByteLengthUnicodeString(random, fixedSize? fixedLength : 1 + random.nextInt(39)));
- int ord = ss.getByValue(bytesValue);
+ int ord = ss.getByValue(bytesValue, new BytesRef());
if (ord >= 0) {
assertTrue(bytesValue
.bytesEquals(ss.getByOrd(ord, bytesRef)));
@@ -151,22 +129,23 @@ public class TestDocValues extends Lucen
final BytesRef firstRef = ss.getByOrd(1, bytesRef);
// random string was before our first
assertTrue(firstRef.compareTo(bytesValue) > 0);
- } else if (insertIndex == numValues) {
- final BytesRef lastRef = ss.getByOrd(numValues-1, bytesRef);
+ } else if (insertIndex == valueCount) {
+ final BytesRef lastRef = ss.getByOrd(valueCount-1, bytesRef);
// random string was after our last
assertTrue(lastRef.compareTo(bytesValue) < 0);
} else {
final BytesRef before = (BytesRef) ss.getByOrd(insertIndex-1, bytesRef)
.clone();
BytesRef after = ss.getByOrd(insertIndex, bytesRef);
- assertTrue(comp.compare(before, bytesValue) < 0);
- assertTrue(comp.compare(bytesValue, after) < 0);
+ assertTrue(COMP.compare(before, bytesValue) < 0);
+ assertTrue(COMP.compare(bytesValue, after) < 0);
}
}
}
}
}
+
r.close();
dir.close();
}
@@ -194,14 +173,6 @@ public class TestDocValues extends Lucen
expectedTypes[i], source.type());
assertEquals(minMax[i][0], source.getInt(0));
assertEquals(minMax[i][1], source.getInt(1));
- ValuesEnum iEnum = getEnum(r);
- assertEquals(i + " with min: " + minMax[i][0] + " max: " + minMax[i][1],
- expectedTypes[i], iEnum.type());
- assertEquals(0, iEnum.nextDoc());
- assertEquals(minMax[i][0], iEnum.intsRef.get());
- assertEquals(1, iEnum.nextDoc());
- assertEquals(minMax[i][1], iEnum.intsRef.get());
- assertEquals(ValuesEnum.NO_MORE_DOCS, iEnum.nextDoc());
r.close();
dir.close();
@@ -308,12 +279,12 @@ public class TestDocValues extends Lucen
float[] sourceArray = new float[] {1,2,3};
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Floats.getWriter(dir, "test", 4, trackBytes, newIOContext(random));
+ Writer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), ValueType.FLOAT_32);
for (int i = 0; i < sourceArray.length; i++) {
w.add(i, sourceArray[i]);
}
w.finish(sourceArray.length);
- IndexDocValues r = Floats.getValues(dir, "test", 3, newIOContext(random));
+ IndexDocValues r = Floats.getValues(dir, "test", 3, newIOContext(random), ValueType.FLOAT_32);
Source source = r.getSource();
assertTrue(source.hasArray());
float[] loaded = ((float[])source.getArray());
@@ -329,12 +300,12 @@ public class TestDocValues extends Lucen
double[] sourceArray = new double[] {1,2,3};
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Floats.getWriter(dir, "test", 8, trackBytes, newIOContext(random));
+ Writer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), ValueType.FLOAT_64);
for (int i = 0; i < sourceArray.length; i++) {
w.add(i, sourceArray[i]);
}
w.finish(sourceArray.length);
- IndexDocValues r = Floats.getValues(dir, "test", 3, newIOContext(random));
+ IndexDocValues r = Floats.getValues(dir, "test", 3, newIOContext(random), ValueType.FLOAT_64);
Source source = r.getSource();
assertTrue(source.hasArray());
double[] loaded = ((double[])source.getArray());
@@ -373,54 +344,23 @@ public class TestDocValues extends Lucen
}
}
- for (int iter = 0; iter < 2; iter++) {
- ValuesEnum iEnum = getEnum(r);
- assertEquals(type, iEnum.type());
- LongsRef ints = iEnum.getInt();
- for (int i = 0; i < NUM_VALUES + additionalDocs; i++) {
- assertEquals(i, iEnum.nextDoc());
- if (i < NUM_VALUES) {
- assertEquals(values[i], ints.get());
- } else {
- assertEquals(0, ints.get());
- }
- }
- assertEquals(ValuesEnum.NO_MORE_DOCS, iEnum.nextDoc());
- iEnum.close();
- }
-
- for (int iter = 0; iter < 2; iter++) {
- ValuesEnum iEnum = getEnum(r);
- assertEquals(type, iEnum.type());
- LongsRef ints = iEnum.getInt();
- for (int i = 0; i < NUM_VALUES + additionalDocs; i += 1 + random.nextInt(25)) {
- assertEquals(i, iEnum.advance(i));
- if (i < NUM_VALUES) {
- assertEquals(values[i], ints.get());
- } else {
- assertEquals(0, ints.get());
- }
- }
- assertEquals(ValuesEnum.NO_MORE_DOCS, iEnum.advance(NUM_VALUES + additionalDocs));
- iEnum.close();
- }
r.close();
dir.close();
}
}
public void testFloats4() throws IOException {
- runTestFloats(4, 0.00001);
+ runTestFloats(ValueType.FLOAT_32, 0.00001);
}
- private void runTestFloats(int precision, double delta) throws IOException {
+ private void runTestFloats(ValueType type, double delta) throws IOException {
Directory dir = newDirectory();
final Counter trackBytes = Counter.newCounter();
- Writer w = Floats.getWriter(dir, "test", precision, trackBytes, newIOContext(random));
+ Writer w = Floats.getWriter(dir, "test", trackBytes, newIOContext(random), type);
final int NUM_VALUES = 777 + random.nextInt(777);;
final double[] values = new double[NUM_VALUES];
for (int i = 0; i < NUM_VALUES; i++) {
- final double v = precision == 4 ? random.nextFloat() : random
+ final double v = type == ValueType.FLOAT_32 ? random.nextFloat() : random
.nextDouble();
values[i] = v;
w.add(i, v);
@@ -429,64 +369,38 @@ public class TestDocValues extends Lucen
w.finish(NUM_VALUES + additionalValues);
assertEquals(0, trackBytes.get());
- IndexDocValues r = Floats.getValues(dir, "test", NUM_VALUES + additionalValues, newIOContext(random));
+ IndexDocValues r = Floats.getValues(dir, "test", NUM_VALUES + additionalValues, newIOContext(random), type);
for (int iter = 0; iter < 2; iter++) {
Source s = getSource(r);
for (int i = 0; i < NUM_VALUES; i++) {
- assertEquals(values[i], s.getFloat(i), 0.0f);
- }
- }
-
- for (int iter = 0; iter < 2; iter++) {
- ValuesEnum fEnum = getEnum(r);
- FloatsRef floats = fEnum.getFloat();
- for (int i = 0; i < NUM_VALUES + additionalValues; i++) {
- assertEquals(i, fEnum.nextDoc());
- if (i < NUM_VALUES) {
- assertEquals(values[i], floats.get(), delta);
- } else {
- assertEquals(0.0d, floats.get(), delta);
- }
+ assertEquals("" + i, values[i], s.getFloat(i), 0.0f);
}
- assertEquals(ValuesEnum.NO_MORE_DOCS, fEnum.nextDoc());
- fEnum.close();
}
- for (int iter = 0; iter < 2; iter++) {
- ValuesEnum fEnum = getEnum(r);
- FloatsRef floats = fEnum.getFloat();
- for (int i = 0; i < NUM_VALUES + additionalValues; i += 1 + random.nextInt(25)) {
- assertEquals(i, fEnum.advance(i));
- if (i < NUM_VALUES) {
- assertEquals(values[i], floats.get(), delta);
- } else {
- assertEquals(0.0d, floats.get(), delta);
- }
- }
- assertEquals(ValuesEnum.NO_MORE_DOCS, fEnum.advance(NUM_VALUES + additionalValues));
- fEnum.close();
- }
-
r.close();
dir.close();
}
public void testFloats8() throws IOException {
- runTestFloats(8, 0.0);
+ runTestFloats(ValueType.FLOAT_64, 0.0);
}
- private ValuesEnum getEnum(IndexDocValues values) throws IOException {
- return random.nextBoolean() ? values.getEnum() : getSource(values).getEnum();
- }
private Source getSource(IndexDocValues values) throws IOException {
// getSource uses cache internally
- return random.nextBoolean() ? values.load() : values.getSource();
+ switch(random.nextInt(5)) {
+ case 3:
+ return values.load();
+ case 2:
+ return values.getDirectSource();
+ case 1:
+ return values.getSource();
+ default:
+ return values.getSource();
+ }
}
-
- private SortedSource getSortedSource(IndexDocValues values,
- Comparator<BytesRef> comparator) throws IOException {
- // getSortedSource uses cache internally
- return random.nextBoolean() ? values.loadSorted(comparator) : values
- .getSortedSorted(comparator);
+
+ private SortedSource getSortedSource(IndexDocValues values) throws IOException {
+ return getSource(values).asSortedSource();
}
+
}