You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ma...@apache.org on 2011/12/09 03:19:48 UTC
svn commit: r1212221 [2/4] - in /lucene/dev/branches/solrcloud: ./
dev-tools/eclipse/ dev-tools/idea/lucene/contrib/
dev-tools/maven/lucene/contrib/misc/ lucene/
lucene/contrib/memory/src/java/org/apache/lucene/index/memory/
lucene/contrib/memory/src/t...
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java Fri Dec 9 02:19:42 2011
@@ -27,8 +27,8 @@ import java.util.TreeMap;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.IndexFileNames;
@@ -950,9 +950,11 @@ public class Lucene3xFields extends Fiel
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
PreDocsEnum docsEnum;
- if (reuse == null || !(reuse instanceof PreDocsEnum)) {
+ if (needsFreqs && fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) {
+ return null;
+ } else if (reuse == null || !(reuse instanceof PreDocsEnum)) {
docsEnum = new PreDocsEnum();
} else {
docsEnum = (PreDocsEnum) reuse;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java Fri Dec 9 02:19:42 2011
@@ -112,7 +112,10 @@ public class SegmentTermDocs {
}
public final int doc() { return doc; }
- public final int freq() { return freq; }
+ public final int freq() {
+ assert indexOptions != IndexOptions.DOCS_ONLY;
+ return freq;
+ }
protected void skippingDoc() throws IOException {
}
@@ -125,7 +128,6 @@ public class SegmentTermDocs {
if (indexOptions == IndexOptions.DOCS_ONLY) {
doc += docCode;
- freq = 1;
} else {
doc += docCode >>> 1; // shift off low bit
if ((docCode & 1) != 0) // if low bit is set
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java Fri Dec 9 02:19:42 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index.codecs.l
*/
import java.io.IOException;
-import java.util.Arrays;
import java.util.Collection;
import org.apache.lucene.index.DocsAndPositionsEnum;
@@ -209,9 +208,11 @@ public class Lucene40PostingsReader exte
}
@Override
- public DocsEnum docs(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
SegmentDocsEnum docsEnum;
- if (reuse == null || !(reuse instanceof SegmentDocsEnum)) {
+ if (needsFreqs && fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) {
+ return null;
+ } else if (reuse == null || !(reuse instanceof SegmentDocsEnum)) {
docsEnum = new SegmentDocsEnum(freqIn);
} else {
docsEnum = (SegmentDocsEnum) reuse;
@@ -277,7 +278,7 @@ public class Lucene40PostingsReader exte
final IndexInput freqIn;
final IndexInput startFreqIn;
- boolean omitTF; // does current field omit term freq?
+ boolean indexOmitsTF; // does current field omit term freq?
boolean storePayloads; // does current field store payloads?
int limit; // number of docs in this posting
@@ -300,12 +301,7 @@ public class Lucene40PostingsReader exte
}
public SegmentDocsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits liveDocs) throws IOException {
- omitTF = fieldInfo.indexOptions == IndexOptions.DOCS_ONLY;
- if (omitTF) {
- freq = 1;
- Arrays.fill(freqs, 1);
- }
-
+ indexOmitsTF = fieldInfo.indexOptions == IndexOptions.DOCS_ONLY;
storePayloads = fieldInfo.storePayloads;
this.liveDocs = liveDocs;
freqOffset = termState.freqOffset;
@@ -331,6 +327,7 @@ public class Lucene40PostingsReader exte
@Override
public int freq() {
+ assert !indexOmitsTF;
return freq;
}
@@ -389,7 +386,7 @@ public class Lucene40PostingsReader exte
count = bufferSize;
ord += bufferSize;
- if (omitTF)
+ if (indexOmitsTF)
fillDocs(bufferSize);
else
fillDocsAndFreqs(bufferSize);
@@ -400,7 +397,7 @@ public class Lucene40PostingsReader exte
private int scanTo(int target) throws IOException {
while (ord++ < limit) {
int code = freqIn.readVInt();
- if (omitTF) {
+ if (indexOmitsTF) {
accum += code;
} else {
accum += code >>> 1; // shift off low bit
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40TermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40TermVectorsReader.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40TermVectorsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40TermVectorsReader.java Fri Dec 9 02:19:42 2011
@@ -505,7 +505,7 @@ public class Lucene40TermVectorsReader e
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs /* ignored */) throws IOException {
TVDocsEnum docsEnum;
if (reuse != null && reuse instanceof TVDocsEnum) {
docsEnum = (TVDocsEnum) reuse;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java Fri Dec 9 02:19:42 2011
@@ -26,8 +26,8 @@ import java.util.TreeMap;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.IndexFileNames;
@@ -36,10 +36,10 @@ import org.apache.lucene.index.SegmentRe
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.codecs.PostingsFormat;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.PostingsConsumer;
+import org.apache.lucene.index.codecs.PostingsFormat;
import org.apache.lucene.index.codecs.TermStats;
import org.apache.lucene.index.codecs.TermsConsumer;
import org.apache.lucene.store.ByteArrayDataInput;
@@ -317,7 +317,6 @@ public class MemoryPostingsFormat extend
docUpto++;
if (indexOptions == IndexOptions.DOCS_ONLY) {
accum += in.readVInt();
- freq = 1;
} else {
final int code = in.readVInt();
accum += code >>> 1;
@@ -371,6 +370,7 @@ public class MemoryPostingsFormat extend
@Override
public int freq() {
+ assert indexOptions != IndexOptions.DOCS_ONLY;
return freq;
}
}
@@ -600,10 +600,13 @@ public class MemoryPostingsFormat extend
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
decodeMetaData();
FSTDocsEnum docsEnum;
- if (reuse == null || !(reuse instanceof FSTDocsEnum)) {
+
+ if (needsFreqs && field.indexOptions == IndexOptions.DOCS_ONLY) {
+ return null;
+ } else if (reuse == null || !(reuse instanceof FSTDocsEnum)) {
docsEnum = new FSTDocsEnum(field.indexOptions, field.storePayloads);
} else {
docsEnum = (FSTDocsEnum) reuse;
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java Fri Dec 9 02:19:42 2011
@@ -178,7 +178,10 @@ public class PulsingPostingsReader exten
}
@Override
- public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ if (needsFreqs && field.indexOptions == IndexOptions.DOCS_ONLY) {
+ return null;
+ }
PulsingTermState termState = (PulsingTermState) _termState;
if (termState.postingsSize != -1) {
PulsingDocsEnum postings;
@@ -202,11 +205,11 @@ public class PulsingPostingsReader exten
return postings.reset(liveDocs, termState);
} else {
if (reuse instanceof PulsingDocsEnum) {
- DocsEnum wrapped = wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, getOther(reuse));
+ DocsEnum wrapped = wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, getOther(reuse), needsFreqs);
setOther(wrapped, reuse); // wrapped.other = reuse
return wrapped;
} else {
- return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, reuse);
+ return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, reuse, needsFreqs);
}
}
}
@@ -283,7 +286,6 @@ public class PulsingPostingsReader exten
docID = -1;
accum = 0;
payloadLength = 0;
- freq = 1;
this.liveDocs = liveDocs;
return this;
}
@@ -342,6 +344,7 @@ public class PulsingPostingsReader exten
@Override
public int freq() {
+ assert indexOptions != IndexOptions.DOCS_ONLY;
return freq;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java Fri Dec 9 02:19:42 2011
@@ -272,7 +272,10 @@ public class SepPostingsReader extends P
}
@Override
- public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ if (needsFreqs && fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) {
+ return null;
+ }
final SepTermState termState = (SepTermState) _termState;
SepDocsEnum docsEnum;
if (reuse == null || !(reuse instanceof SepDocsEnum)) {
@@ -369,8 +372,6 @@ public class SepPostingsReader extends P
if (!omitTF) {
freqIndex.set(termState.freqIndex);
freqIndex.seek(freqReader);
- } else {
- freq = 1;
}
docFreq = termState.docFreq;
@@ -412,6 +413,7 @@ public class SepPostingsReader extends P
@Override
public int freq() {
+ assert !omitTF;
return freq;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java Fri Dec 9 02:19:42 2011
@@ -17,19 +17,24 @@ package org.apache.lucene.index.codecs.s
* limitations under the License.
*/
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.index.codecs.FieldsProducer;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.StringHelper;
@@ -37,13 +42,8 @@ import org.apache.lucene.util.UnicodeUti
import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.BytesRefFSTEnum;
import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.PositiveIntOutputs;
import org.apache.lucene.util.fst.PairOutputs;
-
-import java.io.IOException;
-import java.util.Comparator;
-import java.util.Map;
-import java.util.HashMap;
+import org.apache.lucene.util.fst.PositiveIntOutputs;
class SimpleTextFieldsReader extends FieldsProducer {
@@ -190,14 +190,17 @@ class SimpleTextFieldsReader extends Fie
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ if (needsFreqs && indexOptions == IndexOptions.DOCS_ONLY) {
+ return null;
+ }
SimpleTextDocsEnum docsEnum;
if (reuse != null && reuse instanceof SimpleTextDocsEnum && ((SimpleTextDocsEnum) reuse).canReuse(SimpleTextFieldsReader.this.in)) {
docsEnum = (SimpleTextDocsEnum) reuse;
} else {
docsEnum = new SimpleTextDocsEnum();
}
- return docsEnum.reset(docsStart, liveDocs, indexOptions == IndexOptions.DOCS_ONLY);
+ return docsEnum.reset(docsStart, liveDocs, !needsFreqs);
}
@Override
@@ -245,9 +248,6 @@ class SimpleTextFieldsReader extends Fie
in.seek(fp);
this.omitTF = omitTF;
docID = -1;
- if (omitTF) {
- tf = 1;
- }
return this;
}
@@ -258,6 +258,7 @@ class SimpleTextFieldsReader extends Fie
@Override
public int freq() {
+ assert !omitTF;
return tf;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java Fri Dec 9 02:19:42 2011
@@ -365,10 +365,10 @@ public class SimpleTextTermVectorsReader
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
// TODO: reuse
SimpleTVDocsEnum e = new SimpleTVDocsEnum();
- e.reset(liveDocs, current.getValue().freq);
+ e.reset(liveDocs, needsFreqs ? current.getValue().freq : -1);
return e;
}
@@ -399,6 +399,7 @@ public class SimpleTextTermVectorsReader
@Override
public int freq() {
+ assert freq != -1;
return freq;
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/index/values/IndexDocValues.java Fri Dec 9 02:19:42 2011
@@ -210,18 +210,18 @@ public abstract class IndexDocValues imp
/**
* Returns the internal array representation iff this {@link Source} uses an
- * array as its inner representation, otherwise <code>null</code>.
+ * array as its inner representation, otherwise <code>UOE</code>.
*/
public Object getArray() {
- return null;
+ throw new UnsupportedOperationException("getArray is not supported");
}
/**
* If this {@link Source} is sorted this method will return an instance of
- * {@link SortedSource} otherwise <code>null</code>
+ * {@link SortedSource} otherwise <code>UOE</code>
*/
public SortedSource asSortedSource() {
- return null;
+ throw new UnsupportedOperationException("asSortedSource is not supported");
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/BooleanQuery.java Fri Dec 9 02:19:42 2011
@@ -17,20 +17,21 @@ package org.apache.lucene.search;
* limitations under the License.
*/
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermsEnum;
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.ConjunctionTermScorer.DocsAndFreqs;
-import org.apache.lucene.search.similarities.SimilarityProvider;
-import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
import org.apache.lucene.search.TermQuery.TermWeight;
-
-import java.io.IOException;
-import java.util.*;
+import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
+import org.apache.lucene.search.similarities.SimilarityProvider;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.ToStringUtils;
/** A Query that matches documents matching boolean combinations of other
* queries, e.g. {@link TermQuery}s, {@link PhraseQuery}s or other
@@ -349,6 +350,11 @@ public class BooleanQuery extends Query
private Scorer createConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs)
throws IOException {
+
+ // TODO: fix scorer API to specify "needsScores" up
+ // front, so we can do match-only if caller doesn't
+ // needs scores
+
final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()];
for (int i = 0; i < docsAndFreqs.length; i++) {
final TermWeight weight = (TermWeight) weights.get(i);
@@ -357,12 +363,46 @@ public class BooleanQuery extends Query
return null;
}
final ExactDocScorer docScorer = weight.createDocScorer(context);
- docsAndFreqs[i] = new DocsAndFreqs(termsEnum.docs(
- acceptDocs, null), termsEnum.docFreq(), docScorer);
+ final DocsEnum docsAndFreqsEnum = termsEnum.docs(acceptDocs, null, true);
+ if (docsAndFreqsEnum == null) {
+ // TODO: we could carry over TermState from the
+ // terms we already seek'd to, to save re-seeking
+ // to make the match-only scorer, but it's likely
+ // rare that BQ mixes terms from omitTf and
+ // non-omitTF fields:
+
+ // At least one sub cannot provide freqs; abort
+ // and fallback to full match-only scorer:
+ return createMatchOnlyConjunctionTermScorer(context, acceptDocs);
+ }
+
+ docsAndFreqs[i] = new DocsAndFreqs(docsAndFreqsEnum,
+ docsAndFreqsEnum,
+ termsEnum.docFreq(), docScorer);
}
return new ConjunctionTermScorer(this, disableCoord ? 1.0f : coord(
docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs);
}
+
+ private Scorer createMatchOnlyConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs)
+ throws IOException {
+
+ final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()];
+ for (int i = 0; i < docsAndFreqs.length; i++) {
+ final TermWeight weight = (TermWeight) weights.get(i);
+ final TermsEnum termsEnum = weight.getTermsEnum(context);
+ if (termsEnum == null) {
+ return null;
+ }
+ final ExactDocScorer docScorer = weight.createDocScorer(context);
+ docsAndFreqs[i] = new DocsAndFreqs(null,
+ termsEnum.docs(acceptDocs, null, false),
+ termsEnum.docFreq(), docScorer);
+ }
+
+ return new MatchOnlyConjunctionTermScorer(this, disableCoord ? 1.0f : coord(
+ docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs);
+ }
@Override
public boolean scoresDocsOutOfOrder() {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java Fri Dec 9 02:19:42 2011
@@ -17,17 +17,18 @@ package org.apache.lucene.search;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Comparator;
+
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
import org.apache.lucene.util.ArrayUtil;
-import java.io.IOException;
-import java.util.Comparator;
/** Scorer for conjunctions, sets of terms, all of which are required. */
-final class ConjunctionTermScorer extends Scorer {
- private final float coord;
- private int lastDoc = -1;
- private final DocsAndFreqs[] docsAndFreqs;
+class ConjunctionTermScorer extends Scorer {
+ protected final float coord;
+ protected int lastDoc = -1;
+ protected final DocsAndFreqs[] docsAndFreqs;
private final DocsAndFreqs lead;
ConjunctionTermScorer(Weight weight, float coord,
@@ -39,7 +40,7 @@ final class ConjunctionTermScorer extend
// lead the matching.
ArrayUtil.mergeSort(docsAndFreqs, new Comparator<DocsAndFreqs>() {
public int compare(DocsAndFreqs o1, DocsAndFreqs o2) {
- return o1.freq - o2.freq;
+ return o1.docFreq - o2.docFreq;
}
});
@@ -96,14 +97,16 @@ final class ConjunctionTermScorer extend
}
static final class DocsAndFreqs {
+ final DocsEnum docsAndFreqs;
final DocsEnum docs;
- final int freq;
+ final int docFreq;
final ExactDocScorer docScorer;
int doc = -1;
- DocsAndFreqs(DocsEnum docs, int freq, ExactDocScorer docScorer) {
+ DocsAndFreqs(DocsEnum docsAndFreqs, DocsEnum docs, int docFreq, ExactDocScorer docScorer) {
+ this.docsAndFreqs = docsAndFreqs;
this.docs = docs;
- this.freq = freq;
+ this.docFreq = docFreq;
this.docScorer = docScorer;
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java Fri Dec 9 02:19:42 2011
@@ -342,7 +342,7 @@ class FieldCacheImpl implements FieldCac
break;
}
final byte termval = parser.parseByte(term);
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -415,7 +415,7 @@ class FieldCacheImpl implements FieldCac
break;
}
final short termval = parser.parseShort(term);
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -519,7 +519,7 @@ class FieldCacheImpl implements FieldCac
retArray = new int[maxDoc];
}
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -586,7 +586,7 @@ class FieldCacheImpl implements FieldCac
res = new FixedBitSet(maxDoc);
}
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
// TODO: use bulk API
while (true) {
final int docID = docs.nextDoc();
@@ -669,7 +669,7 @@ class FieldCacheImpl implements FieldCac
retArray = new float[maxDoc];
}
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -757,7 +757,7 @@ class FieldCacheImpl implements FieldCac
retArray = new long[maxDoc];
}
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -846,7 +846,7 @@ class FieldCacheImpl implements FieldCac
retArray = new double[maxDoc];
}
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -1020,7 +1020,7 @@ class FieldCacheImpl implements FieldCac
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
throw new UnsupportedOperationException();
}
@@ -1147,7 +1147,7 @@ class FieldCacheImpl implements FieldCac
termOrdToBytesOffset = termOrdToBytesOffset.resize(ArrayUtil.oversize(1+termOrd, 1));
}
termOrdToBytesOffset.set(termOrd, bytes.copyUsingLengthPrefix(term));
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -1268,7 +1268,7 @@ class FieldCacheImpl implements FieldCac
break;
}
final long pointer = bytes.copyUsingLengthPrefix(term);
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldValueFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldValueFilter.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldValueFilter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FieldValueFilter.java Fri Dec 9 02:19:42 2011
@@ -56,6 +56,22 @@ public class FieldValueFilter extends Fi
this.field = field;
this.negate = negate;
}
+
+ /**
+ * Returns the field this filter is applied on.
+ * @return the field this filter is applied on.
+ */
+ public String field() {
+ return field;
+ }
+
+ /**
+ * Returns <code>true</code> iff this filter is negated, otherwise <code>false</code>
+ * @return <code>true</code> iff this filter is negated, otherwise <code>false</code>
+ */
+ public boolean negate() {
+ return negate;
+ }
@Override
public DocIdSet getDocIdSet(AtomicReaderContext context, Bits acceptDocs)
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java Fri Dec 9 02:19:42 2011
@@ -259,8 +259,8 @@ public final class FuzzyTermsEnum extend
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
- return actualEnum.docs(liveDocs, reuse);
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ return actualEnum.docs(liveDocs, reuse, needsFreqs);
}
@Override
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java Fri Dec 9 02:19:42 2011
@@ -229,7 +229,7 @@ public class MultiPhraseQuery extends Qu
if (postingsEnum == null) {
// term does exist, but has no positions
- assert termsEnum.docs(liveDocs, null) != null: "termstate found but no term exists in reader";
+ assert termsEnum.docs(liveDocs, null, false) != null: "termstate found but no term exists in reader";
throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.text() + ")");
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Fri Dec 9 02:19:42 2011
@@ -105,7 +105,7 @@ public class MultiTermQueryWrapperFilter
do {
// System.out.println(" iter termCount=" + termCount + " term=" +
// enumerator.term().toBytesString());
- docsEnum = termsEnum.docs(acceptDocs, docsEnum);
+ docsEnum = termsEnum.docs(acceptDocs, docsEnum, false);
int docid;
while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
bitSet.set(docid);
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/PhraseQuery.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/PhraseQuery.java Fri Dec 9 02:19:42 2011
@@ -244,7 +244,7 @@ public class PhraseQuery extends Query {
// PhraseQuery on a field that did not index
// positions.
if (postingsEnum == null) {
- assert reader.termDocsEnum(liveDocs, t.field(), t.bytes(), state) != null: "termstate found but no term exists in reader";
+ assert reader.termDocsEnum(liveDocs, t.field(), t.bytes(), state, false) != null: "termstate found but no term exists in reader";
// term does exist, but has no positions
throw new IllegalStateException("field \"" + t.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + t.text() + ")");
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/TermQuery.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/search/TermQuery.java Fri Dec 9 02:19:42 2011
@@ -21,18 +21,18 @@ import java.io.IOException;
import java.util.Set;
import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermState;
-import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
import org.apache.lucene.index.IndexReader.ReaderContext;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.similarities.Similarity;
+import org.apache.lucene.index.TermState;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
+import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ReaderUtil;
+import org.apache.lucene.util.TermContext;
import org.apache.lucene.util.ToStringUtils;
/** A Query that matches documents containing a term.
@@ -83,10 +83,15 @@ public class TermQuery extends Query {
if (termsEnum == null) {
return null;
}
- // TODO should we reuse the DocsEnum here?
- final DocsEnum docs = termsEnum.docs(acceptDocs, null);
- assert docs != null;
- return new TermScorer(this, docs, createDocScorer(context));
+ DocsEnum docs = termsEnum.docs(acceptDocs, null, true);
+ if (docs != null) {
+ return new TermScorer(this, docs, createDocScorer(context));
+ } else {
+ // Index does not store freq info
+ docs = termsEnum.docs(acceptDocs, null, false);
+ assert docs != null;
+ return new MatchOnlyTermScorer(this, docs, createDocScorer(context));
+ }
}
/**
@@ -120,12 +125,11 @@ public class TermQuery extends Query {
@Override
public Explanation explain(AtomicReaderContext context, int doc) throws IOException {
- IndexReader reader = context.reader;
- DocsEnum docs = reader.termDocsEnum(context.reader.getLiveDocs(), term.field(), term.bytes());
- if (docs != null) {
- int newDoc = docs.advance(doc);
+ Scorer scorer = scorer(context, true, false, context.reader.getLiveDocs());
+ if (scorer != null) {
+ int newDoc = scorer.advance(doc);
if (newDoc == doc) {
- int freq = docs.freq();
+ float freq = scorer.freq();
ExactDocScorer docScorer = similarity.exactDocScorer(stats, term.field(), context);
ComplexExplanation result = new ComplexExplanation();
result.setDescription("weight("+getQuery()+" in "+doc+") [" + similarity.getClass().getSimpleName() + "], result of:");
@@ -136,8 +140,7 @@ public class TermQuery extends Query {
return result;
}
}
-
- return new ComplexExplanation(false, 0.0f, "no matching term");
+ return new ComplexExplanation(false, 0.0f, "no matching term");
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/MMapDirectory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/MMapDirectory.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/MMapDirectory.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/java/org/apache/lucene/store/MMapDirectory.java Fri Dec 9 02:19:42 2011
@@ -26,7 +26,7 @@ import java.nio.channels.ClosedChannelEx
import java.nio.channels.FileChannel;
import java.nio.channels.FileChannel.MapMode;
-import java.util.Map;
+import java.util.Set;
import java.util.WeakHashMap;
import java.security.AccessController;
@@ -34,6 +34,7 @@ import java.security.PrivilegedException
import java.security.PrivilegedActionException;
import java.lang.reflect.Method;
+import org.apache.lucene.util.MapBackedSet;
import org.apache.lucene.util.Constants;
/** File-based {@link Directory} implementation that uses
@@ -259,7 +260,7 @@ public class MMapDirectory extends FSDir
private ByteBuffer curBuf; // redundant for speed: buffers[curBufIndex]
private boolean isClone = false;
- private final Map<MMapIndexInput,Boolean> clones = new WeakHashMap<MMapIndexInput,Boolean>();
+ private final Set<MMapIndexInput> clones = new MapBackedSet<MMapIndexInput>(new WeakHashMap<MMapIndexInput,Boolean>());
MMapIndexInput(String resourceDescription, RandomAccessFile raf, long offset, long length, int chunkSizePower) throws IOException {
super(resourceDescription);
@@ -430,7 +431,7 @@ public class MMapDirectory extends FSDir
// register the new clone in our clone list to clean it up on closing:
synchronized(this.clones) {
- this.clones.put(clone, Boolean.TRUE);
+ this.clones.add(clone);
}
return clone;
@@ -449,7 +450,7 @@ public class MMapDirectory extends FSDir
// for extra safety unset also all clones' buffers:
synchronized(this.clones) {
- for (final MMapIndexInput clone : this.clones.keySet()) {
+ for (final MMapIndexInput clone : this.clones) {
assert clone.isClone;
clone.unsetBuffers();
}
Modified: lucene/dev/branches/solrcloud/lucene/src/resources/META-INF/services/org.apache.lucene.index.codecs.Codec
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/resources/META-INF/services/org.apache.lucene.index.codecs.Codec?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/resources/META-INF/services/org.apache.lucene.index.codecs.Codec (original)
+++ lucene/dev/branches/solrcloud/lucene/src/resources/META-INF/services/org.apache.lucene.index.codecs.Codec Fri Dec 9 02:19:42 2011
@@ -16,3 +16,4 @@
org.apache.lucene.index.codecs.lucene40.Lucene40Codec
org.apache.lucene.index.codecs.lucene3x.Lucene3xCodec
org.apache.lucene.index.codecs.simpletext.SimpleTextCodec
+org.apache.lucene.index.codecs.appending.AppendingCodec
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java Fri Dec 9 02:19:42 2011
@@ -383,7 +383,7 @@ public class RAMOnlyPostingsFormat exten
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) {
return new RAMDocsEnum(ramField.termToDocs.get(current), liveDocs);
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCase.java Fri Dec 9 02:19:42 2011
@@ -40,6 +40,7 @@ import org.apache.lucene.document.FieldT
import org.apache.lucene.index.*;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.PostingsFormat;
+import org.apache.lucene.index.codecs.appending.AppendingCodec;
import org.apache.lucene.index.codecs.lucene40.Lucene40Codec;
import org.apache.lucene.index.codecs.preflexrw.PreFlexRWCodec;
import org.apache.lucene.index.codecs.simpletext.SimpleTextCodec;
@@ -148,8 +149,12 @@ public abstract class LuceneTestCase ext
public static final int TEST_ITER_MIN = Integer.parseInt(System.getProperty("tests.iter.min", Integer.toString(TEST_ITER)));
/** Get the random seed for tests */
public static final String TEST_SEED = System.getProperty("tests.seed", "random");
- /** whether or not nightly tests should run */
+ /** whether or not @nightly tests should run */
public static final boolean TEST_NIGHTLY = Boolean.parseBoolean(System.getProperty("tests.nightly", "false"));
+ /** whether or not @weekly tests should run */
+ public static final boolean TEST_WEEKLY = Boolean.parseBoolean(System.getProperty("tests.weekly", "false"));
+ /** whether or not @slow tests should run */
+ public static final boolean TEST_SLOW = Boolean.parseBoolean(System.getProperty("tests.slow", "false"));
/** the line file used by LineFileDocs */
public static final String TEST_LINE_DOCS_FILE = System.getProperty("tests.linedocsfile", "europarl.lines.txt.gz");
/** whether or not to clean threads between test invocations: "false", "perMethod", "perClass" */
@@ -282,6 +287,8 @@ public abstract class LuceneTestCase ext
PREFLEX_IMPERSONATION_IS_ACTIVE = true;
} else if ("SimpleText".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 9)) {
codec = new SimpleTextCodec();
+ } else if ("Appending".equals(TEST_CODEC) || ("random".equals(TEST_CODEC) && randomVal == 8)) {
+ codec = new AppendingCodec();
} else if (!"random".equals(TEST_CODEC)) {
codec = Codec.forName(TEST_CODEC);
} else if ("random".equals(TEST_POSTINGSFORMAT)) {
@@ -1350,6 +1357,22 @@ public abstract class LuceneTestCase ext
public @interface Nightly {}
/**
+ * Annotation for tests that should only be run during weekly builds
+ */
+ @Documented
+ @Inherited
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Weekly{}
+
+ /**
+ * Annotation for tests that are slow and should be run only when specifically asked to run
+ */
+ @Documented
+ @Inherited
+ @Retention(RetentionPolicy.RUNTIME)
+ public @interface Slow{}
+
+ /**
* Annotation for test classes that should only use codecs that are not memory expensive (avoid SimpleText, MemoryCodec).
*/
@Documented
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/LuceneTestCaseRunner.java Fri Dec 9 02:19:42 2011
@@ -17,6 +17,7 @@ package org.apache.lucene.util;
* limitations under the License.
*/
+import java.lang.annotation.Annotation;
import java.lang.reflect.Method;
import java.lang.reflect.Modifier;
import java.util.ArrayList;
@@ -26,6 +27,8 @@ import java.util.List;
import java.util.Random;
import org.apache.lucene.util.LuceneTestCase.Nightly;
+import org.apache.lucene.util.LuceneTestCase.Weekly;
+import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.lucene.util.LuceneTestCase.UseNoMemoryExpensiveCodec;
import org.junit.Ignore;
import org.junit.Test;
@@ -45,8 +48,11 @@ import static org.apache.lucene.util.Luc
import static org.apache.lucene.util.LuceneTestCase.TEST_METHOD;
import static org.apache.lucene.util.LuceneTestCase.TEST_SEED;
import static org.apache.lucene.util.LuceneTestCase.TEST_NIGHTLY;
+import static org.apache.lucene.util.LuceneTestCase.TEST_WEEKLY;
+import static org.apache.lucene.util.LuceneTestCase.TEST_SLOW;
import static org.apache.lucene.util.LuceneTestCase.VERBOSE;
+
/** optionally filters the tests to be run by TEST_METHOD */
public class LuceneTestCaseRunner extends BlockJUnit4ClassRunner {
private List<FrameworkMethod> testMethods;
@@ -89,27 +95,13 @@ public class LuceneTestCaseRunner extend
}
if (TEST_NIGHTLY == false) {
- if (getTestClass().getJavaClass().isAnnotationPresent(Nightly.class)) {
- /* the test class is annotated with nightly, remove all methods */
- String className = getTestClass().getJavaClass().getSimpleName();
- System.err.println("NOTE: Ignoring nightly-only test class '" + className + "'");
- testMethods.clear();
- } else {
- /* remove all nightly-only methods */
- for (int i = 0; i < testMethods.size(); i++) {
- final FrameworkMethod m = testMethods.get(i);
- if (m.getAnnotation(Nightly.class) != null) {
- System.err.println("NOTE: Ignoring nightly-only test method '" + m.getName() + "'");
- testMethods.remove(i--);
- }
- }
- }
- /* dodge a possible "no-runnable methods" exception by adding a fake ignored test */
- if (testMethods.isEmpty()) {
- try {
- testMethods.add(new FrameworkMethod(LuceneTestCase.class.getMethod("alwaysIgnoredTestMethod")));
- } catch (Exception e) { throw new RuntimeException(e); }
- }
+ removeAnnotatedTests(Nightly.class, "@nightly");
+ }
+ if (TEST_WEEKLY == false) {
+ removeAnnotatedTests(Weekly.class, "@weekly");
+ }
+ if (TEST_SLOW == false) {
+ removeAnnotatedTests(Slow.class, "@slow");
}
// sort the test methods first before shuffling them, so that the shuffle is consistent
// across different implementations that might order the methods different originally.
@@ -122,7 +114,31 @@ public class LuceneTestCaseRunner extend
Collections.shuffle(testMethods, r);
return testMethods;
}
-
+
+ private void removeAnnotatedTests(Class<? extends Annotation> annotation, String userFriendlyName) {
+ if (getTestClass().getJavaClass().isAnnotationPresent(annotation)) {
+ /* the test class is annotated with the annotation, remove all methods */
+ String className = getTestClass().getJavaClass().getSimpleName();
+ System.err.println("NOTE: Ignoring " + userFriendlyName + " test class '" + className + "'");
+ testMethods.clear();
+ } else {
+ /* remove all methods with the annotation*/
+ for (int i = 0; i < testMethods.size(); i++) {
+ final FrameworkMethod m = testMethods.get(i);
+ if (m.getAnnotation(annotation) != null) {
+ System.err.println("NOTE: Ignoring " + userFriendlyName + " test method '" + m.getName() + "'");
+ testMethods.remove(i--);
+ }
+ }
+ }
+ /* dodge a possible "no-runnable methods" exception by adding a fake ignored test */
+ if (testMethods.isEmpty()) {
+ try {
+ testMethods.add(new FrameworkMethod(LuceneTestCase.class.getMethod("alwaysIgnoredTestMethod")));
+ } catch (Exception e) { throw new RuntimeException(e); }
+ }
+ }
+
@Override
protected void runChild(FrameworkMethod arg0, RunNotifier arg1) {
if (VERBOSE) {
Modified: lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java Fri Dec 9 02:19:42 2011
@@ -37,12 +37,18 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.Field;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.ConcurrentMergeScheduler;
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexableField;
import org.apache.lucene.index.LogMergePolicy;
import org.apache.lucene.index.MergePolicy;
import org.apache.lucene.index.MergeScheduler;
+import org.apache.lucene.index.MultiFields;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.codecs.Codec;
import org.apache.lucene.index.codecs.PostingsFormat;
@@ -520,4 +526,51 @@ public class _TestUtil {
return doc2;
}
+
+ // Returns a DocsEnum, but randomly sometimes uses a
+ // DocsAndFreqsEnum, DocsAndPositionsEnum. Returns null
+ // if field/term doesn't exist:
+ public static DocsEnum docs(Random random, IndexReader r, String field, BytesRef term, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ final Terms terms = MultiFields.getTerms(r, field);
+ if (terms == null) {
+ return null;
+ }
+ final TermsEnum termsEnum = terms.iterator(null);
+ if (!termsEnum.seekExact(term, random.nextBoolean())) {
+ return null;
+ }
+ if (random.nextBoolean()) {
+ if (random.nextBoolean()) {
+ // TODO: cast re-use to D&PE if we can...?
+ final DocsAndPositionsEnum docsAndPositions = termsEnum.docsAndPositions(liveDocs, null);
+ if (docsAndPositions != null) {
+ return docsAndPositions;
+ }
+ }
+ final DocsEnum docsAndFreqs = termsEnum.docs(liveDocs, reuse, true);
+ if (docsAndFreqs != null) {
+ return docsAndFreqs;
+ }
+ }
+ return termsEnum.docs(liveDocs, reuse, needsFreqs);
+ }
+
+ // Returns a DocsEnum from a positioned TermsEnum, but
+ // randomly sometimes uses a DocsAndFreqsEnum, DocsAndPositionsEnum.
+ public static DocsEnum docs(Random random, TermsEnum termsEnum, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ if (random.nextBoolean()) {
+ if (random.nextBoolean()) {
+ // TODO: cast re-use to D&PE if we can...?
+ final DocsAndPositionsEnum docsAndPositions = termsEnum.docsAndPositions(liveDocs, null);
+ if (docsAndPositions != null) {
+ return docsAndPositions;
+ }
+ }
+ final DocsEnum docsAndFreqs = termsEnum.docs(liveDocs, null, true);
+ if (docsAndFreqs != null) {
+ return docsAndFreqs;
+ }
+ }
+ return termsEnum.docs(liveDocs, null, needsFreqs);
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/Test2BTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/Test2BTerms.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/Test2BTerms.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/Test2BTerms.java Fri Dec 9 02:19:42 2011
@@ -41,11 +41,11 @@ import org.junit.Ignore;
// disk (but, should run successfully). Best to run w/
// -Dtests.codec=Standard, and w/ plenty of RAM, eg:
//
-// ant compile-test
+// ant test -Dtest.slow=true -Dtests.heapsize=8g
//
// java -server -Xmx8g -d64 -cp .:lib/junit-4.7.jar:./build/classes/test:./build/classes/test-framework:./build/classes/java -Dlucene.version=4.0-dev -Dtests.directory=MMapDirectory -DtempDir=build -ea org.junit.runner.JUnitCore org.apache.lucene.index.Test2BTerms
//
-
+@LuceneTestCase.UseNoMemoryExpensiveCodec
public class Test2BTerms extends LuceneTestCase {
private final static int TOKEN_LEN = 10;
@@ -140,13 +140,13 @@ public class Test2BTerms extends LuceneT
}
}
- @Ignore("Takes ~4 hours to run on a fast machine!! And requires that you don't use PreFlex codec.")
+ @Slow
public void test2BTerms() throws IOException {
if ("Lucene3x".equals(Codec.getDefault().getName())) {
- throw new RuntimeException("thist test cannot run with PreFlex codec");
+ throw new RuntimeException("this test cannot run with PreFlex codec");
}
-
+ System.out.println("Starting Test2B");
final long TERM_COUNT = ((long) Integer.MAX_VALUE) + 100000000;
final int TERMS_PER_DOC = _TestUtil.nextInt(random, 100000, 1000000);
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java Fri Dec 9 02:19:42 2011
@@ -27,6 +27,7 @@ import org.apache.lucene.analysis.MockAn
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.IndexDocValuesField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
@@ -44,6 +45,7 @@ import org.apache.lucene.index.codecs.lu
import org.apache.lucene.index.codecs.lucene40.Lucene40StoredFieldsFormat;
import org.apache.lucene.index.codecs.lucene40.Lucene40TermVectorsFormat;
import org.apache.lucene.index.codecs.pulsing.Pulsing40PostingsFormat;
+import org.apache.lucene.index.values.IndexDocValues;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.store.AlreadyClosedException;
@@ -530,7 +532,7 @@ public class TestAddIndexes extends Luce
private void verifyTermDocs(Directory dir, Term term, int numDocs)
throws IOException {
IndexReader reader = IndexReader.open(dir, true);
- DocsEnum docsEnum = MultiFields.getTermDocsEnum(reader, null, term.field, term.bytes);
+ DocsEnum docsEnum = _TestUtil.docs(random, reader, term.field, term.bytes, null, null, false);
int count = 0;
while (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS)
count++;
@@ -1212,5 +1214,48 @@ public class TestAddIndexes extends Luce
}
r3.close();
d3.close();
- }
+ }
+
+ public void testDocValues() throws IOException {
+ assumeFalse("preflex does not support docvalues", Codec.getDefault().getName().equals("Lucene3x"));
+ Directory d1 = newDirectory();
+ RandomIndexWriter w = new RandomIndexWriter(random, d1);
+ Document doc = new Document();
+ doc.add(newField("id", "1", StringField.TYPE_STORED));
+ IndexDocValuesField dv = new IndexDocValuesField("dv");
+ dv.setInt(1);
+ doc.add(dv);
+ w.addDocument(doc);
+ IndexReader r1 = w.getReader();
+ w.close();
+
+ Directory d2 = newDirectory();
+ w = new RandomIndexWriter(random, d2);
+ doc = new Document();
+ doc.add(newField("id", "2", StringField.TYPE_STORED));
+ dv = new IndexDocValuesField("dv");
+ dv.setInt(2);
+ doc.add(dv);
+ w.addDocument(doc);
+ IndexReader r2 = w.getReader();
+ w.close();
+
+ Directory d3 = newDirectory();
+ w = new RandomIndexWriter(random, d3);
+ w.addIndexes(new SlowMultiReaderWrapper(r1), new SlowMultiReaderWrapper(r2));
+ r1.close();
+ d1.close();
+ r2.close();
+ d2.close();
+
+ w.forceMerge(1);
+ IndexReader r3 = w.getReader();
+ w.close();
+ IndexReader sr = getOnlySegmentReader(r3);
+ assertEquals(2, sr.numDocs());
+ IndexDocValues docValues = sr.perDocValues().docValues("dv");
+ assertNotNull(docValues);
+ r3.close();
+ d3.close();
+ }
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java Fri Dec 9 02:19:42 2011
@@ -669,7 +669,7 @@ public class TestBackwardsCompatibility
// should be found exactly
assertEquals(TermsEnum.SeekStatus.FOUND,
terms.seekCeil(aaaTerm));
- assertEquals(35, countDocs(terms.docs(null, null)));
+ assertEquals(35, countDocs(_TestUtil.docs(random, terms, null, null, false)));
assertNull(terms.next());
// should hit end of field
@@ -681,12 +681,12 @@ public class TestBackwardsCompatibility
assertEquals(TermsEnum.SeekStatus.NOT_FOUND,
terms.seekCeil(new BytesRef("a")));
assertTrue(terms.term().bytesEquals(aaaTerm));
- assertEquals(35, countDocs(terms.docs(null, null)));
+ assertEquals(35, countDocs(_TestUtil.docs(random, terms, null, null, false)));
assertNull(terms.next());
assertEquals(TermsEnum.SeekStatus.FOUND,
terms.seekCeil(aaaTerm));
- assertEquals(35, countDocs(terms.docs(null, null)));
+ assertEquals(35, countDocs(_TestUtil.docs(random, terms,null, null, false)));
assertNull(terms.next());
r.close();
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestCodecs.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestCodecs.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestCodecs.java Fri Dec 9 02:19:42 2011
@@ -279,7 +279,7 @@ public class TestCodecs extends LuceneTe
// make sure it properly fully resets (rewinds) its
// internal state:
for(int iter=0;iter<2;iter++) {
- docsEnum = termsEnum.docs(null, docsEnum);
+ docsEnum = _TestUtil.docs(random, termsEnum, null, docsEnum, false);
assertEquals(terms[i].docs[0], docsEnum.nextDoc());
assertEquals(DocsEnum.NO_MORE_DOCS, docsEnum.nextDoc());
}
@@ -479,7 +479,7 @@ public class TestCodecs extends LuceneTe
assertEquals(status, TermsEnum.SeekStatus.FOUND);
assertEquals(term.docs.length, termsEnum.docFreq());
if (field.omitTF) {
- this.verifyDocs(term.docs, term.positions, termsEnum.docs(null, null), false);
+ this.verifyDocs(term.docs, term.positions, _TestUtil.docs(random, termsEnum, null, null, false), false);
} else {
this.verifyDocs(term.docs, term.positions, termsEnum.docsAndPositions(null, null), true);
}
@@ -499,7 +499,7 @@ public class TestCodecs extends LuceneTe
assertTrue(termsEnum.term().bytesEquals(new BytesRef(term.text2)));
assertEquals(term.docs.length, termsEnum.docFreq());
if (field.omitTF) {
- this.verifyDocs(term.docs, term.positions, termsEnum.docs(null, null), false);
+ this.verifyDocs(term.docs, term.positions, _TestUtil.docs(random, termsEnum, null, null, false), false);
} else {
this.verifyDocs(term.docs, term.positions, termsEnum.docsAndPositions(null, null), true);
}
@@ -549,15 +549,22 @@ public class TestCodecs extends LuceneTe
do {
term = field.terms[upto];
if (TestCodecs.random.nextInt(3) == 1) {
- final DocsEnum docs = termsEnum.docs(null, null);
- final DocsAndPositionsEnum postings = termsEnum.docsAndPositions(null, null);
-
- final DocsEnum docsEnum;
- if (postings != null) {
- docsEnum = postings;
+ final DocsEnum docs;
+ final DocsEnum docsAndFreqs;
+ final DocsAndPositionsEnum postings;
+ if (!field.omitTF) {
+ postings = termsEnum.docsAndPositions(null, null);
+ if (postings != null) {
+ docs = docsAndFreqs = postings;
+ } else {
+ docs = docsAndFreqs = _TestUtil.docs(random, termsEnum, null, null, true);
+ }
} else {
- docsEnum = docs;
+ postings = null;
+ docsAndFreqs = null;
+ docs = _TestUtil.docs(random, termsEnum, null, null, false);
}
+ assertNotNull(docs);
int upto2 = -1;
while(upto2 < term.docs.length-1) {
// Maybe skip:
@@ -567,10 +574,10 @@ public class TestCodecs extends LuceneTe
final int inc = 1+TestCodecs.random.nextInt(left-1);
upto2 += inc;
if (TestCodecs.random.nextInt(2) == 1) {
- doc = docsEnum.advance(term.docs[upto2]);
+ doc = docs.advance(term.docs[upto2]);
assertEquals(term.docs[upto2], doc);
} else {
- doc = docsEnum.advance(1+term.docs[upto2]);
+ doc = docs.advance(1+term.docs[upto2]);
if (doc == DocIdSetIterator.NO_MORE_DOCS) {
// skipped past last doc
assert upto2 == term.docs.length-1;
@@ -584,20 +591,20 @@ public class TestCodecs extends LuceneTe
}
}
} else {
- doc = docsEnum.nextDoc();
+ doc = docs.nextDoc();
assertTrue(doc != -1);
upto2++;
}
assertEquals(term.docs[upto2], doc);
if (!field.omitTF) {
- assertEquals(term.positions[upto2].length, docsEnum.freq());
+ assertEquals(term.positions[upto2].length, postings.freq());
if (TestCodecs.random.nextInt(2) == 1) {
this.verifyPositions(term.positions[upto2], postings);
}
}
}
- assertEquals(DocIdSetIterator.NO_MORE_DOCS, docsEnum.nextDoc());
+ assertEquals(DocIdSetIterator.NO_MORE_DOCS, docs.nextDoc());
}
upto++;
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java Fri Dec 9 02:19:42 2011
@@ -17,7 +17,8 @@ package org.apache.lucene.index;
* limitations under the License.
*/
-import org.apache.lucene.util.LuceneTestCase;
+import java.io.IOException;
+import java.util.Random;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -25,9 +26,8 @@ import org.apache.lucene.document.TextFi
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
-
-import java.io.IOException;
-import java.util.Random;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
public class TestDirectoryReader extends LuceneTestCase {
protected Directory dir;
@@ -171,15 +171,18 @@ public class TestDirectoryReader extends
// test mixing up TermDocs and TermEnums from different readers.
TermsEnum te2 = MultiFields.getTerms(mr2, "body").iterator(null);
te2.seekCeil(new BytesRef("wow"));
- DocsEnum td = MultiFields.getTermDocsEnum(mr2,
- MultiFields.getLiveDocs(mr2),
- "body",
- te2.term());
+ DocsEnum td = _TestUtil.docs(random, mr2,
+ "body",
+ te2.term(),
+ MultiFields.getLiveDocs(mr2),
+ null,
+ false);
TermsEnum te3 = MultiFields.getTerms(mr3, "body").iterator(null);
te3.seekCeil(new BytesRef("wow"));
- td = te3.docs(MultiFields.getLiveDocs(mr3),
- td);
+ td = _TestUtil.docs(random, te3, MultiFields.getLiveDocs(mr3),
+ td,
+ false);
int ret = 0;
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocCount.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocCount.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocCount.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocCount.java Fri Dec 9 02:19:42 2011
@@ -68,11 +68,14 @@ public class TestDocCount extends Lucene
String field;
while ((field = e.next()) != null) {
Terms terms = fields.terms(field);
+ if (terms == null) {
+ continue;
+ }
int docCount = terms.getDocCount();
FixedBitSet visited = new FixedBitSet(ir.maxDoc());
TermsEnum te = terms.iterator(null);
while (te.next() != null) {
- DocsEnum de = te.docs(null, null);
+ DocsEnum de = _TestUtil.docs(random, te, null, null, false);
while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
visited.set(de.docID());
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java Fri Dec 9 02:19:42 2011
@@ -22,7 +22,6 @@ import java.util.Arrays;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
@@ -34,6 +33,7 @@ import org.apache.lucene.util.Bits;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.ReaderUtil;
+import org.apache.lucene.util._TestUtil;
public class TestDocsAndPositions extends LuceneTestCase {
private String fieldName;
@@ -99,16 +99,6 @@ public class TestDocsAndPositions extend
return reader.termPositionsEnum(null, fieldName, bytes);
}
- public DocsEnum getDocsEnum(IndexReader reader, BytesRef bytes,
- boolean freqs, Bits liveDocs) throws IOException {
- int randInt = random.nextInt(10);
- if (randInt == 0) { // once in a while throw in a positions enum
- return getDocsAndPositions(reader, bytes, liveDocs);
- } else {
- return reader.termDocsEnum(liveDocs, fieldName, bytes);
- }
- }
-
/**
* this test indexes random numbers within a range into a field and checks
* their occurrences by searching for a number from that range selected at
@@ -232,31 +222,31 @@ public class TestDocsAndPositions extend
AtomicReaderContext[] leaves = ReaderUtil.leaves(topReaderContext);
for (AtomicReaderContext context : leaves) {
int maxDoc = context.reader.maxDoc();
- DocsEnum docsAndPosEnum = getDocsEnum(context.reader, bytes, true, null);
+ DocsEnum docsEnum = _TestUtil.docs(random, context.reader, fieldName, bytes, null, null, true);
if (findNext(freqInDoc, context.docBase, context.docBase + maxDoc) == Integer.MAX_VALUE) {
- assertNull(docsAndPosEnum);
+ assertNull(docsEnum);
continue;
}
- assertNotNull(docsAndPosEnum);
- docsAndPosEnum.nextDoc();
+ assertNotNull(docsEnum);
+ docsEnum.nextDoc();
for (int j = 0; j < maxDoc; j++) {
if (freqInDoc[context.docBase + j] != 0) {
- assertEquals(j, docsAndPosEnum.docID());
- assertEquals(docsAndPosEnum.freq(), freqInDoc[context.docBase +j]);
+ assertEquals(j, docsEnum.docID());
+ assertEquals(docsEnum.freq(), freqInDoc[context.docBase +j]);
if (i % 2 == 0 && random.nextInt(10) == 0) {
int next = findNext(freqInDoc, context.docBase+j+1, context.docBase + maxDoc) - context.docBase;
- int advancedTo = docsAndPosEnum.advance(next);
+ int advancedTo = docsEnum.advance(next);
if (next >= maxDoc) {
assertEquals(DocsEnum.NO_MORE_DOCS, advancedTo);
} else {
assertTrue("advanced to: " +advancedTo + " but should be <= " + next, next >= advancedTo);
}
} else {
- docsAndPosEnum.nextDoc();
+ docsEnum.nextDoc();
}
}
}
- assertEquals("docBase: " + context.docBase + " maxDoc: " + maxDoc + " " + docsAndPosEnum.getClass(), DocsEnum.NO_MORE_DOCS, docsAndPosEnum.docID());
+ assertEquals("docBase: " + context.docBase + " maxDoc: " + maxDoc + " " + docsEnum.getClass(), DocsEnum.NO_MORE_DOCS, docsEnum.docID());
}
}
@@ -343,7 +333,7 @@ public class TestDocsAndPositions extend
writer.addDocument(doc);
IndexReader reader = writer.getReader();
IndexReader r = getOnlySegmentReader(reader);
- DocsEnum disi = r.termDocsEnum(null, "foo", new BytesRef("bar"));
+ DocsEnum disi = _TestUtil.docs(random, r, "foo", new BytesRef("bar"), null, null, false);
int docid = disi.docID();
assertTrue(docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS);
assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
@@ -351,7 +341,7 @@ public class TestDocsAndPositions extend
// now reuse and check again
TermsEnum te = r.terms("foo").iterator(null);
assertTrue(te.seekExact(new BytesRef("bar"), true));
- disi = te.docs(null, disi);
+ disi = _TestUtil.docs(random, te, null, disi, false);
docid = disi.docID();
assertTrue(docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS);
assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java Fri Dec 9 02:19:42 2011
@@ -270,18 +270,42 @@ public class TestDuelingCodecs extends L
assertPositionsSkipping(leftTermsEnum.docFreq(),
leftPositions = leftTermsEnum.docsAndPositions(randomBits, leftPositions),
rightPositions = rightTermsEnum.docsAndPositions(randomBits, rightPositions));
+
+ // with freqs:
+ assertDocsEnum(leftDocs = leftTermsEnum.docs(null, leftDocs, true),
+ rightDocs = rightTermsEnum.docs(null, rightDocs, true),
+ true);
+ assertDocsEnum(leftDocs = leftTermsEnum.docs(randomBits, leftDocs, true),
+ rightDocs = rightTermsEnum.docs(randomBits, rightDocs, true),
+ true);
+
+ // w/o freqs:
+ assertDocsEnum(leftDocs = leftTermsEnum.docs(null, leftDocs, false),
+ rightDocs = rightTermsEnum.docs(null, rightDocs, false),
+ false);
+ assertDocsEnum(leftDocs = leftTermsEnum.docs(randomBits, leftDocs, false),
+ rightDocs = rightTermsEnum.docs(randomBits, rightDocs, false),
+ false);
- assertDocsEnum(leftDocs = leftTermsEnum.docs(null, leftDocs),
- rightDocs = rightTermsEnum.docs(null, rightDocs));
- assertDocsEnum(leftDocs = leftTermsEnum.docs(randomBits, leftDocs),
- rightDocs = rightTermsEnum.docs(randomBits, rightDocs));
-
+ // with freqs:
assertDocsSkipping(leftTermsEnum.docFreq(),
- leftDocs = leftTermsEnum.docs(null, leftDocs),
- rightDocs = rightTermsEnum.docs(null, rightDocs));
+ leftDocs = leftTermsEnum.docs(null, leftDocs, true),
+ rightDocs = rightTermsEnum.docs(null, rightDocs, true),
+ true);
assertDocsSkipping(leftTermsEnum.docFreq(),
- leftDocs = leftTermsEnum.docs(randomBits, leftDocs),
- rightDocs = rightTermsEnum.docs(randomBits, rightDocs));
+ leftDocs = leftTermsEnum.docs(randomBits, leftDocs, true),
+ rightDocs = rightTermsEnum.docs(randomBits, rightDocs, true),
+ true);
+
+ // w/o freqs:
+ assertDocsSkipping(leftTermsEnum.docFreq(),
+ leftDocs = leftTermsEnum.docs(null, leftDocs, false),
+ rightDocs = rightTermsEnum.docs(null, rightDocs, false),
+ false);
+ assertDocsSkipping(leftTermsEnum.docFreq(),
+ leftDocs = leftTermsEnum.docs(randomBits, leftDocs, false),
+ rightDocs = rightTermsEnum.docs(randomBits, rightDocs, false),
+ false);
}
}
assertNull(info, rightTermsEnum.next());
@@ -327,13 +351,19 @@ public class TestDuelingCodecs extends L
/**
* checks docs + freqs, sequentially
*/
- public void assertDocsEnum(DocsEnum leftDocs, DocsEnum rightDocs) throws Exception {
+ public void assertDocsEnum(DocsEnum leftDocs, DocsEnum rightDocs, boolean hasFreqs) throws Exception {
+ if (leftDocs == null) {
+ assertNull(rightDocs);
+ return;
+ }
assertTrue(info, leftDocs.docID() == -1 || leftDocs.docID() == DocIdSetIterator.NO_MORE_DOCS);
assertTrue(info, rightDocs.docID() == -1 || rightDocs.docID() == DocIdSetIterator.NO_MORE_DOCS);
int docid;
while ((docid = leftDocs.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
assertEquals(info, docid, rightDocs.nextDoc());
- assertEquals(info, leftDocs.freq(), rightDocs.freq());
+ if (hasFreqs) {
+ assertEquals(info, leftDocs.freq(), rightDocs.freq());
+ }
}
assertEquals(info, DocIdSetIterator.NO_MORE_DOCS, rightDocs.nextDoc());
}
@@ -341,7 +371,11 @@ public class TestDuelingCodecs extends L
/**
* checks advancing docs
*/
- public void assertDocsSkipping(int docFreq, DocsEnum leftDocs, DocsEnum rightDocs) throws Exception {
+ public void assertDocsSkipping(int docFreq, DocsEnum leftDocs, DocsEnum rightDocs, boolean hasFreqs) throws Exception {
+ if (leftDocs == null) {
+ assertNull(rightDocs);
+ return;
+ }
int docid = -1;
int averageGap = leftReader.maxDoc() / (1+docFreq);
int skipInterval = 16;
@@ -361,7 +395,9 @@ public class TestDuelingCodecs extends L
if (docid == DocIdSetIterator.NO_MORE_DOCS) {
return;
}
- assertEquals(info, leftDocs.freq(), rightDocs.freq());
+ if (hasFreqs) {
+ assertEquals(info, leftDocs.freq(), rightDocs.freq());
+ }
}
}
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexReader.java Fri Dec 9 02:19:42 2011
@@ -320,18 +320,20 @@ public class TestIndexReader extends Luc
Term term,
int expected)
throws IOException {
- DocsEnum tdocs = MultiFields.getTermDocsEnum(reader,
- MultiFields.getLiveDocs(reader),
- term.field(),
- new BytesRef(term.text()));
- int count = 0;
- if (tdocs != null) {
- while(tdocs.nextDoc()!= DocIdSetIterator.NO_MORE_DOCS) {
- count++;
- }
- }
- assertEquals(msg + ", count mismatch", expected, count);
+ DocsEnum tdocs = _TestUtil.docs(random, reader,
+ term.field(),
+ new BytesRef(term.text()),
+ MultiFields.getLiveDocs(reader),
+ null,
+ false);
+ int count = 0;
+ if (tdocs != null) {
+ while(tdocs.nextDoc()!= DocIdSetIterator.NO_MORE_DOCS) {
+ count++;
+ }
}
+ assertEquals(msg + ", count mismatch", expected, count);
+ }
public void testBinaryFields() throws IOException {
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java Fri Dec 9 02:19:42 2011
@@ -546,10 +546,12 @@ public class TestIndexWriter extends Luc
assertEquals(1, reader.numDocs());
Term t = new Term("field", "a");
assertEquals(1, reader.docFreq(t));
- DocsEnum td = MultiFields.getTermDocsEnum(reader,
- MultiFields.getLiveDocs(reader),
- "field",
- new BytesRef("a"));
+ DocsEnum td = _TestUtil.docs(random, reader,
+ "field",
+ new BytesRef("a"),
+ MultiFields.getLiveDocs(reader),
+ null,
+ true);
td.nextDoc();
assertEquals(128*1024, td.freq());
reader.close();
@@ -1338,12 +1340,12 @@ public class TestIndexWriter extends Luc
// test that the terms were indexed.
- assertTrue(MultiFields.getTermDocsEnum(ir, null, "binary", new BytesRef("doc1field1")).nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
- assertTrue(MultiFields.getTermDocsEnum(ir, null, "binary", new BytesRef("doc2field1")).nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
- assertTrue(MultiFields.getTermDocsEnum(ir, null, "binary", new BytesRef("doc3field1")).nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
- assertTrue(MultiFields.getTermDocsEnum(ir, null, "string", new BytesRef("doc1field2")).nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
- assertTrue(MultiFields.getTermDocsEnum(ir, null, "string", new BytesRef("doc2field2")).nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
- assertTrue(MultiFields.getTermDocsEnum(ir, null, "string", new BytesRef("doc3field2")).nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ assertTrue(_TestUtil.docs(random, ir, "binary", new BytesRef("doc1field1"), null, null, false).nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ assertTrue(_TestUtil.docs(random, ir, "binary", new BytesRef("doc2field1"), null, null, false).nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ assertTrue(_TestUtil.docs(random, ir, "binary", new BytesRef("doc3field1"), null, null, false).nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ assertTrue(_TestUtil.docs(random, ir, "string", new BytesRef("doc1field2"), null, null, false).nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ assertTrue(_TestUtil.docs(random, ir, "string", new BytesRef("doc2field2"), null, null, false).nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
+ assertTrue(_TestUtil.docs(random, ir, "string", new BytesRef("doc3field2"), null, null, false).nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
ir.close();
dir.close();
@@ -1415,7 +1417,7 @@ public class TestIndexWriter extends Luc
TermsEnum t = r.fields().terms("field").iterator(null);
int count = 0;
while(t.next() != null) {
- final DocsEnum docs = t.docs(null, null);
+ final DocsEnum docs = _TestUtil.docs(random, t, null, null, false);
assertEquals(0, docs.nextDoc());
assertEquals(DocIdSetIterator.NO_MORE_DOCS, docs.nextDoc());
count++;
Modified: lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java?rev=1212221&r1=1212220&r2=1212221&view=diff
==============================================================================
--- lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java (original)
+++ lucene/dev/branches/solrcloud/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java Fri Dec 9 02:19:42 2011
@@ -500,10 +500,12 @@ public class TestIndexWriterExceptions e
// Make sure the doc that hit the exception was marked
// as deleted:
- DocsEnum tdocs = MultiFields.getTermDocsEnum(reader,
- MultiFields.getLiveDocs(reader),
- t.field(),
- new BytesRef(t.text()));
+ DocsEnum tdocs = _TestUtil.docs(random, reader,
+ t.field(),
+ new BytesRef(t.text()),
+ MultiFields.getLiveDocs(reader),
+ null,
+ false);
int count = 0;
while(tdocs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {