You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2011/12/04 20:18:20 UTC
svn commit: r1210184 [1/3] - in /lucene/dev/branches/lucene3606: ./ lucene/
lucene/contrib/memory/src/java/org/apache/lucene/index/memory/
lucene/contrib/memory/src/test/org/apache/lucene/index/memory/
lucene/contrib/misc/src/java/org/apache/lucene/mis...
Author: uschindler
Date: Sun Dec 4 19:18:17 2011
New Revision: 1210184
URL: http://svn.apache.org/viewvc?rev=1210184&view=rev
Log:
Merge in Mikes huge commit (r1210176)
Added:
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/MatchOnlyConjunctionTermsScorer.java
- copied unchanged from r1210179, lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MatchOnlyConjunctionTermsScorer.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java
- copied unchanged from r1210179, lucene/dev/trunk/lucene/src/java/org/apache/lucene/search/MatchOnlyTermScorer.java
Modified:
lucene/dev/branches/lucene3606/ (props changed)
lucene/dev/branches/lucene3606/lucene/ (props changed)
lucene/dev/branches/lucene3606/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
lucene/dev/branches/lucene3606/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java
lucene/dev/branches/lucene3606/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java
lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java
lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java
lucene/dev/branches/lucene3606/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/CheckIndex.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocTermOrds.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocsEnum.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FieldInfo.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilteredTermsEnum.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/IndexReader.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiFields.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/TermsEnum.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40TermVectorsReader.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/MultiPhraseQuery.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/PhraseQuery.java
lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/TermQuery.java
lucene/dev/branches/lucene3606/lucene/src/test-framework/java/org/apache/lucene/index/codecs/ramonly/RAMOnlyPostingsFormat.java
lucene/dev/branches/lucene3606/lucene/src/test-framework/java/org/apache/lucene/util/_TestUtil.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestAddIndexes.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestBackwardsCompatibility.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestCodecs.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestDirectoryReader.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestDocCount.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestDocsAndPositions.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestDuelingCodecs.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestIndexReader.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestIndexWriter.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestIndexWriterExceptions.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestIndexWriterReader.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestIndexWriterWithThreads.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestLongPostings.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestMultiFields.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestOmitPositions.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestParallelTermEnum.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestPerSegmentDeletes.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestSegmentMerger.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestSegmentReader.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestSegmentTermDocs.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestStressAdvance.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestStressIndexing2.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestTermVectorsReader.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestTermdocPerf.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/TestTermsEnum.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/codecs/pulsing/Test10KPulsings.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/index/codecs/pulsing/TestPulsingReuse.java
lucene/dev/branches/lucene3606/lucene/src/test/org/apache/lucene/search/TestTermVectors.java
lucene/dev/branches/lucene3606/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestKeywordAnalyzer.java
lucene/dev/branches/lucene3606/modules/benchmark/ (props changed)
lucene/dev/branches/lucene3606/modules/benchmark/src/test/org/apache/lucene/benchmark/byTask/TestPerfTasksLogic.java
lucene/dev/branches/lucene3606/modules/facet/ (props changed)
lucene/dev/branches/lucene3606/modules/facet/src/java/org/apache/lucene/facet/search/sampling/TakmiSampleFixer.java
lucene/dev/branches/lucene3606/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyReader.java
lucene/dev/branches/lucene3606/modules/facet/src/java/org/apache/lucene/facet/taxonomy/directory/DirectoryTaxonomyWriter.java
lucene/dev/branches/lucene3606/modules/facet/src/test/org/apache/lucene/facet/FacetTestBase.java
lucene/dev/branches/lucene3606/modules/facet/src/test/org/apache/lucene/facet/search/TestMultipleCategoryLists.java
lucene/dev/branches/lucene3606/modules/queries/src/java/org/apache/lucene/queries/TermsFilter.java
lucene/dev/branches/lucene3606/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java
lucene/dev/branches/lucene3606/modules/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java
lucene/dev/branches/lucene3606/modules/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
lucene/dev/branches/lucene3606/solr/ (props changed)
lucene/dev/branches/lucene3606/solr/core/ (props changed)
lucene/dev/branches/lucene3606/solr/core/src/java/ (props changed)
lucene/dev/branches/lucene3606/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
lucene/dev/branches/lucene3606/solr/core/src/java/org/apache/solr/search/JoinQParserPlugin.java
lucene/dev/branches/lucene3606/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
lucene/dev/branches/lucene3606/solr/core/src/java/org/apache/solr/search/function/FileFloatSource.java
lucene/dev/branches/lucene3606/solr/core/src/test/ (props changed)
lucene/dev/branches/lucene3606/solr/core/src/test/org/apache/solr/search/TestRealTimeGet.java
Modified: lucene/dev/branches/lucene3606/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java (original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/memory/src/java/org/apache/lucene/index/memory/MemoryIndex.java Sun Dec 4 19:18:17 2011
@@ -936,7 +936,7 @@ public class MemoryIndex {
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) {
if (reuse == null || !(reuse instanceof MemoryDocsEnum)) {
reuse = new MemoryDocsEnum();
}
Modified: lucene/dev/branches/lucene3606/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java (original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/memory/src/test/org/apache/lucene/index/memory/MemoryIndexTest.java Sun Dec 4 19:18:17 2011
@@ -188,7 +188,7 @@ public class MemoryIndexTest extends Bas
MemoryIndex memory = new MemoryIndex();
memory.addField("foo", "bar", analyzer);
IndexReader reader = memory.createSearcher().getIndexReader();
- DocsEnum disi = reader.termDocsEnum(null, "foo", new BytesRef("bar"));
+ DocsEnum disi = _TestUtil.docs(random, reader, "foo", new BytesRef("bar"), null, null, false);
int docid = disi.docID();
assertTrue(docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS);
assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
@@ -196,7 +196,7 @@ public class MemoryIndexTest extends Bas
// now reuse and check again
TermsEnum te = reader.terms("foo").iterator(null);
assertTrue(te.seekExact(new BytesRef("bar"), true));
- disi = te.docs(null, disi);
+ disi = te.docs(null, disi, false);
docid = disi.docID();
assertTrue(docid == -1 || docid == DocIdSetIterator.NO_MORE_DOCS);
assertTrue(disi.nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
Modified: lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java (original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/misc/src/java/org/apache/lucene/misc/HighFreqTerms.java Sun Dec 4 19:18:17 2011
@@ -201,7 +201,7 @@ public class HighFreqTerms {
return;
}
}
- DocsEnum de = r.termDocsEnum(liveDocs, field, termText);
+ DocsEnum de = r.termDocsEnum(liveDocs, field, termText, true);
if (de != null) {
while (de.nextDoc() != DocIdSetIterator.NO_MORE_DOCS)
totalTF[0] += de.freq();
Modified: lucene/dev/branches/lucene3606/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java (original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/misc/src/test/org/apache/lucene/index/codecs/appending/TestAppendingCodec.java Sun Dec 4 19:18:17 2011
@@ -29,11 +29,11 @@ import org.apache.lucene.index.Fields;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermsEnum.SeekStatus;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.TieredMergePolicy;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.IOContext;
import org.apache.lucene.store.IndexOutput;
@@ -139,7 +139,7 @@ public class TestAppendingCodec extends
assertEquals(SeekStatus.FOUND, te.seekCeil(new BytesRef("lazy")));
assertEquals(SeekStatus.FOUND, te.seekCeil(new BytesRef("dog")));
assertEquals(SeekStatus.FOUND, te.seekCeil(new BytesRef("the")));
- DocsEnum de = te.docs(null, null);
+ DocsEnum de = te.docs(null, null, true);
assertTrue(de.advance(0) != DocsEnum.NO_MORE_DOCS);
assertEquals(2, de.freq());
assertTrue(de.advance(1) != DocsEnum.NO_MORE_DOCS);
Modified: lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java (original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/java/org/apache/lucene/sandbox/queries/DuplicateFilter.java Sun Dec 4 19:18:17 2011
@@ -93,7 +93,7 @@ public class DuplicateFilter extends Fil
if (currTerm == null) {
break;
} else {
- docs = termsEnum.docs(acceptDocs, docs);
+ docs = termsEnum.docs(acceptDocs, docs, false);
int doc = docs.nextDoc();
if (doc != DocsEnum.NO_MORE_DOCS) {
if (keepMode == KeepMode.KM_USE_FIRST_OCCURRENCE) {
@@ -133,7 +133,7 @@ public class DuplicateFilter extends Fil
} else {
if (termsEnum.docFreq() > 1) {
// unset potential duplicates
- docs = termsEnum.docs(acceptDocs, docs);
+ docs = termsEnum.docs(acceptDocs, docs, false);
int doc = docs.nextDoc();
if (doc != DocsEnum.NO_MORE_DOCS) {
if (keepMode == KeepMode.KM_USE_FIRST_OCCURRENCE) {
Modified: lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java (original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/sandbox/src/test/org/apache/lucene/sandbox/queries/DuplicateFilterTest.java Sun Dec 4 19:18:17 2011
@@ -17,6 +17,9 @@ package org.apache.lucene.sandbox.querie
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.HashSet;
+
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.StringField;
@@ -28,9 +31,7 @@ import org.apache.lucene.search.TermQuer
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
-
-import java.io.IOException;
-import java.util.HashSet;
+import org.apache.lucene.util._TestUtil;
public class DuplicateFilterTest extends LuceneTestCase {
private static final String KEY_FIELD = "url";
@@ -134,10 +135,13 @@ public class DuplicateFilterTest extends
for (ScoreDoc hit : hits) {
Document d = searcher.doc(hit.doc);
String url = d.get(KEY_FIELD);
- DocsEnum td = MultiFields.getTermDocsEnum(reader,
- MultiFields.getLiveDocs(reader),
- KEY_FIELD,
- new BytesRef(url));
+ DocsEnum td = _TestUtil.docs(random, reader,
+ KEY_FIELD,
+ new BytesRef(url),
+ MultiFields.getLiveDocs(reader),
+ null,
+ false);
+
int lastDoc = 0;
while (td.nextDoc() != DocsEnum.NO_MORE_DOCS) {
lastDoc = td.docID();
@@ -155,10 +159,13 @@ public class DuplicateFilterTest extends
for (ScoreDoc hit : hits) {
Document d = searcher.doc(hit.doc);
String url = d.get(KEY_FIELD);
- DocsEnum td = MultiFields.getTermDocsEnum(reader,
- MultiFields.getLiveDocs(reader),
- KEY_FIELD,
- new BytesRef(url));
+ DocsEnum td = _TestUtil.docs(random, reader,
+ KEY_FIELD,
+ new BytesRef(url),
+ MultiFields.getLiveDocs(reader),
+ null,
+ false);
+
int lastDoc = 0;
td.nextDoc();
lastDoc = td.docID();
Modified: lucene/dev/branches/lucene3606/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java (original)
+++ lucene/dev/branches/lucene3606/lucene/contrib/spatial/src/java/org/apache/lucene/spatial/tier/CartesianShapeFilter.java Sun Dec 4 19:18:17 2011
@@ -57,7 +57,7 @@ public class CartesianShapeFilter extend
return new DocIdSet() {
@Override
public DocIdSetIterator iterator() throws IOException {
- return context.reader.termDocsEnum(acceptDocs, fieldName, bytesRef);
+ return context.reader.termDocsEnum(acceptDocs, fieldName, bytesRef, false);
}
@Override
@@ -70,7 +70,7 @@ public class CartesianShapeFilter extend
for (int i =0; i< sz; i++) {
double boxId = area.get(i).doubleValue();
NumericUtils.longToPrefixCoded(NumericUtils.doubleToSortableLong(boxId), 0, bytesRef);
- final DocsEnum docsEnum = context.reader.termDocsEnum(acceptDocs, fieldName, bytesRef);
+ final DocsEnum docsEnum = context.reader.termDocsEnum(acceptDocs, fieldName, bytesRef, false);
if (docsEnum == null) continue;
// iterate through all documents
// which have this boxId
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/BufferedDeletesStream.java Sun Dec 4 19:18:17 2011
@@ -387,7 +387,7 @@ class BufferedDeletesStream {
// System.out.println(" term=" + term);
if (termsEnum.seekExact(term.bytes(), false)) {
- DocsEnum docsEnum = termsEnum.docs(reader.getLiveDocs(), docs);
+ DocsEnum docsEnum = termsEnum.docs(reader.getLiveDocs(), docs, false);
//System.out.println("BDS: got docsEnum=" + docsEnum);
if (docsEnum != null) {
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/CheckIndex.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/CheckIndex.java Sun Dec 4 19:18:17 2011
@@ -683,6 +683,7 @@ public class CheckIndex {
}
DocsEnum docs = null;
+ DocsEnum docsAndFreqs = null;
DocsAndPositionsEnum postings = null;
final FieldsEnum fieldsEnum = fields.iterator();
@@ -740,7 +741,8 @@ public class CheckIndex {
status.totFreq += docFreq;
sumDocFreq += docFreq;
- docs = termsEnum.docs(liveDocs, docs);
+ docs = termsEnum.docs(liveDocs, docs, false);
+ docsAndFreqs = termsEnum.docs(liveDocs, docsAndFreqs, true);
postings = termsEnum.docsAndPositions(liveDocs, postings);
if (hasOrd) {
@@ -762,13 +764,24 @@ public class CheckIndex {
status.termCount++;
final DocsEnum docs2;
+ final DocsEnum docsAndFreqs2;
final boolean hasPositions;
+ final boolean hasFreqs;
if (postings != null) {
docs2 = postings;
+ docsAndFreqs2 = postings;
hasPositions = true;
+ hasFreqs = true;
+ } else if (docsAndFreqs != null) {
+ docs2 = docsAndFreqs;
+ docsAndFreqs2 = docsAndFreqs;
+ hasPositions = false;
+ hasFreqs = true;
} else {
docs2 = docs;
+ docsAndFreqs2 = null;
hasPositions = false;
+ hasFreqs = false;
}
int lastDoc = -1;
@@ -780,9 +793,15 @@ public class CheckIndex {
break;
}
visitedDocs.set(doc);
- final int freq = docs2.freq();
- status.totPos += freq;
- totalTermFreq += freq;
+ int freq = -1;
+ if (hasFreqs) {
+ freq = docsAndFreqs2.freq();
+ if (freq <= 0) {
+ throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
+ }
+ status.totPos += freq;
+ totalTermFreq += freq;
+ }
docCount++;
if (doc <= lastDoc) {
@@ -793,12 +812,9 @@ public class CheckIndex {
}
lastDoc = doc;
- if (freq <= 0) {
- throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
- }
int lastPos = -1;
- if (postings != null) {
+ if (hasPositions) {
for(int j=0;j<freq;j++) {
final int pos = postings.nextPosition();
if (pos < -1) {
@@ -820,13 +836,23 @@ public class CheckIndex {
// Re-count if there are deleted docs:
if (reader.hasDeletions()) {
- final DocsEnum docsNoDel = termsEnum.docs(null, docs);
- docCount = 0;
- totalTermFreq = 0;
- while(docsNoDel.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
- visitedDocs.set(docsNoDel.docID());
- docCount++;
- totalTermFreq += docsNoDel.freq();
+ if (hasFreqs) {
+ final DocsEnum docsNoDel = termsEnum.docs(null, docsAndFreqs, true);
+ docCount = 0;
+ totalTermFreq = 0;
+ while(docsNoDel.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+ visitedDocs.set(docsNoDel.docID());
+ docCount++;
+ totalTermFreq += docsNoDel.freq();
+ }
+ } else {
+ final DocsEnum docsNoDel = termsEnum.docs(null, docs, false);
+ docCount = 0;
+ totalTermFreq = -1;
+ while(docsNoDel.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+ visitedDocs.set(docsNoDel.docID());
+ docCount++;
+ }
}
}
@@ -883,7 +909,7 @@ public class CheckIndex {
} else {
for(int idx=0;idx<7;idx++) {
final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
- docs = termsEnum.docs(liveDocs, docs);
+ docs = termsEnum.docs(liveDocs, docs, false);
final int docID = docs.advance(skipDocID);
if (docID == DocsEnum.NO_MORE_DOCS) {
break;
@@ -986,7 +1012,7 @@ public class CheckIndex {
throw new RuntimeException("seek to existing term " + seekTerms[i] + " failed");
}
- docs = termsEnum.docs(liveDocs, docs);
+ docs = termsEnum.docs(liveDocs, docs, false);
if (docs == null) {
throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]);
}
@@ -1168,6 +1194,7 @@ public class CheckIndex {
// TODO: maybe we can factor out testTermIndex and reuse here?
DocsEnum docs = null;
+ DocsEnum docsAndFreqs = null;
DocsAndPositionsEnum postings = null;
final Bits liveDocs = reader.getLiveDocs();
for (int j = 0; j < info.docCount; ++j) {
@@ -1210,50 +1237,53 @@ public class CheckIndex {
if (totalTermFreq != -1 && totalTermFreq <= 0) {
throw new RuntimeException("totalTermFreq: " + totalTermFreq + " is out of bounds");
}
-
- DocsEnum docsEnum;
- DocsAndPositionsEnum dp = termsEnum.docsAndPositions(null, postings);
- if (dp == null) {
- DocsEnum d = termsEnum.docs(null, docs);
- docsEnum = docs = d;
+
+ postings = termsEnum.docsAndPositions(null, postings);
+ if (postings == null) {
+ docsAndFreqs = termsEnum.docs(null, docsAndFreqs, true);
+ if (docsAndFreqs == null) {
+ docs = termsEnum.docs(null, docs, false);
+ } else {
+ docs = docsAndFreqs;
+ }
} else {
- docsEnum = postings = dp;
+ docs = docsAndFreqs = postings;
}
-
- final int doc = docsEnum.nextDoc();
+
+ final int doc = docs.nextDoc();
if (doc != 0) {
throw new RuntimeException("vector for doc " + j + " didn't return docID=0: got docID=" + doc);
}
-
- final int tf = docsEnum.freq();
- tfvComputedSumTotalTermFreq += tf;
-
- if (tf <= 0) {
- throw new RuntimeException("vector freq " + tf + " is out of bounds");
- }
-
- if (totalTermFreq != -1 && totalTermFreq != tf) {
- throw new RuntimeException("vector totalTermFreq " + totalTermFreq + " != tf " + tf);
- }
+
+ if (docsAndFreqs != null) {
+ final int tf = docsAndFreqs.freq();
+ if (tf <= 0) {
+ throw new RuntimeException("vector freq " + tf + " is out of bounds");
+ }
+ if (totalTermFreq != -1 && totalTermFreq != tf) {
+ throw new RuntimeException("vector totalTermFreq " + totalTermFreq + " != tf " + tf);
+ }
+ tfvComputedSumTotalTermFreq += tf;
- if (dp != null) {
- int lastPosition = -1;
- for (int i = 0; i < tf; i++) {
- int pos = dp.nextPosition();
- if (pos != -1 && pos < 0) {
- throw new RuntimeException("vector position " + pos + " is out of bounds");
- }
+ if (postings != null) {
+ int lastPosition = -1;
+ for (int i = 0; i < tf; i++) {
+ int pos = postings.nextPosition();
+ if (pos != -1 && pos < 0) {
+ throw new RuntimeException("vector position " + pos + " is out of bounds");
+ }
- if (pos < lastPosition) {
- throw new RuntimeException("vector position " + pos + " < lastPos " + lastPosition);
- }
+ if (pos < lastPosition) {
+ throw new RuntimeException("vector position " + pos + " < lastPos " + lastPosition);
+ }
- lastPosition = pos;
+ lastPosition = pos;
+ }
}
}
- if (docsEnum.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
+ if (docs.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
throw new RuntimeException("vector for doc " + j + " references multiple documents!");
}
}
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocTermOrds.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocTermOrds.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocTermOrds.java Sun Dec 4 19:18:17 2011
@@ -314,7 +314,7 @@ public class DocTermOrds {
final int df = te.docFreq();
if (df <= maxTermDocFreq) {
- docsEnum = te.docs(liveDocs, docsEnum);
+ docsEnum = te.docs(liveDocs, docsEnum, false);
// dF, but takes deletions into account
int actualDF = 0;
@@ -650,8 +650,8 @@ public class DocTermOrds {
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
- return termsEnum.docs(liveDocs, reuse);
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ return termsEnum.docs(liveDocs, reuse, needsFreqs);
}
@Override
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocsEnum.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocsEnum.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/DocsEnum.java Sun Dec 4 19:18:17 2011
@@ -20,7 +20,7 @@ package org.apache.lucene.index;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.util.AttributeSource;
-/** Iterates through the documents, term freq and positions.
+/** Iterates through the documents and term freqs.
* NOTE: you must first call {@link #nextDoc} before using
* any of the per-doc methods. */
public abstract class DocsEnum extends DocIdSetIterator {
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FieldInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FieldInfo.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FieldInfo.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FieldInfo.java Sun Dec 4 19:18:17 2011
@@ -44,6 +44,7 @@ public final class FieldInfo {
*/
public static enum IndexOptions {
/** only documents are indexed: term frequencies and positions are omitted */
+ // TODO: maybe rename to just DOCS?
DOCS_ONLY,
/** only documents and term frequencies are indexed: positions are omitted */
DOCS_AND_FREQS,
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilterIndexReader.java Sun Dec 4 19:18:17 2011
@@ -175,8 +175,8 @@ public class FilterIndexReader extends I
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
- return in.docs(liveDocs, reuse);
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ return in.docs(liveDocs, reuse, needsFreqs);
}
@Override
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilteredTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilteredTermsEnum.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilteredTermsEnum.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/FilteredTermsEnum.java Sun Dec 4 19:18:17 2011
@@ -166,8 +166,8 @@ public abstract class FilteredTermsEnum
}
@Override
- public DocsEnum docs(Bits bits, DocsEnum reuse) throws IOException {
- return tenum.docs(bits, reuse);
+ public DocsEnum docs(Bits bits, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ return tenum.docs(bits, reuse, needsFreqs);
}
@Override
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/IndexReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/IndexReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/IndexReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/IndexReader.java Sun Dec 4 19:18:17 2011
@@ -997,7 +997,7 @@ public abstract class IndexReader implem
/** Returns {@link DocsEnum} for the specified field &
* term. This may return null, if either the field or
* term does not exist. */
- public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term) throws IOException {
+ public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, boolean needsFreqs) throws IOException {
assert field != null;
assert term != null;
final Fields fields = fields();
@@ -1006,7 +1006,7 @@ public abstract class IndexReader implem
if (terms != null) {
final TermsEnum termsEnum = terms.iterator(null);
if (termsEnum.seekExact(term, true)) {
- return termsEnum.docs(liveDocs, null);
+ return termsEnum.docs(liveDocs, null, needsFreqs);
}
}
}
@@ -1016,7 +1016,7 @@ public abstract class IndexReader implem
/** Returns {@link DocsAndPositionsEnum} for the specified
* field & term. This may return null, if either the
* field or term does not exist, or, positions were not
- * stored for this term. */
+ * indexed for this field. */
public DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term) throws IOException {
assert field != null;
assert term != null;
@@ -1038,7 +1038,7 @@ public abstract class IndexReader implem
* {@link TermState}. This may return null, if either the field or the term
* does not exists or the {@link TermState} is invalid for the underlying
* implementation.*/
- public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, TermState state) throws IOException {
+ public DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, TermState state, boolean needsFreqs) throws IOException {
assert state != null;
assert field != null;
final Fields fields = fields();
@@ -1047,7 +1047,7 @@ public abstract class IndexReader implem
if (terms != null) {
final TermsEnum termsEnum = terms.iterator(null);
termsEnum.seekExact(term, state);
- return termsEnum.docs(liveDocs, null);
+ return termsEnum.docs(liveDocs, null, needsFreqs);
}
}
return null;
@@ -1057,7 +1057,7 @@ public abstract class IndexReader implem
* Returns {@link DocsAndPositionsEnum} for the specified field and
* {@link TermState}. This may return null, if either the field or the term
* does not exists, the {@link TermState} is invalid for the underlying
- * implementation, or positions were not stored for this term.*/
+ * implementation, or positions were not indexed for this field. */
public DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, TermState state) throws IOException {
assert state != null;
assert field != null;
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiFields.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiFields.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiFields.java Sun Dec 4 19:18:17 2011
@@ -151,14 +151,14 @@ public final class MultiFields extends F
/** Returns {@link DocsEnum} for the specified field &
* term. This may return null if the term does not
* exist. */
- public static DocsEnum getTermDocsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term) throws IOException {
+ public static DocsEnum getTermDocsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, boolean needsFreqs) throws IOException {
assert field != null;
assert term != null;
final Terms terms = getTerms(r, field);
if (terms != null) {
final TermsEnum termsEnum = terms.iterator(null);
if (termsEnum.seekExact(term, true)) {
- return termsEnum.docs(liveDocs, null);
+ return termsEnum.docs(liveDocs, null, needsFreqs);
}
}
return null;
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/MultiTermsEnum.java Sun Dec 4 19:18:17 2011
@@ -347,7 +347,7 @@ public final class MultiTermsEnum extend
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
MultiDocsEnum docsEnum;
// Can only reuse if incoming enum is also a MultiDocsEnum
if (reuse != null && reuse instanceof MultiDocsEnum) {
@@ -397,14 +397,16 @@ public final class MultiTermsEnum extend
}
assert entry.index < docsEnum.subDocsEnum.length: entry.index + " vs " + docsEnum.subDocsEnum.length + "; " + subs.length;
- final DocsEnum subDocsEnum = entry.terms.docs(b, docsEnum.subDocsEnum[entry.index]);
-
+ final DocsEnum subDocsEnum = entry.terms.docs(b, docsEnum.subDocsEnum[entry.index], needsFreqs);
if (subDocsEnum != null) {
docsEnum.subDocsEnum[entry.index] = subDocsEnum;
subDocs[upto].docsEnum = subDocsEnum;
subDocs[upto].slice = entry.subSlice;
-
upto++;
+ } else {
+ // One of our subs cannot provide freqs:
+ assert needsFreqs;
+ return null;
}
}
@@ -475,7 +477,7 @@ public final class MultiTermsEnum extend
subDocsAndPositions[upto].slice = entry.subSlice;
upto++;
} else {
- if (entry.terms.docs(b, null) != null) {
+ if (entry.terms.docs(b, null, false) != null) {
// At least one of our subs does not store
// positions -- we can't correctly produce a
// MultiDocsAndPositions enum
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/TermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/TermsEnum.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/TermsEnum.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/TermsEnum.java Sun Dec 4 19:18:17 2011
@@ -147,12 +147,16 @@ public abstract class TermsEnum {
/** Get {@link DocsEnum} for the current term. Do not
* call this when the enum is unpositioned. This method
- * will not return null.
+ * may return null (if needsFreqs is true but freqs were
+ * not indexed for this field).
*
* @param liveDocs unset bits are documents that should not
* be returned
- * @param reuse pass a prior DocsEnum for possible reuse */
- public abstract DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException;
+ * @param reuse pass a prior DocsEnum for possible reuse
+ * @param needsFreqs true if the caller intends to call
+ * {@link DocsEnum#freq}. If you pass false you must not
+ * call {@link DocsEnum#freq} in the returned DocsEnum. */
+ public abstract DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException;
/** Get {@link DocsAndPositionsEnum} for the current term.
* Do not call this when the enum is unpositioned.
@@ -229,7 +233,7 @@ public abstract class TermsEnum {
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) {
throw new IllegalStateException("this method should never be called");
}
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTermsReader.java Sun Dec 4 19:18:17 2011
@@ -25,8 +25,8 @@ import java.util.TreeMap;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.IndexFileNames;
@@ -689,14 +689,11 @@ public class BlockTermsReader extends Fi
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
//System.out.println("BTR.docs this=" + this);
decodeMetaData();
//System.out.println("BTR.docs: state.docFreq=" + state.docFreq);
- final DocsEnum docsEnum = postingsReader.docs(fieldInfo, state, liveDocs, reuse);
- assert docsEnum != null;
- //System.out.println("BTR.docs: return docsEnum=" + docsEnum);
- return docsEnum;
+ return postingsReader.docs(fieldInfo, state, liveDocs, reuse, needsFreqs);
}
@Override
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/BlockTreeTermsReader.java Sun Dec 4 19:18:17 2011
@@ -875,9 +875,9 @@ public class BlockTreeTermsReader extend
}
@Override
- public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits skipDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
currentFrame.decodeMetaData();
- return postingsReader.docs(fieldInfo, currentFrame.termState, skipDocs, reuse);
+ return postingsReader.docs(fieldInfo, currentFrame.termState, skipDocs, reuse, needsFreqs);
}
@Override
@@ -2082,7 +2082,7 @@ public class BlockTreeTermsReader extend
}
@Override
- public DocsEnum docs(Bits skipDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits skipDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
assert !eof;
//if (DEBUG) {
//System.out.println("BTTR.docs seg=" + segment);
@@ -2091,10 +2091,7 @@ public class BlockTreeTermsReader extend
//if (DEBUG) {
//System.out.println(" state=" + currentFrame.state);
//}
- final DocsEnum docsEnum = postingsReader.docs(fieldInfo, currentFrame.state, skipDocs, reuse);
-
- assert docsEnum != null;
- return docsEnum;
+ return postingsReader.docs(fieldInfo, currentFrame.state, skipDocs, reuse, needsFreqs);
}
@Override
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsConsumer.java Sun Dec 4 19:18:17 2011
@@ -63,7 +63,19 @@ public abstract class PostingsConsumer {
int df = 0;
long totTF = 0;
- if (mergeState.fieldInfo.indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
+ if (mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) {
+ while(true) {
+ final int doc = postings.nextDoc();
+ if (doc == DocIdSetIterator.NO_MORE_DOCS) {
+ break;
+ }
+ visitedDocs.set(doc);
+ this.startDoc(doc, 0);
+ this.finishDoc();
+ df++;
+ }
+ totTF = -1;
+ } else if (mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS) {
while(true) {
final int doc = postings.nextDoc();
if (doc == DocIdSetIterator.NO_MORE_DOCS) {
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/PostingsReaderBase.java Sun Dec 4 19:18:17 2011
@@ -51,7 +51,7 @@ public abstract class PostingsReaderBase
/** Must fully consume state, since after this call that
* TermState may be reused. */
- public abstract DocsEnum docs(FieldInfo fieldInfo, BlockTermState state, Bits skipDocs, DocsEnum reuse) throws IOException;
+ public abstract DocsEnum docs(FieldInfo fieldInfo, BlockTermState state, Bits skipDocs, DocsEnum reuse, boolean needsFreqs) throws IOException;
/** Must fully consume state, since after this call that
* TermState may be reused. */
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/TermsConsumer.java Sun Dec 4 19:18:17 2011
@@ -51,8 +51,9 @@ public abstract class TermsConsumer {
public abstract Comparator<BytesRef> getComparator() throws IOException;
/** Default merge impl */
- private MappingMultiDocsEnum docsEnum = null;
- private MappingMultiDocsAndPositionsEnum postingsEnum = null;
+ private MappingMultiDocsEnum docsEnum;
+ private MappingMultiDocsEnum docsAndFreqsEnum;
+ private MappingMultiDocsAndPositionsEnum postingsEnum;
public void merge(MergeState mergeState, TermsEnum termsEnum) throws IOException {
@@ -63,7 +64,7 @@ public abstract class TermsConsumer {
long sumDFsinceLastAbortCheck = 0;
FixedBitSet visitedDocs = new FixedBitSet(mergeState.mergedDocCount);
- if (mergeState.fieldInfo.indexOptions != IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) {
+ if (mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) {
if (docsEnum == null) {
docsEnum = new MappingMultiDocsEnum();
}
@@ -74,14 +75,14 @@ public abstract class TermsConsumer {
while((term = termsEnum.next()) != null) {
// We can pass null for liveDocs, because the
// mapping enum will skip the non-live docs:
- docsEnumIn = (MultiDocsEnum) termsEnum.docs(null, docsEnumIn);
+ docsEnumIn = (MultiDocsEnum) termsEnum.docs(null, docsEnumIn, false);
if (docsEnumIn != null) {
docsEnum.reset(docsEnumIn);
final PostingsConsumer postingsConsumer = startTerm(term);
final TermStats stats = postingsConsumer.merge(mergeState, docsEnum, visitedDocs);
if (stats.docFreq > 0) {
finishTerm(term, stats);
- sumTotalTermFreq += stats.totalTermFreq;
+ sumTotalTermFreq += stats.docFreq;
sumDFsinceLastAbortCheck += stats.docFreq;
sumDocFreq += stats.docFreq;
if (sumDFsinceLastAbortCheck > 60000) {
@@ -91,7 +92,35 @@ public abstract class TermsConsumer {
}
}
}
+ } else if (mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS) {
+ if (docsAndFreqsEnum == null) {
+ docsAndFreqsEnum = new MappingMultiDocsEnum();
+ }
+ docsAndFreqsEnum.setMergeState(mergeState);
+
+ MultiDocsEnum docsAndFreqsEnumIn = null;
+
+ while((term = termsEnum.next()) != null) {
+ // We can pass null for liveDocs, because the
+ // mapping enum will skip the non-live docs:
+ docsAndFreqsEnumIn = (MultiDocsEnum) termsEnum.docs(null, docsAndFreqsEnumIn, true);
+ assert docsAndFreqsEnumIn != null;
+ docsAndFreqsEnum.reset(docsAndFreqsEnumIn);
+ final PostingsConsumer postingsConsumer = startTerm(term);
+ final TermStats stats = postingsConsumer.merge(mergeState, docsAndFreqsEnum, visitedDocs);
+ if (stats.docFreq > 0) {
+ finishTerm(term, stats);
+ sumTotalTermFreq += stats.totalTermFreq;
+ sumDFsinceLastAbortCheck += stats.docFreq;
+ sumDocFreq += stats.docFreq;
+ if (sumDFsinceLastAbortCheck > 60000) {
+ mergeState.checkAbort.work(sumDFsinceLastAbortCheck/5.0);
+ sumDFsinceLastAbortCheck = 0;
+ }
+ }
+ }
} else {
+ assert mergeState.fieldInfo.indexOptions == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
if (postingsEnum == null) {
postingsEnum = new MappingMultiDocsAndPositionsEnum();
}
@@ -101,27 +130,26 @@ public abstract class TermsConsumer {
// We can pass null for liveDocs, because the
// mapping enum will skip the non-live docs:
postingsEnumIn = (MultiDocsAndPositionsEnum) termsEnum.docsAndPositions(null, postingsEnumIn);
- if (postingsEnumIn != null) {
- postingsEnum.reset(postingsEnumIn);
- // set PayloadProcessor
- if (mergeState.payloadProcessorProvider != null) {
- for (int i = 0; i < mergeState.readers.size(); i++) {
- if (mergeState.dirPayloadProcessor[i] != null) {
- mergeState.currentPayloadProcessor[i] = mergeState.dirPayloadProcessor[i].getProcessor(mergeState.fieldInfo.name, term);
- }
+ assert postingsEnumIn != null;
+ postingsEnum.reset(postingsEnumIn);
+ // set PayloadProcessor
+ if (mergeState.payloadProcessorProvider != null) {
+ for (int i = 0; i < mergeState.readers.size(); i++) {
+ if (mergeState.dirPayloadProcessor[i] != null) {
+ mergeState.currentPayloadProcessor[i] = mergeState.dirPayloadProcessor[i].getProcessor(mergeState.fieldInfo.name, term);
}
}
- final PostingsConsumer postingsConsumer = startTerm(term);
- final TermStats stats = postingsConsumer.merge(mergeState, postingsEnum, visitedDocs);
- if (stats.docFreq > 0) {
- finishTerm(term, stats);
- sumTotalTermFreq += stats.totalTermFreq;
- sumDFsinceLastAbortCheck += stats.docFreq;
- sumDocFreq += stats.docFreq;
- if (sumDFsinceLastAbortCheck > 60000) {
- mergeState.checkAbort.work(sumDFsinceLastAbortCheck/5.0);
- sumDFsinceLastAbortCheck = 0;
- }
+ }
+ final PostingsConsumer postingsConsumer = startTerm(term);
+ final TermStats stats = postingsConsumer.merge(mergeState, postingsEnum, visitedDocs);
+ if (stats.docFreq > 0) {
+ finishTerm(term, stats);
+ sumTotalTermFreq += stats.totalTermFreq;
+ sumDFsinceLastAbortCheck += stats.docFreq;
+ sumDocFreq += stats.docFreq;
+ if (sumDFsinceLastAbortCheck > 60000) {
+ mergeState.checkAbort.work(sumDFsinceLastAbortCheck/5.0);
+ sumDFsinceLastAbortCheck = 0;
}
}
}
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/Lucene3xFields.java Sun Dec 4 19:18:17 2011
@@ -27,8 +27,8 @@ import java.util.TreeMap;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.IndexFileNames;
@@ -950,9 +950,11 @@ public class Lucene3xFields extends Fiel
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
PreDocsEnum docsEnum;
- if (reuse == null || !(reuse instanceof PreDocsEnum)) {
+ if (needsFreqs && fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) {
+ return null;
+ } else if (reuse == null || !(reuse instanceof PreDocsEnum)) {
docsEnum = new PreDocsEnum();
} else {
docsEnum = (PreDocsEnum) reuse;
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene3x/SegmentTermDocs.java Sun Dec 4 19:18:17 2011
@@ -112,7 +112,10 @@ public class SegmentTermDocs {
}
public final int doc() { return doc; }
- public final int freq() { return freq; }
+ public final int freq() {
+ assert indexOptions != IndexOptions.DOCS_ONLY;
+ return freq;
+ }
protected void skippingDoc() throws IOException {
}
@@ -125,7 +128,6 @@ public class SegmentTermDocs {
if (indexOptions == IndexOptions.DOCS_ONLY) {
doc += docCode;
- freq = 1;
} else {
doc += docCode >>> 1; // shift off low bit
if ((docCode & 1) != 0) // if low bit is set
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40PostingsReader.java Sun Dec 4 19:18:17 2011
@@ -18,7 +18,6 @@ package org.apache.lucene.index.codecs.l
*/
import java.io.IOException;
-import java.util.Arrays;
import java.util.Collection;
import org.apache.lucene.index.DocsAndPositionsEnum;
@@ -209,9 +208,11 @@ public class Lucene40PostingsReader exte
}
@Override
- public DocsEnum docs(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(FieldInfo fieldInfo, BlockTermState termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
SegmentDocsEnum docsEnum;
- if (reuse == null || !(reuse instanceof SegmentDocsEnum)) {
+ if (needsFreqs && fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) {
+ return null;
+ } else if (reuse == null || !(reuse instanceof SegmentDocsEnum)) {
docsEnum = new SegmentDocsEnum(freqIn);
} else {
docsEnum = (SegmentDocsEnum) reuse;
@@ -277,7 +278,7 @@ public class Lucene40PostingsReader exte
final IndexInput freqIn;
final IndexInput startFreqIn;
- boolean omitTF; // does current field omit term freq?
+ boolean indexOmitsTF; // does current field omit term freq?
boolean storePayloads; // does current field store payloads?
int limit; // number of docs in this posting
@@ -300,12 +301,7 @@ public class Lucene40PostingsReader exte
}
public SegmentDocsEnum reset(FieldInfo fieldInfo, StandardTermState termState, Bits liveDocs) throws IOException {
- omitTF = fieldInfo.indexOptions == IndexOptions.DOCS_ONLY;
- if (omitTF) {
- freq = 1;
- Arrays.fill(freqs, 1);
- }
-
+ indexOmitsTF = fieldInfo.indexOptions == IndexOptions.DOCS_ONLY;
storePayloads = fieldInfo.storePayloads;
this.liveDocs = liveDocs;
freqOffset = termState.freqOffset;
@@ -331,6 +327,7 @@ public class Lucene40PostingsReader exte
@Override
public int freq() {
+ assert !indexOmitsTF;
return freq;
}
@@ -389,7 +386,7 @@ public class Lucene40PostingsReader exte
count = bufferSize;
ord += bufferSize;
- if (omitTF)
+ if (indexOmitsTF)
fillDocs(bufferSize);
else
fillDocsAndFreqs(bufferSize);
@@ -400,7 +397,7 @@ public class Lucene40PostingsReader exte
private int scanTo(int target) throws IOException {
while (ord++ < limit) {
int code = freqIn.readVInt();
- if (omitTF) {
+ if (indexOmitsTF) {
accum += code;
} else {
accum += code >>> 1; // shift off low bit
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40TermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40TermVectorsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40TermVectorsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/lucene40/Lucene40TermVectorsReader.java Sun Dec 4 19:18:17 2011
@@ -505,7 +505,7 @@ public class Lucene40TermVectorsReader e
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs /* ignored */) throws IOException {
TVDocsEnum docsEnum;
if (reuse != null && reuse instanceof TVDocsEnum) {
docsEnum = (TVDocsEnum) reuse;
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/memory/MemoryPostingsFormat.java Sun Dec 4 19:18:17 2011
@@ -26,8 +26,8 @@ import java.util.TreeMap;
import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfo.IndexOptions;
+import org.apache.lucene.index.FieldInfo;
import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FieldsEnum;
import org.apache.lucene.index.IndexFileNames;
@@ -36,10 +36,10 @@ import org.apache.lucene.index.SegmentRe
import org.apache.lucene.index.SegmentWriteState;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.codecs.PostingsFormat;
import org.apache.lucene.index.codecs.FieldsConsumer;
import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.index.codecs.PostingsConsumer;
+import org.apache.lucene.index.codecs.PostingsFormat;
import org.apache.lucene.index.codecs.TermStats;
import org.apache.lucene.index.codecs.TermsConsumer;
import org.apache.lucene.store.ByteArrayDataInput;
@@ -317,7 +317,6 @@ public class MemoryPostingsFormat extend
docUpto++;
if (indexOptions == IndexOptions.DOCS_ONLY) {
accum += in.readVInt();
- freq = 1;
} else {
final int code = in.readVInt();
accum += code >>> 1;
@@ -371,6 +370,7 @@ public class MemoryPostingsFormat extend
@Override
public int freq() {
+ assert indexOptions != IndexOptions.DOCS_ONLY;
return freq;
}
}
@@ -600,10 +600,13 @@ public class MemoryPostingsFormat extend
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
decodeMetaData();
FSTDocsEnum docsEnum;
- if (reuse == null || !(reuse instanceof FSTDocsEnum)) {
+
+ if (needsFreqs && field.indexOptions == IndexOptions.DOCS_ONLY) {
+ return null;
+ } else if (reuse == null || !(reuse instanceof FSTDocsEnum)) {
docsEnum = new FSTDocsEnum(field.indexOptions, field.storePayloads);
} else {
docsEnum = (FSTDocsEnum) reuse;
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/pulsing/PulsingPostingsReader.java Sun Dec 4 19:18:17 2011
@@ -178,7 +178,10 @@ public class PulsingPostingsReader exten
}
@Override
- public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ if (needsFreqs && field.indexOptions == IndexOptions.DOCS_ONLY) {
+ return null;
+ }
PulsingTermState termState = (PulsingTermState) _termState;
if (termState.postingsSize != -1) {
PulsingDocsEnum postings;
@@ -202,11 +205,11 @@ public class PulsingPostingsReader exten
return postings.reset(liveDocs, termState);
} else {
if (reuse instanceof PulsingDocsEnum) {
- DocsEnum wrapped = wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, getOther(reuse));
+ DocsEnum wrapped = wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, getOther(reuse), needsFreqs);
setOther(wrapped, reuse); // wrapped.other = reuse
return wrapped;
} else {
- return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, reuse);
+ return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, reuse, needsFreqs);
}
}
}
@@ -283,7 +286,6 @@ public class PulsingPostingsReader exten
docID = -1;
accum = 0;
payloadLength = 0;
- freq = 1;
this.liveDocs = liveDocs;
return this;
}
@@ -342,6 +344,7 @@ public class PulsingPostingsReader exten
@Override
public int freq() {
+ assert indexOptions != IndexOptions.DOCS_ONLY;
return freq;
}
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/sep/SepPostingsReader.java Sun Dec 4 19:18:17 2011
@@ -272,7 +272,10 @@ public class SepPostingsReader extends P
}
@Override
- public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ if (needsFreqs && fieldInfo.indexOptions == IndexOptions.DOCS_ONLY) {
+ return null;
+ }
final SepTermState termState = (SepTermState) _termState;
SepDocsEnum docsEnum;
if (reuse == null || !(reuse instanceof SepDocsEnum)) {
@@ -369,8 +372,6 @@ public class SepPostingsReader extends P
if (!omitTF) {
freqIndex.set(termState.freqIndex);
freqIndex.seek(freqReader);
- } else {
- freq = 1;
}
docFreq = termState.docFreq;
@@ -412,6 +413,7 @@ public class SepPostingsReader extends P
@Override
public int freq() {
+ assert !omitTF;
return freq;
}
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextFieldsReader.java Sun Dec 4 19:18:17 2011
@@ -17,19 +17,24 @@ package org.apache.lucene.index.codecs.s
* limitations under the License.
*/
-import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.index.codecs.FieldsProducer;
+import java.io.IOException;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.lucene.index.DocsAndPositionsEnum;
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.FieldInfo.IndexOptions;
-import org.apache.lucene.index.SegmentReadState;
+import org.apache.lucene.index.FieldInfos;
import org.apache.lucene.index.FieldsEnum;
+import org.apache.lucene.index.SegmentReadState;
import org.apache.lucene.index.Terms;
-import org.apache.lucene.index.DocsEnum;
-import org.apache.lucene.index.DocsAndPositionsEnum;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.index.FieldInfos;
+import org.apache.lucene.index.codecs.FieldsProducer;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.OpenBitSet;
import org.apache.lucene.util.StringHelper;
@@ -37,13 +42,8 @@ import org.apache.lucene.util.UnicodeUti
import org.apache.lucene.util.fst.Builder;
import org.apache.lucene.util.fst.BytesRefFSTEnum;
import org.apache.lucene.util.fst.FST;
-import org.apache.lucene.util.fst.PositiveIntOutputs;
import org.apache.lucene.util.fst.PairOutputs;
-
-import java.io.IOException;
-import java.util.Comparator;
-import java.util.Map;
-import java.util.HashMap;
+import org.apache.lucene.util.fst.PositiveIntOutputs;
class SimpleTextFieldsReader extends FieldsProducer {
@@ -190,14 +190,17 @@ class SimpleTextFieldsReader extends Fie
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ if (needsFreqs && indexOptions == IndexOptions.DOCS_ONLY) {
+ return null;
+ }
SimpleTextDocsEnum docsEnum;
if (reuse != null && reuse instanceof SimpleTextDocsEnum && ((SimpleTextDocsEnum) reuse).canReuse(SimpleTextFieldsReader.this.in)) {
docsEnum = (SimpleTextDocsEnum) reuse;
} else {
docsEnum = new SimpleTextDocsEnum();
}
- return docsEnum.reset(docsStart, liveDocs, indexOptions == IndexOptions.DOCS_ONLY);
+ return docsEnum.reset(docsStart, liveDocs, !needsFreqs);
}
@Override
@@ -245,9 +248,6 @@ class SimpleTextFieldsReader extends Fie
in.seek(fp);
this.omitTF = omitTF;
docID = -1;
- if (omitTF) {
- tf = 1;
- }
return this;
}
@@ -258,6 +258,7 @@ class SimpleTextFieldsReader extends Fie
@Override
public int freq() {
+ assert !omitTF;
return tf;
}
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/index/codecs/simpletext/SimpleTextTermVectorsReader.java Sun Dec 4 19:18:17 2011
@@ -365,10 +365,10 @@ public class SimpleTextTermVectorsReader
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
// TODO: reuse
SimpleTVDocsEnum e = new SimpleTVDocsEnum();
- e.reset(liveDocs, current.getValue().freq);
+ e.reset(liveDocs, needsFreqs ? current.getValue().freq : -1);
return e;
}
@@ -399,6 +399,7 @@ public class SimpleTextTermVectorsReader
@Override
public int freq() {
+ assert freq != -1;
return freq;
}
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/BooleanQuery.java Sun Dec 4 19:18:17 2011
@@ -17,20 +17,21 @@ package org.apache.lucene.search;
* limitations under the License.
*/
-import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermsEnum;
+import java.io.IOException;
+import java.util.*;
+
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader.AtomicReaderContext;
+import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.util.Bits;
-import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.ConjunctionTermScorer.DocsAndFreqs;
-import org.apache.lucene.search.similarities.SimilarityProvider;
-import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
import org.apache.lucene.search.TermQuery.TermWeight;
-
-import java.io.IOException;
-import java.util.*;
+import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
+import org.apache.lucene.search.similarities.SimilarityProvider;
+import org.apache.lucene.util.Bits;
+import org.apache.lucene.util.ToStringUtils;
/** A Query that matches documents matching boolean combinations of other
* queries, e.g. {@link TermQuery}s, {@link PhraseQuery}s or other
@@ -349,6 +350,11 @@ public class BooleanQuery extends Query
private Scorer createConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs)
throws IOException {
+
+ // TODO: fix scorer API to specify "needsScores" up
+ // front, so we can do match-only if caller doesn't
+ // needs scores
+
final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()];
for (int i = 0; i < docsAndFreqs.length; i++) {
final TermWeight weight = (TermWeight) weights.get(i);
@@ -357,12 +363,46 @@ public class BooleanQuery extends Query
return null;
}
final ExactDocScorer docScorer = weight.createDocScorer(context);
- docsAndFreqs[i] = new DocsAndFreqs(termsEnum.docs(
- acceptDocs, null), termsEnum.docFreq(), docScorer);
+ final DocsEnum docsAndFreqsEnum = termsEnum.docs(acceptDocs, null, true);
+ if (docsAndFreqsEnum == null) {
+ // TODO: we could carry over TermState from the
+ // terms we already seek'd to, to save re-seeking
+ // to make the match-only scorer, but it's likely
+ // rare that BQ mixes terms from omitTf and
+ // non-omitTF fields:
+
+ // At least one sub cannot provide freqs; abort
+ // and fallback to full match-only scorer:
+ return createMatchOnlyConjunctionTermScorer(context, acceptDocs);
+ }
+
+ docsAndFreqs[i] = new DocsAndFreqs(docsAndFreqsEnum,
+ docsAndFreqsEnum,
+ termsEnum.docFreq(), docScorer);
}
return new ConjunctionTermScorer(this, disableCoord ? 1.0f : coord(
docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs);
}
+
+ private Scorer createMatchOnlyConjunctionTermScorer(AtomicReaderContext context, Bits acceptDocs)
+ throws IOException {
+
+ final DocsAndFreqs[] docsAndFreqs = new DocsAndFreqs[weights.size()];
+ for (int i = 0; i < docsAndFreqs.length; i++) {
+ final TermWeight weight = (TermWeight) weights.get(i);
+ final TermsEnum termsEnum = weight.getTermsEnum(context);
+ if (termsEnum == null) {
+ return null;
+ }
+ final ExactDocScorer docScorer = weight.createDocScorer(context);
+ docsAndFreqs[i] = new DocsAndFreqs(null,
+ termsEnum.docs(acceptDocs, null, false),
+ termsEnum.docFreq(), docScorer);
+ }
+
+ return new MatchOnlyConjunctionTermScorer(this, disableCoord ? 1.0f : coord(
+ docsAndFreqs.length, docsAndFreqs.length), docsAndFreqs);
+ }
@Override
public boolean scoresDocsOutOfOrder() {
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/ConjunctionTermScorer.java Sun Dec 4 19:18:17 2011
@@ -17,17 +17,18 @@ package org.apache.lucene.search;
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.Comparator;
+
import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.search.similarities.Similarity.ExactDocScorer;
import org.apache.lucene.util.ArrayUtil;
-import java.io.IOException;
-import java.util.Comparator;
/** Scorer for conjunctions, sets of terms, all of which are required. */
-final class ConjunctionTermScorer extends Scorer {
- private final float coord;
- private int lastDoc = -1;
- private final DocsAndFreqs[] docsAndFreqs;
+class ConjunctionTermScorer extends Scorer {
+ protected final float coord;
+ protected int lastDoc = -1;
+ protected final DocsAndFreqs[] docsAndFreqs;
private final DocsAndFreqs lead;
ConjunctionTermScorer(Weight weight, float coord,
@@ -39,7 +40,7 @@ final class ConjunctionTermScorer extend
// lead the matching.
ArrayUtil.mergeSort(docsAndFreqs, new Comparator<DocsAndFreqs>() {
public int compare(DocsAndFreqs o1, DocsAndFreqs o2) {
- return o1.freq - o2.freq;
+ return o1.docFreq - o2.docFreq;
}
});
@@ -96,14 +97,16 @@ final class ConjunctionTermScorer extend
}
static final class DocsAndFreqs {
+ final DocsEnum docsAndFreqs;
final DocsEnum docs;
- final int freq;
+ final int docFreq;
final ExactDocScorer docScorer;
int doc = -1;
- DocsAndFreqs(DocsEnum docs, int freq, ExactDocScorer docScorer) {
+ DocsAndFreqs(DocsEnum docsAndFreqs, DocsEnum docs, int docFreq, ExactDocScorer docScorer) {
+ this.docsAndFreqs = docsAndFreqs;
this.docs = docs;
- this.freq = freq;
+ this.docFreq = docFreq;
this.docScorer = docScorer;
}
}
Modified: lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1210184&r1=1210183&r2=1210184&view=diff
==============================================================================
--- lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/lucene3606/lucene/src/java/org/apache/lucene/search/FieldCacheImpl.java Sun Dec 4 19:18:17 2011
@@ -342,7 +342,7 @@ class FieldCacheImpl implements FieldCac
break;
}
final byte termval = parser.parseByte(term);
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -415,7 +415,7 @@ class FieldCacheImpl implements FieldCac
break;
}
final short termval = parser.parseShort(term);
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -519,7 +519,7 @@ class FieldCacheImpl implements FieldCac
retArray = new int[maxDoc];
}
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -586,7 +586,7 @@ class FieldCacheImpl implements FieldCac
res = new FixedBitSet(maxDoc);
}
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
// TODO: use bulk API
while (true) {
final int docID = docs.nextDoc();
@@ -669,7 +669,7 @@ class FieldCacheImpl implements FieldCac
retArray = new float[maxDoc];
}
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -757,7 +757,7 @@ class FieldCacheImpl implements FieldCac
retArray = new long[maxDoc];
}
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -846,7 +846,7 @@ class FieldCacheImpl implements FieldCac
retArray = new double[maxDoc];
}
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -1020,7 +1020,7 @@ class FieldCacheImpl implements FieldCac
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
throw new UnsupportedOperationException();
}
@@ -1147,7 +1147,7 @@ class FieldCacheImpl implements FieldCac
termOrdToBytesOffset = termOrdToBytesOffset.resize(ArrayUtil.oversize(1+termOrd, 1));
}
termOrdToBytesOffset.set(termOrd, bytes.copyUsingLengthPrefix(term));
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {
@@ -1268,7 +1268,7 @@ class FieldCacheImpl implements FieldCac
break;
}
final long pointer = bytes.copyUsingLengthPrefix(term);
- docs = termsEnum.docs(null, docs);
+ docs = termsEnum.docs(null, docs, false);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocsEnum.NO_MORE_DOCS) {