You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/07/31 22:59:01 UTC
svn commit: r1367777 [8/14] - in /lucene/dev/branches/pforcodec_3892: ./
dev-tools/ dev-tools/eclipse/ dev-tools/maven/ dev-tools/scripts/ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/ar/ l...
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/pulsing/PulsingPostingsReader.java Tue Jul 31 20:58:32 2012
@@ -178,7 +178,7 @@ public class PulsingPostingsReader exten
}
@Override
- public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ public DocsEnum docs(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
PulsingTermState termState = (PulsingTermState) _termState;
if (termState.postingsSize != -1) {
PulsingDocsEnum postings;
@@ -202,18 +202,18 @@ public class PulsingPostingsReader exten
return postings.reset(liveDocs, termState);
} else {
if (reuse instanceof PulsingDocsEnum) {
- DocsEnum wrapped = wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, getOther(reuse), needsFreqs);
+ DocsEnum wrapped = wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, getOther(reuse), flags);
setOther(wrapped, reuse); // wrapped.other = reuse
return wrapped;
} else {
- return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, reuse, needsFreqs);
+ return wrappedPostingsReader.docs(field, termState.wrappedTermState, liveDocs, reuse, flags);
}
}
}
@Override
public DocsAndPositionsEnum docsAndPositions(FieldInfo field, BlockTermState _termState, Bits liveDocs, DocsAndPositionsEnum reuse,
- boolean needsOffsets) throws IOException {
+ int flags) throws IOException {
final PulsingTermState termState = (PulsingTermState) _termState;
@@ -240,11 +240,11 @@ public class PulsingPostingsReader exten
} else {
if (reuse instanceof PulsingDocsAndPositionsEnum) {
DocsAndPositionsEnum wrapped = wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, liveDocs, (DocsAndPositionsEnum) getOther(reuse),
- needsOffsets);
+ flags);
setOther(wrapped, reuse); // wrapped.other = reuse
return wrapped;
} else {
- return wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, liveDocs, reuse, needsOffsets);
+ return wrappedPostingsReader.docsAndPositions(field, termState.wrappedTermState, liveDocs, reuse, flags);
}
}
}
@@ -282,6 +282,7 @@ public class PulsingPostingsReader exten
postings.reset(postingsBytes, 0, termState.postingsSize);
docID = -1;
accum = 0;
+ freq = 1;
payloadLength = 0;
this.liveDocs = liveDocs;
return this;
@@ -349,7 +350,6 @@ public class PulsingPostingsReader exten
@Override
public int freq() throws IOException {
- assert indexOptions != IndexOptions.DOCS_ONLY;
return freq;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/sep/SepPostingsReader.java Tue Jul 31 20:58:32 2012
@@ -258,7 +258,7 @@ public class SepPostingsReader extends P
}
@Override
- public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ public DocsEnum docs(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
final SepTermState termState = (SepTermState) _termState;
SepDocsEnum docsEnum;
if (reuse == null || !(reuse instanceof SepDocsEnum)) {
@@ -278,13 +278,9 @@ public class SepPostingsReader extends P
@Override
public DocsAndPositionsEnum docsAndPositions(FieldInfo fieldInfo, BlockTermState _termState, Bits liveDocs,
- DocsAndPositionsEnum reuse, boolean needsOffsets)
+ DocsAndPositionsEnum reuse, int flags)
throws IOException {
- if (needsOffsets) {
- return null;
- }
-
assert fieldInfo.getIndexOptions() == IndexOptions.DOCS_AND_FREQS_AND_POSITIONS;
final SepTermState termState = (SepTermState) _termState;
SepDocsAndPositionsEnum postingsEnum;
@@ -370,6 +366,7 @@ public class SepPostingsReader extends P
count = 0;
doc = -1;
accum = 0;
+ freq = 1;
skipped = false;
return this;
@@ -403,7 +400,6 @@ public class SepPostingsReader extends P
@Override
public int freq() throws IOException {
- assert !omitTF;
return freq;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextFieldsReader.java Tue Jul 31 20:58:32 2012
@@ -194,33 +194,24 @@ class SimpleTextFieldsReader extends Fie
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
- if (needsFreqs && indexOptions == IndexOptions.DOCS_ONLY) {
- return null;
- }
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
SimpleTextDocsEnum docsEnum;
if (reuse != null && reuse instanceof SimpleTextDocsEnum && ((SimpleTextDocsEnum) reuse).canReuse(SimpleTextFieldsReader.this.in)) {
docsEnum = (SimpleTextDocsEnum) reuse;
} else {
docsEnum = new SimpleTextDocsEnum();
}
- return docsEnum.reset(docsStart, liveDocs, !needsFreqs);
+ return docsEnum.reset(docsStart, liveDocs, indexOptions == IndexOptions.DOCS_ONLY);
}
@Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
if (indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) < 0) {
// Positions were not indexed
return null;
}
- if (needsOffsets &&
- indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) < 0) {
- // Offsets were not indexed
- return null;
- }
-
SimpleTextDocsAndPositionsEnum docsAndPositionsEnum;
if (reuse != null && reuse instanceof SimpleTextDocsAndPositionsEnum && ((SimpleTextDocsAndPositionsEnum) reuse).canReuse(SimpleTextFieldsReader.this.in)) {
docsAndPositionsEnum = (SimpleTextDocsAndPositionsEnum) reuse;
@@ -260,6 +251,7 @@ class SimpleTextFieldsReader extends Fie
in.seek(fp);
this.omitTF = omitTF;
docID = -1;
+ tf = 1;
return this;
}
@@ -270,7 +262,6 @@ class SimpleTextFieldsReader extends Fie
@Override
public int freq() throws IOException {
- assert !omitTF;
return tf;
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/codecs/simpletext/SimpleTextTermVectorsReader.java Tue Jul 31 20:58:32 2012
@@ -357,22 +357,19 @@ public class SimpleTextTermVectorsReader
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
// TODO: reuse
SimpleTVDocsEnum e = new SimpleTVDocsEnum();
- e.reset(liveDocs, needsFreqs ? current.getValue().freq : -1);
+ e.reset(liveDocs, (flags & DocsEnum.FLAG_FREQS) == 0 ? 1 : current.getValue().freq);
return e;
}
@Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
SimpleTVPostings postings = current.getValue();
if (postings.positions == null && postings.startOffsets == null) {
return null;
}
- if (needsOffsets && (postings.startOffsets == null || postings.endOffsets == null)) {
- return null;
- }
// TODO: reuse
SimpleTVDocsAndPositionsEnum e = new SimpleTVDocsAndPositionsEnum();
e.reset(liveDocs, postings.positions, postings.startOffsets, postings.endOffsets);
@@ -507,12 +504,20 @@ public class SimpleTextTermVectorsReader
@Override
public int startOffset() {
- return startOffsets[nextPos-1];
+ if (startOffsets == null) {
+ return -1;
+ } else {
+ return startOffsets[nextPos-1];
+ }
}
@Override
public int endOffset() {
- return endOffsets[nextPos-1];
+ if (endOffsets == null) {
+ return -1;
+ } else {
+ return endOffsets[nextPos-1];
+ }
}
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/AtomicReader.java Tue Jul 31 20:58:32 2012
@@ -116,9 +116,19 @@ public abstract class AtomicReader exten
}
/** Returns {@link DocsEnum} for the specified field &
- * term. This may return null, if either the field or
+ * term. This will return null if either the field or
* term does not exist. */
- public final DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, boolean needsFreqs) throws IOException {
+ public final DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term) throws IOException {
+ return termDocsEnum(liveDocs, field, term, DocsEnum.FLAG_FREQS);
+ }
+
+ /** Returns {@link DocsEnum} for the specified field &
+ * term, with control over whether freqs are required.
+ * Some codecs may be able to optimize their
+ * implementation when freqs are not required. This will
+ * return null if the field or term does not
+ * exist. See {@link TermsEnum#docs(Bits,DocsEnum,int)}. */
+ public final DocsEnum termDocsEnum(Bits liveDocs, String field, BytesRef term, int flags) throws IOException {
assert field != null;
assert term != null;
final Fields fields = fields();
@@ -127,18 +137,30 @@ public abstract class AtomicReader exten
if (terms != null) {
final TermsEnum termsEnum = terms.iterator(null);
if (termsEnum.seekExact(term, true)) {
- return termsEnum.docs(liveDocs, null, needsFreqs);
+ return termsEnum.docs(liveDocs, null, flags);
}
}
}
return null;
}
+
+ /** Returns {@link DocsAndPositionsEnum} for the specified
+ * field & term. This will return null if the
+ * field or term does not exist or positions weren't indexed.
+ * @see #termPositionsEnum(Bits, String, BytesRef, int) */
+ public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term) throws IOException {
+ return termPositionsEnum(liveDocs, field, term, DocsAndPositionsEnum.FLAG_OFFSETS | DocsAndPositionsEnum.FLAG_PAYLOADS);
+ }
+
/** Returns {@link DocsAndPositionsEnum} for the specified
- * field & term. This may return null, if either the
- * field or term does not exist, or needsOffsets is
- * true but offsets were not indexed for this field. */
- public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, boolean needsOffsets) throws IOException {
+ * field & term, with control over whether offsets and payloads are
+ * required. Some codecs may be able to optimize their
+ * implementation when offsets and/or payloads are not required.
+ * This will return null if the field or term
+ * does not exist or positions weren't indexed. See
+ * {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)}. */
+ public final DocsAndPositionsEnum termPositionsEnum(Bits liveDocs, String field, BytesRef term, int flags) throws IOException {
assert field != null;
assert term != null;
final Fields fields = fields();
@@ -147,7 +169,7 @@ public abstract class AtomicReader exten
if (terms != null) {
final TermsEnum termsEnum = terms.iterator(null);
if (termsEnum.seekExact(term, true)) {
- return termsEnum.docsAndPositions(liveDocs, null, needsOffsets);
+ return termsEnum.docsAndPositions(liveDocs, null, flags);
}
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/BufferedDeletesStream.java Tue Jul 31 20:58:32 2012
@@ -396,7 +396,8 @@ class BufferedDeletesStream {
// System.out.println(" term=" + term);
if (termsEnum.seekExact(term.bytes(), false)) {
- DocsEnum docsEnum = termsEnum.docs(rld.getLiveDocs(), docs, false);
+ // we don't need term frequencies for this
+ DocsEnum docsEnum = termsEnum.docs(rld.getLiveDocs(), docs, 0);
//System.out.println("BDS: got docsEnum=" + docsEnum);
if (docsEnum != null) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/CheckIndex.java Tue Jul 31 20:58:32 2012
@@ -34,6 +34,7 @@ import org.apache.lucene.document.Docume
import org.apache.lucene.document.FieldType; // for javadocs
import org.apache.lucene.index.DocValues.SortedSource;
import org.apache.lucene.index.DocValues.Source;
+import org.apache.lucene.index.FieldInfo.IndexOptions;
import org.apache.lucene.search.DocIdSetIterator;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
@@ -682,7 +683,6 @@ public class CheckIndex {
DocsEnum docs = null;
DocsEnum docsAndFreqs = null;
DocsAndPositionsEnum postings = null;
- DocsAndPositionsEnum offsets = null;
String lastField = null;
final FieldsEnum fieldsEnum = fields.iterator();
@@ -699,11 +699,11 @@ public class CheckIndex {
// check that the field is in fieldinfos, and is indexed.
// TODO: add a separate test to check this for different reader impls
- FieldInfo fi = fieldInfos.fieldInfo(field);
- if (fi == null) {
+ FieldInfo fieldInfo = fieldInfos.fieldInfo(field);
+ if (fieldInfo == null) {
throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, no fieldInfos for: " + field);
}
- if (!fi.isIndexed()) {
+ if (!fieldInfo.isIndexed()) {
throw new RuntimeException("fieldsEnum inconsistent with fieldInfos, isIndexed == false for: " + field);
}
@@ -755,10 +755,8 @@ public class CheckIndex {
status.totFreq += docFreq;
sumDocFreq += docFreq;
- docs = termsEnum.docs(liveDocs, docs, false);
- docsAndFreqs = termsEnum.docs(liveDocs, docsAndFreqs, true);
- postings = termsEnum.docsAndPositions(liveDocs, postings, false);
- offsets = termsEnum.docsAndPositions(liveDocs, offsets, true);
+ docs = termsEnum.docs(liveDocs, docs);
+ postings = termsEnum.docsAndPositions(liveDocs, postings);
if (hasOrd) {
long ord = -1;
@@ -779,34 +777,17 @@ public class CheckIndex {
status.termCount++;
final DocsEnum docs2;
- final DocsEnum docsAndFreqs2;
final boolean hasPositions;
- final boolean hasFreqs;
- final boolean hasOffsets;
- if (offsets != null) {
- docs2 = postings = offsets;
- docsAndFreqs2 = postings = offsets;
- hasOffsets = true;
- hasPositions = true;
- hasFreqs = true;
- } else if (postings != null) {
+ // if we are checking vectors, we have freqs implicitly
+ final boolean hasFreqs = isVectors || fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
+ // if we are checking vectors, offsets are a free-for-all anyway
+ final boolean hasOffsets = isVectors || fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS) >= 0;
+ if (postings != null) {
docs2 = postings;
- docsAndFreqs2 = postings;
- hasOffsets = false;
hasPositions = true;
- hasFreqs = true;
- } else if (docsAndFreqs != null) {
- docs2 = docsAndFreqs;
- docsAndFreqs2 = docsAndFreqs;
- hasOffsets = false;
- hasPositions = false;
- hasFreqs = true;
} else {
docs2 = docs;
- docsAndFreqs2 = null;
- hasOffsets = false;
hasPositions = false;
- hasFreqs = false;
}
int lastDoc = -1;
@@ -820,7 +801,7 @@ public class CheckIndex {
visitedDocs.set(doc);
int freq = -1;
if (hasFreqs) {
- freq = docsAndFreqs2.freq();
+ freq = docs2.freq();
if (freq <= 0) {
throw new RuntimeException("term " + term + ": doc " + doc + ": freq " + freq + " is out of bounds");
}
@@ -886,12 +867,12 @@ public class CheckIndex {
}
final long totalTermFreq2 = termsEnum.totalTermFreq();
- final boolean hasTotalTermFreq = postings != null && totalTermFreq2 != -1;
+ final boolean hasTotalTermFreq = hasFreqs && totalTermFreq2 != -1;
// Re-count if there are deleted docs:
if (liveDocs != null) {
if (hasFreqs) {
- final DocsEnum docsNoDel = termsEnum.docs(null, docsAndFreqs, true);
+ final DocsEnum docsNoDel = termsEnum.docs(null, docsAndFreqs);
docCount = 0;
totalTermFreq = 0;
while(docsNoDel.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
@@ -900,7 +881,7 @@ public class CheckIndex {
totalTermFreq += docsNoDel.freq();
}
} else {
- final DocsEnum docsNoDel = termsEnum.docs(null, docs, false);
+ final DocsEnum docsNoDel = termsEnum.docs(null, docs, 0);
docCount = 0;
totalTermFreq = -1;
while(docsNoDel.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
@@ -927,7 +908,7 @@ public class CheckIndex {
if (hasPositions) {
for(int idx=0;idx<7;idx++) {
final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
- postings = termsEnum.docsAndPositions(liveDocs, postings, hasOffsets);
+ postings = termsEnum.docsAndPositions(liveDocs, postings);
final int docID = postings.advance(skipDocID);
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
break;
@@ -992,7 +973,7 @@ public class CheckIndex {
} else {
for(int idx=0;idx<7;idx++) {
final int skipDocID = (int) (((idx+1)*(long) maxDoc)/8);
- docs = termsEnum.docs(liveDocs, docs, false);
+ docs = termsEnum.docs(liveDocs, docs, 0);
final int docID = docs.advance(skipDocID);
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
break;
@@ -1062,7 +1043,7 @@ public class CheckIndex {
}
int expectedDocFreq = termsEnum.docFreq();
- DocsEnum d = termsEnum.docs(null, null, false);
+ DocsEnum d = termsEnum.docs(null, null, 0);
int docFreq = 0;
while (d.nextDoc() != DocIdSetIterator.NO_MORE_DOCS) {
docFreq++;
@@ -1103,7 +1084,7 @@ public class CheckIndex {
throw new RuntimeException("seek to existing term " + seekTerms[i] + " failed");
}
- docs = termsEnum.docs(liveDocs, docs, false);
+ docs = termsEnum.docs(liveDocs, docs, 0);
if (docs == null) {
throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]);
}
@@ -1121,7 +1102,7 @@ public class CheckIndex {
}
totDocFreq += termsEnum.docFreq();
- docs = termsEnum.docs(null, docs, false);
+ docs = termsEnum.docs(null, docs, 0);
if (docs == null) {
throw new RuntimeException("null DocsEnum from to existing term " + seekTerms[i]);
}
@@ -1450,6 +1431,7 @@ public class CheckIndex {
if (crossCheckTermVectors) {
Terms terms = tfv.terms(field);
termsEnum = terms.iterator(termsEnum);
+ final boolean postingsHasFreq = fieldInfo.getIndexOptions().compareTo(IndexOptions.DOCS_AND_FREQS) >= 0;
Terms postingsTerms = postingsFields.terms(field);
if (postingsTerms == null) {
@@ -1460,44 +1442,20 @@ public class CheckIndex {
BytesRef term = null;
while ((term = termsEnum.next()) != null) {
- final boolean hasPositions;
- final boolean hasOffsets;
- final boolean hasFreqs;
-
- // TODO: really we need a reflection/query
- // API so we can just ask what was indexed
- // instead of "probing"...
+ final boolean hasProx;
- // Try offsets:
- postings = termsEnum.docsAndPositions(null, postings, true);
+ // Try positions:
+ postings = termsEnum.docsAndPositions(null, postings);
if (postings == null) {
- hasOffsets = false;
- // Try only positions:
- postings = termsEnum.docsAndPositions(null, postings, false);
- if (postings == null) {
- hasPositions = false;
- // Try docIDs & freqs:
- docs = termsEnum.docs(null, docs, true);
- if (docs == null) {
- // OK, only docIDs:
- hasFreqs = false;
- docs = termsEnum.docs(null, docs, false);
- } else {
- hasFreqs = true;
- }
- } else {
- hasPositions = true;
- hasFreqs = true;
- }
+ hasProx = false;
+ // Try docIDs & freqs:
+ docs = termsEnum.docs(null, docs);
} else {
- hasOffsets = true;
- // NOTE: may be a lie... but we accept -1
- hasPositions = true;
- hasFreqs = true;
+ hasProx = true;
}
final DocsEnum docs2;
- if (hasPositions || hasOffsets) {
+ if (hasProx) {
assert postings != null;
docs2 = postings;
} else {
@@ -1506,30 +1464,16 @@ public class CheckIndex {
}
final DocsEnum postingsDocs2;
- final boolean postingsHasFreq;
if (!postingsTermsEnum.seekExact(term, true)) {
throw new RuntimeException("vector term=" + term + " field=" + field + " does not exist in postings; doc=" + j);
}
- postingsPostings = postingsTermsEnum.docsAndPositions(null, postingsPostings, true);
+ postingsPostings = postingsTermsEnum.docsAndPositions(null, postingsPostings);
if (postingsPostings == null) {
- // Term vectors were indexed w/ offsets but postings were not
- postingsPostings = postingsTermsEnum.docsAndPositions(null, postingsPostings, false);
- if (postingsPostings == null) {
- postingsDocs = postingsTermsEnum.docs(null, postingsDocs, true);
- if (postingsDocs == null) {
- postingsHasFreq = false;
- postingsDocs = postingsTermsEnum.docs(null, postingsDocs, false);
- if (postingsDocs == null) {
- throw new RuntimeException("vector term=" + term + " field=" + field + " does not exist in postings; doc=" + j);
- }
- } else {
- postingsHasFreq = true;
- }
- } else {
- postingsHasFreq = true;
+ // Term vectors were indexed w/ pos but postings were not
+ postingsDocs = postingsTermsEnum.docs(null, postingsDocs);
+ if (postingsDocs == null) {
+ throw new RuntimeException("vector term=" + term + " field=" + field + " does not exist in postings; doc=" + j);
}
- } else {
- postingsHasFreq = true;
}
if (postingsPostings != null) {
@@ -1549,13 +1493,13 @@ public class CheckIndex {
throw new RuntimeException("vector for doc " + j + " didn't return docID=0: got docID=" + doc);
}
- if (hasFreqs) {
+ if (postingsHasFreq) {
final int tf = docs2.freq();
if (postingsHasFreq && postingsDocs2.freq() != tf) {
throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": freq=" + tf + " differs from postings freq=" + postingsDocs2.freq());
}
- if (hasPositions || hasOffsets) {
+ if (hasProx) {
for (int i = 0; i < tf; i++) {
int pos = postings.nextPosition();
if (postingsPostings != null) {
@@ -1565,32 +1509,30 @@ public class CheckIndex {
}
}
- if (hasOffsets) {
- // Call the methods to at least make
- // sure they don't throw exc:
- final int startOffset = postings.startOffset();
- final int endOffset = postings.endOffset();
- // TODO: these are too anal...?
- /*
- if (endOffset < startOffset) {
- throw new RuntimeException("vector startOffset=" + startOffset + " is > endOffset=" + endOffset);
- }
- if (startOffset < lastStartOffset) {
- throw new RuntimeException("vector startOffset=" + startOffset + " is < prior startOffset=" + lastStartOffset);
- }
- lastStartOffset = startOffset;
- */
-
- if (postingsPostings != null) {
- final int postingsStartOffset = postingsPostings.startOffset();
-
- final int postingsEndOffset = postingsPostings.endOffset();
- if (startOffset != -1 && postingsStartOffset != -1 && startOffset != postingsStartOffset) {
- throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": startOffset=" + startOffset + " differs from postings startOffset=" + postingsStartOffset);
- }
- if (endOffset != -1 && postingsEndOffset != -1 && endOffset != postingsEndOffset) {
- throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": endOffset=" + endOffset + " differs from postings endOffset=" + postingsEndOffset);
- }
+ // Call the methods to at least make
+ // sure they don't throw exc:
+ final int startOffset = postings.startOffset();
+ final int endOffset = postings.endOffset();
+ // TODO: these are too anal...?
+ /*
+ if (endOffset < startOffset) {
+ throw new RuntimeException("vector startOffset=" + startOffset + " is > endOffset=" + endOffset);
+ }
+ if (startOffset < lastStartOffset) {
+ throw new RuntimeException("vector startOffset=" + startOffset + " is < prior startOffset=" + lastStartOffset);
+ }
+ lastStartOffset = startOffset;
+ */
+
+ if (postingsPostings != null) {
+ final int postingsStartOffset = postingsPostings.startOffset();
+
+ final int postingsEndOffset = postingsPostings.endOffset();
+ if (startOffset != -1 && postingsStartOffset != -1 && startOffset != postingsStartOffset) {
+ throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": startOffset=" + startOffset + " differs from postings startOffset=" + postingsStartOffset);
+ }
+ if (endOffset != -1 && postingsEndOffset != -1 && endOffset != postingsEndOffset) {
+ throw new RuntimeException("vector term=" + term + " field=" + field + " doc=" + j + ": endOffset=" + endOffset + " differs from postings endOffset=" + postingsEndOffset);
}
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/ConcurrentMergeScheduler.java Tue Jul 31 20:58:32 2012
@@ -243,27 +243,34 @@ public class ConcurrentMergeScheduler ex
sync();
}
- /** Wait for any running merge threads to finish */
+ /** Wait for any running merge threads to finish. This call is not interruptible as used by {@link #close()}. */
public void sync() {
- while (true) {
- MergeThread toSync = null;
- synchronized (this) {
- for (MergeThread t : mergeThreads) {
- if (t.isAlive()) {
- toSync = t;
- break;
+ boolean interrupted = false;
+ try {
+ while (true) {
+ MergeThread toSync = null;
+ synchronized (this) {
+ for (MergeThread t : mergeThreads) {
+ if (t.isAlive()) {
+ toSync = t;
+ break;
+ }
}
}
- }
- if (toSync != null) {
- try {
- toSync.join();
- } catch (InterruptedException ie) {
- throw new ThreadInterruptedException(ie);
+ if (toSync != null) {
+ try {
+ toSync.join();
+ } catch (InterruptedException ie) {
+ // ignore this Exception, we will retry until all threads are dead
+ interrupted = true;
+ }
+ } else {
+ break;
}
- } else {
- break;
}
+ } finally {
+ // finally, restore interrupt status:
+ if (interrupted) Thread.currentThread().interrupt();
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocTermOrds.java Tue Jul 31 20:58:32 2012
@@ -334,7 +334,7 @@ public class DocTermOrds {
final int df = te.docFreq();
if (df <= maxTermDocFreq) {
- docsEnum = te.docs(liveDocs, docsEnum, false);
+ docsEnum = te.docs(liveDocs, docsEnum, 0);
// dF, but takes deletions into account
int actualDF = 0;
@@ -668,13 +668,13 @@ public class DocTermOrds {
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
- return termsEnum.docs(liveDocs, reuse, needsFreqs);
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
+ return termsEnum.docs(liveDocs, reuse, flags);
}
@Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
- return termsEnum.docsAndPositions(liveDocs, reuse, needsOffsets);
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
+ return termsEnum.docsAndPositions(liveDocs, reuse, flags);
}
@Override
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsAndPositionsEnum.java Tue Jul 31 20:58:32 2012
@@ -23,7 +23,14 @@ import org.apache.lucene.util.BytesRef;
/** Also iterates through positions. */
public abstract class DocsAndPositionsEnum extends DocsEnum {
-
+ /** Flag to pass to {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)}
+ * if you require offsets in the returned enum. */
+ public static final int FLAG_OFFSETS = 0x1;
+
+ /** Flag to pass to {@link TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)}
+ * if you require payloads in the returned enum. */
+ public static final int FLAG_PAYLOADS = 0x2;
+
/** Returns the next position. You should only call this
* up to {@link DocsEnum#freq()} times else
* the behavior is not defined. If positions were not
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/DocsEnum.java Tue Jul 31 20:58:32 2012
@@ -27,6 +27,10 @@ import org.apache.lucene.util.AttributeS
* any of the per-doc methods. */
public abstract class DocsEnum extends DocIdSetIterator {
+ /** Flag to pass to {@link TermsEnum#docs(Bits,DocsEnum,int)}
+ * if you require term frequencies in the returned enum. */
+ public static final int FLAG_FREQS = 0x1;
+
private AttributeSource atts = null;
/** Returns term frequency in the current document. Do
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilterAtomicReader.java Tue Jul 31 20:58:32 2012
@@ -181,13 +181,13 @@ public class FilterAtomicReader extends
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
- return in.docs(liveDocs, reuse, needsFreqs);
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
+ return in.docs(liveDocs, reuse, flags);
}
@Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
- return in.docsAndPositions(liveDocs, reuse, needsOffsets);
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
+ return in.docsAndPositions(liveDocs, reuse, flags);
}
@Override
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/FilteredTermsEnum.java Tue Jul 31 20:58:32 2012
@@ -166,13 +166,13 @@ public abstract class FilteredTermsEnum
}
@Override
- public DocsEnum docs(Bits bits, DocsEnum reuse, boolean needsFreqs) throws IOException {
- return tenum.docs(bits, reuse, needsFreqs);
+ public DocsEnum docs(Bits bits, DocsEnum reuse, int flags) throws IOException {
+ return tenum.docs(bits, reuse, flags);
}
@Override
- public DocsAndPositionsEnum docsAndPositions(Bits bits, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
- return tenum.docsAndPositions(bits, reuse, needsOffsets);
+ public DocsAndPositionsEnum docsAndPositions(Bits bits, DocsAndPositionsEnum reuse, int flags) throws IOException {
+ return tenum.docsAndPositions(bits, reuse, flags);
}
/** This enum does not support seeking!
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileDeleter.java Tue Jul 31 20:58:32 2012
@@ -25,6 +25,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.regex.Matcher;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.NoSuchDirectoryException;
@@ -146,57 +147,61 @@ final class IndexFileDeleter {
// it means the directory is empty, so ignore it.
files = new String[0];
}
-
- for (String fileName : files) {
-
- if (!fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)) {
-
- // Add this file to refCounts with initial count 0:
- getRefCount(fileName);
-
- if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
-
- // This is a commit (segments or segments_N), and
- // it's valid (<= the max gen). Load it, then
- // incref all files it refers to:
- if (infoStream.isEnabled("IFD")) {
- infoStream.message("IFD", "init: load commit \"" + fileName + "\"");
- }
- SegmentInfos sis = new SegmentInfos();
- try {
- sis.read(directory, fileName);
- } catch (FileNotFoundException e) {
- // LUCENE-948: on NFS (and maybe others), if
- // you have writers switching back and forth
- // between machines, it's very likely that the
- // dir listing will be stale and will claim a
- // file segments_X exists when in fact it
- // doesn't. So, we catch this and handle it
- // as if the file does not exist
+
+ if (currentSegmentsFile != null) {
+ Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
+ for (String fileName : files) {
+ m.reset(fileName);
+ if (!fileName.endsWith("write.lock") && !fileName.equals(IndexFileNames.SEGMENTS_GEN)
+ && (m.matches() || fileName.startsWith(IndexFileNames.SEGMENTS))) {
+
+ // Add this file to refCounts with initial count 0:
+ getRefCount(fileName);
+
+ if (fileName.startsWith(IndexFileNames.SEGMENTS)) {
+
+ // This is a commit (segments or segments_N), and
+ // it's valid (<= the max gen). Load it, then
+ // incref all files it refers to:
if (infoStream.isEnabled("IFD")) {
- infoStream.message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
+ infoStream.message("IFD", "init: load commit \"" + fileName + "\"");
}
- sis = null;
- } catch (IOException e) {
- if (SegmentInfos.generationFromSegmentsFileName(fileName) <= currentGen && directory.fileLength(fileName) > 0) {
- throw e;
- } else {
- // Most likely we are opening an index that
- // has an aborted "future" commit, so suppress
- // exc in this case
+ SegmentInfos sis = new SegmentInfos();
+ try {
+ sis.read(directory, fileName);
+ } catch (FileNotFoundException e) {
+ // LUCENE-948: on NFS (and maybe others), if
+ // you have writers switching back and forth
+ // between machines, it's very likely that the
+ // dir listing will be stale and will claim a
+ // file segments_X exists when in fact it
+ // doesn't. So, we catch this and handle it
+ // as if the file does not exist
+ if (infoStream.isEnabled("IFD")) {
+ infoStream.message("IFD", "init: hit FileNotFoundException when loading commit \"" + fileName + "\"; skipping this commit point");
+ }
sis = null;
+ } catch (IOException e) {
+ if (SegmentInfos.generationFromSegmentsFileName(fileName) <= currentGen && directory.fileLength(fileName) > 0) {
+ throw e;
+ } else {
+ // Most likely we are opening an index that
+ // has an aborted "future" commit, so suppress
+ // exc in this case
+ sis = null;
+ }
}
- }
- if (sis != null) {
- final CommitPoint commitPoint = new CommitPoint(commitsToDelete, directory, sis);
- if (sis.getGeneration() == segmentInfos.getGeneration()) {
- currentCommitPoint = commitPoint;
- }
- commits.add(commitPoint);
- incRef(sis, true);
-
- if (lastSegmentInfos == null || sis.getGeneration() > lastSegmentInfos.getGeneration()) {
- lastSegmentInfos = sis;
+ if (sis != null) {
+ final CommitPoint commitPoint = new CommitPoint(commitsToDelete, directory, sis);
+ if (sis.getGeneration() == segmentInfos.getGeneration()) {
+ currentCommitPoint = commitPoint;
+ }
+ commits.add(commitPoint);
+ incRef(sis, true);
+
+ if (lastSegmentInfos == null || sis.getGeneration() > lastSegmentInfos.getGeneration()) {
+ lastSegmentInfos = sis;
+ }
}
}
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexFileNames.java Tue Jul 31 20:58:32 2012
@@ -17,6 +17,8 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.util.regex.Pattern;
+
import org.apache.lucene.codecs.Codec;
// TODO: put all files under codec and remove all the static extensions here
@@ -189,4 +191,8 @@ public final class IndexFileNames {
}
return filename;
}
+
+ // All files created by codecs much match this pattern (we
+ // check this in SegmentInfo.java):
+ static final Pattern CODEC_FILE_PATTERN = Pattern.compile("_[a-z0-9]+(_.*)?\\..*");
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/IndexWriter.java Tue Jul 31 20:58:32 2012
@@ -842,7 +842,7 @@ public class IndexWriter implements Clos
if (hitOOM) {
rollbackInternal();
} else {
- closeInternal(waitForMerges, !hitOOM);
+ closeInternal(waitForMerges, true);
}
}
}
@@ -870,7 +870,7 @@ public class IndexWriter implements Clos
}
private void closeInternal(boolean waitForMerges, boolean doFlush) throws IOException {
-
+ boolean interrupted = false;
try {
if (pendingCommit != null) {
@@ -883,26 +883,57 @@ public class IndexWriter implements Clos
docWriter.close();
- // Only allow a new merge to be triggered if we are
- // going to wait for merges:
- if (doFlush) {
- flush(waitForMerges, true);
- } else {
- docWriter.abort(); // already closed
- }
-
- if (waitForMerges)
- // Give merge scheduler last chance to run, in case
- // any pending merges are waiting:
- mergeScheduler.merge(this);
-
- mergePolicy.close();
-
- synchronized(this) {
- finishMerges(waitForMerges);
- stopMerges = true;
+ try {
+ // Only allow a new merge to be triggered if we are
+ // going to wait for merges:
+ if (doFlush) {
+ flush(waitForMerges, true);
+ } else {
+ docWriter.abort(); // already closed
+ }
+
+ } finally {
+ try {
+ // clean up merge scheduler in all cases, although flushing may have failed:
+ interrupted = Thread.interrupted();
+
+ if (waitForMerges) {
+ try {
+ // Give merge scheduler last chance to run, in case
+ // any pending merges are waiting:
+ mergeScheduler.merge(this);
+ } catch (ThreadInterruptedException tie) {
+ // ignore any interruption, does not matter
+ interrupted = true;
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "interrupted while waiting for final merges");
+ }
+ }
+ }
+
+ synchronized(this) {
+ for (;;) {
+ try {
+ finishMerges(waitForMerges && !interrupted);
+ break;
+ } catch (ThreadInterruptedException tie) {
+ // by setting the interrupted status, the
+ // next call to finishMerges will pass false,
+ // so it will not wait
+ interrupted = true;
+ if (infoStream.isEnabled("IW")) {
+ infoStream.message("IW", "interrupted while waiting for merges to finish");
+ }
+ }
+ }
+ stopMerges = true;
+ }
+
+ } finally {
+ // shutdown policy, scheduler and all threads (this call is not interruptible):
+ IOUtils.closeWhileHandlingException(mergePolicy, mergeScheduler);
+ }
}
- mergeScheduler.close();
if (infoStream.isEnabled("IW")) {
infoStream.message("IW", "now call final commit()");
@@ -943,6 +974,8 @@ public class IndexWriter implements Clos
}
}
}
+ // finally, restore interrupt status:
+ if (interrupted) Thread.currentThread().interrupt();
}
}
@@ -3052,7 +3085,7 @@ public class IndexWriter implements Clos
checkpoint();
if (infoStream.isEnabled("IW")) {
- infoStream.message("IW", "after commit: " + segString());
+ infoStream.message("IW", "after commitMerge: " + segString());
}
if (merge.maxNumSegments != -1 && !dropSegment) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MergeScheduler.java Tue Jul 31 20:58:32 2012
@@ -17,6 +17,7 @@ package org.apache.lucene.index;
* limitations under the License.
*/
+import java.io.Closeable;
import java.io.IOException;
/** <p>Expert: {@link IndexWriter} uses an instance
@@ -26,7 +27,7 @@ import java.io.IOException;
*
* @lucene.experimental
*/
-public abstract class MergeScheduler {
+public abstract class MergeScheduler implements Closeable {
/** Run the merges provided by {@link IndexWriter#getNextMerge()}. */
public abstract void merge(IndexWriter writer) throws IOException;
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiFields.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiFields.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiFields.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiFields.java Tue Jul 31 20:58:32 2012
@@ -122,32 +122,54 @@ public final class MultiFields extends F
}
/** Returns {@link DocsEnum} for the specified field &
- * term. This may return null if the term does not
- * exist. */
- public static DocsEnum getTermDocsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, boolean needsFreqs) throws IOException {
+ * term. This will return null if the field or term does
+ * not exist. */
+ public static DocsEnum getTermDocsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term) throws IOException {
+ return getTermDocsEnum(r, liveDocs, field, term);
+ }
+
+ /** Returns {@link DocsEnum} for the specified field &
+ * term, with control over whether freqs are required.
+ * Some codecs may be able to optimize their
+ * implementation when freqs are not required. This will
+ * return null if the field or term does not exist. See {@link
+ * TermsEnum#docs(Bits,DocsEnum,int)}.*/
+ public static DocsEnum getTermDocsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, int flags) throws IOException {
assert field != null;
assert term != null;
final Terms terms = getTerms(r, field);
if (terms != null) {
final TermsEnum termsEnum = terms.iterator(null);
if (termsEnum.seekExact(term, true)) {
- return termsEnum.docs(liveDocs, null, needsFreqs);
+ return termsEnum.docs(liveDocs, null, flags);
}
}
return null;
}
/** Returns {@link DocsAndPositionsEnum} for the specified
- * field & term. This may return null if the term does
- * not exist or positions were not indexed. */
- public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, boolean needsOffsets) throws IOException {
+ * field & term. This will return null if the field or
+ * term does not exist or positions were not indexed.
+ * @see #getTermPositionsEnum(IndexReader, Bits, String, BytesRef, int) */
+ public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term) throws IOException {
+ return getTermPositionsEnum(r, liveDocs, field, term, DocsAndPositionsEnum.FLAG_OFFSETS | DocsAndPositionsEnum.FLAG_PAYLOADS);
+ }
+
+ /** Returns {@link DocsAndPositionsEnum} for the specified
+ * field & term, with control over whether offsets and payloads are
+ * required. Some codecs may be able to optimize
+ * their implementation when offsets and/or payloads are not
+ * required. This will return null if the field or term does not
+ * exist or positions were not indexed. See {@link
+ * TermsEnum#docsAndPositions(Bits,DocsAndPositionsEnum,int)}. */
+ public static DocsAndPositionsEnum getTermPositionsEnum(IndexReader r, Bits liveDocs, String field, BytesRef term, int flags) throws IOException {
assert field != null;
assert term != null;
final Terms terms = getTerms(r, field);
if (terms != null) {
final TermsEnum termsEnum = terms.iterator(null);
if (termsEnum.seekExact(term, true)) {
- return termsEnum.docsAndPositions(liveDocs, null, needsOffsets);
+ return termsEnum.docsAndPositions(liveDocs, null, flags);
}
}
return null;
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/MultiTermsEnum.java Tue Jul 31 20:58:32 2012
@@ -344,7 +344,7 @@ public final class MultiTermsEnum extend
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
MultiDocsEnum docsEnum;
// Can only reuse if incoming enum is also a MultiDocsEnum
if (reuse != null && reuse instanceof MultiDocsEnum) {
@@ -394,16 +394,15 @@ public final class MultiTermsEnum extend
}
assert entry.index < docsEnum.subDocsEnum.length: entry.index + " vs " + docsEnum.subDocsEnum.length + "; " + subs.length;
- final DocsEnum subDocsEnum = entry.terms.docs(b, docsEnum.subDocsEnum[entry.index], needsFreqs);
+ final DocsEnum subDocsEnum = entry.terms.docs(b, docsEnum.subDocsEnum[entry.index], flags);
if (subDocsEnum != null) {
docsEnum.subDocsEnum[entry.index] = subDocsEnum;
subDocs[upto].docsEnum = subDocsEnum;
subDocs[upto].slice = entry.subSlice;
upto++;
} else {
- // One of our subs cannot provide freqs:
- assert needsFreqs;
- return null;
+ // One of our subs cannot provide a docsenum:
+ assert false;
}
}
@@ -415,7 +414,7 @@ public final class MultiTermsEnum extend
}
@Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
MultiDocsAndPositionsEnum docsAndPositionsEnum;
// Can only reuse if incoming enum is also a MultiDocsAndPositionsEnum
if (reuse != null && reuse instanceof MultiDocsAndPositionsEnum) {
@@ -466,7 +465,7 @@ public final class MultiTermsEnum extend
}
assert entry.index < docsAndPositionsEnum.subDocsAndPositionsEnum.length: entry.index + " vs " + docsAndPositionsEnum.subDocsAndPositionsEnum.length + "; " + subs.length;
- final DocsAndPositionsEnum subPostings = entry.terms.docsAndPositions(b, docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index], needsOffsets);
+ final DocsAndPositionsEnum subPostings = entry.terms.docsAndPositions(b, docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index], flags);
if (subPostings != null) {
docsAndPositionsEnum.subDocsAndPositionsEnum[entry.index] = subPostings;
@@ -474,7 +473,7 @@ public final class MultiTermsEnum extend
subDocsAndPositions[upto].slice = entry.subSlice;
upto++;
} else {
- if (entry.terms.docs(b, null, false) != null) {
+ if (entry.terms.docs(b, null, 0) != null) {
// At least one of our subs does not store
// offsets or positions -- we can't correctly
// produce a MultiDocsAndPositions enum
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/SegmentInfo.java Tue Jul 31 20:58:32 2012
@@ -24,6 +24,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
+import java.util.regex.Matcher;
import org.apache.lucene.codecs.Codec;
import org.apache.lucene.store.Directory;
@@ -242,16 +243,31 @@ public final class SegmentInfo {
private Set<String> setFiles;
public void setFiles(Set<String> files) {
+ checkFileNames(files);
setFiles = files;
sizeInBytes = -1;
}
public void addFiles(Collection<String> files) {
+ checkFileNames(files);
setFiles.addAll(files);
+ sizeInBytes = -1;
}
public void addFile(String file) {
+ checkFileNames(Collections.singleton(file));
setFiles.add(file);
+ sizeInBytes = -1;
+ }
+
+ private void checkFileNames(Collection<String> files) {
+ Matcher m = IndexFileNames.CODEC_FILE_PATTERN.matcher("");
+ for (String file : files) {
+ m.reset(file);
+ if (!m.matches()) {
+ throw new IllegalArgumentException("invalid codec filename '" + file + "', must match: " + IndexFileNames.CODEC_FILE_PATTERN.pattern());
+ }
+ }
}
/**
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/index/TermsEnum.java Tue Jul 31 20:58:32 2012
@@ -140,26 +140,55 @@ public abstract class TermsEnum implemen
/** Get {@link DocsEnum} for the current term. Do not
* call this when the enum is unpositioned. This method
- * may return null (if needsFreqs is true but freqs were
- * not indexed for this field).
+ * will not return null.
+ *
+ * @param liveDocs unset bits are documents that should not
+ * be returned
+ * @param reuse pass a prior DocsEnum for possible reuse */
+ public final DocsEnum docs(Bits liveDocs, DocsEnum reuse) throws IOException {
+ return docs(liveDocs, reuse, DocsEnum.FLAG_FREQS);
+ }
+
+ /** Get {@link DocsEnum} for the current term, with
+ * control over whether freqs are required. Do not
+ * call this when the enum is unpositioned. This method
+ * will not return null.
*
* @param liveDocs unset bits are documents that should not
* be returned
* @param reuse pass a prior DocsEnum for possible reuse
- * @param needsFreqs true if the caller intends to call
- * {@link DocsEnum#freq}. If you pass false you must not
- * call {@link DocsEnum#freq} in the returned DocsEnum. */
- public abstract DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException;
+ * @param flags specifies which optional per-document values
+ * you require; see {@link DocsEnum#FLAG_FREQS}
+ * @see #docs(Bits, DocsEnum, int) */
+ public abstract DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException;
/** Get {@link DocsAndPositionsEnum} for the current term.
- * Do not call this when the enum is unpositioned.
- * This method will only return null if needsOffsets is
- * true but offsets were not indexed.
+ * Do not call this when the enum is unpositioned. This
+ * method will return null if positions were not
+ * indexed.
+ *
+ * @param liveDocs unset bits are documents that should not
+ * be returned
+ * @param reuse pass a prior DocsAndPositionsEnum for possible reuse
+ * @see #docsAndPositions(Bits, DocsAndPositionsEnum, int) */
+ public final DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse) throws IOException {
+ return docsAndPositions(liveDocs, reuse, DocsAndPositionsEnum.FLAG_OFFSETS | DocsAndPositionsEnum.FLAG_PAYLOADS);
+ }
+
+ /** Get {@link DocsAndPositionsEnum} for the current term,
+ * with control over whether offsets and payloads are
+ * required. Some codecs may be able to optimize their
+ * implementation when offsets and/or payloads are not required.
+ * Do not call this when the enum is unpositioned. This
+ * will return null if positions were not indexed.
+
* @param liveDocs unset bits are documents that should not
* be returned
* @param reuse pass a prior DocsAndPositionsEnum for possible reuse
- * @param needsOffsets true if offsets are required */
- public abstract DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException;
+ * @param flags specifies which optional per-position values you
+ * require; see {@link DocsAndPositionsEnum#FLAG_OFFSETS} and
+ * {@link DocsAndPositionsEnum#FLAG_PAYLOADS}. */
+ public abstract DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException;
/**
* Expert: Returns the TermsEnums internal state to position the TermsEnum
@@ -220,12 +249,12 @@ public abstract class TermsEnum implemen
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) {
throw new IllegalStateException("this method should never be called");
}
@Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) {
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) {
throw new IllegalStateException("this method should never be called");
}
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/BooleanQuery.java Tue Jul 31 20:58:32 2012
@@ -361,11 +361,9 @@ public class BooleanQuery extends Query
final Scorer scorer = weight.scorer(context, true, false, acceptDocs);
if (scorer == null) {
return null;
- }
- if (scorer instanceof TermScorer) {
- docsAndFreqs[i] = new DocsAndFreqs((TermScorer) scorer);
} else {
- docsAndFreqs[i] = new DocsAndFreqs((MatchOnlyTermScorer) scorer);
+ assert scorer instanceof TermScorer;
+ docsAndFreqs[i] = new DocsAndFreqs((TermScorer) scorer);
}
}
return new ConjunctionTermScorer(this, disableCoord ? 1.0f : coord(
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/ConjunctionTermScorer.java Tue Jul 31 20:58:32 2012
@@ -121,10 +121,6 @@ class ConjunctionTermScorer extends Scor
this(termScorer, termScorer.getDocsEnum(), termScorer.getDocFreq());
}
- DocsAndFreqs(MatchOnlyTermScorer termScorer) {
- this(termScorer, termScorer.getDocsEnum(), termScorer.getDocFreq());
- }
-
DocsAndFreqs(Scorer scorer, DocsEnum docs, int docFreq) {
this.docs = docs;
this.docFreq = docFreq;
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FieldCacheImpl.java Tue Jul 31 20:58:32 2012
@@ -364,7 +364,7 @@ class FieldCacheImpl implements FieldCac
break;
}
final byte termval = parser.parseByte(term);
- docs = termsEnum.docs(null, docs, false);
+ docs = termsEnum.docs(null, docs, 0);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -437,7 +437,7 @@ class FieldCacheImpl implements FieldCac
break;
}
final short termval = parser.parseShort(term);
- docs = termsEnum.docs(null, docs, false);
+ docs = termsEnum.docs(null, docs, 0);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -541,7 +541,7 @@ class FieldCacheImpl implements FieldCac
retArray = new int[maxDoc];
}
- docs = termsEnum.docs(null, docs, false);
+ docs = termsEnum.docs(null, docs, 0);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -608,7 +608,7 @@ class FieldCacheImpl implements FieldCac
res = new FixedBitSet(maxDoc);
}
- docs = termsEnum.docs(null, docs, false);
+ docs = termsEnum.docs(null, docs, 0);
// TODO: use bulk API
while (true) {
final int docID = docs.nextDoc();
@@ -691,7 +691,7 @@ class FieldCacheImpl implements FieldCac
retArray = new float[maxDoc];
}
- docs = termsEnum.docs(null, docs, false);
+ docs = termsEnum.docs(null, docs, 0);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -779,7 +779,7 @@ class FieldCacheImpl implements FieldCac
retArray = new long[maxDoc];
}
- docs = termsEnum.docs(null, docs, false);
+ docs = termsEnum.docs(null, docs, 0);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -868,7 +868,7 @@ class FieldCacheImpl implements FieldCac
retArray = new double[maxDoc];
}
- docs = termsEnum.docs(null, docs, false);
+ docs = termsEnum.docs(null, docs, 0);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -1042,12 +1042,12 @@ class FieldCacheImpl implements FieldCac
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
throw new UnsupportedOperationException();
}
@Override
- public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
+ public DocsAndPositionsEnum docsAndPositions(Bits liveDocs, DocsAndPositionsEnum reuse, int flags) throws IOException {
throw new UnsupportedOperationException();
}
@@ -1167,7 +1167,7 @@ class FieldCacheImpl implements FieldCac
termOrdToBytesOffset = termOrdToBytesOffset.resize(ArrayUtil.oversize(1+termOrd, 1));
}
termOrdToBytesOffset.set(termOrd, bytes.copyUsingLengthPrefix(term));
- docs = termsEnum.docs(null, docs, false);
+ docs = termsEnum.docs(null, docs, 0);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
@@ -1288,7 +1288,7 @@ class FieldCacheImpl implements FieldCac
break;
}
final long pointer = bytes.copyUsingLengthPrefix(term);
- docs = termsEnum.docs(null, docs, false);
+ docs = termsEnum.docs(null, docs, 0);
while (true) {
final int docID = docs.nextDoc();
if (docID == DocIdSetIterator.NO_MORE_DOCS) {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/FuzzyTermsEnum.java Tue Jul 31 20:58:32 2012
@@ -272,14 +272,14 @@ public class FuzzyTermsEnum extends Term
}
@Override
- public DocsEnum docs(Bits liveDocs, DocsEnum reuse, boolean needsFreqs) throws IOException {
- return actualEnum.docs(liveDocs, reuse, needsFreqs);
+ public DocsEnum docs(Bits liveDocs, DocsEnum reuse, int flags) throws IOException {
+ return actualEnum.docs(liveDocs, reuse, flags);
}
@Override
public DocsAndPositionsEnum docsAndPositions(Bits liveDocs,
- DocsAndPositionsEnum reuse, boolean needsOffsets) throws IOException {
- return actualEnum.docsAndPositions(liveDocs, reuse, needsOffsets);
+ DocsAndPositionsEnum reuse, int flags) throws IOException {
+ return actualEnum.docsAndPositions(liveDocs, reuse, flags);
}
@Override
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java Tue Jul 31 20:58:32 2012
@@ -226,11 +226,11 @@ public class MultiPhraseQuery extends Qu
return null;
}
termsEnum.seekExact(term.bytes(), termState);
- postingsEnum = termsEnum.docsAndPositions(liveDocs, null, false);
+ postingsEnum = termsEnum.docsAndPositions(liveDocs, null, 0);
if (postingsEnum == null) {
// term does exist, but has no positions
- assert termsEnum.docs(liveDocs, null, false) != null: "termstate found but no term exists in reader";
+ assert termsEnum.docs(liveDocs, null, 0) != null: "termstate found but no term exists in reader";
throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.text() + ")");
}
@@ -484,7 +484,7 @@ class UnionDocsAndPositionsEnum extends
continue;
}
termsEnum.seekExact(term.bytes(), termState);
- DocsAndPositionsEnum postings = termsEnum.docsAndPositions(liveDocs, null, false);
+ DocsAndPositionsEnum postings = termsEnum.docsAndPositions(liveDocs, null, 0);
if (postings == null) {
// term does exist, but has no positions
throw new IllegalStateException("field \"" + term.field() + "\" was indexed without position data; cannot run PhraseQuery (term=" + term.text() + ")");
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Tue Jul 31 20:58:32 2012
@@ -106,7 +106,7 @@ public class MultiTermQueryWrapperFilter
do {
// System.out.println(" iter termCount=" + termCount + " term=" +
// enumerator.term().toBytesString());
- docsEnum = termsEnum.docs(acceptDocs, docsEnum, false);
+ docsEnum = termsEnum.docs(acceptDocs, docsEnum, 0);
int docid;
while ((docid = docsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
bitSet.set(docid);
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java Tue Jul 31 20:58:32 2012
@@ -264,7 +264,7 @@ public class PhraseQuery extends Query {
return null;
}
te.seekExact(t.bytes(), state);
- DocsAndPositionsEnum postingsEnum = te.docsAndPositions(liveDocs, null, false);
+ DocsAndPositionsEnum postingsEnum = te.docsAndPositions(liveDocs, null, 0);
// PhraseQuery on a field that did not index
// positions.
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/TermQuery.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/TermQuery.java Tue Jul 31 20:58:32 2012
@@ -83,15 +83,9 @@ public class TermQuery extends Query {
if (termsEnum == null) {
return null;
}
- DocsEnum docs = termsEnum.docs(acceptDocs, null, true);
- if (docs != null) {
- return new TermScorer(this, docs, similarity.exactSimScorer(stats, context), termsEnum.docFreq());
- } else {
- // Index does not store freq info
- docs = termsEnum.docs(acceptDocs, null, false);
- assert docs != null;
- return new MatchOnlyTermScorer(this, docs, similarity.exactSimScorer(stats, context), termsEnum.docFreq());
- }
+ DocsEnum docs = termsEnum.docs(acceptDocs, null);
+ assert docs != null;
+ return new TermScorer(this, docs, similarity.exactSimScorer(stats, context), termsEnum.docFreq());
}
/**
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/AveragePayloadFunction.java Tue Jul 31 20:58:32 2012
@@ -1,6 +1,5 @@
package org.apache.lucene.search.payloads;
-import org.apache.lucene.search.Explanation;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -36,14 +35,6 @@ public class AveragePayloadFunction exte
public float docScore(int docId, String field, int numPayloadsSeen, float payloadScore) {
return numPayloadsSeen > 0 ? (payloadScore / numPayloadsSeen) : 1;
}
- @Override
- public Explanation explain(int doc, int numPayloadsSeen, float payloadScore) {
- Explanation payloadBoost = new Explanation();
- float avgPayloadScore = (numPayloadsSeen > 0 ? (payloadScore / numPayloadsSeen) : 1);
- payloadBoost.setValue(avgPayloadScore);
- payloadBoost.setDescription("AveragePayloadFunction(...)");
- return payloadBoost;
- }
@Override
public int hashCode() {
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MaxPayloadFunction.java Tue Jul 31 20:58:32 2012
@@ -1,6 +1,5 @@
package org.apache.lucene.search.payloads;
-import org.apache.lucene.search.Explanation;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -41,14 +40,6 @@ public class MaxPayloadFunction extends
}
@Override
- public Explanation explain(int doc, int numPayloadsSeen, float payloadScore) {
- Explanation expl = new Explanation();
- float maxPayloadScore = (numPayloadsSeen > 0 ? payloadScore : 1);
- expl.setValue(maxPayloadScore);
- expl.setDescription("MaxPayloadFunction(...)");
- return expl;
- }
- @Override
public int hashCode() {
final int prime = 31;
int result = 1;
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/MinPayloadFunction.java Tue Jul 31 20:58:32 2012
@@ -1,6 +1,5 @@
package org.apache.lucene.search.payloads;
-import org.apache.lucene.search.Explanation;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -39,14 +38,6 @@ public class MinPayloadFunction extends
}
@Override
- public Explanation explain(int doc, int numPayloadsSeen, float payloadScore) {
- Explanation expl = new Explanation();
- float minPayloadScore = (numPayloadsSeen > 0 ? payloadScore : 1);
- expl.setValue(minPayloadScore);
- expl.setDescription("MinPayloadFunction(...)");
- return expl;
- }
- @Override
public int hashCode() {
final int prime = 31;
int result = 1;
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadFunction.java Tue Jul 31 20:58:32 2012
@@ -55,10 +55,10 @@ public abstract class PayloadFunction {
*/
public abstract float docScore(int docId, String field, int numPayloadsSeen, float payloadScore);
- public Explanation explain(int docId, int numPayloadsSeen, float payloadScore){
+ public Explanation explain(int docId, String field, int numPayloadsSeen, float payloadScore){
Explanation result = new Explanation();
- result.setDescription("Unimpl Payload Function Explain");
- result.setValue(1);
+ result.setDescription(getClass().getSimpleName() + ".docScore()");
+ result.setValue(docScore(docId, field, numPayloadsSeen, payloadScore));
return result;
};
Modified: lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java?rev=1367777&r1=1367776&r2=1367777&view=diff
==============================================================================
--- lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java (original)
+++ lucene/dev/branches/pforcodec_3892/lucene/core/src/java/org/apache/lucene/search/payloads/PayloadNearQuery.java Tue Jul 31 20:58:32 2012
@@ -167,8 +167,9 @@ public class PayloadNearQuery extends Sp
Explanation scoreExplanation = docScorer.explain(doc, new Explanation(freq, "phraseFreq=" + freq));
expl.addDetail(scoreExplanation);
expl.setValue(scoreExplanation.getValue());
+ String field = ((SpanQuery)getQuery()).getField();
// now the payloads part
- Explanation payloadExpl = function.explain(doc, scorer.payloadsSeen, scorer.payloadScore);
+ Explanation payloadExpl = function.explain(doc, field, scorer.payloadsSeen, scorer.payloadScore);
// combined
ComplexExplanation result = new ComplexExplanation();
result.addDetail(expl);