You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by mi...@apache.org on 2009/10/13 22:44:59 UTC
svn commit: r824918 [9/11] - in /lucene/java/branches/flex_1458:
contrib/analyzers/common/src/java/org/apache/lucene/analysis/query/
contrib/benchmark/src/java/org/apache/lucene/benchmark/quality/utils/
contrib/benchmark/src/test/org/apache/lucene/benc...
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheImpl.java Tue Oct 13 20:44:51 2009
@@ -29,9 +29,13 @@
import org.apache.lucene.document.NumericField; // javadoc
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.TermDocs; // deprecated
+import org.apache.lucene.index.TermRef;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.FieldCacheSanityChecker;
@@ -264,22 +268,28 @@
return wrapper.getBytes(reader, field, FieldCache.DEFAULT_BYTE_PARSER);
}
final byte[] retArray = new byte[reader.maxDoc()];
- TermDocs termDocs = reader.termDocs();
- TermEnum termEnum = reader.terms (new Term (field));
- try {
- do {
- Term term = termEnum.term();
- if (term==null || term.field() != field) break;
- byte termval = parser.parseByte(term.text());
- termDocs.seek (termEnum);
- while (termDocs.next()) {
- retArray[termDocs.doc()] = termval;
- }
- } while (termEnum.next());
- } catch (StopFillCacheException stop) {
- } finally {
- termDocs.close();
- termEnum.close();
+ Terms terms = reader.fields().terms(field);
+ if (terms != null) {
+ final TermsEnum termsEnum = terms.iterator();
+ final Bits delDocs = reader.getDeletedDocs();
+ try {
+ while(true) {
+ final TermRef term = termsEnum.next();
+ if (term == null) {
+ break;
+ }
+ final byte termval = parser.parseByte(term);
+ final DocsEnum docs = termsEnum.docs(delDocs);
+ while (true) {
+ final int docID = docs.next();
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+ retArray[docID] = termval;
+ }
+ }
+ } catch (StopFillCacheException stop) {
+ }
}
return retArray;
}
@@ -310,22 +320,28 @@
return wrapper.getShorts(reader, field, FieldCache.DEFAULT_SHORT_PARSER);
}
final short[] retArray = new short[reader.maxDoc()];
- TermDocs termDocs = reader.termDocs();
- TermEnum termEnum = reader.terms (new Term (field));
- try {
- do {
- Term term = termEnum.term();
- if (term==null || term.field() != field) break;
- short termval = parser.parseShort(term.text());
- termDocs.seek (termEnum);
- while (termDocs.next()) {
- retArray[termDocs.doc()] = termval;
- }
- } while (termEnum.next());
- } catch (StopFillCacheException stop) {
- } finally {
- termDocs.close();
- termEnum.close();
+ Terms terms = reader.fields().terms(field);
+ if (terms != null) {
+ final TermsEnum termsEnum = terms.iterator();
+ final Bits delDocs = reader.getDeletedDocs();
+ try {
+ while(true) {
+ final TermRef term = termsEnum.next();
+ if (term == null) {
+ break;
+ }
+ final short termval = parser.parseShort(term);
+ final DocsEnum docs = termsEnum.docs(delDocs);
+ while (true) {
+ final int docID = docs.next();
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+ retArray[docID] = termval;
+ }
+ }
+ } catch (StopFillCacheException stop) {
+ }
}
return retArray;
}
@@ -360,27 +376,40 @@
}
}
int[] retArray = null;
- TermDocs termDocs = reader.termDocs();
- TermEnum termEnum = reader.terms (new Term (field));
- try {
- do {
- Term term = termEnum.term();
- if (term==null || term.field() != field) break;
- int termval = parser.parseInt(term.text());
- if (retArray == null) // late init
- retArray = new int[reader.maxDoc()];
- termDocs.seek (termEnum);
- while (termDocs.next()) {
- retArray[termDocs.doc()] = termval;
- }
- } while (termEnum.next());
- } catch (StopFillCacheException stop) {
- } finally {
- termDocs.close();
- termEnum.close();
+
+ Terms terms = reader.fields().terms(field);
+ if (terms != null) {
+ final TermsEnum termsEnum = terms.iterator();
+ final Bits delDocs = reader.getDeletedDocs();
+ try {
+ while(true) {
+ final TermRef term = termsEnum.next();
+ if (term == null) {
+ break;
+ }
+ final int termval = parser.parseInt(term);
+ if (retArray == null) {
+ // late init so numeric fields don't double allocate
+ retArray = new int[reader.maxDoc()];
+ }
+
+ final DocsEnum docs = termsEnum.docs(delDocs);
+ while (true) {
+ final int docID = docs.next();
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+ retArray[docID] = termval;
+ }
+ }
+ } catch (StopFillCacheException stop) {
+ }
}
- if (retArray == null) // no values
+
+ if (retArray == null) {
+ // no values
retArray = new int[reader.maxDoc()];
+ }
return retArray;
}
};
@@ -415,29 +444,42 @@
} catch (NumberFormatException ne) {
return wrapper.getFloats(reader, field, NUMERIC_UTILS_FLOAT_PARSER);
}
- }
+ }
float[] retArray = null;
- TermDocs termDocs = reader.termDocs();
- TermEnum termEnum = reader.terms (new Term (field));
- try {
- do {
- Term term = termEnum.term();
- if (term==null || term.field() != field) break;
- float termval = parser.parseFloat(term.text());
- if (retArray == null) // late init
- retArray = new float[reader.maxDoc()];
- termDocs.seek (termEnum);
- while (termDocs.next()) {
- retArray[termDocs.doc()] = termval;
- }
- } while (termEnum.next());
- } catch (StopFillCacheException stop) {
- } finally {
- termDocs.close();
- termEnum.close();
+
+ Terms terms = reader.fields().terms(field);
+ if (terms != null) {
+ final TermsEnum termsEnum = terms.iterator();
+ final Bits delDocs = reader.getDeletedDocs();
+ try {
+ while(true) {
+ final TermRef term = termsEnum.next();
+ if (term == null) {
+ break;
+ }
+ final float termval = parser.parseFloat(term);
+ if (retArray == null) {
+ // late init so numeric fields don't double allocate
+ retArray = new float[reader.maxDoc()];
+ }
+
+ final DocsEnum docs = termsEnum.docs(delDocs);
+ while (true) {
+ final int docID = docs.next();
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+ retArray[docID] = termval;
+ }
+ }
+ } catch (StopFillCacheException stop) {
+ }
}
- if (retArray == null) // no values
+
+ if (retArray == null) {
+ // no values
retArray = new float[reader.maxDoc()];
+ }
return retArray;
}
};
@@ -471,27 +513,39 @@
}
}
long[] retArray = null;
- TermDocs termDocs = reader.termDocs();
- TermEnum termEnum = reader.terms (new Term(field));
- try {
- do {
- Term term = termEnum.term();
- if (term==null || term.field() != field) break;
- long termval = parser.parseLong(term.text());
- if (retArray == null) // late init
- retArray = new long[reader.maxDoc()];
- termDocs.seek (termEnum);
- while (termDocs.next()) {
- retArray[termDocs.doc()] = termval;
- }
- } while (termEnum.next());
- } catch (StopFillCacheException stop) {
- } finally {
- termDocs.close();
- termEnum.close();
+ Terms terms = reader.fields().terms(field);
+ if (terms != null) {
+ final TermsEnum termsEnum = terms.iterator();
+ final Bits delDocs = reader.getDeletedDocs();
+ try {
+ while(true) {
+ final TermRef term = termsEnum.next();
+ if (term == null) {
+ break;
+ }
+ final long termval = parser.parseLong(term);
+ if (retArray == null) {
+ // late init so numeric fields don't double allocate
+ retArray = new long[reader.maxDoc()];
+ }
+
+ final DocsEnum docs = termsEnum.docs(delDocs);
+ while (true) {
+ final int docID = docs.next();
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+ retArray[docID] = termval;
+ }
+ }
+ } catch (StopFillCacheException stop) {
+ }
}
- if (retArray == null) // no values
+
+ if (retArray == null) {
+ // no values
retArray = new long[reader.maxDoc()];
+ }
return retArray;
}
};
@@ -526,24 +580,33 @@
}
}
double[] retArray = null;
- TermDocs termDocs = reader.termDocs();
- TermEnum termEnum = reader.terms (new Term (field));
- try {
- do {
- Term term = termEnum.term();
- if (term==null || term.field() != field) break;
- double termval = parser.parseDouble(term.text());
- if (retArray == null) // late init
- retArray = new double[reader.maxDoc()];
- termDocs.seek (termEnum);
- while (termDocs.next()) {
- retArray[termDocs.doc()] = termval;
- }
- } while (termEnum.next());
- } catch (StopFillCacheException stop) {
- } finally {
- termDocs.close();
- termEnum.close();
+ Terms terms = reader.fields().terms(field);
+ if (terms != null) {
+ final TermsEnum termsEnum = terms.iterator();
+ final Bits delDocs = reader.getDeletedDocs();
+ try {
+ while(true) {
+ final TermRef term = termsEnum.next();
+ if (term == null) {
+ break;
+ }
+ final double termval = parser.parseDouble(term);
+ if (retArray == null) {
+ // late init so numeric fields don't double allocate
+ retArray = new double[reader.maxDoc()];
+ }
+
+ final DocsEnum docs = termsEnum.docs(delDocs);
+ while (true) {
+ final int docID = docs.next();
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+ retArray[docID] = termval;
+ }
+ }
+ } catch (StopFillCacheException stop) {
+ }
}
if (retArray == null) // no values
retArray = new double[reader.maxDoc()];
@@ -566,21 +629,26 @@
throws IOException {
String field = StringHelper.intern((String) entryKey.field);
final String[] retArray = new String[reader.maxDoc()];
- TermDocs termDocs = reader.termDocs();
- TermEnum termEnum = reader.terms (new Term (field));
- try {
- do {
- Term term = termEnum.term();
- if (term==null || term.field() != field) break;
- String termval = term.text();
- termDocs.seek (termEnum);
- while (termDocs.next()) {
- retArray[termDocs.doc()] = termval;
- }
- } while (termEnum.next());
- } finally {
- termDocs.close();
- termEnum.close();
+
+ Terms terms = reader.fields().terms(field);
+ if (terms != null) {
+ final TermsEnum termsEnum = terms.iterator();
+ final Bits delDocs = reader.getDeletedDocs();
+ while(true) {
+ final TermRef term = termsEnum.next();
+ if (term == null) {
+ break;
+ }
+ final DocsEnum docs = termsEnum.docs(delDocs);
+ final String termval = term.toString();
+ while (true) {
+ final int docID = docs.next();
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+ retArray[docID] = termval;
+ }
+ }
}
return retArray;
}
@@ -602,8 +670,9 @@
String field = StringHelper.intern((String) entryKey.field);
final int[] retArray = new int[reader.maxDoc()];
String[] mterms = new String[reader.maxDoc()+1];
- TermDocs termDocs = reader.termDocs();
- TermEnum termEnum = reader.terms (new Term (field));
+
+ Terms terms = reader.fields().terms(field);
+
int t = 0; // current term number
// an entry for documents that have no terms in this field
@@ -612,28 +681,34 @@
// needs to change as well.
mterms[t++] = null;
- try {
- do {
- Term term = termEnum.term();
- if (term==null || term.field() != field) break;
+ if (terms != null) {
+ final TermsEnum termsEnum = terms.iterator();
+ final Bits delDocs = reader.getDeletedDocs();
+ while(true) {
+ final TermRef term = termsEnum.next();
+ if (term == null) {
+ break;
+ }
// store term text
// we expect that there is at most one term per document
- if (t >= mterms.length) throw new RuntimeException ("there are more terms than " +
- "documents in field \"" + field + "\", but it's impossible to sort on " +
- "tokenized fields");
- mterms[t] = term.text();
-
- termDocs.seek (termEnum);
- while (termDocs.next()) {
- retArray[termDocs.doc()] = t;
+ if (t >= mterms.length) {
+ throw new RuntimeException ("there are more terms than " +
+ "documents in field \"" + field + "\", but it's impossible to sort on " +
+ "tokenized fields");
+ }
+ mterms[t] = term.toString();
+
+ final DocsEnum docs = termsEnum.docs(delDocs);
+ while (true) {
+ final int docID = docs.next();
+ if (docID == DocsEnum.NO_MORE_DOCS) {
+ break;
+ }
+ retArray[docID] = t;
}
-
t++;
- } while (termEnum.next());
- } finally {
- termDocs.close();
- termEnum.close();
+ }
}
if (t == 0) {
@@ -643,16 +718,16 @@
} else if (t < mterms.length) {
// if there are less terms than documents,
// trim off the dead array space
- String[] terms = new String[t];
- System.arraycopy (mterms, 0, terms, 0, t);
- mterms = terms;
+ String[] newTerms = new String[t];
+ System.arraycopy (mterms, 0, newTerms, 0, t);
+ mterms = newTerms;
}
StringIndex value = new StringIndex (retArray, mterms);
return value;
}
};
-
+
private volatile PrintStream infoStream;
public void setInfoStream(PrintStream stream) {
@@ -662,5 +737,29 @@
public PrintStream getInfoStream() {
return infoStream;
}
+
+ // Directly parses a numeric value from UTF8 bytes
+ // nocommit -- whitespace? +e syntax?
+ final static long parseLong(TermRef term) {
+ int upto = term.offset;
+ final int negMul;
+ if (term.bytes[upto] == '-') {
+ negMul = -1;
+ upto++;
+ } else {
+ negMul = 1;
+ }
+ final int end = term.offset + term.length;
+ long number = 0;
+ while(upto < end) {
+ final int b = term.bytes[upto++];
+ if (b >= '0' && b <= '9') {
+ number = 10*number + (int) (b-'0');
+ } else {
+ throw new NumberFormatException("could not parse \"" + term + "\" to a number");
+ }
+ }
+ return negMul * number;
+ }
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FieldCacheRangeFilter.java Tue Oct 13 20:44:51 2009
@@ -19,8 +19,8 @@
import java.io.IOException;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermDocs;
import org.apache.lucene.util.NumericUtils;
+import org.apache.lucene.util.Bits;
import org.apache.lucene.document.NumericField; // for javadocs
/**
@@ -117,9 +117,9 @@
assert inclusiveLowerPoint > 0 && inclusiveUpperPoint > 0;
- // for this DocIdSet, we never need to use TermDocs,
+ // for this DocIdSet, we can ignore deleted docs
// because deleted docs have an order of 0 (null entry in StringIndex)
- return new FieldCacheDocIdSet(reader, false) {
+ return new FieldCacheDocIdSet(reader, true) {
final boolean matchDoc(int doc) {
return fcsi.order[doc] >= inclusiveLowerPoint && fcsi.order[doc] <= inclusiveUpperPoint;
}
@@ -167,8 +167,8 @@
return DocIdSet.EMPTY_DOCIDSET;
final byte[] values = FieldCache.DEFAULT.getBytes(reader, field, (FieldCache.ByteParser) parser);
- // we only request the usage of termDocs, if the range contains 0
- return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) {
+ // we only respect deleted docs if the range contains 0
+ return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) {
boolean matchDoc(int doc) {
return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint;
}
@@ -216,8 +216,8 @@
return DocIdSet.EMPTY_DOCIDSET;
final short[] values = FieldCache.DEFAULT.getShorts(reader, field, (FieldCache.ShortParser) parser);
- // we only request the usage of termDocs, if the range contains 0
- return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) {
+ // ignore deleted docs if range doesn't contain 0
+ return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) {
boolean matchDoc(int doc) {
return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint;
}
@@ -265,8 +265,8 @@
return DocIdSet.EMPTY_DOCIDSET;
final int[] values = FieldCache.DEFAULT.getInts(reader, field, (FieldCache.IntParser) parser);
- // we only request the usage of termDocs, if the range contains 0
- return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) {
+ // ignore deleted docs if range doesn't contain 0
+ return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0 && inclusiveUpperPoint >= 0)) {
boolean matchDoc(int doc) {
return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint;
}
@@ -314,8 +314,8 @@
return DocIdSet.EMPTY_DOCIDSET;
final long[] values = FieldCache.DEFAULT.getLongs(reader, field, (FieldCache.LongParser) parser);
- // we only request the usage of termDocs, if the range contains 0
- return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0L && inclusiveUpperPoint >= 0L)) {
+ // ignore deleted docs if range doesn't contain 0
+ return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0L && inclusiveUpperPoint >= 0L)) {
boolean matchDoc(int doc) {
return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint;
}
@@ -367,8 +367,8 @@
return DocIdSet.EMPTY_DOCIDSET;
final float[] values = FieldCache.DEFAULT.getFloats(reader, field, (FieldCache.FloatParser) parser);
- // we only request the usage of termDocs, if the range contains 0
- return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0.0f && inclusiveUpperPoint >= 0.0f)) {
+ // ignore deleted docs if range doesn't contain 0
+ return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0.0f && inclusiveUpperPoint >= 0.0f)) {
boolean matchDoc(int doc) {
return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint;
}
@@ -420,8 +420,8 @@
return DocIdSet.EMPTY_DOCIDSET;
final double[] values = FieldCache.DEFAULT.getDoubles(reader, field, (FieldCache.DoubleParser) parser);
- // we only request the usage of termDocs, if the range contains 0
- return new FieldCacheDocIdSet(reader, (inclusiveLowerPoint <= 0.0 && inclusiveUpperPoint >= 0.0)) {
+ // ignore deleted docs if range doesn't contain 0
+ return new FieldCacheDocIdSet(reader, !(inclusiveLowerPoint <= 0.0 && inclusiveUpperPoint >= 0.0)) {
boolean matchDoc(int doc) {
return values[doc] >= inclusiveLowerPoint && values[doc] <= inclusiveUpperPoint;
}
@@ -467,91 +467,77 @@
static abstract class FieldCacheDocIdSet extends DocIdSet {
private final IndexReader reader;
- private boolean mayUseTermDocs;
-
- FieldCacheDocIdSet(IndexReader reader, boolean mayUseTermDocs) {
+ private boolean canIgnoreDeletedDocs;
+
+ FieldCacheDocIdSet(IndexReader reader, boolean canIgnoreDeletedDocs) {
this.reader = reader;
- this.mayUseTermDocs = mayUseTermDocs;
+ this.canIgnoreDeletedDocs = canIgnoreDeletedDocs;
}
-
- /** this method checks, if a doc is a hit, should throw AIOBE, when position invalid */
+
+ /**
+ * this method checks, if a doc is a hit, should throw AIOBE, when position
+ * invalid
+ */
abstract boolean matchDoc(int doc) throws ArrayIndexOutOfBoundsException;
-
- /** this DocIdSet is cacheable, if it works solely with FieldCache and no TermDocs */
+
+ /**
+ * this DocIdSet is cacheable, if it can ignore deletions
+ */
public boolean isCacheable() {
- return !(mayUseTermDocs && reader.hasDeletions());
+ return canIgnoreDeletedDocs || !reader.hasDeletions();
}
public DocIdSetIterator iterator() throws IOException {
// Synchronization needed because deleted docs BitVector
// can change after call to hasDeletions until TermDocs creation.
- // We only use an iterator with termDocs, when this was requested (e.g. range contains 0)
+ // We only use an iterator with termDocs, when this was requested (e.g.
+ // range contains 0)
// and the index has deletions
- final TermDocs termDocs;
- synchronized(reader) {
- termDocs = isCacheable() ? null : reader.termDocs(null);
+
+ final Bits skipDocs;
+ synchronized (reader) {
+ if (isCacheable()) {
+ skipDocs = null;
+ } else {
+ skipDocs = reader.getDeletedDocs();
+ }
}
- if (termDocs != null) {
- // a DocIdSetIterator using TermDocs to iterate valid docIds
- return new DocIdSetIterator() {
- private int doc = -1;
-
- public int docID() {
- return doc;
- }
-
- public int nextDoc() throws IOException {
+ final int maxDoc = reader.maxDoc();
+
+ // a DocIdSetIterator generating docIds by
+ // incrementing a variable & checking skipDocs -
+ return new DocIdSetIterator() {
+ private int doc = -1;
+
+ public int docID() {
+ return doc;
+ }
+
+ public int nextDoc() {
+ try {
do {
- if (!termDocs.next())
- return doc = NO_MORE_DOCS;
- } while (!matchDoc(doc = termDocs.doc()));
+ doc++;
+ } while ((skipDocs != null && doc < maxDoc && skipDocs.get(doc))
+ || !matchDoc(doc));
return doc;
+ } catch (ArrayIndexOutOfBoundsException e) {
+ return doc = NO_MORE_DOCS;
}
-
- public int advance(int target) throws IOException {
- if (!termDocs.skipTo(target))
- return doc = NO_MORE_DOCS;
- while (!matchDoc(doc = termDocs.doc())) {
- if (!termDocs.next())
- return doc = NO_MORE_DOCS;
+ }
+
+ public int advance(int target) {
+ try {
+ doc = target;
+ while (!matchDoc(doc)) {
+ doc++;
}
return doc;
+ } catch (ArrayIndexOutOfBoundsException e) {
+ return doc = NO_MORE_DOCS;
}
- };
- } else {
- // a DocIdSetIterator generating docIds by incrementing a variable -
- // this one can be used if there are no deletions are on the index
- return new DocIdSetIterator() {
- private int doc = -1;
-
- public int docID() {
- return doc;
- }
-
- public int nextDoc() {
- try {
- do {
- doc++;
- } while (!matchDoc(doc));
- return doc;
- } catch (ArrayIndexOutOfBoundsException e) {
- return doc = NO_MORE_DOCS;
- }
- }
-
- public int advance(int target) {
- try {
- doc = target;
- while (!matchDoc(doc)) {
- doc++;
- }
- return doc;
- } catch (ArrayIndexOutOfBoundsException e) {
- return doc = NO_MORE_DOCS;
- }
- }
- };
- }
+
+ }
+ };
}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermEnum.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FilteredTermEnum.java Tue Oct 13 20:44:51 2009
@@ -24,7 +24,10 @@
/** Abstract class for enumerating a subset of all terms.
<p>Term enumerations are always ordered by Term.compareTo(). Each term in
- the enumeration is greater than all that precede it. */
+ the enumeration is greater than all that precede it.
+
+ @deprecated Switch to {@link FilteredTermsEnum} instead.
+*/
public abstract class FilteredTermEnum extends TermEnum {
/** the current term */
protected Term currentTerm = null;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyQuery.java Tue Oct 13 20:44:51 2009
@@ -109,10 +109,15 @@
return prefixLength;
}
+ // @deprecated see #getTermsEnum
protected FilteredTermEnum getEnum(IndexReader reader) throws IOException {
return new FuzzyTermEnum(reader, getTerm(), minimumSimilarity, prefixLength);
}
+ protected FilteredTermsEnum getTermsEnum(IndexReader reader) throws IOException {
+ return new FuzzyTermsEnum(reader, getTerm(), minimumSimilarity, prefixLength);
+ }
+
/**
* Returns the pattern term.
*/
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyTermEnum.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyTermEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/FuzzyTermEnum.java Tue Oct 13 20:44:51 2009
@@ -27,6 +27,8 @@
*
* <p>Term enumerations are always ordered by Term.compareTo(). Each term in
* the enumeration is greater than all that precede it.
+ *
+ * @deprecated Please use {@link FuzzyTermsEnum} instead.
*/
public final class FuzzyTermEnum extends FilteredTermEnum {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MatchAllDocsQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MatchAllDocsQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MatchAllDocsQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MatchAllDocsQuery.java Tue Oct 13 20:44:51 2009
@@ -19,8 +19,8 @@
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.util.Bits;
import java.util.Set;
import java.io.IOException;
@@ -45,16 +45,18 @@
}
private class MatchAllScorer extends Scorer {
- final TermDocs termDocs;
final float score;
final byte[] norms;
private int doc = -1;
+ private final int maxDoc;
+ private final Bits delDocs;
MatchAllScorer(IndexReader reader, Similarity similarity, Weight w,
byte[] norms) throws IOException {
super(similarity);
- this.termDocs = reader.termDocs(null);
+ delDocs = reader.getDeletedDocs();
score = w.getValue();
+ maxDoc = reader.maxDoc();
this.norms = norms;
}
@@ -67,7 +69,14 @@
}
public int nextDoc() throws IOException {
- return doc = termDocs.next() ? termDocs.doc() : NO_MORE_DOCS;
+ doc++;
+ while(delDocs != null && doc < maxDoc && delDocs.get(doc)) {
+ doc++;
+ }
+ if (doc == maxDoc) {
+ doc = NO_MORE_DOCS;
+ }
+ return doc;
}
public float score() {
@@ -75,7 +84,8 @@
}
public int advance(int target) throws IOException {
- return doc = termDocs.skipTo(target) ? termDocs.doc() : NO_MORE_DOCS;
+ doc = target-1;
+ return nextDoc();
}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiPhraseQuery.java Tue Oct 13 20:44:51 2009
@@ -21,10 +21,13 @@
import java.util.*;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.MultipleTermPositions;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.index.DocsEnum;
+import org.apache.lucene.index.PositionsEnum;
+import org.apache.lucene.index.TermRef;
import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.util.PriorityQueue;
+import org.apache.lucene.util.Bits;
/**
* MultiPhraseQuery is a generalized version of PhraseQuery, with an added
@@ -113,7 +116,7 @@
}
// inherit javadoc
- public void extractTerms(Set<Term> terms) {
+ public void extractTerms(Set terms) {
for (Iterator iter = termArrays.iterator(); iter.hasNext();) {
Term[] arr = (Term[])iter.next();
for (int i=0; i<arr.length; i++) {
@@ -162,27 +165,31 @@
if (termArrays.size() == 0) // optimize zero-term case
return null;
- TermPositions[] tps = new TermPositions[termArrays.size()];
- for (int i=0; i<tps.length; i++) {
+ DocsEnum[] docs = new DocsEnum[termArrays.size()];
+ for (int i=0; i<docs.length; i++) {
Term[] terms = (Term[])termArrays.get(i);
- TermPositions p;
- if (terms.length > 1)
- p = new MultipleTermPositions(reader, terms);
- else
- p = reader.termPositions(terms[0]);
+ final DocsEnum docsEnum;
+ if (terms.length > 1) {
+ docsEnum = new UnionDocsEnum(reader, terms);
+ } else {
+ docsEnum = reader.termDocsEnum(reader.getDeletedDocs(),
+ terms[0].field(),
+ new TermRef(terms[0].text()));
+ }
- if (p == null)
+ if (docsEnum == null) {
return null;
+ }
- tps[i] = p;
+ docs[i] = docsEnum;
}
if (slop == 0)
- return new ExactPhraseScorer(this, tps, getPositions(), similarity,
+ return new ExactPhraseScorer(this, docs, getPositions(), similarity,
reader.norms(field));
else
- return new SloppyPhraseScorer(this, tps, getPositions(), similarity,
+ return new SloppyPhraseScorer(this, docs, getPositions(), similarity,
slop, reader.norms(field));
}
@@ -371,3 +378,187 @@
return true;
}
}
+
+/**
+ * Takes the logical union of multiple DocsEnum iterators.
+ */
+
+class UnionDocsEnum extends DocsEnum {
+
+ private final static class DocsEnumWrapper {
+ int doc;
+ final DocsEnum docsEnum;
+ public DocsEnumWrapper(DocsEnum docsEnum) {
+ this.docsEnum = docsEnum;
+ }
+ }
+
+ private static final class DocsQueue extends PriorityQueue {
+ DocsQueue(List docsEnums) throws IOException {
+ initialize(docsEnums.size());
+
+ Iterator i = docsEnums.iterator();
+ while (i.hasNext()) {
+ DocsEnumWrapper docs = (DocsEnumWrapper) i.next();
+ docs.doc = docs.docsEnum.next();
+ if (docs.doc != DocsEnum.NO_MORE_DOCS) {
+ add(docs);
+ }
+ }
+ }
+
+ final public DocsEnumWrapper peek() {
+ return (DocsEnumWrapper) top();
+ }
+
+ public final boolean lessThan(Object a, Object b) {
+ return ((DocsEnumWrapper) a).doc < ((DocsEnumWrapper) b).doc;
+ }
+ }
+
+ private static final class IntQueue {
+ private int _arraySize = 16;
+ private int _index = 0;
+ private int _lastIndex = 0;
+ private int[] _array = new int[_arraySize];
+
+ final void add(int i) {
+ if (_lastIndex == _arraySize)
+ growArray();
+
+ _array[_lastIndex++] = i;
+ }
+
+ final int next() {
+ return _array[_index++];
+ }
+
+ final void sort() {
+ Arrays.sort(_array, _index, _lastIndex);
+ }
+
+ final void clear() {
+ _index = 0;
+ _lastIndex = 0;
+ }
+
+ final int size() {
+ return (_lastIndex - _index);
+ }
+
+ private void growArray() {
+ int[] newArray = new int[_arraySize * 2];
+ System.arraycopy(_array, 0, newArray, 0, _arraySize);
+ _array = newArray;
+ _arraySize *= 2;
+ }
+ }
+
+ private int _doc;
+ private int _freq;
+ private DocsQueue _queue;
+ private IntQueue _posList;
+
+ private final UnionPositionsEnum unionPositionsEnum;
+
+ public UnionDocsEnum(IndexReader indexReader, Term[] terms) throws IOException {
+ List docsEnums = new LinkedList();
+ final Bits delDocs = indexReader.getDeletedDocs();
+
+ for (int i = 0; i < terms.length; i++) {
+ DocsEnum docs = indexReader.termDocsEnum(delDocs,
+ terms[i].field(),
+ new TermRef(terms[i].text()));
+ if (docs != null) {
+ docsEnums.add(new DocsEnumWrapper(docs));
+ }
+ }
+
+ _queue = new DocsQueue(docsEnums);
+ _posList = new IntQueue();
+ unionPositionsEnum = new UnionPositionsEnum();
+ }
+
+ public PositionsEnum positions() {
+ return unionPositionsEnum;
+ }
+
+ public final int next() throws IOException {
+ if (_queue.size() == 0) {
+ return NO_MORE_DOCS;
+ }
+
+ // TODO: move this init into positions(): if the search
+ // doesn't need the positions for this doc then don't
+ // waste CPU merging them:
+ _posList.clear();
+ _doc = _queue.peek().doc;
+
+ // merge sort all positions together
+ DocsEnumWrapper docs;
+ do {
+ docs = _queue.peek();
+ final PositionsEnum positions = docs.docsEnum.positions();
+
+ final int freq = docs.docsEnum.freq();
+ for (int i = 0; i < freq; i++) {
+ _posList.add(positions.next());
+ }
+
+ docs.doc = docs.docsEnum.next();
+
+ if (docs.doc != NO_MORE_DOCS) {
+ _queue.updateTop();
+ } else {
+ _queue.pop();
+ }
+ } while (_queue.size() > 0 && _queue.peek().doc == _doc);
+
+ _posList.sort();
+ _freq = _posList.size();
+
+ return _doc;
+ }
+
+ private class UnionPositionsEnum extends PositionsEnum {
+
+ public int next() {
+ return _posList.next();
+ }
+
+ public int getPayloadLength() {
+ throw new UnsupportedOperationException();
+ }
+
+ public byte[] getPayload(byte[] data, int offset) {
+ throw new UnsupportedOperationException();
+ }
+
+ public boolean hasPayload() {
+ throw new UnsupportedOperationException();
+ }
+ }
+
+ public final int advance(int target) throws IOException {
+ while (_queue.peek() != null && target > _queue.peek().doc) {
+ DocsEnumWrapper docs = (DocsEnumWrapper) _queue.pop();
+ docs.doc = docs.docsEnum.advance(target);
+ if (docs.doc != NO_MORE_DOCS) {
+ _queue.add(docs);
+ }
+ }
+ return next();
+ }
+
+ public final int freq() {
+ return _freq;
+ }
+
+ /**
+ * Not implemented.
+ * @throws UnsupportedOperationException
+ */
+ public int read(int[] arg0, int[] arg1) throws IOException {
+ throw new UnsupportedOperationException();
+ }
+}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQuery.java Tue Oct 13 20:44:51 2009
@@ -25,6 +25,8 @@
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermRef;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.util.ToStringUtils;
import org.apache.lucene.queryParser.QueryParser; // for javadoc
@@ -96,24 +98,49 @@
private static class ScoringBooleanQueryRewrite extends RewriteMethod implements Serializable {
public Query rewrite(IndexReader reader, MultiTermQuery query) throws IOException {
- FilteredTermEnum enumerator = query.getEnum(reader);
- BooleanQuery result = new BooleanQuery(true);
- int count = 0;
- try {
- do {
- Term t = enumerator.term();
- if (t != null) {
- TermQuery tq = new TermQuery(t); // found a match
- tq.setBoost(query.getBoost() * enumerator.difference()); // set the boost
+ FilteredTermsEnum termsEnum = query.getTermsEnum(reader);
+ if (termsEnum != null) {
+
+ // nocommit -- if no terms we'd want to return NullQuery
+ BooleanQuery result = new BooleanQuery(true);
+ if (!termsEnum.empty()) {
+ final String field = termsEnum.field();
+ assert field != null;
+ int count = 0;
+ TermRef term = termsEnum.term();
+ // first term must exist since termsEnum wasn't null
+ assert term != null;
+ do {
+ TermQuery tq = new TermQuery(new Term(field, term.toString())); // found a match
+ tq.setBoost(query.getBoost() * termsEnum.difference()); // set the boost
result.add(tq, BooleanClause.Occur.SHOULD); // add to query
count++;
- }
- } while (enumerator.next());
- } finally {
- enumerator.close();
+ term = termsEnum.next();
+ } while(term != null);
+ query.incTotalNumberOfTerms(count);
+ }
+ return result;
+ } else {
+ // deprecated case
+ FilteredTermEnum enumerator = query.getEnum(reader);
+ BooleanQuery result = new BooleanQuery(true);
+ int count = 0;
+ try {
+ do {
+ Term t = enumerator.term();
+ if (t != null) {
+ TermQuery tq = new TermQuery(t); // found a match
+ tq.setBoost(query.getBoost() * enumerator.difference()); // set the boost
+ result.add(tq, BooleanClause.Occur.SHOULD); // add to query
+ count++;
+ }
+ } while (enumerator.next());
+ } finally {
+ enumerator.close();
+ }
+ query.incTotalNumberOfTerms(count);
+ return result;
}
- query.incTotalNumberOfTerms(count);
- return result;
}
// Make sure we are still a singleton even after deserializing
@@ -213,6 +240,7 @@
}
public Query rewrite(IndexReader reader, MultiTermQuery query) throws IOException {
+
// Get the enum and start visiting terms. If we
// exhaust the enum before hitting either of the
// cutoffs, we use ConstantBooleanQueryRewrite; else,
@@ -222,53 +250,97 @@
final int termCountLimit = Math.min(BooleanQuery.getMaxClauseCount(), termCountCutoff);
int docVisitCount = 0;
- FilteredTermEnum enumerator = query.getEnum(reader);
- try {
- while(true) {
- Term t = enumerator.term();
- if (t != null) {
- pendingTerms.add(t);
+ FilteredTermsEnum termsEnum = query.getTermsEnum(reader);
+ if (termsEnum != null) {
+ if (!termsEnum.empty()) {
+ final String field = termsEnum.field();
+ assert field != null;
+ TermRef term = termsEnum.term();
+ // first term must exist since termsEnum wasn't null
+ assert term != null;
+ do {
+ pendingTerms.add(term.clone());
+ if (pendingTerms.size() >= termCountLimit || docVisitCount >= docCountCutoff) {
+ // Too many terms -- cut our losses now and make a filter.
+ Query result = new ConstantScoreQuery(new MultiTermQueryWrapperFilter(query));
+ result.setBoost(query.getBoost());
+ return result;
+ }
// Loading the TermInfo from the terms dict here
// should not be costly, because 1) the
// query/filter will load the TermInfo when it
// runs, and 2) the terms dict has a cache:
- docVisitCount += reader.docFreq(t);
+ docVisitCount += reader.docFreq(field, term);
+ term = termsEnum.next();
+ } while(term != null);
+
+ // Enumeration is done, and we hit a small
+ // enough number of terms & docs -- just make a
+ // BooleanQuery, now
+ Iterator it = pendingTerms.iterator();
+ BooleanQuery bq = new BooleanQuery(true);
+ while(it.hasNext()) {
+ TermQuery tq = new TermQuery(new Term(field, ((TermRef) it.next()).toString()));
+ bq.add(tq, BooleanClause.Occur.SHOULD);
}
+ // Strip scores
+ Query result = new ConstantScoreQuery(new QueryWrapperFilter(bq));
+ result.setBoost(query.getBoost());
+ query.incTotalNumberOfTerms(pendingTerms.size());
+ return result;
+ } else {
+ // nocommit -- need NullQuery here
+ return new BooleanQuery();
+ }
+ } else {
- if (pendingTerms.size() >= termCountLimit || docVisitCount >= docCountCutoff) {
- // Too many terms -- make a filter.
- Query result = new ConstantScoreQuery(new MultiTermQueryWrapperFilter(query));
- result.setBoost(query.getBoost());
- return result;
- } else if (!enumerator.next()) {
- // Enumeration is done, and we hit a small
- // enough number of terms & docs -- just make a
- // BooleanQuery, now
- Iterator it = pendingTerms.iterator();
- BooleanQuery bq = new BooleanQuery(true);
- while(it.hasNext()) {
- TermQuery tq = new TermQuery((Term) it.next());
- bq.add(tq, BooleanClause.Occur.SHOULD);
+ // deprecated case
+ FilteredTermEnum enumerator = query.getEnum(reader);
+ try {
+ while(true) {
+ Term t = enumerator.term();
+ if (t != null) {
+ pendingTerms.add(t);
+ // Loading the TermInfo from the terms dict here
+ // should not be costly, because 1) the
+ // query/filter will load the TermInfo when it
+ // runs, and 2) the terms dict has a cache:
+ docVisitCount += reader.docFreq(t);
+ }
+
+ if (pendingTerms.size() >= termCountLimit || docVisitCount >= docCountCutoff) {
+ // Too many terms -- make a filter.
+ Query result = new ConstantScoreQuery(new MultiTermQueryWrapperFilter(query));
+ result.setBoost(query.getBoost());
+ return result;
+ } else if (!enumerator.next()) {
+ // Enumeration is done, and we hit a small
+ // enough number of terms & docs -- just make a
+ // BooleanQuery, now
+ Iterator it = pendingTerms.iterator();
+ BooleanQuery bq = new BooleanQuery(true);
+ while(it.hasNext()) {
+ TermQuery tq = new TermQuery((Term) it.next());
+ bq.add(tq, BooleanClause.Occur.SHOULD);
+ }
+ // Strip scores
+ Query result = new ConstantScoreQuery(new QueryWrapperFilter(bq));
+ result.setBoost(query.getBoost());
+ query.incTotalNumberOfTerms(pendingTerms.size());
+ return result;
}
- // Strip scores
- Query result = new ConstantScoreQuery(new QueryWrapperFilter(bq));
- result.setBoost(query.getBoost());
- query.incTotalNumberOfTerms(pendingTerms.size());
- return result;
}
+ } finally {
+ enumerator.close();
}
- } finally {
- enumerator.close();
}
}
- @Override
public int hashCode() {
final int prime = 1279;
return (int) (prime * termCountCutoff + Double.doubleToLongBits(docCountPercent));
}
- @Override
public boolean equals(Object obj) {
if (this == obj)
return true;
@@ -326,9 +398,25 @@
public MultiTermQuery() {
}
- /** Construct the enumeration to be used, expanding the pattern term. */
- protected abstract FilteredTermEnum getEnum(IndexReader reader)
- throws IOException;
+ /** Construct the enumeration to be used, expanding the
+ * pattern term.
+ * @deprecated Please override {@link #getTermsEnum} instead */
+ protected FilteredTermEnum getEnum(IndexReader reader)
+ throws IOException {
+ return null;
+ }
+
+ /** Construct the enumeration to be used, expanding the
+ * pattern term. This method must return null if no
+ * terms fall in the range; else, it must return a
+ * TermsEnum already positioned to the first matching
+ * term.
+ *
+ * nocommit in 3.x this will become abstract */
+ protected FilteredTermsEnum getTermsEnum(IndexReader reader)
+ throws IOException {
+ return null;
+ }
/**
* Expert: Return the number of unique terms visited during execution of the query.
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/MultiTermQueryWrapperFilter.java Tue Oct 13 20:44:51 2009
@@ -23,7 +23,11 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermDocs;
import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.index.TermRef;
+import org.apache.lucene.index.TermsEnum;
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.util.OpenBitSet;
+import org.apache.lucene.util.Bits;
/**
* A wrapper for {@link MultiTermQuery}, that exposes its
@@ -95,6 +99,7 @@
}
abstract class TermGenerator {
+ // @deprecated
public void generate(IndexReader reader, TermEnum enumerator) throws IOException {
final int[] docs = new int[32];
final int[] freqs = new int[32];
@@ -125,6 +130,38 @@
termDocs.close();
}
}
+
+ public void generate(IndexReader reader, TermsEnum enumerator) throws IOException {
+ //System.out.println("mtq.filter generate");
+ final int[] docs = new int[32];
+ final int[] freqs = new int[32];
+ int termCount = 0;
+ final Bits delDocs = reader.getDeletedDocs();
+ while(true) {
+ termCount++;
+ //System.out.println(" iter termCount=" + termCount + " term=" + enumerator.term().toBytesString());
+ DocsEnum docsEnum = enumerator.docs(delDocs);
+ while (true) {
+ final int count = docsEnum.read(docs, freqs);
+ if (count != 0) {
+ for(int i=0;i<count;i++) {
+ handleDoc(docs[i]);
+ }
+ } else {
+ break;
+ }
+ }
+ TermRef term = enumerator.next();
+ if (term == null) {
+ break;
+ }
+ //System.out.println(" enum next term=" + term.toBytesString());
+ assert term.termEquals(enumerator.term());
+ }
+ //System.out.println(" done termCount=" + termCount);
+
+ query.incTotalNumberOfTerms(termCount);
+ }
abstract public void handleDoc(int doc);
}
@@ -134,21 +171,37 @@
*/
@Override
public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
- final TermEnum enumerator = query.getEnum(reader);
- try {
- // if current term in enum is null, the enum is empty -> shortcut
- if (enumerator.term() == null)
+ final FilteredTermsEnum termsEnum = query.getTermsEnum(reader);
+ if (termsEnum != null) {
+ if (!termsEnum.empty()) {
+ // fill into a OpenBitSet
+ final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc());
+ new TermGenerator() {
+ public void handleDoc(int doc) {
+ bitSet.set(doc);
+ }
+ }.generate(reader, termsEnum);
+ return bitSet;
+ } else {
return DocIdSet.EMPTY_DOCIDSET;
- // else fill into a OpenBitSet
- final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc());
- new TermGenerator() {
- public void handleDoc(int doc) {
- bitSet.set(doc);
- }
- }.generate(reader, enumerator);
- return bitSet;
- } finally {
- enumerator.close();
+ }
+ } else {
+ final TermEnum enumerator = query.getEnum(reader);
+ try {
+ // if current term in enum is null, the enum is empty -> shortcut
+ if (enumerator.term() == null)
+ return DocIdSet.EMPTY_DOCIDSET;
+ // else fill into a OpenBitSet
+ final OpenBitSet bitSet = new OpenBitSet(reader.maxDoc());
+ new TermGenerator() {
+ public void handleDoc(int doc) {
+ bitSet.set(doc);
+ }
+ }.generate(reader, enumerator);
+ return bitSet;
+ } finally {
+ enumerator.close();
+ }
}
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/NumericRangeQuery.java Tue Oct 13 20:44:51 2009
@@ -27,6 +27,8 @@
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermRef;
+import org.apache.lucene.index.Terms;
/**
* <p>A {@link Query} that matches numeric values within a
@@ -305,6 +307,10 @@
return new NumericRangeTermEnum(reader);
}
+ protected FilteredTermsEnum getTermsEnum(final IndexReader reader) throws IOException {
+ return new NumericRangeTermsEnum(reader);
+ }
+
/** Returns the field name for this query */
public String getField() { return field; }
@@ -378,7 +384,11 @@
* The ordering depends on how {@link NumericUtils#splitLongRange} and
* {@link NumericUtils#splitIntRange} generates the sub-ranges. For
* {@link MultiTermQuery} ordering is not relevant.
+ *
+ * @deprecated use NumericRangeTermsEnum instead
*/
+ // nocommit -- can we remove this? only back compat
+ // concern would be subclasses of NRQ that invoke getEnum
private final class NumericRangeTermEnum extends FilteredTermEnum {
private final IndexReader reader;
@@ -532,5 +542,174 @@
}
}
+
+
+ /**
+ * Subclass of FilteredTermsEnum for enumerating all terms that match the
+ * sub-ranges for trie range queries, using flex API.
+ * <p>
+ * WARNING: This term enumeration is not guaranteed to be always ordered by
+ * {@link Term#compareTo}.
+ * The ordering depends on how {@link NumericUtils#splitLongRange} and
+ * {@link NumericUtils#splitIntRange} generates the sub-ranges. For
+ * {@link MultiTermQuery} ordering is not relevant.
+ */
+ private final class NumericRangeTermsEnum extends FilteredTermsEnum {
+
+ private final IndexReader reader;
+ private final LinkedList<String> rangeBounds = new LinkedList<String>();
+ private TermRef currentUpperBound = null;
+ private final boolean empty;
+
+ NumericRangeTermsEnum(final IndexReader reader) throws IOException {
+ this.reader = reader;
+
+ switch (valSize) {
+ case 64: {
+ // lower
+ long minBound = Long.MIN_VALUE;
+ if (min instanceof Long) {
+ minBound = min.longValue();
+ } else if (min instanceof Double) {
+ minBound = NumericUtils.doubleToSortableLong(min.doubleValue());
+ }
+ if (!minInclusive && min != null) {
+ if (minBound == Long.MAX_VALUE) break;
+ minBound++;
+ }
+
+ // upper
+ long maxBound = Long.MAX_VALUE;
+ if (max instanceof Long) {
+ maxBound = max.longValue();
+ } else if (max instanceof Double) {
+ maxBound = NumericUtils.doubleToSortableLong(max.doubleValue());
+ }
+ if (!maxInclusive && max != null) {
+ if (maxBound == Long.MIN_VALUE) break;
+ maxBound--;
+ }
+
+ NumericUtils.splitLongRange(new NumericUtils.LongRangeBuilder() {
+ //@Override
+ public final void addRange(String minPrefixCoded, String maxPrefixCoded) {
+ rangeBounds.add(minPrefixCoded);
+ rangeBounds.add(maxPrefixCoded);
+ }
+ }, precisionStep, minBound, maxBound);
+ break;
+ }
+
+ case 32: {
+ // lower
+ int minBound = Integer.MIN_VALUE;
+ if (min instanceof Integer) {
+ minBound = min.intValue();
+ } else if (min instanceof Float) {
+ minBound = NumericUtils.floatToSortableInt(min.floatValue());
+ }
+ if (!minInclusive && min != null) {
+ if (minBound == Integer.MAX_VALUE) break;
+ minBound++;
+ }
+
+ // upper
+ int maxBound = Integer.MAX_VALUE;
+ if (max instanceof Integer) {
+ maxBound = max.intValue();
+ } else if (max instanceof Float) {
+ maxBound = NumericUtils.floatToSortableInt(max.floatValue());
+ }
+ if (!maxInclusive && max != null) {
+ if (maxBound == Integer.MIN_VALUE) break;
+ maxBound--;
+ }
+
+ NumericUtils.splitIntRange(new NumericUtils.IntRangeBuilder() {
+ //@Override
+ public final void addRange(String minPrefixCoded, String maxPrefixCoded) {
+ rangeBounds.add(minPrefixCoded);
+ rangeBounds.add(maxPrefixCoded);
+ }
+ }, precisionStep, minBound, maxBound);
+ break;
+ }
+
+ default:
+ // should never happen
+ throw new IllegalArgumentException("valSize must be 32 or 64");
+ }
+
+ // seek to first term
+ empty = next() == null;
+ }
+
+ @Override
+ public float difference() {
+ return 1.0f;
+ }
+
+ /** this is a dummy, it is not used by this class. */
+ @Override
+ public boolean empty() {
+ return empty;
+ }
+
+ public String field() {
+ return field;
+ }
+
+ /**
+ * Compares if current upper bound is reached,
+ * this also updates the term count for statistics.
+ * In contrast to {@link FilteredTermEnum}, a return value
+ * of <code>false</code> ends iterating the current enum
+ * and forwards to the next sub-range.
+ */
+ @Override
+ protected boolean accept(TermRef term) {
+ return (term.compareTerm(currentUpperBound) <= 0);
+ }
+
+ /** Increments the enumeration to the next element. True if one exists. */
+ @Override
+ public TermRef next() throws IOException {
+ //System.out.println("nrq.next");
+ // if the actual enum is initialized, try change to
+ // next term, if no such term exists, fall-through
+ if (actualEnum != null) {
+ TermRef term = actualEnum.next();
+ if (term != null && accept(term)) {
+ //System.out.println(" return term=" + term.toBytesString());
+ return term;
+ }
+ }
+
+ //System.out.println(" ranges = " + rangeBounds.size());
+
+ // if all above fails, we go forward to the next enum,
+ // if one is available
+ if (rangeBounds.size() < 2) {
+ assert rangeBounds.size() == 0;
+ //System.out.println(" return null0");
+ return null;
+ }
+
+ final TermRef lowerBound = new TermRef(rangeBounds.removeFirst());
+ this.currentUpperBound = new TermRef(rangeBounds.removeFirst());
+
+ // this call recursively uses next(), if no valid term in
+ // next enum found.
+ // if this behavior is changed/modified in the superclass,
+ // this enum will not work anymore!
+ Terms terms = reader.fields().terms(field);
+ if (terms != null) {
+ return setEnum(terms.iterator(), lowerBound);
+ } else {
+ //System.out.println(" return null");
+ return null;
+ }
+ }
+ }
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhrasePositions.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhrasePositions.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhrasePositions.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhrasePositions.java Tue Oct 13 20:44:51 2009
@@ -28,40 +28,43 @@
int position; // position in doc
int count; // remaining pos in this doc
int offset; // position in phrase
- TermPositions tp; // stream of positions
- PhrasePositions next; // used to make lists
+ final DocsEnum docs; // stream of docs
+ PositionsEnum positions; // positions in current doc
+ PhrasePositions next; // used to make lists
boolean repeats; // there's other pp for same term (e.g. query="1st word 2nd word"~1)
- PhrasePositions(TermPositions t, int o) {
- tp = t;
+ PhrasePositions(DocsEnum docs, int o) {
+ this.docs = docs;
offset = o;
}
final boolean next() throws IOException { // increments to next doc
- if (!tp.next()) {
- tp.close(); // close stream
- doc = Integer.MAX_VALUE; // sentinel value
+ doc = docs.next();
+ if (doc == docs.NO_MORE_DOCS) {
return false;
}
- doc = tp.doc();
- position = 0;
+ positions = docs.positions();
+
+ // nocommit -- really needed?
+ //position = 0;
+
return true;
}
final boolean skipTo(int target) throws IOException {
- if (!tp.skipTo(target)) {
- tp.close(); // close stream
- doc = Integer.MAX_VALUE; // sentinel value
+ doc = docs.advance(target);
+ if (doc == docs.NO_MORE_DOCS) {
return false;
}
- doc = tp.doc();
- position = 0;
+ // nocommit -- really needed?
+ // position = 0;
return true;
}
final void firstPosition() throws IOException {
- count = tp.freq(); // read first pos
+ count = docs.freq(); // read first pos
+ positions = docs.positions();
nextPosition();
}
@@ -73,7 +76,7 @@
*/
final boolean nextPosition() throws IOException {
if (count-- > 0) { // read subsequent pos's
- position = tp.nextPosition() - offset;
+ position = positions.next() - offset;
return true;
} else
return false;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseQuery.java Tue Oct 13 20:44:51 2009
@@ -22,10 +22,12 @@
import java.util.ArrayList;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.index.TermRef;
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Explanation.IDFExplanation;
import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.util.Bits;
/** A Query that matches documents containing a particular sequence of terms.
* A PhraseQuery is built by QueryParser for input like <code>"new york"</code>.
@@ -143,20 +145,25 @@
if (terms.size() == 0) // optimize zero-term case
return null;
- TermPositions[] tps = new TermPositions[terms.size()];
+ DocsEnum[] docs = new DocsEnum[terms.size()];
+ final Bits delDocs = reader.getDeletedDocs();
for (int i = 0; i < terms.size(); i++) {
- TermPositions p = reader.termPositions((Term)terms.get(i));
- if (p == null)
+ final Term t = (Term) terms.get(i);
+ DocsEnum docsEnum = reader.termDocsEnum(delDocs,
+ t.field(),
+ new TermRef(t.text()));
+ if (docsEnum == null) {
return null;
- tps[i] = p;
+ }
+ docs[i] = docsEnum;
}
if (slop == 0) // optimize exact case
- return new ExactPhraseScorer(this, tps, getPositions(), similarity,
+ return new ExactPhraseScorer(this, docs, getPositions(), similarity,
reader.norms(field));
else
return
- new SloppyPhraseScorer(this, tps, getPositions(), similarity, slop,
+ new SloppyPhraseScorer(this, docs, getPositions(), similarity, slop,
reader.norms(field));
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseScorer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseScorer.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseScorer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PhraseScorer.java Tue Oct 13 20:44:51 2009
@@ -19,7 +19,7 @@
import java.io.IOException;
-import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.index.DocsEnum;
/** Expert: Scoring functionality for phrase queries.
* <br>A document is considered matching if it contains the phrase-query terms
@@ -43,7 +43,7 @@
private float freq; //phrase frequency in current doc as computed by phraseFreq().
- PhraseScorer(Weight weight, TermPositions[] tps, int[] offsets,
+ PhraseScorer(Weight weight, DocsEnum[] docs, int[] offsets,
Similarity similarity, byte[] norms) {
super(similarity);
this.norms = norms;
@@ -55,8 +55,8 @@
// reflects the phrase offset: pp.pos = tp.pos - offset.
// this allows to easily identify a matching (exact) phrase
// when all PhrasePositions have exactly the same position.
- for (int i = 0; i < tps.length; i++) {
- PhrasePositions pp = new PhrasePositions(tps[i], offsets[i]);
+ for (int i = 0; i < docs.length; i++) {
+ PhrasePositions pp = new PhrasePositions(docs[i], offsets[i]);
if (last != null) { // add next to end of list
last.next = pp;
} else {
@@ -65,7 +65,7 @@
last = pp;
}
- pq = new PhraseQueue(tps.length); // construct empty pq
+ pq = new PhraseQueue(docs.length); // construct empty pq
first.doc = -1;
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixQuery.java Tue Oct 13 20:44:51 2009
@@ -40,8 +40,8 @@
/** Returns the prefix of this query. */
public Term getPrefix() { return prefix; }
- protected FilteredTermEnum getEnum(IndexReader reader) throws IOException {
- return new PrefixTermEnum(reader, prefix);
+ protected FilteredTermsEnum getTermsEnum(IndexReader reader) throws IOException {
+ return new PrefixTermsEnum(reader, prefix);
}
/** Prints a user-readable version of this query. */
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixTermEnum.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixTermEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/PrefixTermEnum.java Tue Oct 13 20:44:51 2009
@@ -29,6 +29,7 @@
* Term enumerations are always ordered by Term.compareTo(). Each term in
* the enumeration is greater than all that precede it.
*
+ * @deprecated Use {@link PrefixTermsEnum} instead.
*/
public class PrefixTermEnum extends FilteredTermEnum {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SloppyPhraseScorer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SloppyPhraseScorer.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SloppyPhraseScorer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/SloppyPhraseScorer.java Tue Oct 13 20:44:51 2009
@@ -17,7 +17,7 @@
* limitations under the License.
*/
-import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.index.DocsEnum;
import java.io.IOException;
import java.util.HashMap;
@@ -28,9 +28,9 @@
private PhrasePositions tmpPos[]; // for flipping repeating pps.
private boolean checkedRepeats;
- SloppyPhraseScorer(Weight weight, TermPositions[] tps, int[] offsets, Similarity similarity,
+ SloppyPhraseScorer(Weight weight, DocsEnum[] docs, int[] offsets, Similarity similarity,
int slop, byte[] norms) {
- super(weight, tps, offsets, similarity, norms);
+ super(weight, docs, offsets, similarity, norms);
this.slop = slop;
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermQuery.java Tue Oct 13 20:44:51 2009
@@ -20,8 +20,9 @@
import java.io.IOException;
import java.util.Set;
+import org.apache.lucene.index.DocsEnum;
import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.TermRef;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.Explanation.IDFExplanation;
import org.apache.lucene.util.ToStringUtils;
@@ -64,12 +65,12 @@
}
public Scorer scorer(IndexReader reader, boolean scoreDocsInOrder, boolean topScorer) throws IOException {
- TermDocs termDocs = reader.termDocs(term);
-
- if (termDocs == null)
+ DocsEnum docs = reader.termDocsEnum(reader.getDeletedDocs(), term.field(), new TermRef(term.text()));
+ if (docs == null) {
return null;
+ }
- return new TermScorer(this, termDocs, similarity, reader.norms(term.field()));
+ return new TermScorer(this, docs, similarity, reader.norms(term.field()));
}
public Explanation explain(IndexReader reader, int doc)
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeQuery.java Tue Oct 13 20:44:51 2009
@@ -135,6 +135,17 @@
upperTerm, includeLower, includeUpper, collator);
}
+ public String field() {
+ return field;
+ }
+
+ protected FilteredTermsEnum getTermsEnum(IndexReader reader) throws IOException {
+ return new TermRangeTermsEnum(reader, field,
+ lowerTerm, upperTerm,
+ includeLower, includeUpper,
+ collator);
+ }
+
/** Prints a user-readable version of this query. */
public String toString(String field) {
StringBuilder buffer = new StringBuilder();
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeTermEnum.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeTermEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermRangeTermEnum.java Tue Oct 13 20:44:51 2009
@@ -31,6 +31,7 @@
* Term enumerations are always ordered by Term.compareTo(). Each term in
* the enumeration is greater than all that precede it.
* @since 2.9
+ * @deprecated Please switch to {@link TermRangeTermsEnum}
*/
public class TermRangeTermEnum extends FilteredTermEnum {
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermScorer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermScorer.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermScorer.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/TermScorer.java Tue Oct 13 20:44:51 2009
@@ -19,7 +19,7 @@
import java.io.IOException;
-import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.DocsEnum;
/** Expert: A <code>Scorer</code> for documents matching a <code>Term</code>.
*/
@@ -28,7 +28,7 @@
private static final float[] SIM_NORM_DECODER = Similarity.getNormDecoder();
private Weight weight;
- private TermDocs termDocs;
+ private DocsEnum docsEnum;
private byte[] norms;
private float weightValue;
private int doc = -1;
@@ -54,10 +54,10 @@
* @param norms
* The field norms of the document fields for the <code>Term</code>.
*/
- TermScorer(Weight weight, TermDocs td, Similarity similarity, byte[] norms) {
+ TermScorer(Weight weight, DocsEnum td, Similarity similarity, byte[] norms) {
super(similarity);
this.weight = weight;
- this.termDocs = td;
+ this.docsEnum = td;
this.norms = norms;
this.weightValue = weight.getValue();
@@ -71,17 +71,17 @@
// firstDocID is ignored since nextDoc() sets 'doc'
protected boolean score(Collector c, int end, int firstDocID) throws IOException {
+ //System.out.println("top score " + firstDocID + " max=" + pointerMax);
c.setScorer(this);
while (doc < end) { // for docs in window
c.collect(doc); // collect score
-
+ //System.out.println("done collect");
if (++pointer >= pointerMax) {
- pointerMax = termDocs.read(docs, freqs); // refill buffers
+ pointerMax = docsEnum.read(docs, freqs); // refill buffers
if (pointerMax != 0) {
pointer = 0;
} else {
- termDocs.close(); // close stream
- doc = Integer.MAX_VALUE; // set to sentinel value
+ doc = NO_MORE_DOCS; // set to sentinel value
return false;
}
}
@@ -97,25 +97,28 @@
* The iterator over the matching documents is buffered using
* {@link TermDocs#read(int[],int[])}.
*
- * @return the document matching the query or -1 if there are no more documents.
+ * @return the document matching the query or NO_MORE_DOCS if there are no more documents.
*/
public int nextDoc() throws IOException {
+ //System.out.println("ts.nextDoc pointer=" + pointer + " max=" + pointerMax + " this=" + this + " docsEnum=" + docsEnum);
pointer++;
if (pointer >= pointerMax) {
- pointerMax = termDocs.read(docs, freqs); // refill buffer
+ pointerMax = docsEnum.read(docs, freqs); // refill buffer
+ //System.out.println("ts set max=" + pointerMax);
if (pointerMax != 0) {
pointer = 0;
} else {
- termDocs.close(); // close stream
+ //System.out.println("ts no more docs");
return doc = NO_MORE_DOCS;
}
}
doc = docs[pointer];
+ assert doc != NO_MORE_DOCS;
return doc;
}
public float score() {
- assert doc != -1;
+ assert doc != NO_MORE_DOCS;
int f = freqs[pointer];
float raw = // compute tf(f)*weight
f < SCORE_CACHE_SIZE // check cache
@@ -128,11 +131,11 @@
/**
* Advances to the first match beyond the current whose document number is
* greater than or equal to a given target. <br>
- * The implementation uses {@link TermDocs#skipTo(int)}.
+ * The implementation uses {@link DocsEnum#adnvace(int)}.
*
* @param target
* The target document number.
- * @return the matching document or -1 if none exist.
+ * @return the matching document or NO_MORE_DOCS if none exist.
*/
public int advance(int target) throws IOException {
// first scan in cache
@@ -142,13 +145,14 @@
}
}
- // not found in cache, seek underlying stream
- boolean result = termDocs.skipTo(target);
- if (result) {
+ // not found in readahead cache, seek underlying stream
+ int newDoc = docsEnum.advance(target);
+ //System.out.println("ts.advance docsEnum=" + docsEnum);
+ if (newDoc != DocsEnum.NO_MORE_DOCS) {
pointerMax = 1;
pointer = 0;
- docs[pointer] = doc = termDocs.doc();
- freqs[pointer] = termDocs.freq();
+ docs[pointer] = doc = newDoc;
+ freqs[pointer] = docsEnum.freq();
} else {
doc = NO_MORE_DOCS;
}
@@ -168,15 +172,11 @@
pointer++;
}
if (tf == 0) {
- if (termDocs.skipTo(doc))
- {
- if (termDocs.doc() == doc)
- {
- tf = termDocs.freq();
- }
- }
+ int newDoc = docsEnum.advance(doc);
+ if (newDoc == doc) {
+ tf = docsEnum.freq();
+ }
}
- termDocs.close();
tfExplanation.setValue(getSimilarity().tf(tf));
tfExplanation.setDescription("tf(termFreq("+query.getTerm()+")="+tf+")");
@@ -184,5 +184,6 @@
}
/** Returns a string representation of this <code>TermScorer</code>. */
- public String toString() { return "scorer(" + weight + ")"; }
+ // nocommit
+ //public String toString() { return "scorer(" + weight + ")"; }
}
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardQuery.java Tue Oct 13 20:44:51 2009
@@ -34,7 +34,7 @@
* MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}
* rewrite method.
*
- * @see WildcardTermEnum */
+ * @see WildcardTermEnums */
public class WildcardQuery extends MultiTermQuery {
private boolean termContainsWildcard;
private boolean termIsPrefix;
@@ -49,7 +49,13 @@
&& (text.indexOf('?') == -1)
&& (text.indexOf('*') == text.length() - 1);
}
-
+
+ // nocommit: needs singletermenum stuff
+ protected FilteredTermsEnum getTermsEnum(IndexReader reader) throws IOException {
+ return new WildcardTermsEnum(reader, getTerm());
+ }
+
+ // @deprecated see getTermsEnum
protected FilteredTermEnum getEnum(IndexReader reader) throws IOException {
if (termContainsWildcard)
return new WildcardTermEnum(reader, getTerm());
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardTermEnum.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardTermEnum.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/WildcardTermEnum.java Tue Oct 13 20:44:51 2009
@@ -28,6 +28,7 @@
* <p>
* Term enumerations are always ordered by Term.compareTo(). Each term in
* the enumeration is greater than all that precede it.
+ * @deprecated Please use {@link WildcardTermsEnum} instead.
*/
public class WildcardTermEnum extends FilteredTermEnum {
final Term searchTerm;
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/ValueSourceQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/ValueSourceQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/ValueSourceQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/function/ValueSourceQuery.java Tue Oct 13 20:44:51 2009
@@ -18,10 +18,10 @@
*/
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.Term;
-import org.apache.lucene.index.TermDocs;
import org.apache.lucene.search.*;
+import org.apache.lucene.index.Term;
import org.apache.lucene.util.ToStringUtils;
+import org.apache.lucene.util.Bits;
import java.io.IOException;
import java.util.Set;
@@ -58,7 +58,7 @@
return this;
}
- /*(non-Javadoc) @see org.apache.lucene.search.Query#extractTerms(java.util.Set) */
+ /*(non-Javadoc) @see org.apache.lucene.search.Query#extractTerms(Set) */
public void extractTerms(Set<Term> terms) {
// no terms involved here
}
@@ -114,7 +114,8 @@
private final ValueSourceWeight weight;
private final float qWeight;
private final DocValues vals;
- private final TermDocs termDocs;
+ private final Bits delDocs;
+ private final int maxDoc;
private int doc = -1;
// constructor
@@ -124,24 +125,33 @@
this.qWeight = w.getValue();
// this is when/where the values are first created.
vals = valSrc.getValues(reader);
- termDocs = reader.termDocs(null);
+ delDocs = reader.getDeletedDocs();
+ maxDoc = reader.maxDoc();
}
public int nextDoc() throws IOException {
- return doc = termDocs.next() ? termDocs.doc() : NO_MORE_DOCS;
+ doc++;
+ while (delDocs != null && doc < maxDoc && delDocs.get(doc)) {
+ doc++;
+ }
+ if (doc == maxDoc) {
+ doc = NO_MORE_DOCS;
+ }
+ return doc;
}
-
+
public int docID() {
return doc;
}
-
+
public int advance(int target) throws IOException {
- return doc = termDocs.skipTo(target) ? termDocs.doc() : NO_MORE_DOCS;
+ doc = target - 1;
+ return nextDoc();
}
/*(non-Javadoc) @see org.apache.lucene.search.Scorer#score() */
public float score() throws IOException {
- return qWeight * vals.floatVal(termDocs.doc());
+ return qWeight * vals.floatVal(doc);
}
/*(non-Javadoc) @see org.apache.lucene.search.Scorer#explain(int) */
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/payloads/PayloadTermQuery.java Tue Oct 13 20:44:51 2009
@@ -19,7 +19,7 @@
import org.apache.lucene.index.Term;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.index.TermPositions;
+import org.apache.lucene.index.PositionsEnum;
import org.apache.lucene.search.Searcher;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.Weight;
@@ -80,14 +80,14 @@
protected class PayloadTermSpanScorer extends SpanScorer {
// TODO: is this the best way to allocate this?
protected byte[] payload = new byte[256];
- protected TermPositions positions;
protected float payloadScore;
protected int payloadsSeen;
+ private final TermSpans termSpans;
public PayloadTermSpanScorer(TermSpans spans, Weight weight,
Similarity similarity, byte[] norms) throws IOException {
super(spans, weight, similarity, norms);
- positions = spans.getPositions();
+ termSpans = spans;
}
protected boolean setFreqCurrentDoc() throws IOException {
@@ -112,7 +112,8 @@
}
protected void processPayload(Similarity similarity) throws IOException {
- if (positions.isPayloadAvailable()) {
+ final PositionsEnum positions = termSpans.getPositions();
+ if (positions.hasPayload()) {
payload = positions.getPayload(payload, 0);
payloadScore = function.currentScore(doc, term.field(),
spans.start(), spans.end(), payloadsSeen, payloadScore,
Modified: lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/SpanTermQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/SpanTermQuery.java?rev=824918&r1=824917&r2=824918&view=diff
==============================================================================
--- lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/SpanTermQuery.java (original)
+++ lucene/java/branches/flex_1458/src/java/org/apache/lucene/search/spans/SpanTermQuery.java Tue Oct 13 20:44:51 2009
@@ -19,6 +19,7 @@
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermRef;
import org.apache.lucene.util.ToStringUtils;
import java.io.IOException;
@@ -76,7 +77,9 @@
}
public Spans getSpans(final IndexReader reader) throws IOException {
- return new TermSpans(reader.termPositions(term), term);
+ return new TermSpans(reader.termDocsEnum(reader.getDeletedDocs(),
+ term.field(),
+ new TermRef(term.text())), term);
}
}