You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by us...@apache.org on 2009/07/04 22:08:56 UTC
svn commit: r791173 [3/4] - in
/lucene/java/branches/lucene_2_4_back_compat_tests: ./
contrib/highlighter/src/test/org/apache/lucene/search/highlight/
src/java/org/apache/lucene/analysis/
src/java/org/apache/lucene/analysis/standard/ src/java/org/apach...
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeFilter.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeFilter.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeFilter.java Sat Jul 4 20:08:54 2009
@@ -18,6 +18,10 @@
*/
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermDocs;
+import org.apache.lucene.index.TermEnum;
+import org.apache.lucene.util.OpenBitSet;
import java.io.IOException;
import java.util.BitSet;
@@ -31,9 +35,6 @@
* This code borrows heavily from {@link RangeQuery}, but is implemented as a Filter
*
* </p>
- *
- * If you construct a large number of range filters with different ranges but on the
- * same field, {@link FieldCacheRangeFilter} may have significantly better performance.
*/
public class RangeFilter extends Filter {
@@ -43,7 +44,6 @@
private boolean includeLower;
private boolean includeUpper;
private Collator collator;
- private RangeQuery rangeQuery;
/**
* @param fieldName The field this range applies to
@@ -75,7 +75,6 @@
throw new IllegalArgumentException
("The upper bound must be non-null to be inclusive");
}
- initRangeQuery();
}
/**
@@ -100,11 +99,6 @@
Collator collator) {
this(fieldName, lowerTerm, upperTerm, includeLower, includeUpper);
this.collator = collator;
- initRangeQuery();
- }
-
- private void initRangeQuery() {
- rangeQuery = new RangeQuery(fieldName, lowerTerm, upperTerm, includeLower, includeUpper, collator);
}
/**
@@ -130,7 +124,81 @@
* @deprecated Use {@link #getDocIdSet(IndexReader)} instead.
*/
public BitSet bits(IndexReader reader) throws IOException {
- return rangeQuery.getFilter().bits(reader);
+ BitSet bits = new BitSet(reader.maxDoc());
+ TermEnum enumerator =
+ (null != lowerTerm && collator == null
+ ? reader.terms(new Term(fieldName, lowerTerm))
+ : reader.terms(new Term(fieldName)));
+
+ try {
+
+ if (enumerator.term() == null) {
+ return bits;
+ }
+
+ TermDocs termDocs = reader.termDocs();
+ try {
+ if (collator != null) {
+ do {
+ Term term = enumerator.term();
+ if (term != null && term.field().equals(fieldName)) {
+ if ((lowerTerm == null
+ || (includeLower
+ ? collator.compare(term.text(), lowerTerm) >= 0
+ : collator.compare(term.text(), lowerTerm) > 0))
+ && (upperTerm == null
+ || (includeUpper
+ ? collator.compare(term.text(), upperTerm) <= 0
+ : collator.compare(term.text(), upperTerm) < 0))) {
+ /* we have a good term, find the docs */
+ termDocs.seek(enumerator.term());
+ while (termDocs.next()) {
+ bits.set(termDocs.doc());
+ }
+ }
+ }
+ }
+ while (enumerator.next());
+ } else { // collator is null - use Unicode code point ordering
+ boolean checkLower = false;
+ if (!includeLower) // make adjustments to set to exclusive
+ checkLower = true;
+
+ do {
+ Term term = enumerator.term();
+ if (term != null && term.field().equals(fieldName)) {
+ if (!checkLower || null==lowerTerm || term.text().compareTo(lowerTerm) > 0) {
+ checkLower = false;
+ if (upperTerm != null) {
+ int compare = upperTerm.compareTo(term.text());
+ /* if beyond the upper term, or is exclusive and
+ * this is equal to the upper term, break out */
+ if ((compare < 0) ||
+ (!includeUpper && compare==0)) {
+ break;
+ }
+ }
+ /* we have a good term, find the docs */
+
+ termDocs.seek(enumerator.term());
+ while (termDocs.next()) {
+ bits.set(termDocs.doc());
+ }
+ }
+ } else {
+ break;
+ }
+ }
+ while (enumerator.next());
+ }
+ } finally {
+ termDocs.close();
+ }
+ } finally {
+ enumerator.close();
+ }
+
+ return bits;
}
/**
@@ -138,7 +206,84 @@
* permitted in search results.
*/
public DocIdSet getDocIdSet(IndexReader reader) throws IOException {
- return rangeQuery.getFilter().getDocIdSet(reader);
+ OpenBitSet bits = new OpenBitSet(reader.maxDoc());
+
+ TermEnum enumerator =
+ (null != lowerTerm && collator == null
+ ? reader.terms(new Term(fieldName, lowerTerm))
+ : reader.terms(new Term(fieldName)));
+
+ try {
+
+ if (enumerator.term() == null) {
+ return bits;
+ }
+
+ TermDocs termDocs = reader.termDocs();
+
+ try {
+ if (collator != null) {
+ do {
+ Term term = enumerator.term();
+ if (term != null && term.field().equals(fieldName)) {
+ if ((lowerTerm == null
+ || (includeLower
+ ? collator.compare(term.text(), lowerTerm) >= 0
+ : collator.compare(term.text(), lowerTerm) > 0))
+ && (upperTerm == null
+ || (includeUpper
+ ? collator.compare(term.text(), upperTerm) <= 0
+ : collator.compare(term.text(), upperTerm) < 0))) {
+ /* we have a good term, find the docs */
+ termDocs.seek(enumerator.term());
+ while (termDocs.next()) {
+ bits.set(termDocs.doc());
+ }
+ }
+ }
+ }
+ while (enumerator.next());
+ } else { // collator is null - use Unicode code point ordering
+ boolean checkLower = false;
+ if (!includeLower) // make adjustments to set to exclusive
+ checkLower = true;
+
+ do {
+ Term term = enumerator.term();
+ if (term != null && term.field().equals(fieldName)) {
+ if (!checkLower || null==lowerTerm || term.text().compareTo(lowerTerm) > 0) {
+ checkLower = false;
+ if (upperTerm != null) {
+ int compare = upperTerm.compareTo(term.text());
+ /* if beyond the upper term, or is exclusive and
+ * this is equal to the upper term, break out */
+ if ((compare < 0) ||
+ (!includeUpper && compare==0)) {
+ break;
+ }
+ }
+ /* we have a good term, find the docs */
+
+ termDocs.seek(enumerator.term());
+ while (termDocs.next()) {
+ bits.set(termDocs.doc());
+ }
+ }
+ } else {
+ break;
+ }
+ }
+ while (enumerator.next());
+ }
+
+ } finally {
+ termDocs.close();
+ }
+ } finally {
+ enumerator.close();
+ }
+
+ return bits;
}
public String toString() {
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeQuery.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeQuery.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/RangeQuery.java Sat Jul 4 20:08:54 2009
@@ -21,214 +21,234 @@
import java.text.Collator;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.TermEnum;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.util.ToStringUtils;
/**
* A Query that matches documents within an exclusive range. A RangeQuery
* is built by QueryParser for input like <code>[010 TO 120]</code> but only if the QueryParser has
* the useOldRangeQuery property set to true. The QueryParser default behaviour is to use
- * the newer ConstantScore mode. This is generally preferable because:
+ * the newer ConstantScoreRangeQuery class. This is generally preferable because:
* <ul>
- * <li>It is faster than the standard RangeQuery mode</li>
- * <li>Unlike the RangeQuery mode, it does not cause a BooleanQuery.TooManyClauses exception if the range of values is large</li>
- * <li>Unlike the RangeQuery mode, it does not influence scoring based on the scarcity of individual terms that may match</li>
+ * <li>It is faster than RangeQuery</li>
+ * <li>Unlike RangeQuery, it does not cause a BooleanQuery.TooManyClauses exception if the range of values is large</li>
+ * <li>Unlike RangeQuery it does not influence scoring based on the scarcity of individual terms that may match</li>
* </ul>
*
+ *
+ * @see ConstantScoreRangeQuery
+ *
*
* @version $Id$
*/
-public class RangeQuery extends MultiTermQuery {
- private Term lowerTerm;
- private Term upperTerm;
- private Collator collator;
- private String field;
- private boolean includeLower;
- private boolean includeUpper;
-
-
- /**
- * Constructs a query selecting all terms greater/equal than <code>lowerTerm</code>
- * but less/equal than <code>upperTerm</code>.
- *
- * <p>
- * If an endpoint is null, it is said
- * to be "open". Either or both endpoints may be open. Open endpoints may not
- * be exclusive (you can't select all but the first or last term without
- * explicitly specifying the term to exclude.)
- *
- * @param field The field that holds both lower and upper terms.
- * @param lowerTerm
- * The term text at the lower end of the range
- * @param upperTerm
- * The term text at the upper end of the range
- * @param includeLower
- * If true, the <code>lowerTerm</code> is
- * included in the range.
- * @param includeUpper
- * If true, the <code>upperTerm</code> is
- * included in the range.
- */
- public RangeQuery(String field, String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper) {
- init(new Term(field, lowerTerm), new Term(field, upperTerm), includeLower, includeUpper, null);
- }
-
- /** Constructs a query selecting all terms greater/equal than
- * <code>lowerTerm</code> but less/equal than <code>upperTerm</code>.
- * <p>
- * If an endpoint is null, it is said
- * to be "open". Either or both endpoints may be open. Open endpoints may not
- * be exclusive (you can't select all but the first or last term without
- * explicitly specifying the term to exclude.)
- * <p>
- * If <code>collator</code> is not null, it will be used to decide whether
- * index terms are within the given range, rather than using the Unicode code
- * point order in which index terms are stored.
- * <p>
- * <strong>WARNING:</strong> Using this constructor and supplying a non-null
- * value in the <code>collator</code> parameter will cause every single
- * index Term in the Field referenced by lowerTerm and/or upperTerm to be
- * examined. Depending on the number of index Terms in this Field, the
- * operation could be very slow.
- *
- * @param lowerTerm The Term text at the lower end of the range
- * @param upperTerm The Term text at the upper end of the range
- * @param includeLower
- * If true, the <code>lowerTerm</code> is
- * included in the range.
- * @param includeUpper
- * If true, the <code>upperTerm</code> is
- * included in the range.
- * @param collator The collator to use to collate index Terms, to determine
- * their membership in the range bounded by <code>lowerTerm</code> and
- * <code>upperTerm</code>.
- */
- public RangeQuery(String field, String lowerTerm, String upperTerm, boolean includeLower, boolean includeUpper,
- Collator collator) {
- init(new Term(field, lowerTerm), new Term(field,upperTerm), includeLower, includeUpper, collator);
- }
-
- /** @deprecated Please use {@link #RangeQuery(String,
- * String, String, boolean, boolean, Collator)} instead */
- public RangeQuery(Term lowerTerm, Term upperTerm, boolean inclusive,
- Collator collator) {
- init(lowerTerm, upperTerm, inclusive, inclusive, collator);
- }
-
- /** @deprecated Please use {@link #RangeQuery(String,
- * String, String, boolean, boolean)} instead */
- public RangeQuery(Term lowerTerm, Term upperTerm, boolean inclusive) {
- init(lowerTerm, upperTerm, inclusive, inclusive, null);
- }
-
- private void init(Term lowerTerm, Term upperTerm, boolean includeLower, boolean includeUpper, Collator collator) {
- if (lowerTerm == null && upperTerm == null)
- throw new IllegalArgumentException("At least one term must be non-null");
- if (lowerTerm != null && upperTerm != null && lowerTerm.field() != upperTerm.field())
- throw new IllegalArgumentException("Both terms must be for the same field");
-
- if (lowerTerm == null)
- this.field = upperTerm.field();
- else
- this.field = lowerTerm.field();
- this.lowerTerm = lowerTerm;
- this.upperTerm = upperTerm;
- this.includeLower = includeLower;
- this.includeUpper = includeUpper;
- this.collator = collator;
- }
-
- /** Returns the field name for this query */
- public String getField() {
- return field;
- }
-
- /** Returns the lower term of this range query.
- * @deprecated Use {@link #getLowerTermText} instead. */
- public Term getLowerTerm() { return lowerTerm; }
-
- /** Returns the upper term of this range query.
- * @deprecated Use {@link #getUpperTermText} instead. */
- public Term getUpperTerm() { return upperTerm; }
-
- /** Returns the lower value of this range query */
- public String getLowerTermText() { return lowerTerm == null ? null : lowerTerm.text(); }
-
- /** Returns the upper value of this range query */
- public String getUpperTermText() { return upperTerm == null ? null : upperTerm.text(); }
-
- /** Returns <code>true</code> if the lower endpoint is inclusive */
- public boolean includesLower() { return includeLower; }
-
- /** Returns <code>true</code> if the upper endpoint is inclusive */
- public boolean includesUpper() { return includeUpper; }
-
- /** Returns <code>true</code> if the range query is inclusive
- * @deprecated Use {@link #includesLower}, {@link #includesUpper} instead.
- */
- public boolean isInclusive() { return includeUpper && includeLower; }
-
- /** Returns the collator used to determine range inclusion, if any. */
- public Collator getCollator() { return collator; }
-
- protected FilteredTermEnum getEnum(IndexReader reader) throws IOException {
- //TODO: when the deprecated 'Term' constructors are removed we can remove these null checks
- return new RangeTermEnum(reader, collator, getField(), lowerTerm == null ? null : lowerTerm.text(),
- upperTerm == null ? null : upperTerm.text(), includeLower, includeUpper);
- }
-
- /** Prints a user-readable version of this query. */
- public String toString(String field) {
- StringBuffer buffer = new StringBuffer();
- if (!getField().equals(field)) {
- buffer.append(getField());
- buffer.append(":");
- }
- buffer.append(includeLower ? '[' : '{');
- buffer.append(lowerTerm != null ? lowerTerm.text() : "*");
- buffer.append(" TO ");
- buffer.append(upperTerm != null ? upperTerm.text() : "*");
- buffer.append(includeUpper ? ']' : '}');
- if (getBoost() != 1.0f) {
- buffer.append("^");
- buffer.append(Float.toString(getBoost()));
- }
- return buffer.toString();
- }
-
- /** Returns true iff <code>o</code> is equal to this. */
- public boolean equals(Object o) {
- if (this == o) return true;
- if (!(o instanceof RangeQuery)) return false;
- RangeQuery other = (RangeQuery) o;
-
- if (this.field != other.field // interned comparison
- || this.includeLower != other.includeLower
- || this.includeUpper != other.includeUpper
- || (this.collator != null && ! this.collator.equals(other.collator))
- ) { return false; }
- String lowerVal = this.lowerTerm == null ? null : lowerTerm.text();
- String upperVal = this.upperTerm == null ? null : upperTerm.text();
- String olowerText = other.lowerTerm == null ? null : other.lowerTerm.text();
- String oupperText = other.upperTerm == null ? null : other.upperTerm.text();
- if (lowerVal != null ? !lowerVal.equals(olowerText) : olowerText != null) return false;
- if (upperVal != null ? !upperVal.equals(oupperText) : oupperText != null) return false;
- return this.getBoost() == other.getBoost();
- }
-
- /** Returns a hash code value for this object.*/
- public int hashCode() {
- int h = Float.floatToIntBits(getBoost()) ^ field.hashCode();
- String lowerVal = this.lowerTerm == null ? null : lowerTerm.text();
- String upperVal = this.upperTerm == null ? null : upperTerm.text();
- // hashCode of "" is 0, so don't use that for null...
- h ^= lowerVal != null ? lowerVal.hashCode() : 0x965a965a;
- // don't just XOR upperVal with out mixing either it or h, as it will cancel
- // out lowerVal if they are equal.
- h ^= (h << 17) | (h >>> 16); // a reversible (one to one) 32 bit mapping mix
- h ^= (upperVal != null ? (upperVal.hashCode()) : 0x5a695a69);
- h ^= (includeLower ? 0x665599aa : 0)
- ^ (includeUpper ? 0x99aa5566 : 0);
- h ^= collator != null ? collator.hashCode() : 0;
- return h;
- }
+public class RangeQuery extends Query
+{
+ private Term lowerTerm;
+ private Term upperTerm;
+ private boolean inclusive;
+ private Collator collator;
+
+ /** Constructs a query selecting all terms greater than
+ * <code>lowerTerm</code> but less than <code>upperTerm</code>.
+ * There must be at least one term and either term may be null,
+ * in which case there is no bound on that side, but if there are
+ * two terms, both terms <b>must</b> be for the same field.
+ *
+ * @param lowerTerm The Term at the lower end of the range
+ * @param upperTerm The Term at the upper end of the range
+ * @param inclusive If true, both <code>lowerTerm</code> and
+ * <code>upperTerm</code> will themselves be included in the range.
+ */
+ public RangeQuery(Term lowerTerm, Term upperTerm, boolean inclusive)
+ {
+ if (lowerTerm == null && upperTerm == null)
+ {
+ throw new IllegalArgumentException("At least one term must be non-null");
+ }
+ if (lowerTerm != null && upperTerm != null && lowerTerm.field() != upperTerm.field())
+ {
+ throw new IllegalArgumentException("Both terms must be for the same field");
+ }
+
+ // if we have a lowerTerm, start there. otherwise, start at beginning
+ if (lowerTerm != null) {
+ this.lowerTerm = lowerTerm;
+ }
+ else {
+ this.lowerTerm = new Term(upperTerm.field());
+ }
+
+ this.upperTerm = upperTerm;
+ this.inclusive = inclusive;
+ }
+
+ /** Constructs a query selecting all terms greater than
+ * <code>lowerTerm</code> but less than <code>upperTerm</code>.
+ * There must be at least one term and either term may be null,
+ * in which case there is no bound on that side, but if there are
+ * two terms, both terms <b>must</b> be for the same field.
+ * <p>
+ * If <code>collator</code> is not null, it will be used to decide whether
+ * index terms are within the given range, rather than using the Unicode code
+ * point order in which index terms are stored.
+ * <p>
+ * <strong>WARNING:</strong> Using this constructor and supplying a non-null
+ * value in the <code>collator</code> parameter will cause every single
+ * index Term in the Field referenced by lowerTerm and/or upperTerm to be
+ * examined. Depending on the number of index Terms in this Field, the
+ * operation could be very slow.
+ *
+ * @param lowerTerm The Term at the lower end of the range
+ * @param upperTerm The Term at the upper end of the range
+ * @param inclusive If true, both <code>lowerTerm</code> and
+ * <code>upperTerm</code> will themselves be included in the range.
+ * @param collator The collator to use to collate index Terms, to determine
+ * their membership in the range bounded by <code>lowerTerm</code> and
+ * <code>upperTerm</code>.
+ */
+ public RangeQuery(Term lowerTerm, Term upperTerm, boolean inclusive,
+ Collator collator)
+ {
+ this(lowerTerm, upperTerm, inclusive);
+ this.collator = collator;
+ }
+
+ public Query rewrite(IndexReader reader) throws IOException {
+
+ BooleanQuery query = new BooleanQuery(true);
+ String testField = getField();
+ if (collator != null) {
+ TermEnum enumerator = reader.terms(new Term(testField, ""));
+ String lowerTermText = lowerTerm != null ? lowerTerm.text() : null;
+ String upperTermText = upperTerm != null ? upperTerm.text() : null;
+
+ try {
+ do {
+ Term term = enumerator.term();
+ if (term != null && term.field() == testField) { // interned comparison
+ if ((lowerTermText == null
+ || (inclusive ? collator.compare(term.text(), lowerTermText) >= 0
+ : collator.compare(term.text(), lowerTermText) > 0))
+ && (upperTermText == null
+ || (inclusive ? collator.compare(term.text(), upperTermText) <= 0
+ : collator.compare(term.text(), upperTermText) < 0))) {
+ addTermToQuery(term, query);
+ }
+ }
+ }
+ while (enumerator.next());
+ }
+ finally {
+ enumerator.close();
+ }
+ }
+ else { // collator is null
+ TermEnum enumerator = reader.terms(lowerTerm);
+
+ try {
+
+ boolean checkLower = false;
+ if (!inclusive) // make adjustments to set to exclusive
+ checkLower = true;
+
+ do {
+ Term term = enumerator.term();
+ if (term != null && term.field() == testField) { // interned comparison
+ if (!checkLower || term.text().compareTo(lowerTerm.text()) > 0) {
+ checkLower = false;
+ if (upperTerm != null) {
+ int compare = upperTerm.text().compareTo(term.text());
+ /* if beyond the upper term, or is exclusive and
+ * this is equal to the upper term, break out */
+ if ((compare < 0) || (!inclusive && compare == 0))
+ break;
+ }
+ addTermToQuery(term, query); // Found a match
+ }
+ }
+ else {
+ break;
+ }
+ }
+ while (enumerator.next());
+ }
+ finally {
+ enumerator.close();
+ }
+ }
+ return query;
+ }
+
+ private void addTermToQuery(Term term, BooleanQuery query) {
+ TermQuery tq = new TermQuery(term);
+ tq.setBoost(getBoost()); // set the boost
+ query.add(tq, BooleanClause.Occur.SHOULD); // add to query
+ }
+
+ /** Returns the field name for this query */
+ public String getField() {
+ return (lowerTerm != null ? lowerTerm.field() : upperTerm.field());
+ }
+
+ /** Returns the lower term of this range query */
+ public Term getLowerTerm() { return lowerTerm; }
+
+ /** Returns the upper term of this range query */
+ public Term getUpperTerm() { return upperTerm; }
+
+ /** Returns <code>true</code> if the range query is inclusive */
+ public boolean isInclusive() { return inclusive; }
+
+ /** Returns the collator used to determine range inclusion, if any. */
+ public Collator getCollator() { return collator; }
+
+
+ /** Prints a user-readable version of this query. */
+ public String toString(String field)
+ {
+ StringBuffer buffer = new StringBuffer();
+ if (!getField().equals(field))
+ {
+ buffer.append(getField());
+ buffer.append(":");
+ }
+ buffer.append(inclusive ? "[" : "{");
+ buffer.append(lowerTerm != null ? lowerTerm.text() : "null");
+ buffer.append(" TO ");
+ buffer.append(upperTerm != null ? upperTerm.text() : "null");
+ buffer.append(inclusive ? "]" : "}");
+ buffer.append(ToStringUtils.boost(getBoost()));
+ return buffer.toString();
+ }
+
+ /** Returns true iff <code>o</code> is equal to this. */
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (!(o instanceof RangeQuery)) return false;
+
+ final RangeQuery other = (RangeQuery) o;
+ if (this.getBoost() != other.getBoost()) return false;
+ if (this.inclusive != other.inclusive) return false;
+ if (this.collator != null && ! this.collator.equals(other.collator))
+ return false;
+
+ // one of lowerTerm and upperTerm can be null
+ if (this.lowerTerm != null ? !this.lowerTerm.equals(other.lowerTerm) : other.lowerTerm != null) return false;
+ if (this.upperTerm != null ? !this.upperTerm.equals(other.upperTerm) : other.upperTerm != null) return false;
+ return true;
+ }
+
+ /** Returns a hash code value for this object.*/
+ public int hashCode() {
+ int h = Float.floatToIntBits(getBoost());
+ h ^= lowerTerm != null ? lowerTerm.hashCode() : 0;
+ // reversible mix to make lower and upper position dependent and
+ // to prevent them from cancelling out.
+ h ^= (h << 25) | (h >>> 8);
+ h ^= upperTerm != null ? upperTerm.hashCode() : 0;
+ h ^= this.inclusive ? 0x2742E74A : 0;
+ h ^= collator != null ? collator.hashCode() : 0;
+ return h;
+ }
}
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardQuery.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardQuery.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardQuery.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardQuery.java Sat Jul 4 20:08:54 2009
@@ -48,11 +48,12 @@
return false;
}
-
+
public Query rewrite(IndexReader reader) throws IOException {
- if (!termContainsWildcard)
+ if (this.termContainsWildcard) {
+ return super.rewrite(reader);
+ }
+
return new TermQuery(getTerm());
- else
- return super.rewrite(reader);
}
}
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardTermEnum.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardTermEnum.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardTermEnum.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/java/org/apache/lucene/search/WildcardTermEnum.java Sat Jul 4 20:08:54 2009
@@ -40,7 +40,9 @@
boolean endEnum = false;
/**
- * Creates a new <code>WildcardTermEnum</code>.
+ * Creates a new <code>WildcardTermEnum</code>. Passing in a
+ * {@link org.apache.lucene.index.Term Term} that does not contain a
+ * <code>WILDCARD_CHAR</code> will cause an exception to be thrown.
* <p>
* After calling the constructor the enumeration is already pointing to the first
* valid term if such a term exists.
@@ -60,12 +62,8 @@
else if (cidx >= 0) {
idx = Math.min(idx, cidx);
}
- if (idx != -1) {
- pre = searchTerm.text().substring(0,idx);
- } else {
- pre = "";
- }
+ pre = searchTerm.text().substring(0,idx);
preLen = pre.length();
text = text.substring(preLen);
setEnum(reader.terms(new Term(searchTerm.field(), pre)));
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TeeSinkTokenTest.java Sat Jul 4 20:08:54 2009
@@ -18,9 +18,6 @@
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.English;
import org.apache.lucene.util.LuceneTestCase;
@@ -43,8 +40,7 @@
super(s);
}
- protected void setUp() throws Exception {
- super.setUp();
+ protected void setUp() {
tokens1 = new String[]{"The", "quick", "Burgundy", "Fox", "jumped", "over", "the", "lazy", "Red", "Dogs"};
tokens2 = new String[]{"The", "Lazy", "Dogs", "should", "stay", "on", "the", "porch"};
buffer1 = new StringBuffer();
@@ -66,29 +62,24 @@
public void test() throws IOException {
SinkTokenizer sink1 = new SinkTokenizer(null) {
- public void add(AttributeSource a) throws IOException {
- TermAttribute termAtt = null;
- if (a.hasAttribute(TermAttribute.class)) {
- termAtt = (TermAttribute) a.getAttribute(TermAttribute.class);
- }
- if (termAtt != null && termAtt.term().equalsIgnoreCase("The")) {
- super.add(a);
+ public void add(Token t) {
+ if (t != null && t.term().equalsIgnoreCase("The")) {
+ super.add(t);
}
}
};
TokenStream source = new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer1.toString())), sink1);
int i = 0;
- TermAttribute termAtt = (TermAttribute) source.getAttribute(TermAttribute.class);
- while (source.incrementToken()) {
- assertTrue(termAtt.term() + " is not equal to " + tokens1[i], termAtt.term().equals(tokens1[i]) == true);
+ final Token reusableToken = new Token();
+ for (Token nextToken = source.next(reusableToken); nextToken != null; nextToken = source.next(reusableToken)) {
+ assertTrue(nextToken.term() + " is not equal to " + tokens1[i], nextToken.term().equals(tokens1[i]) == true);
i++;
}
assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
assertTrue("sink1 Size: " + sink1.getTokens().size() + " is not: " + 2, sink1.getTokens().size() == 2);
i = 0;
- termAtt = (TermAttribute) sink1.getAttribute(TermAttribute.class);
- while (sink1.incrementToken()) {
- assertTrue(termAtt.term() + " is not equal to " + "The", termAtt.term().equalsIgnoreCase("The") == true);
+ for (Token token = sink1.next(reusableToken); token != null; token = sink1.next(reusableToken)) {
+ assertTrue(token.term() + " is not equal to " + "The", token.term().equalsIgnoreCase("The") == true);
i++;
}
assertTrue(i + " does not equal: " + sink1.getTokens().size(), i == sink1.getTokens().size());
@@ -96,67 +87,55 @@
public void testMultipleSources() throws Exception {
SinkTokenizer theDetector = new SinkTokenizer(null) {
- public void add(AttributeSource a) throws IOException {
- TermAttribute termAtt = null;
- if (a.hasAttribute(TermAttribute.class)) {
- termAtt = (TermAttribute) a.getAttribute(TermAttribute.class);
- }
- if (termAtt != null && termAtt.term().equalsIgnoreCase("The")) {
- super.add(a);
+ public void add(Token t) {
+ if (t != null && t.term().equalsIgnoreCase("The")) {
+ super.add(t);
}
}
};
- SinkTokenizer dogDetector = new SinkTokenizer(null) {
- public void add(AttributeSource a) throws IOException {
- TermAttribute termAtt = null;
- if (a.hasAttribute(TermAttribute.class)) {
- termAtt = (TermAttribute) a.getAttribute(TermAttribute.class);
- }
- if (termAtt != null && termAtt.term().equalsIgnoreCase("Dogs")) {
- super.add(a);
+ SinkTokenizer dogDetector = new SinkTokenizer(null) {
+ public void add(Token t) {
+ if (t != null && t.term().equalsIgnoreCase("Dogs")) {
+ super.add(t);
}
}
};
TokenStream source1 = new CachingTokenFilter(new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer1.toString())), theDetector), dogDetector));
TokenStream source2 = new TeeTokenFilter(new TeeTokenFilter(new WhitespaceTokenizer(new StringReader(buffer2.toString())), theDetector), dogDetector);
int i = 0;
- TermAttribute termAtt = (TermAttribute) source1.getAttribute(TermAttribute.class);
- while (source1.incrementToken()) {
- assertTrue(termAtt.term() + " is not equal to " + tokens1[i], termAtt.term().equals(tokens1[i]) == true);
+ final Token reusableToken = new Token();
+ for (Token nextToken = source1.next(reusableToken); nextToken != null; nextToken = source1.next(reusableToken)) {
+ assertTrue(nextToken.term() + " is not equal to " + tokens1[i], nextToken.term().equals(tokens1[i]) == true);
i++;
}
assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
assertTrue("theDetector Size: " + theDetector.getTokens().size() + " is not: " + 2, theDetector.getTokens().size() == 2);
assertTrue("dogDetector Size: " + dogDetector.getTokens().size() + " is not: " + 1, dogDetector.getTokens().size() == 1);
i = 0;
- termAtt = (TermAttribute) source2.getAttribute(TermAttribute.class);
- while (source2.incrementToken()) {
- assertTrue(termAtt.term() + " is not equal to " + tokens2[i], termAtt.term().equals(tokens2[i]) == true);
+ for (Token nextToken = source2.next(reusableToken); nextToken != null; nextToken = source2.next(reusableToken)) {
+ assertTrue(nextToken.term() + " is not equal to " + tokens2[i], nextToken.term().equals(tokens2[i]) == true);
i++;
}
assertTrue(i + " does not equal: " + tokens2.length, i == tokens2.length);
assertTrue("theDetector Size: " + theDetector.getTokens().size() + " is not: " + 4, theDetector.getTokens().size() == 4);
assertTrue("dogDetector Size: " + dogDetector.getTokens().size() + " is not: " + 2, dogDetector.getTokens().size() == 2);
i = 0;
- termAtt = (TermAttribute) theDetector.getAttribute(TermAttribute.class);
- while (theDetector.incrementToken()) {
- assertTrue(termAtt.term() + " is not equal to " + "The", termAtt.term().equalsIgnoreCase("The") == true);
+ for (Token nextToken = theDetector.next(reusableToken); nextToken != null; nextToken = theDetector.next(reusableToken)) {
+ assertTrue(nextToken.term() + " is not equal to " + "The", nextToken.term().equalsIgnoreCase("The") == true);
i++;
}
assertTrue(i + " does not equal: " + theDetector.getTokens().size(), i == theDetector.getTokens().size());
i = 0;
- termAtt = (TermAttribute) dogDetector.getAttribute(TermAttribute.class);
- while (dogDetector.incrementToken()) {
- assertTrue(termAtt.term() + " is not equal to " + "Dogs", termAtt.term().equalsIgnoreCase("Dogs") == true);
+ for (Token nextToken = dogDetector.next(reusableToken); nextToken != null; nextToken = dogDetector.next(reusableToken)) {
+ assertTrue(nextToken.term() + " is not equal to " + "Dogs", nextToken.term().equalsIgnoreCase("Dogs") == true);
i++;
}
assertTrue(i + " does not equal: " + dogDetector.getTokens().size(), i == dogDetector.getTokens().size());
source1.reset();
TokenStream lowerCasing = new LowerCaseFilter(source1);
i = 0;
- termAtt = (TermAttribute) lowerCasing.getAttribute(TermAttribute.class);
- while (lowerCasing.incrementToken()) {
- assertTrue(termAtt.term() + " is not equal to " + tokens1[i].toLowerCase(), termAtt.term().equals(tokens1[i].toLowerCase()) == true);
+ for (Token nextToken = lowerCasing.next(reusableToken); nextToken != null; nextToken = lowerCasing.next(reusableToken)) {
+ assertTrue(nextToken.term() + " is not equal to " + tokens1[i].toLowerCase(), nextToken.term().equals(tokens1[i].toLowerCase()) == true);
i++;
}
assertTrue(i + " does not equal: " + tokens1.length, i == tokens1.length);
@@ -178,20 +157,21 @@
}
//make sure we produce the same tokens
ModuloSinkTokenizer sink = new ModuloSinkTokenizer(tokCount[k], 100);
+ final Token reusableToken = new Token();
TokenStream stream = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), sink);
- while (stream.incrementToken()) {
+ while (stream.next(reusableToken) != null) {
}
stream = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), 100);
List tmp = new ArrayList();
- while (stream.incrementToken()) {
- tmp.add(stream.captureState());
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ tmp.add(nextToken.clone());
}
List sinkList = sink.getTokens();
assertTrue("tmp Size: " + tmp.size() + " is not: " + sinkList.size(), tmp.size() == sinkList.size());
for (int i = 0; i < tmp.size(); i++) {
- AttributeSource tfTok = (AttributeSource) tmp.get(i);
- AttributeSource sinkTok = (AttributeSource) sinkList.get(i);
- assertTrue(tfTok + " is not equal to " + sinkTok + " at token: " + i, tfTok.equals(sinkTok) == true);
+ Token tfTok = (Token) tmp.get(i);
+ Token sinkTok = (Token) sinkList.get(i);
+ assertTrue(tfTok.term() + " is not equal to " + sinkTok.term() + " at token: " + i, tfTok.term().equals(sinkTok.term()) == true);
}
//simulate two fields, each being analyzed once, for 20 documents
@@ -200,14 +180,12 @@
long start = System.currentTimeMillis();
for (int i = 0; i < 20; i++) {
stream = new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString())));
- PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) stream.getAttribute(PositionIncrementAttribute.class);
- while (stream.incrementToken()) {
- tfPos += posIncrAtt.getPositionIncrement();
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ tfPos += nextToken.getPositionIncrement();
}
stream = new ModuloTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), modCounts[j]);
- posIncrAtt = (PositionIncrementAttribute) stream.getAttribute(PositionIncrementAttribute.class);
- while (stream.incrementToken()) {
- tfPos += posIncrAtt.getPositionIncrement();
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ tfPos += nextToken.getPositionIncrement();
}
}
long finish = System.currentTimeMillis();
@@ -218,15 +196,13 @@
for (int i = 0; i < 20; i++) {
sink = new ModuloSinkTokenizer(tokCount[k], modCounts[j]);
stream = new TeeTokenFilter(new StandardFilter(new StandardTokenizer(new StringReader(buffer.toString()))), sink);
- PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) stream.getAttribute(PositionIncrementAttribute.class);
- while (stream.incrementToken()) {
- sinkPos += posIncrAtt.getPositionIncrement();
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ sinkPos += nextToken.getPositionIncrement();
}
//System.out.println("Modulo--------");
stream = sink;
- posIncrAtt = (PositionIncrementAttribute) stream.getAttribute(PositionIncrementAttribute.class);
- while (stream.incrementToken()) {
- sinkPos += posIncrAtt.getPositionIncrement();
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ sinkPos += nextToken.getPositionIncrement();
}
}
finish = System.currentTimeMillis();
@@ -252,15 +228,15 @@
int count = 0;
//return every 100 tokens
- public boolean incrementToken() throws IOException {
- boolean hasNext;
- for (hasNext = input.incrementToken();
- hasNext && count % modCount != 0;
- hasNext = input.incrementToken()) {
+ public Token next(final Token reusableToken) throws IOException {
+ Token nextToken = null;
+ for (nextToken = input.next(reusableToken);
+ nextToken != null && count % modCount != 0;
+ nextToken = input.next(reusableToken)) {
count++;
}
count++;
- return hasNext;
+ return nextToken;
}
}
@@ -274,9 +250,9 @@
lst = new ArrayList(numToks % mc);
}
- public void add(AttributeSource a) throws IOException {
- if (a != null && count % modCount == 0) {
- super.add(a);
+ public void add(Token t) {
+ if (t != null && count % modCount == 0) {
+ super.add(t);
}
count++;
}
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestAnalyzers.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestAnalyzers.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestAnalyzers.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestAnalyzers.java Sat Jul 4 20:08:54 2009
@@ -19,10 +19,10 @@
import java.io.IOException;
import java.io.StringReader;
+import java.util.LinkedList;
+import java.util.List;
import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.index.Payload;
import org.apache.lucene.util.LuceneTestCase;
@@ -36,12 +36,13 @@
String input,
String[] output) throws Exception {
TokenStream ts = a.tokenStream("dummy", new StringReader(input));
- TermAttribute termAtt = (TermAttribute) ts.getAttribute(TermAttribute.class);
+ final Token reusableToken = new Token();
for (int i=0; i<output.length; i++) {
- assertTrue(ts.incrementToken());
- assertEquals(termAtt.term(), output[i]);
+ Token nextToken = ts.next(reusableToken);
+ assertNotNull(nextToken);
+ assertEquals(nextToken.term(), output[i]);
}
- assertFalse(ts.incrementToken());
+ assertNull(ts.next(reusableToken));
ts.close();
}
@@ -94,13 +95,14 @@
}
void verifyPayload(TokenStream ts) throws IOException {
- PayloadAttribute payloadAtt = (PayloadAttribute) ts.getAttribute(PayloadAttribute.class);
+ final Token reusableToken = new Token();
for(byte b=1;;b++) {
- boolean hasNext = ts.incrementToken();
- if (!hasNext) break;
+ reusableToken.clear();
+ Token nextToken = ts.next(reusableToken);
+ if (nextToken==null) break;
// System.out.println("id="+System.identityHashCode(nextToken) + " " + t);
// System.out.println("payload=" + (int)nextToken.getPayload().toByteArray()[0]);
- assertEquals(b, payloadAtt.getPayload().toByteArray()[0]);
+ assertEquals(b, nextToken.getPayload().toByteArray()[0]);
}
}
@@ -109,11 +111,13 @@
String s = "how now brown cow";
TokenStream ts;
ts = new WhitespaceTokenizer(new StringReader(s));
+ ts = new BuffTokenFilter(ts);
ts = new PayloadSetter(ts);
verifyPayload(ts);
ts = new WhitespaceTokenizer(new StringReader(s));
ts = new PayloadSetter(ts);
+ ts = new BuffTokenFilter(ts);
verifyPayload(ts);
}
@@ -132,21 +136,38 @@
}
}
+class BuffTokenFilter extends TokenFilter {
+ List lst;
+
+ public BuffTokenFilter(TokenStream input) {
+ super(input);
+ }
+
+ public Token next(final Token reusableToken) throws IOException {
+ if (lst == null) {
+ lst = new LinkedList();
+ for(Token nextToken = input.next(reusableToken); nextToken != null; nextToken = input.next(reusableToken)) {
+ lst.add(nextToken.clone());
+ }
+ }
+ return lst.size()==0 ? null : (Token)lst.remove(0);
+ }
+}
+
class PayloadSetter extends TokenFilter {
- PayloadAttribute payloadAtt;
public PayloadSetter(TokenStream input) {
super(input);
- payloadAtt = (PayloadAttribute) addAttribute(PayloadAttribute.class);
}
byte[] data = new byte[1];
Payload p = new Payload(data,0,1);
- public boolean incrementToken() throws IOException {
- boolean hasNext = input.incrementToken();
- if (!hasNext) return false;
- payloadAtt.setPayload(p); // reuse the payload / byte[]
+ public Token next(final Token reusableToken) throws IOException {
+ assert reusableToken != null;
+ Token nextToken = input.next(reusableToken);
+ if (nextToken==null) return null;
+ nextToken.setPayload(p); // reuse the payload / byte[]
data[0]++;
- return true;
+ return nextToken;
}
}
\ No newline at end of file
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestCachingTokenFilter.java Sat Jul 4 20:08:54 2009
@@ -22,8 +22,6 @@
import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Field.TermVector;
@@ -43,16 +41,13 @@
Document doc = new Document();
TokenStream stream = new TokenStream() {
private int index = 0;
- private TermAttribute termAtt = (TermAttribute) addAttribute(TermAttribute.class);
- private OffsetAttribute offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
- public boolean incrementToken() throws IOException {
+ public Token next(final Token reusableToken) throws IOException {
+ assert reusableToken != null;
if (index == tokens.length) {
- return false;
+ return null;
} else {
- termAtt.setTermBuffer(tokens[index++]);
- offsetAtt.setOffset(0, 0);
- return true;
+ return reusableToken.reinit(tokens[index++], 0, 0);
}
}
@@ -97,12 +92,10 @@
private void checkTokens(TokenStream stream) throws IOException {
int count = 0;
-
- TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
- assertNotNull(termAtt);
- while (stream.incrementToken()) {
+ final Token reusableToken = new Token();
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
assertTrue(count < tokens.length);
- assertEquals(tokens[count], termAtt.term());
+ assertEquals(tokens[count], nextToken.term());
count++;
}
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestISOLatin1AccentFilter.java Sat Jul 4 20:08:54 2009
@@ -17,7 +17,6 @@
* limitations under the License.
*/
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.util.LuceneTestCase;
import java.io.StringReader;
@@ -26,87 +25,82 @@
public void testU() throws Exception {
TokenStream stream = new WhitespaceTokenizer(new StringReader("Des mot clés à LA CHAÃNE à à à à à Ã
à à à à à à à à à à IJ à à à à à à à à Šà à à à à à Ÿ à á â ã ä Ã¥ æ ç è é ê ë ì à î ï ij ð ñ ò ó ô õ ö ø Šà þ ù ú û ü ý ÿ ï¬ ï¬"));
ISOLatin1AccentFilter filter = new ISOLatin1AccentFilter(stream);
- TermAttribute termAtt = (TermAttribute) filter.getAttribute(TermAttribute.class);
- assertTermEquals("Des", filter, termAtt);
- assertTermEquals("mot", filter, termAtt);
- assertTermEquals("cles", filter, termAtt);
- assertTermEquals("A", filter, termAtt);
- assertTermEquals("LA", filter, termAtt);
- assertTermEquals("CHAINE", filter, termAtt);
- assertTermEquals("A", filter, termAtt);
- assertTermEquals("A", filter, termAtt);
- assertTermEquals("A", filter, termAtt);
- assertTermEquals("A", filter, termAtt);
- assertTermEquals("A", filter, termAtt);
- assertTermEquals("A", filter, termAtt);
- assertTermEquals("AE", filter, termAtt);
- assertTermEquals("C", filter, termAtt);
- assertTermEquals("E", filter, termAtt);
- assertTermEquals("E", filter, termAtt);
- assertTermEquals("E", filter, termAtt);
- assertTermEquals("E", filter, termAtt);
- assertTermEquals("I", filter, termAtt);
- assertTermEquals("I", filter, termAtt);
- assertTermEquals("I", filter, termAtt);
- assertTermEquals("I", filter, termAtt);
- assertTermEquals("IJ", filter, termAtt);
- assertTermEquals("D", filter, termAtt);
- assertTermEquals("N", filter, termAtt);
- assertTermEquals("O", filter, termAtt);
- assertTermEquals("O", filter, termAtt);
- assertTermEquals("O", filter, termAtt);
- assertTermEquals("O", filter, termAtt);
- assertTermEquals("O", filter, termAtt);
- assertTermEquals("O", filter, termAtt);
- assertTermEquals("OE", filter, termAtt);
- assertTermEquals("TH", filter, termAtt);
- assertTermEquals("U", filter, termAtt);
- assertTermEquals("U", filter, termAtt);
- assertTermEquals("U", filter, termAtt);
- assertTermEquals("U", filter, termAtt);
- assertTermEquals("Y", filter, termAtt);
- assertTermEquals("Y", filter, termAtt);
- assertTermEquals("a", filter, termAtt);
- assertTermEquals("a", filter, termAtt);
- assertTermEquals("a", filter, termAtt);
- assertTermEquals("a", filter, termAtt);
- assertTermEquals("a", filter, termAtt);
- assertTermEquals("a", filter, termAtt);
- assertTermEquals("ae", filter, termAtt);
- assertTermEquals("c", filter, termAtt);
- assertTermEquals("e", filter, termAtt);
- assertTermEquals("e", filter, termAtt);
- assertTermEquals("e", filter, termAtt);
- assertTermEquals("e", filter, termAtt);
- assertTermEquals("i", filter, termAtt);
- assertTermEquals("i", filter, termAtt);
- assertTermEquals("i", filter, termAtt);
- assertTermEquals("i", filter, termAtt);
- assertTermEquals("ij", filter, termAtt);
- assertTermEquals("d", filter, termAtt);
- assertTermEquals("n", filter, termAtt);
- assertTermEquals("o", filter, termAtt);
- assertTermEquals("o", filter, termAtt);
- assertTermEquals("o", filter, termAtt);
- assertTermEquals("o", filter, termAtt);
- assertTermEquals("o", filter, termAtt);
- assertTermEquals("o", filter, termAtt);
- assertTermEquals("oe", filter, termAtt);
- assertTermEquals("ss", filter, termAtt);
- assertTermEquals("th", filter, termAtt);
- assertTermEquals("u", filter, termAtt);
- assertTermEquals("u", filter, termAtt);
- assertTermEquals("u", filter, termAtt);
- assertTermEquals("u", filter, termAtt);
- assertTermEquals("y", filter, termAtt);
- assertTermEquals("y", filter, termAtt);
- assertTermEquals("fi", filter, termAtt);
- assertTermEquals("fl", filter, termAtt);
- assertFalse(filter.incrementToken());
- }
-
- void assertTermEquals(String expected, TokenStream stream, TermAttribute termAtt) throws Exception {
- assertTrue(stream.incrementToken());
- assertEquals(expected, termAtt.term());
+ final Token reusableToken = new Token();
+ assertEquals("Des", filter.next(reusableToken).term());
+ assertEquals("mot", filter.next(reusableToken).term());
+ assertEquals("cles", filter.next(reusableToken).term());
+ assertEquals("A", filter.next(reusableToken).term());
+ assertEquals("LA", filter.next(reusableToken).term());
+ assertEquals("CHAINE", filter.next(reusableToken).term());
+ assertEquals("A", filter.next(reusableToken).term());
+ assertEquals("A", filter.next(reusableToken).term());
+ assertEquals("A", filter.next(reusableToken).term());
+ assertEquals("A", filter.next(reusableToken).term());
+ assertEquals("A", filter.next(reusableToken).term());
+ assertEquals("A", filter.next(reusableToken).term());
+ assertEquals("AE", filter.next(reusableToken).term());
+ assertEquals("C", filter.next(reusableToken).term());
+ assertEquals("E", filter.next(reusableToken).term());
+ assertEquals("E", filter.next(reusableToken).term());
+ assertEquals("E", filter.next(reusableToken).term());
+ assertEquals("E", filter.next(reusableToken).term());
+ assertEquals("I", filter.next(reusableToken).term());
+ assertEquals("I", filter.next(reusableToken).term());
+ assertEquals("I", filter.next(reusableToken).term());
+ assertEquals("I", filter.next(reusableToken).term());
+ assertEquals("IJ", filter.next(reusableToken).term());
+ assertEquals("D", filter.next(reusableToken).term());
+ assertEquals("N", filter.next(reusableToken).term());
+ assertEquals("O", filter.next(reusableToken).term());
+ assertEquals("O", filter.next(reusableToken).term());
+ assertEquals("O", filter.next(reusableToken).term());
+ assertEquals("O", filter.next(reusableToken).term());
+ assertEquals("O", filter.next(reusableToken).term());
+ assertEquals("O", filter.next(reusableToken).term());
+ assertEquals("OE", filter.next(reusableToken).term());
+ assertEquals("TH", filter.next(reusableToken).term());
+ assertEquals("U", filter.next(reusableToken).term());
+ assertEquals("U", filter.next(reusableToken).term());
+ assertEquals("U", filter.next(reusableToken).term());
+ assertEquals("U", filter.next(reusableToken).term());
+ assertEquals("Y", filter.next(reusableToken).term());
+ assertEquals("Y", filter.next(reusableToken).term());
+ assertEquals("a", filter.next(reusableToken).term());
+ assertEquals("a", filter.next(reusableToken).term());
+ assertEquals("a", filter.next(reusableToken).term());
+ assertEquals("a", filter.next(reusableToken).term());
+ assertEquals("a", filter.next(reusableToken).term());
+ assertEquals("a", filter.next(reusableToken).term());
+ assertEquals("ae", filter.next(reusableToken).term());
+ assertEquals("c", filter.next(reusableToken).term());
+ assertEquals("e", filter.next(reusableToken).term());
+ assertEquals("e", filter.next(reusableToken).term());
+ assertEquals("e", filter.next(reusableToken).term());
+ assertEquals("e", filter.next(reusableToken).term());
+ assertEquals("i", filter.next(reusableToken).term());
+ assertEquals("i", filter.next(reusableToken).term());
+ assertEquals("i", filter.next(reusableToken).term());
+ assertEquals("i", filter.next(reusableToken).term());
+ assertEquals("ij", filter.next(reusableToken).term());
+ assertEquals("d", filter.next(reusableToken).term());
+ assertEquals("n", filter.next(reusableToken).term());
+ assertEquals("o", filter.next(reusableToken).term());
+ assertEquals("o", filter.next(reusableToken).term());
+ assertEquals("o", filter.next(reusableToken).term());
+ assertEquals("o", filter.next(reusableToken).term());
+ assertEquals("o", filter.next(reusableToken).term());
+ assertEquals("o", filter.next(reusableToken).term());
+ assertEquals("oe", filter.next(reusableToken).term());
+ assertEquals("ss", filter.next(reusableToken).term());
+ assertEquals("th", filter.next(reusableToken).term());
+ assertEquals("u", filter.next(reusableToken).term());
+ assertEquals("u", filter.next(reusableToken).term());
+ assertEquals("u", filter.next(reusableToken).term());
+ assertEquals("u", filter.next(reusableToken).term());
+ assertEquals("y", filter.next(reusableToken).term());
+ assertEquals("y", filter.next(reusableToken).term());
+ assertEquals("fi", filter.next(reusableToken).term());
+ assertEquals("fl", filter.next(reusableToken).term());
+ assertNull(filter.next(reusableToken));
}
}
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestKeywordAnalyzer.java Sat Jul 4 20:08:54 2009
@@ -19,7 +19,6 @@
import java.io.StringReader;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexReader;
@@ -89,9 +88,9 @@
// LUCENE-1441
public void testOffsets() throws Exception {
TokenStream stream = new KeywordAnalyzer().tokenStream("field", new StringReader("abcd"));
- OffsetAttribute offsetAtt = (OffsetAttribute) stream.addAttribute(OffsetAttribute.class);
- assertTrue(stream.incrementToken());
- assertEquals(0, offsetAtt.startOffset());
- assertEquals(4, offsetAtt.endOffset());
+ Token token = new Token();
+ assertTrue(stream.next(token) != null);
+ assertEquals(0, token.startOffset);
+ assertEquals(4, token.endOffset);
}
}
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestLengthFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestLengthFilter.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestLengthFilter.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestLengthFilter.java Sat Jul 4 20:08:54 2009
@@ -17,7 +17,6 @@
* limitations under the License.
*/
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.util.LuceneTestCase;
import java.io.StringReader;
@@ -28,15 +27,11 @@
TokenStream stream = new WhitespaceTokenizer(
new StringReader("short toolong evenmuchlongertext a ab toolong foo"));
LengthFilter filter = new LengthFilter(stream, 2, 6);
- TermAttribute termAtt = (TermAttribute) filter.getAttribute(TermAttribute.class);
-
- assertTrue(filter.incrementToken());
- assertEquals("short", termAtt.term());
- assertTrue(filter.incrementToken());
- assertEquals("ab", termAtt.term());
- assertTrue(filter.incrementToken());
- assertEquals("foo", termAtt.term());
- assertFalse(filter.incrementToken());
+ final Token reusableToken = new Token();
+ assertEquals("short", filter.next(reusableToken).term());
+ assertEquals("ab", filter.next(reusableToken).term());
+ assertEquals("foo", filter.next(reusableToken).term());
+ assertNull(filter.next(reusableToken));
}
}
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestPerFieldAnalzyerWrapper.java Sat Jul 4 20:08:54 2009
@@ -1,9 +1,7 @@
package org.apache.lucene.analysis;
-import java.io.StringReader;
-
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.util.LuceneTestCase;
+import java.io.StringReader;
/**
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -31,19 +29,17 @@
TokenStream tokenStream = analyzer.tokenStream("field",
new StringReader(text));
- TermAttribute termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
-
- assertTrue(tokenStream.incrementToken());
+ final Token reusableToken = new Token();
+ Token nextToken = tokenStream.next(reusableToken);
assertEquals("WhitespaceAnalyzer does not lowercase",
"Qwerty",
- termAtt.term());
+ nextToken.term());
tokenStream = analyzer.tokenStream("special",
new StringReader(text));
- termAtt = (TermAttribute) tokenStream.getAttribute(TermAttribute.class);
- assertTrue(tokenStream.incrementToken());
+ nextToken = tokenStream.next(reusableToken);
assertEquals("SimpleAnalyzer lowercases",
"qwerty",
- termAtt.term());
+ nextToken.term());
}
}
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStandardAnalyzer.java Sat Jul 4 20:08:54 2009
@@ -1,10 +1,6 @@
package org.apache.lucene.analysis;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
-import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.LuceneTestCase;
import java.io.StringReader;
@@ -39,25 +35,19 @@
public void assertAnalyzesTo(Analyzer a, String input, String[] expectedImages, String[] expectedTypes, int[] expectedPosIncrs) throws Exception {
TokenStream ts = a.tokenStream("dummy", new StringReader(input));
- // TODO Java 1.5
- //final TypeAttribute typeAtt = reusableToken.getAttribute(TypeAttribute.class);
- //final PositionIncrementAttribute posIncrAtt = reusableToken.getAttribute(PositionIncrementAttribute.class);
-
- final TermAttribute termAtt = (TermAttribute) ts.getAttribute(TermAttribute.class);
- final TypeAttribute typeAtt = (TypeAttribute) ts.getAttribute(TypeAttribute.class);
- final PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) ts.getAttribute(PositionIncrementAttribute.class);
-
+ final Token reusableToken = new Token();
for (int i = 0; i < expectedImages.length; i++) {
- assertTrue(ts.incrementToken());
- assertEquals(expectedImages[i], new String(termAtt.termBuffer(), 0, termAtt.termLength()));
+ Token nextToken = ts.next(reusableToken);
+ assertNotNull(nextToken);
+ assertEquals(expectedImages[i], nextToken.term());
if (expectedTypes != null) {
- assertEquals(expectedTypes[i], typeAtt.type());
+ assertEquals(expectedTypes[i], nextToken.type());
}
if (expectedPosIncrs != null) {
- assertEquals(expectedPosIncrs[i], posIncrAtt.getPositionIncrement());
+ assertEquals(expectedPosIncrs[i], nextToken.getPositionIncrement());
}
}
- assertFalse(ts.incrementToken());
+ assertNull(ts.next(reusableToken));
ts.close();
}
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopAnalyzer.java Sat Jul 4 20:08:54 2009
@@ -17,8 +17,6 @@
* limitations under the License.
*/
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.util.LuceneTestCase;
import java.io.StringReader;
@@ -47,10 +45,9 @@
StringReader reader = new StringReader("This is a test of the english stop analyzer");
TokenStream stream = stop.tokenStream("test", reader);
assertTrue(stream != null);
- TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
-
- while (stream.incrementToken()) {
- assertFalse(inValidTokens.contains(termAtt.term()));
+ final Token reusableToken = new Token();
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ assertFalse(inValidTokens.contains(nextToken.term()));
}
}
@@ -63,13 +60,11 @@
StringReader reader = new StringReader("This is a good test of the english stop analyzer");
TokenStream stream = newStop.tokenStream("test", reader);
assertNotNull(stream);
- TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
- PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) stream.addAttribute(PositionIncrementAttribute.class);
-
- while (stream.incrementToken()) {
- String text = termAtt.term();
+ final Token reusableToken = new Token();
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ String text = nextToken.term();
assertFalse(stopWordsSet.contains(text));
- assertEquals(1,posIncrAtt.getPositionIncrement()); // by default stop tokenizer does not apply increments.
+ assertEquals(1,nextToken.getPositionIncrement()); // by default stop tokenizer does not apply increments.
}
}
@@ -87,13 +82,11 @@
TokenStream stream = newStop.tokenStream("test", reader);
assertNotNull(stream);
int i = 0;
- TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
- PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) stream.addAttribute(PositionIncrementAttribute.class);
-
- while (stream.incrementToken()) {
- String text = termAtt.term();
+ final Token reusableToken = new Token();
+ for (Token nextToken = stream.next(reusableToken); nextToken != null; nextToken = stream.next(reusableToken)) {
+ String text = nextToken.term();
assertFalse(stopWordsSet.contains(text));
- assertEquals(expectedIncr[i++],posIncrAtt.getPositionIncrement());
+ assertEquals(expectedIncr[i++],nextToken.getPositionIncrement());
}
} finally {
StopFilter.setEnablePositionIncrementsDefault(defaultEnable);
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopFilter.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopFilter.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestStopFilter.java Sat Jul 4 20:08:54 2009
@@ -16,8 +16,6 @@
* limitations under the License.
*/
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.util.English;
import org.apache.lucene.util.LuceneTestCase;
@@ -37,22 +35,19 @@
StringReader reader = new StringReader("Now is The Time");
String[] stopWords = new String[] { "is", "the", "Time" };
TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopWords);
- final TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
- assertTrue(stream.incrementToken());
- assertEquals("Now", termAtt.term());
- assertTrue(stream.incrementToken());
- assertEquals("The", termAtt.term());
- assertFalse(stream.incrementToken());
+ final Token reusableToken = new Token();
+ assertEquals("Now", stream.next(reusableToken).term());
+ assertEquals("The", stream.next(reusableToken).term());
+ assertEquals(null, stream.next(reusableToken));
}
public void testIgnoreCase() throws IOException {
StringReader reader = new StringReader("Now is The Time");
String[] stopWords = new String[] { "is", "the", "Time" };
TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopWords, true);
- final TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
- assertTrue(stream.incrementToken());
- assertEquals("Now", termAtt.term());
- assertFalse(stream.incrementToken());
+ final Token reusableToken = new Token();
+ assertEquals("Now", stream.next(reusableToken).term());
+ assertEquals(null,stream.next(reusableToken));
}
public void testStopFilt() throws IOException {
@@ -60,12 +55,10 @@
String[] stopWords = new String[] { "is", "the", "Time" };
Set stopSet = StopFilter.makeStopSet(stopWords);
TokenStream stream = new StopFilter(new WhitespaceTokenizer(reader), stopSet);
- final TermAttribute termAtt = (TermAttribute) stream.getAttribute(TermAttribute.class);
- assertTrue(stream.incrementToken());
- assertEquals("Now", termAtt.term());
- assertTrue(stream.incrementToken());
- assertEquals("The", termAtt.term());
- assertFalse(stream.incrementToken());
+ final Token reusableToken = new Token();
+ assertEquals("Now", stream.next(reusableToken).term());
+ assertEquals("The", stream.next(reusableToken).term());
+ assertEquals(null, stream.next(reusableToken));
}
/**
@@ -117,16 +110,15 @@
private void doTestStopPositons(StopFilter stpf, boolean enableIcrements) throws IOException {
log("---> test with enable-increments-"+(enableIcrements?"enabled":"disabled"));
stpf.setEnablePositionIncrements(enableIcrements);
- TermAttribute termAtt = (TermAttribute) stpf.getAttribute(TermAttribute.class);
- PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) stpf.getAttribute(PositionIncrementAttribute.class);
+ final Token reusableToken = new Token();
for (int i=0; i<20; i+=3) {
- assertTrue(stpf.incrementToken());
- log("Token "+i+": "+stpf);
+ Token nextToken = stpf.next(reusableToken);
+ log("Token "+i+": "+nextToken);
String w = English.intToEnglish(i).trim();
- assertEquals("expecting token "+i+" to be "+w,w,termAtt.term());
- assertEquals("all but first token must have position increment of 3",enableIcrements?(i==0?1:3):1,posIncrAtt.getPositionIncrement());
+ assertEquals("expecting token "+i+" to be "+w,w,nextToken.term());
+ assertEquals("all but first token must have position increment of 3",enableIcrements?(i==0?1:3):1,nextToken.getPositionIncrement());
}
- assertFalse(stpf.incrementToken());
+ assertNull(stpf.next(reusableToken));
}
// print debug info depending on VERBOSE
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestToken.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestToken.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestToken.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/analysis/TestToken.java Sat Jul 4 20:08:54 2009
@@ -19,7 +19,6 @@
import org.apache.lucene.util.LuceneTestCase;
-/** @deprecated */
public class TestToken extends LuceneTestCase {
public TestToken(String name) {
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestDocumentWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestDocumentWriter.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestDocumentWriter.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestDocumentWriter.java Sat Jul 4 20:08:54 2009
@@ -22,14 +22,12 @@
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.SimpleAnalyzer;
+import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.WhitespaceAnalyzer;
import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.Fieldable;
@@ -37,7 +35,6 @@
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.document.Field.TermVector;
import org.apache.lucene.store.RAMDirectory;
-import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
@@ -141,38 +138,33 @@
public TokenStream tokenStream(String fieldName, Reader reader) {
return new TokenFilter(new WhitespaceTokenizer(reader)) {
boolean first=true;
- AttributeSource state;
+ Token buffered;
- public boolean incrementToken() throws IOException {
- if (state != null) {
- state.restoreState(this);
- payloadAtt.setPayload(null);
- posIncrAtt.setPositionIncrement(0);
- termAtt.setTermBuffer(new char[]{'b'}, 0, 1);
- state = null;
- return true;
+ public Token next(final Token reusableToken) throws IOException {
+ if (buffered != null) {
+ Token nextToken = buffered;
+ buffered=null;
+ return nextToken;
}
-
- boolean hasNext = input.incrementToken();
- if (!hasNext) return false;
- if (Character.isDigit(termAtt.termBuffer()[0])) {
- posIncrAtt.setPositionIncrement(termAtt.termBuffer()[0] - '0');
+ Token nextToken = input.next(reusableToken);
+ if (nextToken==null) return null;
+ if (Character.isDigit(nextToken.termBuffer()[0])) {
+ nextToken.setPositionIncrement(nextToken.termBuffer()[0] - '0');
}
if (first) {
// set payload on first position only
- payloadAtt.setPayload(new Payload(new byte[]{100}));
+ nextToken.setPayload(new Payload(new byte[]{100}));
first = false;
}
// index a "synonym" for every token
- state = captureState();
- return true;
+ buffered = (Token)nextToken.clone();
+ buffered.setPayload(null);
+ buffered.setPositionIncrement(0);
+ buffered.setTermBuffer(new char[]{'b'}, 0, 1);
+ return nextToken;
}
-
- TermAttribute termAtt = (TermAttribute) addAttribute(TermAttribute.class);
- PayloadAttribute payloadAtt = (PayloadAttribute) addAttribute(PayloadAttribute.class);
- PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) addAttribute(PositionIncrementAttribute.class);
};
}
};
@@ -209,14 +201,12 @@
private String[] tokens = new String[] {"term1", "term2", "term3", "term2"};
private int index = 0;
- private TermAttribute termAtt = (TermAttribute) addAttribute(TermAttribute.class);
-
- public boolean incrementToken() throws IOException {
+ public Token next(final Token reusableToken) throws IOException {
+ assert reusableToken != null;
if (index == tokens.length) {
- return false;
+ return null;
} else {
- termAtt.setTermBuffer(tokens[index++]);
- return true;
+ return reusableToken.reinit(tokens[index++], 0, 0);
}
}
Modified: lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestIndexWriter.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestIndexWriter.java?rev=791173&r1=791172&r2=791173&view=diff
==============================================================================
--- lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestIndexWriter.java (original)
+++ lucene/java/branches/lucene_2_4_back_compat_tests/src/test/org/apache/lucene/index/TestIndexWriter.java Sat Jul 4 20:08:54 2009
@@ -17,48 +17,48 @@
* limitations under the License.
*/
-import java.io.ByteArrayOutputStream;
-import java.io.File;
import java.io.IOException;
-import java.io.PrintStream;
import java.io.Reader;
-import java.util.ArrayList;
+import java.io.File;
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
import java.util.Arrays;
+import java.util.ArrayList;
import java.util.List;
import java.util.Random;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.UnicodeUtil;
+
+import org.apache.lucene.analysis.WhitespaceAnalyzer;
+import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.SinkTokenizer;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.WhitespaceAnalyzer;
-import org.apache.lucene.analysis.WhitespaceTokenizer;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.standard.StandardTokenizer;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
-import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+import org.apache.lucene.analysis.Token;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.search.IndexSearcher;
-import org.apache.lucene.search.PhraseQuery;
-import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TermQuery;
+import org.apache.lucene.search.Query;
import org.apache.lucene.search.spans.SpanTermQuery;
-import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
+import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.store.IndexInput;
import org.apache.lucene.store.IndexOutput;
-import org.apache.lucene.store.Lock;
-import org.apache.lucene.store.LockFactory;
+import org.apache.lucene.store.AlreadyClosedException;
+import org.apache.lucene.util._TestUtil;
+
import org.apache.lucene.store.MockRAMDirectory;
-import org.apache.lucene.store.RAMDirectory;
+import org.apache.lucene.store.LockFactory;
+import org.apache.lucene.store.Lock;
import org.apache.lucene.store.SingleInstanceLockFactory;
-import org.apache.lucene.util.AttributeSource;
-import org.apache.lucene.util.LuceneTestCase;
-import org.apache.lucene.util.UnicodeUtil;
-import org.apache.lucene.util._TestUtil;
/**
*
@@ -1793,11 +1793,11 @@
return new TokenFilter(new StandardTokenizer(reader)) {
private int count = 0;
- public boolean incrementToken() throws IOException {
+ public Token next(final Token reusableToken) throws IOException {
if (count++ == 5) {
throw new IOException();
}
- return input.incrementToken();
+ return input.next(reusableToken);
}
};
}
@@ -1916,10 +1916,10 @@
this.fieldName = fieldName;
}
- public boolean incrementToken() throws IOException {
+ public Token next(final Token reusableToken) throws IOException {
if (this.fieldName.equals("crash") && count++ >= 4)
throw new IOException("I'm experiencing problems");
- return input.incrementToken();
+ return input.next(reusableToken);
}
public void reset() throws IOException {
@@ -3577,47 +3577,21 @@
}
}
- private static class MyAnalyzer extends Analyzer {
-
- public TokenStream tokenStream(String fieldName, Reader reader) {
- TokenStream s = new WhitespaceTokenizer(reader);
- s.addAttribute(PositionIncrementAttribute.class);
- return s;
- }
-
- }
-
// LUCENE-1255
public void testNegativePositions() throws Throwable {
SinkTokenizer tokens = new SinkTokenizer();
- tokens.addAttribute(TermAttribute.class);
- tokens.addAttribute(PositionIncrementAttribute.class);
-
- AttributeSource state = new AttributeSource();
- TermAttribute termAtt = (TermAttribute) state.addAttribute(TermAttribute.class);
- PositionIncrementAttribute posIncrAtt = (PositionIncrementAttribute) state.addAttribute(PositionIncrementAttribute.class);
- termAtt.setTermBuffer("a");
- posIncrAtt.setPositionIncrement(0);
- tokens.add(state);
-
- state = new AttributeSource();
- termAtt = (TermAttribute) state.addAttribute(TermAttribute.class);
- posIncrAtt = (PositionIncrementAttribute) state.addAttribute(PositionIncrementAttribute.class);
-
- termAtt.setTermBuffer("b");
- posIncrAtt.setPositionIncrement(1);
- tokens.add(state);
-
- state = new AttributeSource();
- termAtt = (TermAttribute) state.addAttribute(TermAttribute.class);
- posIncrAtt = (PositionIncrementAttribute) state.addAttribute(PositionIncrementAttribute.class);
-
- termAtt.setTermBuffer("c");
- posIncrAtt.setPositionIncrement(1);
- tokens.add(state);
+ Token t = new Token();
+ t.setTermBuffer("a");
+ t.setPositionIncrement(0);
+ tokens.add(t);
+ t.setTermBuffer("b");
+ t.setPositionIncrement(1);
+ tokens.add(t);
+ t.setTermBuffer("c");
+ tokens.add(t);
MockRAMDirectory dir = new MockRAMDirectory();
- IndexWriter w = new IndexWriter(dir, new MyAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
+ IndexWriter w = new IndexWriter(dir, new WhitespaceAnalyzer(), true, IndexWriter.MaxFieldLength.UNLIMITED);
w.setAllowMinus1Position();
Document doc = new Document();
doc.add(new Field("field", tokens));