You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2011/01/19 13:27:20 UTC
svn commit: r1060784 [1/2] - in /lucene/dev/branches/branch_3x: lucene/
lucene/contrib/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/
lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/
lucene/contrib/queryp...
Author: uschindler
Date: Wed Jan 19 12:27:19 2011
New Revision: 1060784
URL: http://svn.apache.org/viewvc?rev=1060784&view=rev
Log:
LUCENE-2374: Added Attribute reflection API: It's now possible to inspect the contents of AttributeImpl and AttributeSource using a well-defined API
Added:
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/config/
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/config/TestAttributes.java (with props)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeReflector.java (with props)
Modified:
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/contrib/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/AllowLeadingWildcardAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/AnalyzerAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/BoostAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DateResolutionAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DefaultOperatorAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DefaultPhraseSlopAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FieldBoostMapAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FieldDateResolutionMapAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FuzzyAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/LocaleAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/LowercaseExpandedTermsAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/MultiFieldAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/MultiTermRewriteMethodAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/PositionIncrementsAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/RangeCollatorAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/spans/UniqueFieldAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Token.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeImpl.java
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeSource.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestToken.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpls.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/_TestUtil.java
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/AnalysisRequestHandler.java
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java
lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java
lucene/dev/branches/branch_3x/solr/src/webapp/web/admin/analysis.jsp
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Wed Jan 19 12:27:19 2011
@@ -38,9 +38,9 @@ Changes in backwards compatibility polic
* LUCENE-2302: The new interface for term attributes, CharTermAttribute,
now implements CharSequence. This requires the toString() methods of
CharTermAttribute, deprecated TermAttribute, and Token to return only
- the term text and no other attribute contents.
- TODO: Point to new attribute inspection API coming with LUCENE-2374.
- (Uwe Schindler, Robert Muir)
+ the term text and no other attribute contents. LUCENE-2374 implements
+ an attribute reflection API to no longer rely on toString() for attribute
+ inspection. (Uwe Schindler, Robert Muir)
* LUCENE-2372, LUCENE-2389: StandardAnalyzer, KeywordAnalyzer,
PerFieldAnalyzerWrapper, WhitespaceTokenizer are now final. Also removed
@@ -268,6 +268,23 @@ API Changes
to ensure that the norm is encoded with your Similarity.
(Robert Muir, Mike McCandless)
+* LUCENE-2374: Added Attribute reflection API: It's now possible to inspect the
+ contents of AttributeImpl and AttributeSource using a well-defined API.
+ This is e.g. used by Solr's AnalysisRequestHandlers to display all attributes
+ in a structured way.
+ There are also some backwards incompatible changes in toString() output,
+ as LUCENE-2302 introduced the CharSequence interface to CharTermAttribute
+ leading to changed toString() return values. The new API allows to get a
+ string representation in a well-defined way using a new method
+ reflectAsString(). For backwards compatibility reasons, when toString()
+ was implemented by implementation subclasses, the default implementation of
+ AttributeImpl.reflectWith() uses toString()s output instead to report the
+ Attribute's properties. Otherwise, reflectWith() uses Java's reflection
+ (like toString() did before) to get the attribute properties.
+ In addition, the mandatory equals() and hashCode() are no longer required
+ for AttributeImpls, but can still be provided (if needed).
+ (Uwe Schindler)
+
Bug fixes
* LUCENE-2249: ParallelMultiSearcher should shut down thread pool on
Modified: lucene/dev/branches/branch_3x/lucene/contrib/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/icu/src/java/org/apache/lucene/analysis/icu/tokenattributes/ScriptAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -20,6 +20,7 @@ package org.apache.lucene.analysis.icu.t
import java.io.Serializable;
import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeReflector;
import com.ibm.icu.lang.UScript;
@@ -77,7 +78,7 @@ public class ScriptAttributeImpl extends
}
@Override
- public String toString() {
- return "script=" + getName();
+ public void reflectWith(AttributeReflector reflector) {
+ reflector.reflect(ScriptAttribute.class, "script", getName());
}
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/AllowLeadingWildcardAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/AllowLeadingWildcardAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/AllowLeadingWildcardAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/AllowLeadingWildcardAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -33,6 +33,8 @@ public class AllowLeadingWildcardAttribu
private static final long serialVersionUID = -2804763012723049527L;
+ { enableBackwards = false; }
+
private boolean allowLeadingWildcard = false; // default in 2.9
public void setAllowLeadingWildcard(boolean allowLeadingWildcard) {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/AnalyzerAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/AnalyzerAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/AnalyzerAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/AnalyzerAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -35,6 +35,8 @@ public class AnalyzerAttributeImpl exten
private static final long serialVersionUID = -6804760312723049526L;
+ { enableBackwards = false; }
+
private Analyzer analyzer;
public AnalyzerAttributeImpl() {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/BoostAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/BoostAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/BoostAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/BoostAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -35,6 +35,8 @@ public class BoostAttributeImpl extends
private static final long serialVersionUID = -2104763012523049527L;
+ { enableBackwards = false; }
+
private float boost = 1.0f;
public BoostAttributeImpl() {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DateResolutionAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DateResolutionAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DateResolutionAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DateResolutionAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -35,6 +35,8 @@ public class DateResolutionAttributeImpl
private static final long serialVersionUID = -6804360312723049526L;
+ { enableBackwards = false; }
+
private DateTools.Resolution dateResolution = null;
public DateResolutionAttributeImpl() {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DefaultOperatorAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DefaultOperatorAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DefaultOperatorAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DefaultOperatorAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -34,6 +34,8 @@ public class DefaultOperatorAttributeImp
private static final long serialVersionUID = -6804760312723049526L;
+ { enableBackwards = false; }
+
private Operator operator = Operator.OR;
public DefaultOperatorAttributeImpl() {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DefaultPhraseSlopAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DefaultPhraseSlopAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DefaultPhraseSlopAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/DefaultPhraseSlopAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -34,6 +34,8 @@ public class DefaultPhraseSlopAttributeI
private static final long serialVersionUID = -2104763012527049527L;
+ { enableBackwards = false; }
+
private int defaultPhraseSlop = 0;
public DefaultPhraseSlopAttributeImpl() {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FieldBoostMapAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FieldBoostMapAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FieldBoostMapAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FieldBoostMapAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -38,6 +38,8 @@ public class FieldBoostMapAttributeImpl
private static final long serialVersionUID = -2104763012523049527L;
+ { enableBackwards = false; }
+
private Map<String, Float> boosts = new LinkedHashMap<String, Float>();
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FieldDateResolutionMapAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FieldDateResolutionMapAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FieldDateResolutionMapAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FieldDateResolutionMapAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -35,6 +35,8 @@ public class FieldDateResolutionMapAttri
private static final long serialVersionUID = -2104763012523049527L;
+ { enableBackwards = false; }
+
private Map<CharSequence, DateTools.Resolution> dateRes = new HashMap<CharSequence, DateTools.Resolution>();
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FuzzyAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FuzzyAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FuzzyAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/FuzzyAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -35,6 +35,8 @@ public class FuzzyAttributeImpl extends
private static final long serialVersionUID = -2104763012527049527L;
+ { enableBackwards = false; }
+
private int prefixLength = FuzzyQuery.defaultPrefixLength;
private float minSimilarity = FuzzyQuery.defaultMinSimilarity;
@@ -90,7 +92,7 @@ public class FuzzyAttributeImpl extends
@Override
public String toString() {
- return "<fuzzyAttribute prefixLength=" + this.prefixLength + "/>";
+ return "<fuzzyAttribute prefixLength=" + this.prefixLength + ",minSimilarity=" + this.minSimilarity + "/>";
}
}
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/LocaleAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/LocaleAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/LocaleAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/LocaleAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -35,6 +35,8 @@ public class LocaleAttributeImpl extends
private static final long serialVersionUID = -6804760312720049526L;
+ { enableBackwards = false; }
+
private Locale locale = Locale.getDefault();
public LocaleAttributeImpl() {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/LowercaseExpandedTermsAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/LowercaseExpandedTermsAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/LowercaseExpandedTermsAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/LowercaseExpandedTermsAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -35,6 +35,8 @@ public class LowercaseExpandedTermsAttri
private static final long serialVersionUID = -2804760312723049527L;
+ { enableBackwards = false; }
+
private boolean lowercaseExpandedTerms = true;
public LowercaseExpandedTermsAttributeImpl() {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/MultiFieldAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/MultiFieldAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/MultiFieldAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/MultiFieldAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -35,6 +35,8 @@ public class MultiFieldAttributeImpl ext
private static final long serialVersionUID = -6809760312720049526L;
+ { enableBackwards = false; }
+
private CharSequence[] fields;
public MultiFieldAttributeImpl() {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/MultiTermRewriteMethodAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/MultiTermRewriteMethodAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/MultiTermRewriteMethodAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/MultiTermRewriteMethodAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -36,6 +36,8 @@ public class MultiTermRewriteMethodAttri
private static final long serialVersionUID = -2104763012723049527L;
+ { enableBackwards = false; }
+
private MultiTermQuery.RewriteMethod multiTermRewriteMethod = MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT;
public MultiTermRewriteMethodAttributeImpl() {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/PositionIncrementsAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/PositionIncrementsAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/PositionIncrementsAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/PositionIncrementsAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -33,6 +33,8 @@ public class PositionIncrementsAttribute
private static final long serialVersionUID = -2804763012793049527L;
+ { enableBackwards = false; }
+
private boolean positionIncrementsEnabled = false;
public PositionIncrementsAttributeImpl() {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/RangeCollatorAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/RangeCollatorAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/RangeCollatorAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/java/org/apache/lucene/queryParser/standard/config/RangeCollatorAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -37,6 +37,8 @@ public class RangeCollatorAttributeImpl
private static final long serialVersionUID = -6804360312723049526L;
+ { enableBackwards = false; }
+
private Collator rangeCollator;
public RangeCollatorAttributeImpl() {
Modified: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/spans/UniqueFieldAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/spans/UniqueFieldAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/spans/UniqueFieldAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/spans/UniqueFieldAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -33,6 +33,8 @@ public class UniqueFieldAttributeImpl ex
private static final long serialVersionUID = 8553318595851064232L;
+ { enableBackwards = false; }
+
private CharSequence uniqueField;
public UniqueFieldAttributeImpl() {
Added: lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/config/TestAttributes.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/config/TestAttributes.java?rev=1060784&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/config/TestAttributes.java (added)
+++ lucene/dev/branches/branch_3x/lucene/contrib/queryparser/src/test/org/apache/lucene/queryParser/standard/config/TestAttributes.java Wed Jan 19 12:27:19 2011
@@ -0,0 +1,67 @@
+package org.apache.lucene.queryParser.standard.config;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.util._TestUtil;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.search.FuzzyQuery;
+import org.apache.lucene.search.MultiTermQuery;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Locale;
+
+public class TestAttributes extends LuceneTestCase {
+
+ // this checks using reflection API if the defaults are correct
+ public void testAttributes() {
+ _TestUtil.assertAttributeReflection(new AllowLeadingWildcardAttributeImpl(),
+ Collections.singletonMap(AllowLeadingWildcardAttribute.class.getName()+"#allowLeadingWildcard", false));
+ _TestUtil.assertAttributeReflection(new AnalyzerAttributeImpl(),
+ Collections.singletonMap(AnalyzerAttribute.class.getName()+"#analyzer", null));
+ _TestUtil.assertAttributeReflection(new BoostAttributeImpl(),
+ Collections.singletonMap(BoostAttribute.class.getName()+"#boost", 1.0f));
+ _TestUtil.assertAttributeReflection(new DateResolutionAttributeImpl(),
+ Collections.singletonMap(DateResolutionAttribute.class.getName()+"#dateResolution", null));
+ _TestUtil.assertAttributeReflection(new DefaultOperatorAttributeImpl(),
+ Collections.singletonMap(DefaultOperatorAttribute.class.getName()+"#operator", DefaultOperatorAttribute.Operator.OR));
+ _TestUtil.assertAttributeReflection(new DefaultPhraseSlopAttributeImpl(),
+ Collections.singletonMap(DefaultPhraseSlopAttribute.class.getName()+"#defaultPhraseSlop", 0));
+ _TestUtil.assertAttributeReflection(new FieldBoostMapAttributeImpl(),
+ Collections.singletonMap(FieldBoostMapAttribute.class.getName()+"#boosts", Collections.emptyMap()));
+ _TestUtil.assertAttributeReflection(new FieldDateResolutionMapAttributeImpl(),
+ Collections.singletonMap(FieldDateResolutionMapAttribute.class.getName()+"#dateRes", Collections.emptyMap()));
+ _TestUtil.assertAttributeReflection(new FuzzyAttributeImpl(), new HashMap<String,Object>() {{
+ put(FuzzyAttribute.class.getName()+"#prefixLength", FuzzyQuery.defaultPrefixLength);
+ put(FuzzyAttribute.class.getName()+"#minSimilarity", FuzzyQuery.defaultMinSimilarity);
+ }});
+ _TestUtil.assertAttributeReflection(new LocaleAttributeImpl(),
+ Collections.singletonMap(LocaleAttribute.class.getName()+"#locale", Locale.getDefault()));
+ _TestUtil.assertAttributeReflection(new LowercaseExpandedTermsAttributeImpl(),
+ Collections.singletonMap(LowercaseExpandedTermsAttribute.class.getName()+"#lowercaseExpandedTerms", true));
+ _TestUtil.assertAttributeReflection(new MultiFieldAttributeImpl(),
+ Collections.singletonMap(MultiFieldAttribute.class.getName()+"#fields", null));
+ _TestUtil.assertAttributeReflection(new MultiTermRewriteMethodAttributeImpl(),
+ Collections.singletonMap(MultiTermRewriteMethodAttribute.class.getName()+"#multiTermRewriteMethod", MultiTermQuery.CONSTANT_SCORE_AUTO_REWRITE_DEFAULT));
+ _TestUtil.assertAttributeReflection(new PositionIncrementsAttributeImpl(),
+ Collections.singletonMap(PositionIncrementsAttribute.class.getName()+"#positionIncrementsEnabled", false));
+ _TestUtil.assertAttributeReflection(new RangeCollatorAttributeImpl(),
+ Collections.singletonMap(RangeCollatorAttribute.class.getName()+"#rangeCollator", null));
+ }
+
+}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Token.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Token.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Token.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/Token.java Wed Jan 19 12:27:19 2011
@@ -28,6 +28,7 @@ import org.apache.lucene.index.TermPosit
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeReflector;
/**
A Token is an occurrence of a term from the text of a field. It consists of
@@ -589,6 +590,17 @@ public class Token extends TermAttribute
}
}
+ @Override
+ public void reflectWith(AttributeReflector reflector) {
+ super.reflectWith(reflector);
+ reflector.reflect(OffsetAttribute.class, "startOffset", startOffset);
+ reflector.reflect(OffsetAttribute.class, "endOffset", endOffset);
+ reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", positionIncrement);
+ reflector.reflect(PayloadAttribute.class, "payload", payload);
+ reflector.reflect(FlagsAttribute.class, "flags", flags);
+ reflector.reflect(TypeAttribute.class, "type", type);
+ }
+
/** Convenience factory that returns <code>Token</code> as implementation for the basic
* attributes and return the default impl (with "Impl" appended) for all other
* attributes.
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/analysis/tokenattributes/CharTermAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -22,6 +22,7 @@ import java.nio.CharBuffer;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util.AttributeReflector;
import org.apache.lucene.util.RamUsageEstimator;
/**
@@ -286,6 +287,11 @@ public class CharTermAttributeImpl exten
}
@Override
+ public void reflectWith(AttributeReflector reflector) {
+ reflector.reflect(CharTermAttribute.class, "term", toString());
+ }
+
+ @Override
public void copyTo(AttributeImpl target) {
if (target instanceof CharTermAttribute) {
CharTermAttribute t = (CharTermAttribute) target;
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -20,6 +20,10 @@ package org.apache.lucene.util;
import java.io.Serializable;
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
+import java.lang.ref.WeakReference;
+import java.util.LinkedList;
+
+import org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl; // deprecated
/**
* Base class for Attributes that can be added to a
@@ -37,72 +41,144 @@ public abstract class AttributeImpl impl
public abstract void clear();
/**
- * The default implementation of this method accesses all declared
- * fields of this object and prints the values in the following syntax:
+ * Returns a string representation of the object. In general, the {@code toString} method
+ * returns a string that "textually represents" this object.
+ *
+ * <p><b>WARNING:</b> For backwards compatibility this method is implemented as
+ * {@code return reflectAsString(false)}. In Lucene 4.0 this default implementation
+ * will be removed. The reason for this is the problem of
+ * {@link CharTermAttributeImpl#toString} that must return a string representation
+ * of the term's char sequence.
+ *
+ * <p>It is recommeneded to use {@link #reflectAsString} or {@link #reflectWith}
+ * to get a well-defined output of AttributeImpl's internals.
+ */
+ // TODO: @deprecated remove this method in 4.0
+ @Override
+ public String toString() {
+ return reflectAsString(false);
+ }
+
+ /**
+ * This method returns the current attribute values as a string in the following format
+ * by calling the {@link #reflectWith(AttributeReflector)} method:
*
+ * <ul>
+ * <li><em>iff {@code prependAttClass=true}:</em> {@code "AttributeClass#key=value,AttributeClass#key=value"}
+ * <li><em>iff {@code prependAttClass=false}:</em> {@code "key=value,key=value"}
+ * </ul>
+ *
+ * @see #reflectWith(AttributeReflector)
+ * @see #toString()
+ */
+ public final String reflectAsString(final boolean prependAttClass) {
+ final StringBuilder buffer = new StringBuilder();
+ reflectWith(new AttributeReflector() {
+ public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
+ if (buffer.length() > 0) {
+ buffer.append(',');
+ }
+ if (prependAttClass) {
+ buffer.append(attClass.getName()).append('#');
+ }
+ buffer.append(key).append('=').append((value == null) ? "null" : value);
+ }
+ });
+ return buffer.toString();
+ }
+
+ /**
+ * @deprecated this will be removed in Lucene 4.0
+ */
+ @Deprecated
+ private static final VirtualMethod<AttributeImpl> toStringMethod =
+ new VirtualMethod<AttributeImpl>(AttributeImpl.class, "toString");
+
+ /**
+ * When {@code true} (default), if a subclass overrides {@link #toString()},
+ * its output is parsed by {@link #reflectWith} and used for attribute reflection.
+ * This is added to enable attribute implementations from Lucene 2.9 or 3.0 to
+ * work correctly with reflection.
+ * @deprecated this will be removed in Lucene 4.0.
+ */
+ @Deprecated
+ protected boolean enableBackwards = true;
+
+ /**
+ * @deprecated this will be removed in Lucene 4.0
+ */
+ @Deprecated
+ private boolean assertExternalClass(Class<? extends AttributeImpl> clazz) {
+ final String name = clazz.getName();
+ return (!name.startsWith("org.apache.lucene.") && !name.startsWith("org.apache.solr."))
+ || name.equals("org.apache.lucene.util.TestAttributeSource$TestAttributeImpl");
+ }
+
+ /**
+ * This method is for introspection of attributes, it should simply
+ * add the key/values this attribute holds to the given {@link AttributeReflector}.
+ *
+ * <p>The default implementation calls {@link AttributeReflector#reflect} for all
+ * non-static fields from the implementing class, using the field name as key
+ * and the field value as value. The Attribute class is also determined by reflection.
+ * Please note that the default implementation can only handle single-Attribute
+ * implementations.
+ *
+ * <p>Custom implementations look like this (e.g. for a combined attribute implementation):
* <pre>
- * public String toString() {
- * return "start=" + startOffset + ",end=" + endOffset;
+ * public void reflectWith(AttributeReflector reflector) {
+ * reflector.reflect(CharTermAttribute.class, "term", term());
+ * reflector.reflect(PositionIncrementAttribute.class, "positionIncrement", getPositionIncrement());
* }
* </pre>
- *
- * This method may be overridden by subclasses.
- */
- @Override
- public String toString() {
- StringBuilder buffer = new StringBuilder();
- Class<?> clazz = this.getClass();
- Field[] fields = clazz.getDeclaredFields();
+ *
+ * <p>If you implement this method, make sure that for each invocation, the same set of {@link Attribute}
+ * interfaces and keys are passed to {@link AttributeReflector#reflect} in the same order, but possibly
+ * different values. So don't automatically exclude e.g. {@code null} properties!
+ *
+ * @see #reflectAsString(boolean)
+ */
+ public void reflectWith(AttributeReflector reflector) {
+ final Class<? extends AttributeImpl> clazz = this.getClass();
+ final LinkedList<WeakReference<Class<? extends Attribute>>> interfaces = AttributeSource.getAttributeInterfaces(clazz);
+ if (interfaces.size() != 1) {
+ throw new UnsupportedOperationException(clazz.getName() +
+ " implements more than one Attribute interface, the default reflectWith() implementation cannot handle this.");
+ }
+ final Class<? extends Attribute> interf = interfaces.getFirst().get();
+
+ // TODO: @deprecated sophisticated(TM) backwards
+ if (enableBackwards && toStringMethod.isOverriddenAsOf(clazz)) {
+ assert assertExternalClass(clazz) : "no Lucene/Solr classes should fallback to toString() parsing";
+ // this class overrides toString and for backwards compatibility we try to parse the string returned by this method:
+ for (String part : toString().split(",")) {
+ final int pos = part.indexOf('=');
+ if (pos < 0) {
+ throw new UnsupportedOperationException("The backwards compatibility layer to support reflectWith() " +
+ "on old AtributeImpls expects the toString() implementation to return a correct format as specified for method reflectAsString(false)");
+ }
+ reflector.reflect(interf, part.substring(0, pos).trim(), part.substring(pos + 1));
+ }
+ return;
+ }
+ // end sophisticated(TM) backwards
+
+ final Field[] fields = clazz.getDeclaredFields();
try {
for (int i = 0; i < fields.length; i++) {
- Field f = fields[i];
+ final Field f = fields[i];
if (Modifier.isStatic(f.getModifiers())) continue;
f.setAccessible(true);
- Object value = f.get(this);
- if (buffer.length()>0) {
- buffer.append(',');
- }
- if (value == null) {
- buffer.append(f.getName() + "=null");
- } else {
- buffer.append(f.getName() + "=" + value);
- }
+ reflector.reflect(interf, f.getName(), f.get(this));
}
} catch (IllegalAccessException e) {
// this should never happen, because we're just accessing fields
// from 'this'
throw new RuntimeException(e);
}
-
- return buffer.toString();
}
/**
- * Subclasses must implement this method and should compute
- * a hashCode similar to this:
- * <pre>
- * public int hashCode() {
- * int code = startOffset;
- * code = code * 31 + endOffset;
- * return code;
- * }
- * </pre>
- *
- * see also {@link #equals(Object)}
- */
- @Override
- public abstract int hashCode();
-
- /**
- * All values used for computation of {@link #hashCode()}
- * should be checked here for equality.
- *
- * see also {@link Object#equals(Object)}
- */
- @Override
- public abstract boolean equals(Object other);
-
- /**
* Copies the values from this Attribute into the passed-in
* target attribute. The target implementation must support all the
* Attributes this implementation supports.
Added: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeReflector.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeReflector.java?rev=1060784&view=auto
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeReflector.java (added)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeReflector.java Wed Jan 19 12:27:19 2011
@@ -0,0 +1,34 @@
+package org.apache.lucene.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * This interface is used to reflect contents of {@link AttributeSource} or {@link AttributeImpl}.
+ */
+public interface AttributeReflector {
+
+ /**
+ * This method gets called for every property in an {@link AttributeImpl}/{@link AttributeSource}
+ * passing the class name of the {@link Attribute}, a key and the actual value.
+ * E.g., an invocation of {@link org.apache.lucene.analysis.tokenattributes.CharTermAttributeImpl#reflectWith}
+ * would call this method once using {@code org.apache.lucene.analysis.tokenattributes.CharTermAttribute.class}
+ * as attribute class, {@code "term"} as key and the actual value as a String.
+ */
+ public void reflect(Class<? extends Attribute> attClass, String key, Object value);
+
+}
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeSource.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeSource.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeSource.java Wed Jan 19 12:27:19 2011
@@ -188,20 +188,9 @@ public class AttributeSource {
private static final WeakHashMap<Class<? extends AttributeImpl>,LinkedList<WeakReference<Class<? extends Attribute>>>> knownImplClasses =
new WeakHashMap<Class<? extends AttributeImpl>,LinkedList<WeakReference<Class<? extends Attribute>>>>();
- /** <b>Expert:</b> Adds a custom AttributeImpl instance with one or more Attribute interfaces.
- * <p><font color="red"><b>Please note:</b> It is not guaranteed, that <code>att</code> is added to
- * the <code>AttributeSource</code>, because the provided attributes may already exist.
- * You should always retrieve the wanted attributes using {@link #getAttribute} after adding
- * with this method and cast to your class.
- * The recommended way to use custom implementations is using an {@link AttributeFactory}.
- * </font></p>
- */
- public void addAttributeImpl(final AttributeImpl att) {
- final Class<? extends AttributeImpl> clazz = att.getClass();
- if (attributeImpls.containsKey(clazz)) return;
- LinkedList<WeakReference<Class<? extends Attribute>>> foundInterfaces;
+ static LinkedList<WeakReference<Class<? extends Attribute>>> getAttributeInterfaces(final Class<? extends AttributeImpl> clazz) {
synchronized(knownImplClasses) {
- foundInterfaces = knownImplClasses.get(clazz);
+ LinkedList<WeakReference<Class<? extends Attribute>>> foundInterfaces = knownImplClasses.get(clazz);
if (foundInterfaces == null) {
// we have a strong reference to the class instance holding all interfaces in the list (parameter "att"),
// so all WeakReferences are never evicted by GC
@@ -218,7 +207,23 @@ public class AttributeSource {
actClazz = actClazz.getSuperclass();
} while (actClazz != null);
}
+ return foundInterfaces;
}
+ }
+
+ /** <b>Expert:</b> Adds a custom AttributeImpl instance with one or more Attribute interfaces.
+ * <p><font color="red"><b>Please note:</b> It is not guaranteed, that <code>att</code> is added to
+ * the <code>AttributeSource</code>, because the provided attributes may already exist.
+ * You should always retrieve the wanted attributes using {@link #getAttribute} after adding
+ * with this method and cast to your class.
+ * The recommended way to use custom implementations is using an {@link AttributeFactory}.
+ * </font></p>
+ */
+ public void addAttributeImpl(final AttributeImpl att) {
+ final Class<? extends AttributeImpl> clazz = att.getClass();
+ if (attributeImpls.containsKey(clazz)) return;
+ final LinkedList<WeakReference<Class<? extends Attribute>>> foundInterfaces =
+ getAttributeInterfaces(clazz);
// add all interfaces of this AttributeImpl to the maps
for (WeakReference<Class<? extends Attribute>> curInterfaceRef : foundInterfaces) {
@@ -439,9 +444,21 @@ public class AttributeSource {
return false;
}
+ /**
+ * Returns a string representation of the object. In general, the {@code toString} method
+ * returns a string that "textually represents" this object.
+ *
+ * <p><b>WARNING:</b> For backwards compatibility this method is implemented as
+ * in Lucene 2.9/3.0. In Lucene 4.0 this default implementation
+ * will be removed.
+ *
+ * <p>It is recommeneded to use {@link #reflectAsString} or {@link #reflectWith}
+ * to get a well-defined output of AttributeSource's internals.
+ */
+ // TODO: @deprecated remove this method in 4.0
@Override
public String toString() {
- StringBuilder sb = new StringBuilder().append('(');
+ final StringBuilder sb = new StringBuilder().append('(');
if (hasAttributes()) {
if (currentState == null) {
computeCurrentState();
@@ -455,6 +472,53 @@ public class AttributeSource {
}
/**
+ * This method returns the current attribute values as a string in the following format
+ * by calling the {@link #reflectWith(AttributeReflector)} method:
+ *
+ * <ul>
+ * <li><em>iff {@code prependAttClass=true}:</em> {@code "AttributeClass#key=value,AttributeClass#key=value"}
+ * <li><em>iff {@code prependAttClass=false}:</em> {@code "key=value,key=value"}
+ * </ul>
+ *
+ * @see #reflectWith(AttributeReflector)
+ */
+ public final String reflectAsString(final boolean prependAttClass) {
+ final StringBuilder buffer = new StringBuilder();
+ reflectWith(new AttributeReflector() {
+ public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
+ if (buffer.length() > 0) {
+ buffer.append(',');
+ }
+ if (prependAttClass) {
+ buffer.append(attClass.getName()).append('#');
+ }
+ buffer.append(key).append('=').append((value == null) ? "null" : value);
+ }
+ });
+ return buffer.toString();
+ }
+
+ /**
+ * This method is for introspection of attributes, it should simply
+ * add the key/values this AttributeSource holds to the given {@link AttributeReflector}.
+ *
+ * <p>This method iterates over all Attribute implementations and calls the
+ * corresponding {@link AttributeImpl#reflectWith} method.</p>
+ *
+ * @see AttributeImpl#reflectWith
+ */
+ public final void reflectWith(AttributeReflector reflector) {
+ if (hasAttributes()) {
+ if (currentState == null) {
+ computeCurrentState();
+ }
+ for (State state = currentState; state != null; state = state.next) {
+ state.attribute.reflectWith(reflector);
+ }
+ }
+ }
+
+ /**
* Performs a clone of all {@link AttributeImpl} instances returned in a new
* {@code AttributeSource} instance. This method can be used to e.g. create another TokenStream
* with exactly the same attributes (using {@link #AttributeSource(AttributeSource)}).
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestToken.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestToken.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestToken.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/TestToken.java Wed Jan 19 12:27:19 2011
@@ -22,8 +22,10 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.Attribute;
import org.apache.lucene.util.AttributeImpl;
+import org.apache.lucene.util._TestUtil;
import java.io.StringReader;
+import java.util.HashMap;
public class TestToken extends LuceneTestCase {
@@ -253,4 +255,19 @@ public class TestToken extends LuceneTes
assertTrue("TypeAttribute is not implemented by Token",
ts.addAttribute(TypeAttribute.class) instanceof Token);
}
+
+ public void testAttributeReflection() throws Exception {
+ Token t = new Token("foobar", 6, 22, 8);
+ _TestUtil.assertAttributeReflection(t,
+ new HashMap<String,Object>() {{
+ put(CharTermAttribute.class.getName() + "#term", "foobar");
+ put(OffsetAttribute.class.getName() + "#startOffset", 6);
+ put(OffsetAttribute.class.getName() + "#endOffset", 22);
+ put(PositionIncrementAttribute.class.getName() + "#positionIncrement", 1);
+ put(PayloadAttribute.class.getName() + "#payload", null);
+ put(TypeAttribute.class.getName() + "#type", TypeAttribute.DEFAULT_TYPE);
+ put(FlagsAttribute.class.getName() + "#flags", 8);
+ }});
+ }
+
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestCharTermAttributeImpl.java Wed Jan 19 12:27:19 2011
@@ -18,7 +18,9 @@ package org.apache.lucene.analysis.token
*/
import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util._TestUtil;
import java.nio.CharBuffer;
+import java.util.Collections;
import java.util.Formatter;
import java.util.Locale;
import java.util.regex.Pattern;
@@ -125,6 +127,13 @@ public class TestCharTermAttributeImpl e
assertNotSame(buf, copy.buffer());
}
+ public void testAttributeReflection() throws Exception {
+ CharTermAttributeImpl t = new CharTermAttributeImpl();
+ t.append("foobar");
+ _TestUtil.assertAttributeReflection(t,
+ Collections.singletonMap(CharTermAttribute.class.getName() + "#term", "foobar"));
+ }
+
public void testCharSequenceInterface() {
final String s = "0123456789";
final CharTermAttributeImpl t = new CharTermAttributeImpl();
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpls.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpls.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpls.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/analysis/tokenattributes/TestSimpleAttributeImpls.java Wed Jan 19 12:27:19 2011
@@ -21,6 +21,10 @@ import org.apache.lucene.index.Payload;
import org.apache.lucene.util.AttributeImpl;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.AttributeSource.AttributeFactory;
+import org.apache.lucene.util._TestUtil;
+
+import java.util.Collections;
+import java.util.HashMap;
@Deprecated
public class TestSimpleAttributeImpls extends LuceneTestCase {
@@ -40,6 +44,9 @@ public class TestSimpleAttributeImpls ex
att.clear();
assertEquals(0, att.getFlags());
+
+ _TestUtil.assertAttributeReflection(att,
+ Collections.singletonMap(FlagsAttribute.class.getName() + "#flags", att.getFlags()));
}
public void testPositionIncrementAttribute() throws Exception {
@@ -57,6 +64,9 @@ public class TestSimpleAttributeImpls ex
att.clear();
assertEquals(1, att.getPositionIncrement());
+
+ _TestUtil.assertAttributeReflection(att,
+ Collections.singletonMap(PositionIncrementAttribute.class.getName() + "#positionIncrement", att.getPositionIncrement()));
}
public void testTypeAttribute() throws Exception {
@@ -74,6 +84,9 @@ public class TestSimpleAttributeImpls ex
att.clear();
assertEquals(TypeAttribute.DEFAULT_TYPE, att.type());
+
+ _TestUtil.assertAttributeReflection(att,
+ Collections.singletonMap(TypeAttribute.class.getName() + "#type", att.type()));
}
public void testPayloadAttribute() throws Exception {
@@ -82,6 +95,9 @@ public class TestSimpleAttributeImpls ex
Payload pl = new Payload(new byte[]{1,2,3,4});
att.setPayload(pl);
+
+ _TestUtil.assertAttributeReflection(att,
+ Collections.singletonMap(PayloadAttribute.class.getName() + "#payload", pl));
PayloadAttributeImpl att2 = (PayloadAttributeImpl) assertCloneIsEqual(att);
assertEquals(pl, att2.getPayload());
@@ -102,6 +118,12 @@ public class TestSimpleAttributeImpls ex
att.setOffset(12, 34);
// no string test here, because order unknown
+
+ _TestUtil.assertAttributeReflection(att,
+ new HashMap<String,Object>() {{
+ put(OffsetAttribute.class.getName() + "#startOffset", 12);
+ put(OffsetAttribute.class.getName() + "#endOffset", 34);
+ }});
OffsetAttributeImpl att2 = (OffsetAttributeImpl) assertCloneIsEqual(att);
assertEquals(12, att2.startOffset());
@@ -133,6 +155,9 @@ public class TestSimpleAttributeImpls ex
att.copyTo(assertCloneIsEqual);
assertTrue(assertCloneIsEqual.isKeyword());
assertTrue(att.isKeyword());
+
+ _TestUtil.assertAttributeReflection(att,
+ Collections.singletonMap(KeywordAttribute.class.getName() + "#keyword", att.isKeyword()));
}
public static final AttributeImpl assertCloneIsEqual(AttributeImpl att) {
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java Wed Jan 19 12:27:19 2011
@@ -21,6 +21,8 @@ import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.tokenattributes.*;
import java.util.Iterator;
+import java.util.HashMap;
+import java.util.Map;
public class TestAttributeSource extends LuceneTestCase {
@@ -175,4 +177,70 @@ public class TestAttributeSource extends
fail("Should throw IllegalArgumentException");
} catch (IllegalArgumentException iae) {}
}
+
+ // this class is included in external class check, so no assertion errors occur
+ @Deprecated
+ static class TestAttributeImpl extends AttributeImpl implements FlagsAttribute {
+
+ private int flags = 0;
+
+ public int getFlags() { return flags; }
+ public void setFlags(int flags) { this.flags = flags; }
+
+ @Override
+ public void clear() { flags = 0; }
+
+ @Override
+ public void copyTo(AttributeImpl target) {
+ FlagsAttribute t = (FlagsAttribute) target;
+ t.setFlags(flags);
+ }
+
+ @Override
+ public String toString() {
+ return "foo=bar,moo=mae";
+ }
+
+ }
+
+ // this class is excluded in external class check, so assertion on calling reflectWith should occur
+ @Deprecated
+ static class TestAttributeImpl2 extends TestAttributeImpl {}
+
+ @Deprecated
+ public void testReflectionOfToString() throws Exception {
+ final AttributeSource src = new AttributeSource();
+ final AttributeImpl att = new TestAttributeImpl();
+ src.addAttributeImpl(att);
+
+ assertSame("FlagsAttribute is not implemented by same instance of TestAttributeImpl",
+ att, src.addAttribute(FlagsAttribute.class));
+
+ final Map<String,Object> map = new HashMap<String,Object>();
+ final AttributeReflector reflector = new AttributeReflector() {
+ public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
+ assertSame(FlagsAttribute.class, attClass);
+ map.put(key, value);
+ }
+ };
+ att.reflectWith(reflector);
+ assertEquals(2, map.size());
+ assertEquals("bar", map.get("foo"));
+ assertEquals("mae", map.get("moo"));
+
+ map.clear();
+ src.reflectWith(reflector);
+ assertEquals(2, map.size());
+ assertEquals("bar", map.get("foo"));
+ assertEquals("mae", map.get("moo"));
+
+ map.clear();
+ try {
+ new TestAttributeImpl2().reflectWith(reflector);
+ fail("TestAttributeImpl2 should fail assertion on toString() parsing");
+ } catch (AssertionError e) {
+ // pass
+ }
+ }
+
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/_TestUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/_TestUtil.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/_TestUtil.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/_TestUtil.java Wed Jan 19 12:27:19 2011
@@ -22,6 +22,10 @@ import java.io.File;
import java.io.IOException;
import java.io.PrintStream;
import java.util.Random;
+import java.util.Map;
+import java.util.HashMap;
+
+import org.junit.Assert;
import org.apache.lucene.index.CheckIndex;
import org.apache.lucene.index.ConcurrentMergeScheduler;
@@ -241,4 +245,17 @@ public class _TestUtil {
((ConcurrentMergeScheduler) ms).setMaxMergeCount(3);
}
}
+
+ /** Checks some basic behaviour of an AttributeImpl
+ * @param reflectedValues contains a map with "AttributeClass#key" as values
+ */
+ public static <T> void assertAttributeReflection(final AttributeImpl att, Map<String,T> reflectedValues) {
+ final Map<String,Object> map = new HashMap<String,Object>();
+ att.reflectWith(new AttributeReflector() {
+ public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
+ map.put(attClass.getName() + '#' + key, value);
+ }
+ });
+ Assert.assertEquals("Reflection does not produce same map", reflectedValues, map);
+ }
}
Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/AnalysisRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/AnalysisRequestHandler.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/AnalysisRequestHandler.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/AnalysisRequestHandler.java Wed Jan 19 12:27:19 2011
@@ -23,6 +23,8 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeReflector;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.params.SolrParams;
@@ -45,6 +47,9 @@ import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
/**
*
@@ -132,25 +137,33 @@ public class AnalysisRequestHandler exte
}
}
}
+
+ // a static mapping of the reflected attribute keys to the names used in Solr 1.3/1.4
+ static Map<String,String> ATTRIBUTE_MAPPING = Collections.unmodifiableMap(new HashMap<String,String>() {{
+ put(CharTermAttribute.class.getName() + "#term", "value");
+ put(OffsetAttribute.class.getName() + "#startOffset", "start");
+ put(OffsetAttribute.class.getName() + "#endOffset", "end");
+ put(PositionIncrementAttribute.class.getName() + "#positionIncrement", "posInc");
+ put(TypeAttribute.class.getName() + "#type", "type");
+ }});
static NamedList<NamedList<Object>> getTokens(TokenStream tstream) throws IOException {
// outer is namedList since order of tokens is important
NamedList<NamedList<Object>> tokens = new NamedList<NamedList<Object>>();
- // TODO: support custom attributes
- final CharTermAttribute termAtt = tstream.getAttribute(CharTermAttribute.class);
- final OffsetAttribute offsetAtt = tstream.addAttribute(OffsetAttribute.class);
- final TypeAttribute typeAtt = tstream.addAttribute(TypeAttribute.class);
- final PositionIncrementAttribute posIncAtt = tstream.addAttribute(PositionIncrementAttribute.class);
while (tstream.incrementToken()) {
- NamedList<Object> token = new SimpleOrderedMap<Object>();
+ final NamedList<Object> token = new SimpleOrderedMap<Object>();
tokens.add("token", token);
- token.add("value", termAtt.toString());
- token.add("start", offsetAtt.startOffset());
- token.add("end", offsetAtt.endOffset());
- token.add("posInc", posIncAtt.getPositionIncrement());
- token.add("type", typeAtt.type());
- //TODO: handle payloads
+ tstream.reflectWith(new AttributeReflector() {
+ public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
+ String k = attClass.getName() + '#' + key;
+ // map keys for "standard attributes":
+ if (ATTRIBUTE_MAPPING.containsKey(k)) {
+ k = ATTRIBUTE_MAPPING.get(k);
+ }
+ token.add(k, value);
+ }
+ });
}
return tokens;
}
Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/AnalysisRequestHandlerBase.java Wed Jan 19 12:27:19 2011
@@ -20,14 +20,16 @@ package org.apache.solr.handler;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.CharStream;
-import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
-import org.apache.lucene.analysis.tokenattributes.PayloadAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.index.Payload;
+import org.apache.lucene.util.Attribute;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.AttributeReflector;
+import org.apache.lucene.util.SorterTemplate;
import org.apache.solr.analysis.CharFilterFactory;
import org.apache.solr.analysis.TokenFilterFactory;
import org.apache.solr.analysis.TokenizerChain;
@@ -42,6 +44,7 @@ import org.apache.solr.schema.FieldType;
import java.io.IOException;
import java.io.StringReader;
import java.util.*;
+import java.math.BigInteger;
/**
* A base class for all analysis request handlers.
@@ -109,7 +112,7 @@ public abstract class AnalysisRequestHan
}
TokenStream tokenStream = tfac.create(tokenizerChain.charStream(new StringReader(value)));
- List<Token> tokens = analyzeTokenStream(tokenStream);
+ List<AttributeSource> tokens = analyzeTokenStream(tokenStream);
namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(tokens, context));
@@ -117,7 +120,7 @@ public abstract class AnalysisRequestHan
for (TokenFilterFactory tokenFilterFactory : filtfacs) {
tokenStream = tokenFilterFactory.create(listBasedTokenStream);
- List<Token> tokenList = analyzeTokenStream(tokenStream);
+ List<AttributeSource> tokenList = analyzeTokenStream(tokenStream);
namedList.add(tokenStream.getClass().getName(), convertTokensToNamedLists(tokenList, context));
listBasedTokenStream = new ListBasedTokenStream(tokenList);
}
@@ -132,40 +135,54 @@ public abstract class AnalysisRequestHan
* @param analyzer The analyzer to use.
*
* @return The produces token list.
+ * @deprecated This method is no longer used by Solr
+ * @see #getQueryTokenSet
*/
- protected List<Token> analyzeValue(String value, Analyzer analyzer) {
+ @Deprecated
+ protected List<AttributeSource> analyzeValue(String value, Analyzer analyzer) {
TokenStream tokenStream = analyzer.tokenStream("", new StringReader(value));
return analyzeTokenStream(tokenStream);
}
/**
+ * Analyzes the given text using the given analyzer and returns the produced tokens.
+ *
+ * @param query The query to analyze.
+ * @param analyzer The analyzer to use.
+ */
+ protected Set<String> getQueryTokenSet(String query, Analyzer analyzer) {
+ final Set<String> tokens = new HashSet<String>();
+ final TokenStream tokenStream = analyzer.tokenStream("", new StringReader(query));
+ final CharTermAttribute termAtt = tokenStream.addAttribute(CharTermAttribute.class);
+ try {
+ tokenStream.reset();
+ while (tokenStream.incrementToken()) {
+ tokens.add(termAtt.toString());
+ }
+ } catch (IOException ioe) {
+ throw new RuntimeException("Error occured while iterating over tokenstream", ioe);
+ }
+ return tokens;
+ }
+
+ /**
* Analyzes the given TokenStream, collecting the Tokens it produces.
*
* @param tokenStream TokenStream to analyze
*
* @return List of tokens produced from the TokenStream
*/
- private List<Token> analyzeTokenStream(TokenStream tokenStream) {
- List<Token> tokens = new ArrayList<Token>();
-
- // TODO change this API to support custom attributes
- final CharTermAttribute termAtt = tokenStream.getAttribute(CharTermAttribute.class);
- final OffsetAttribute offsetAtt = tokenStream.addAttribute(OffsetAttribute.class);
- final TypeAttribute typeAtt = tokenStream.addAttribute(TypeAttribute.class);
- final PositionIncrementAttribute posIncAtt = tokenStream.addAttribute(PositionIncrementAttribute.class);
- final FlagsAttribute flagsAtt = tokenStream.addAttribute(FlagsAttribute.class);
- final PayloadAttribute payloadAtt = tokenStream.addAttribute(PayloadAttribute.class);
-
+ private List<AttributeSource> analyzeTokenStream(TokenStream tokenStream) {
+ List<AttributeSource> tokens = new ArrayList<AttributeSource>();
+ // for backwards compatibility, add all "common" attributes
+ tokenStream.addAttribute(CharTermAttribute.class);
+ tokenStream.addAttribute(PositionIncrementAttribute.class);
+ tokenStream.addAttribute(OffsetAttribute.class);
+ tokenStream.addAttribute(TypeAttribute.class);
try {
+ tokenStream.reset();
while (tokenStream.incrementToken()) {
- Token token = new Token();
- token.setEmpty().append(termAtt);
- token.setOffset(offsetAtt.startOffset(), offsetAtt.endOffset());
- token.setType(typeAtt.type());
- token.setFlags(flagsAtt.getFlags());
- token.setPayload(payloadAtt.getPayload());
- token.setPositionIncrement(posIncAtt.getPositionIncrement());
- tokens.add((Token) token.clone());
+ tokens.add(tokenStream.cloneAttributes());
}
} catch (IOException ioe) {
throw new RuntimeException("Error occured while iterating over tokenstream", ioe);
@@ -174,6 +191,13 @@ public abstract class AnalysisRequestHan
return tokens;
}
+ // a static mapping of the reflected attribute keys to the names used in Solr 1.4
+ static Map<String,String> ATTRIBUTE_MAPPING = Collections.unmodifiableMap(new HashMap<String,String>() {{
+ put(OffsetAttribute.class.getName() + "#startOffset", "start");
+ put(OffsetAttribute.class.getName() + "#endOffset", "end");
+ put(TypeAttribute.class.getName() + "#type", "type");
+ }});
+
/**
* Converts the list of Tokens to a list of NamedLists representing the tokens.
*
@@ -182,41 +206,95 @@ public abstract class AnalysisRequestHan
*
* @return List of NamedLists containing the relevant information taken from the tokens
*/
- private List<NamedList> convertTokensToNamedLists(List<Token> tokens, AnalysisContext context) {
- List<NamedList> tokensNamedLists = new ArrayList<NamedList>();
+ private List<NamedList> convertTokensToNamedLists(final List<AttributeSource> tokens, AnalysisContext context) {
+ final List<NamedList> tokensNamedLists = new ArrayList<NamedList>();
- Collections.sort(tokens, new Comparator<Token>() {
- public int compare(Token o1, Token o2) {
- return o1.endOffset() - o2.endOffset();
+ final int[] positions = new int[tokens.size()];
+ int position = 0;
+ for (int i = 0, c = tokens.size(); i < c; i++) {
+ AttributeSource token = tokens.get(i);
+ position += token.addAttribute(PositionIncrementAttribute.class).getPositionIncrement();
+ positions[i] = position;
+ }
+
+ // sort the tokens by absoulte position
+ new SorterTemplate() {
+ @Override
+ protected void swap(int i, int j) {
+ Collections.swap(tokens, i, j);
+ }
+
+ @Override
+ protected int compare(int i, int j) {
+ return positions[i] - positions[j];
}
- });
- int position = 0;
+ @Override
+ protected void setPivot(int i) {
+ pivot = positions[i];
+ }
+
+ @Override
+ protected int comparePivot(int j) {
+ return pivot - positions[j];
+ }
+
+ private int pivot;
+ }.mergeSort(0, tokens.size() - 1);
FieldType fieldType = context.getFieldType();
- for (Token token : tokens) {
- NamedList<Object> tokenNamedList = new SimpleOrderedMap<Object>();
+ for (int i = 0, c = tokens.size(); i < c; i++) {
+ AttributeSource token = tokens.get(i);
+ final NamedList<Object> tokenNamedList = new SimpleOrderedMap<Object>();
+ final String rawText = token.addAttribute(CharTermAttribute.class).toString();
- String text = fieldType.indexedToReadable(token.toString());
+ String text = fieldType.indexedToReadable(rawText);
tokenNamedList.add("text", text);
- if (!text.equals(token.toString())) {
- tokenNamedList.add("raw_text", token.toString());
+ if (!text.equals(rawText)) {
+ tokenNamedList.add("raw_text", rawText);
}
- tokenNamedList.add("type", token.type());
- tokenNamedList.add("start", token.startOffset());
- tokenNamedList.add("end", token.endOffset());
- position += token.getPositionIncrement();
- tokenNamedList.add("position", position);
-
- if (context.getTermsToMatch().contains(token.toString())) {
+ if (context.getTermsToMatch().contains(rawText)) {
tokenNamedList.add("match", true);
}
- if (token.getPayload() != null) {
- tokenNamedList.add("payload", token.getPayload());
- }
+ tokenNamedList.add("position", positions[i]);
+
+ token.reflectWith(new AttributeReflector() {
+ public void reflect(Class<? extends Attribute> attClass, String key, Object value) {
+ // leave out position and term
+ if (CharTermAttribute.class.isAssignableFrom(attClass))
+ return;
+ if (PositionIncrementAttribute.class.isAssignableFrom(attClass))
+ return;
+
+ String k = attClass.getName() + '#' + key;
+
+ // map keys for "standard attributes":
+ if (ATTRIBUTE_MAPPING.containsKey(k)) {
+ k = ATTRIBUTE_MAPPING.get(k);
+ }
+
+ // TODO: special handling for payloads - move this to ResponseWriter?
+ if (value instanceof Payload) {
+ Payload p = (Payload) value;
+ if( null != p ) {
+ BigInteger bi = new BigInteger( p.getData() );
+ String ret = bi.toString( 16 );
+ if (ret.length() % 2 != 0) {
+ // Pad with 0
+ ret = "0"+ret;
+ }
+ value = ret;
+ } else {
+ value = null;
+ }
+ }
+
+ tokenNamedList.add(k, value);
+ }
+ });
tokensNamedLists.add(tokenNamedList);
}
@@ -250,38 +328,27 @@ public abstract class AnalysisRequestHan
*/
// TODO refactor to support custom attributes
protected final static class ListBasedTokenStream extends TokenStream {
- private final List<Token> tokens;
- private Iterator<Token> tokenIterator;
+ private final List<AttributeSource> tokens;
+ private Iterator<AttributeSource> tokenIterator;
- private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
- private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
- private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
- private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);
- private final PayloadAttribute payloadAtt = addAttribute(PayloadAttribute.class);
- private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
/**
* Creates a new ListBasedTokenStream which uses the given tokens as its token source.
*
* @param tokens Source of tokens to be used
*/
- ListBasedTokenStream(List<Token> tokens) {
+ ListBasedTokenStream(List<AttributeSource> tokens) {
this.tokens = tokens;
tokenIterator = tokens.iterator();
}
- /**
- * {@inheritDoc}
- */
@Override
public boolean incrementToken() throws IOException {
if (tokenIterator.hasNext()) {
- Token next = tokenIterator.next();
- termAtt.copyBuffer(next.buffer(), 0, next.length());
- typeAtt.setType(next.type());
- offsetAtt.setOffset(next.startOffset(), next.endOffset());
- flagsAtt.setFlags(next.getFlags());
- payloadAtt.setPayload(next.getPayload());
- posIncAtt.setPositionIncrement(next.getPositionIncrement());
+ AttributeSource next = tokenIterator.next();
+ Iterator<Class<? extends Attribute>> atts = next.getAttributeClassesIterator();
+ while (atts.hasNext()) // make sure all att impls in the token exist here
+ addAttribute(atts.next());
+ next.copyTo(this);
return true;
} else {
return false;
Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/DocumentAnalysisRequestHandler.java Wed Jan 19 12:27:19 2011
@@ -19,7 +19,8 @@ package org.apache.solr.handler;
import org.apache.commons.io.IOUtils;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.AttributeSource;
import org.apache.solr.client.solrj.request.DocumentAnalysisRequest;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrInputDocument;
@@ -216,21 +217,20 @@ public class DocumentAnalysisRequestHand
FieldType fieldType = schema.getFieldType(name);
- Set<String> termsToMatch = new HashSet<String>();
- if (request.getQuery() != null && request.isShowMatch()) {
- try {
- List<Token> tokens = analyzeValue(request.getQuery(), fieldType.getQueryAnalyzer());
- for (Token token : tokens) {
- termsToMatch.add(token.toString());
- }
- } catch (Exception e) {
- // ignore analysis exceptions since we are applying arbitrary text to all fields
- }
+ final String queryValue = request.getQuery();
+ Set<String> termsToMatch;
+ try {
+ termsToMatch = (queryValue != null && request.isShowMatch())
+ ? getQueryTokenSet(queryValue, fieldType.getQueryAnalyzer())
+ : Collections.<String>emptySet();
+ } catch (Exception e) {
+ // ignore analysis exceptions since we are applying arbitrary text to all fields
+ termsToMatch = Collections.<String>emptySet();
}
if (request.getQuery() != null) {
try {
- AnalysisContext analysisContext = new AnalysisContext(fieldType, fieldType.getQueryAnalyzer(), Collections.EMPTY_SET);
+ AnalysisContext analysisContext = new AnalysisContext(fieldType, fieldType.getQueryAnalyzer(), Collections.<String>emptySet());
NamedList<List<NamedList>> tokens = analyzeValue(request.getQuery(), analysisContext);
fieldTokens.add("query", tokens);
} catch (Exception e) {
Modified: lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java?rev=1060784&r1=1060783&r2=1060784&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java (original)
+++ lucene/dev/branches/branch_3x/solr/src/java/org/apache/solr/handler/FieldAnalysisRequestHandler.java Wed Jan 19 12:27:19 2011
@@ -17,7 +17,8 @@
package org.apache.solr.handler;
-import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.util.AttributeSource;
import org.apache.solr.client.solrj.request.FieldAnalysisRequest;
import org.apache.solr.common.params.AnalysisParams;
import org.apache.solr.common.params.CommonParams;
@@ -30,10 +31,7 @@ import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.IndexSchema;
import org.apache.commons.io.IOUtils;
-import java.util.Arrays;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
+import java.util.*;
import java.io.Reader;
import java.io.IOException;
@@ -222,14 +220,10 @@ public class FieldAnalysisRequestHandler
*/
private NamedList<NamedList> analyzeValues(FieldAnalysisRequest analysisRequest, FieldType fieldType, String fieldName) {
- Set<String> termsToMatch = new HashSet<String>();
- String queryValue = analysisRequest.getQuery();
- if (queryValue != null && analysisRequest.isShowMatch()) {
- List<Token> tokens = analyzeValue(queryValue, fieldType.getQueryAnalyzer());
- for (Token token : tokens) {
- termsToMatch.add(token.toString());
- }
- }
+ final String queryValue = analysisRequest.getQuery();
+ final Set<String> termsToMatch = (queryValue != null && analysisRequest.isShowMatch())
+ ? getQueryTokenSet(queryValue, fieldType.getQueryAnalyzer())
+ : Collections.<String>emptySet();
NamedList<NamedList> analyzeResults = new SimpleOrderedMap<NamedList>();
if (analysisRequest.getFieldValue() != null) {