You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by us...@apache.org on 2009/08/10 23:34:46 UTC
svn commit: r802930 - in /lucene/java/trunk/src:
java/org/apache/lucene/analysis/ java/org/apache/lucene/analysis/standard/
java/org/apache/lucene/util/ test/org/apache/lucene/util/
Author: uschindler
Date: Mon Aug 10 21:34:46 2009
New Revision: 802930
URL: http://svn.apache.org/viewvc?rev=802930&view=rev
Log:
LUCENE-1796: Speed up repeated TokenStream init by caching isMethodOverridden results. Also speed up AttributeSource.clearAttributes().
Modified:
lucene/java/trunk/src/java/org/apache/lucene/analysis/CharTokenizer.java
lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java
lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java
lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
lucene/java/trunk/src/java/org/apache/lucene/util/AttributeSource.java
lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java
Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/CharTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/CharTokenizer.java?rev=802930&r1=802929&r2=802930&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/CharTokenizer.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/CharTokenizer.java Mon Aug 10 21:34:46 2009
@@ -53,9 +53,9 @@
}
public final boolean incrementToken() throws IOException {
- clearAttributes();
int length = 0;
int start = bufferIndex;
+ termAtt.clear();
char[] buffer = termAtt.termBuffer();
while (true) {
Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java?rev=802930&r1=802929&r2=802930&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/KeywordTokenizer.java Mon Aug 10 21:34:46 2009
@@ -44,13 +44,13 @@
this.done = false;
termAtt = (TermAttribute) addAttribute(TermAttribute.class);
offsetAtt = (OffsetAttribute) addAttribute(OffsetAttribute.class);
+ termAtt.resizeTermBuffer(bufferSize);
}
public final boolean incrementToken() throws IOException {
if (!done) {
done = true;
int upto = 0;
- termAtt.clear();
char[] buffer = termAtt.termBuffer();
while (true) {
final int length = input.read(buffer, upto, buffer.length-upto);
Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java?rev=802930&r1=802929&r2=802930&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/TokenStream.java Mon Aug 10 21:34:46 2009
@@ -18,6 +18,7 @@
*/
import java.io.IOException;
+import java.util.IdentityHashMap;
import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
@@ -75,34 +76,53 @@
= new TokenWrapperAttributeFactory(AttributeFactory.DEFAULT_ATTRIBUTE_FACTORY);
/** @deprecated Remove this when old API is removed! */
- private static final Class[] METHOD_NO_PARAMS = new Class[0];
-
- /** @deprecated Remove this when old API is removed! */
- private static final Class[] METHOD_TOKEN_PARAM = new Class[]{Token.class};
-
- /** @deprecated Remove this when old API is removed! */
private final TokenWrapper tokenWrapper;
/** @deprecated Remove this when old API is removed! */
private static boolean onlyUseNewAPI = false;
/** @deprecated Remove this when old API is removed! */
- private final boolean
- hasIncrementToken = isMethodOverridden("incrementToken", METHOD_NO_PARAMS),
- hasReusableNext = onlyUseNewAPI ? false : isMethodOverridden("next", METHOD_TOKEN_PARAM),
- hasNext = onlyUseNewAPI ? false : isMethodOverridden("next", METHOD_NO_PARAMS);
-
+ private final MethodSupport supportedMethods = getSupportedMethods(this.getClass());
+
/** @deprecated Remove this when old API is removed! */
- private boolean isMethodOverridden(String name, Class[] params) {
- try {
- return this.getClass().getMethod(name, params).getDeclaringClass() != TokenStream.class;
- } catch (NoSuchMethodException e) {
- // should not happen
- throw new RuntimeException(e);
+ private static final class MethodSupport {
+ final boolean hasIncrementToken, hasReusableNext, hasNext;
+
+ MethodSupport(Class clazz) {
+ hasIncrementToken = isMethodOverridden(clazz, "incrementToken", METHOD_NO_PARAMS);
+ hasReusableNext = isMethodOverridden(clazz, "next", METHOD_TOKEN_PARAM);
+ hasNext = isMethodOverridden(clazz, "next", METHOD_NO_PARAMS);
}
+
+ private static boolean isMethodOverridden(Class clazz, String name, Class[] params) {
+ try {
+ return clazz.getMethod(name, params).getDeclaringClass() != TokenStream.class;
+ } catch (NoSuchMethodException e) {
+ // should not happen
+ throw new RuntimeException(e);
+ }
+ }
+
+ private static final Class[] METHOD_NO_PARAMS = new Class[0];
+ private static final Class[] METHOD_TOKEN_PARAM = new Class[]{Token.class};
}
+
+ /** @deprecated Remove this when old API is removed! */
+ private static final IdentityHashMap/*<Class<? extends TokenStream>,MethodSupport>*/ knownMethodSupport = new IdentityHashMap();
/** @deprecated Remove this when old API is removed! */
+ private static MethodSupport getSupportedMethods(Class clazz) {
+ MethodSupport supportedMethods;
+ synchronized(knownMethodSupport) {
+ supportedMethods = (MethodSupport) knownMethodSupport.get(clazz);
+ if (supportedMethods == null) {
+ knownMethodSupport.put(clazz, supportedMethods = new MethodSupport(clazz));
+ }
+ }
+ return supportedMethods;
+ }
+
+ /** @deprecated Remove this when old API is removed! */
private static final class TokenWrapperAttributeFactory extends AttributeFactory {
private final AttributeFactory delegate;
@@ -193,12 +213,12 @@
/** @deprecated Remove this when old API is removed! */
private void check() {
- if (onlyUseNewAPI && !hasIncrementToken) {
+ if (onlyUseNewAPI && !supportedMethods.hasIncrementToken) {
throw new UnsupportedOperationException(getClass().getName()+" does not implement incrementToken() which is needed for onlyUseNewAPI.");
}
// a TokenStream subclass must at least implement one of the methods!
- if (!(hasIncrementToken || hasNext || hasReusableNext)) {
+ if (!(supportedMethods.hasIncrementToken || supportedMethods.hasNext || supportedMethods.hasReusableNext)) {
throw new UnsupportedOperationException(getClass().getName()+" does not implement any of incrementToken(), next(Token), next().");
}
}
@@ -257,10 +277,10 @@
assert !onlyUseNewAPI && tokenWrapper != null;
final Token token;
- if (hasReusableNext) {
+ if (supportedMethods.hasReusableNext) {
token = next(tokenWrapper.delegate);
} else {
- assert hasNext;
+ assert supportedMethods.hasNext;
token = next();
}
if (token == null) return false;
@@ -321,11 +341,11 @@
if (onlyUseNewAPI)
throw new UnsupportedOperationException("This TokenStream only supports the new Attributes API.");
- if (hasIncrementToken) {
+ if (supportedMethods.hasIncrementToken) {
tokenWrapper.delegate = reusableToken;
return incrementToken() ? tokenWrapper.delegate : null;
} else {
- assert hasNext;
+ assert supportedMethods.hasNext;
final Token token = next();
if (token == null) return null;
tokenWrapper.delegate = token;
@@ -344,10 +364,10 @@
if (onlyUseNewAPI)
throw new UnsupportedOperationException("This TokenStream only supports the new Attributes API.");
- if (hasIncrementToken) {
+ if (supportedMethods.hasIncrementToken) {
return incrementToken() ? ((Token) tokenWrapper.delegate.clone()) : null;
} else {
- assert hasReusableNext;
+ assert supportedMethods.hasReusableNext;
final Token token = next(tokenWrapper.delegate);
if (token == null) return null;
tokenWrapper.delegate = token;
Modified: lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=802930&r1=802929&r2=802930&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Mon Aug 10 21:34:46 2009
@@ -158,7 +158,6 @@
}
if (scanner.yylength() <= maxTokenLength) {
- termAtt.clear();
posIncrAtt.setPositionIncrement(posIncr);
scanner.getText(termAtt);
final int start = scanner.yychar();
Modified: lucene/java/trunk/src/java/org/apache/lucene/util/AttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/java/org/apache/lucene/util/AttributeSource.java?rev=802930&r1=802929&r2=802930&view=diff
==============================================================================
--- lucene/java/trunk/src/java/org/apache/lucene/util/AttributeSource.java (original)
+++ lucene/java/trunk/src/java/org/apache/lucene/util/AttributeSource.java Mon Aug 10 21:34:46 2009
@@ -17,8 +17,9 @@
* limitations under the License.
*/
-import java.util.Iterator;
import java.util.Collections;
+import java.util.NoSuchElementException;
+import java.util.Iterator;
import java.util.LinkedHashMap;
import java.util.IdentityHashMap;
import java.util.LinkedList;
@@ -137,7 +138,33 @@
* if one instance implements more than one Attribute interface.
*/
public Iterator/*<AttributeImpl>*/ getAttributeImplsIterator() {
- return Collections.unmodifiableCollection(attributeImpls.values()).iterator();
+ if (hasAttributes()) {
+ if (currentState == null) {
+ computeCurrentState();
+ }
+ final State initState = currentState;
+ return new Iterator() {
+ private State state = initState;
+
+ public void remove() {
+ throw new UnsupportedOperationException();
+ }
+
+ public Object next() {
+ if (state == null)
+ throw new NoSuchElementException();
+ final AttributeImpl att = state.attribute;
+ state = state.next;
+ return att;
+ }
+
+ public boolean hasNext() {
+ return state != null;
+ }
+ };
+ } else {
+ return Collections.EMPTY_SET.iterator();
+ }
}
/** a cache that stores all interfaces for known implementation classes for performance (slow reflection) */
@@ -226,17 +253,6 @@
}
/**
- * Resets all Attributes in this AttributeSource by calling
- * {@link AttributeImpl#clear()} on each Attribute implementation.
- */
- public void clearAttributes() {
- Iterator it = getAttributeImplsIterator();
- while (it.hasNext()) {
- ((AttributeImpl) it.next()).clear();
- }
- }
-
- /**
* This class holds the state of an AttributeSource.
* @see #captureState
* @see #restoreState
@@ -262,7 +278,7 @@
private void computeCurrentState() {
currentState = new State();
State c = currentState;
- Iterator it = getAttributeImplsIterator();
+ Iterator it = attributeImpls.values().iterator();
c.attribute = (AttributeImpl) it.next();
while (it.hasNext()) {
c.next = new State();
@@ -272,6 +288,21 @@
}
/**
+ * Resets all Attributes in this AttributeSource by calling
+ * {@link AttributeImpl#clear()} on each Attribute implementation.
+ */
+ public void clearAttributes() {
+ if (hasAttributes()) {
+ if (currentState == null) {
+ computeCurrentState();
+ }
+ for (State state = currentState; state != null; state = state.next) {
+ state.attribute.clear();
+ }
+ }
+ }
+
+ /**
* Captures the state of all Attributes. The return value can be passed to
* {@link #restoreState} to restore the state of this or another AttributeSource.
*/
@@ -316,9 +347,11 @@
public int hashCode() {
int code = 0;
if (hasAttributes()) {
- Iterator it = getAttributeImplsIterator();
- while (it.hasNext()) {
- code = code * 31 + it.next().hashCode();
+ if (currentState == null) {
+ computeCurrentState();
+ }
+ for (State state = currentState; state != null; state = state.next) {
+ code = code * 31 + state.attribute.hashCode();
}
}
@@ -343,14 +376,20 @@
}
// it is only equal if all attribute impls are the same in the same order
- Iterator thisIt = this.getAttributeImplsIterator();
- Iterator otherIt = other.getAttributeImplsIterator();
- while (thisIt.hasNext() && otherIt.hasNext()) {
- AttributeImpl thisAtt = (AttributeImpl) thisIt.next();
- AttributeImpl otherAtt = (AttributeImpl) otherIt.next();
- if (otherAtt.getClass() != thisAtt.getClass() || !otherAtt.equals(thisAtt)) {
+ if (this.currentState == null) {
+ this.computeCurrentState();
+ }
+ State thisState = this.currentState;
+ if (other.currentState == null) {
+ other.computeCurrentState();
+ }
+ State otherState = other.currentState;
+ while (thisState != null && otherState != null) {
+ if (otherState.attribute.getClass() != thisState.attribute.getClass() || !otherState.attribute.equals(thisState.attribute)) {
return false;
}
+ thisState = thisState.next;
+ otherState = otherState.next;
}
return true;
} else {
@@ -365,13 +404,12 @@
sb.append('(');
if (hasAttributes()) {
- Iterator it = getAttributeImplsIterator();
- if (it.hasNext()) {
- sb.append(it.next().toString());
- }
- while (it.hasNext()) {
- sb.append(',');
- sb.append(it.next().toString());
+ if (currentState == null) {
+ computeCurrentState();
+ }
+ for (State state = currentState; state != null; state = state.next) {
+ if (state != currentState) sb.append(',');
+ sb.append(state.attribute.toString());
}
}
sb.append(')');
@@ -387,10 +425,13 @@
AttributeSource clone = new AttributeSource(this.factory);
// first clone the impls
- Iterator/*<AttributeImpl>*/ implIt = getAttributeImplsIterator();
- while (implIt.hasNext()) {
- AttributeImpl impl = (AttributeImpl) implIt.next();
- clone.attributeImpls.put(impl.getClass(), impl.clone());
+ if (hasAttributes()) {
+ if (currentState == null) {
+ computeCurrentState();
+ }
+ for (State state = currentState; state != null; state = state.next) {
+ clone.attributeImpls.put(state.attribute.getClass(), state.attribute.clone());
+ }
}
// now the interfaces
Modified: lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java?rev=802930&r1=802929&r2=802930&view=diff
==============================================================================
--- lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java (original)
+++ lucene/java/trunk/src/test/org/apache/lucene/util/TestAttributeSource.java Mon Aug 10 21:34:46 2009
@@ -104,6 +104,12 @@
termAtt.setTermBuffer("TestTerm");
typeAtt.setType("TestType");
assertEquals("Attributes should appear in original order", "("+termAtt.toString()+","+typeAtt.toString()+")", src.toString());
+ Iterator it = src.getAttributeImplsIterator();
+ assertTrue("Iterator should have 2 attributes left", it.hasNext());
+ assertSame("First AttributeImpl from iterator should be termAtt", termAtt, it.next());
+ assertTrue("Iterator should have 1 attributes left", it.hasNext());
+ assertSame("Second AttributeImpl from iterator should be typeAtt", typeAtt, it.next());
+ assertFalse("Iterator should have 0 attributes left", it.hasNext());
src = new AttributeSource();
src.addAttributeImpl(new Token());
@@ -111,7 +117,7 @@
termAtt = (TermAttribute) src.addAttribute(TermAttribute.class);
assertTrue("TermAttribute should be implemented by Token", termAtt instanceof Token);
// get the Token attribute and check, that it is the only one
- final Iterator it = src.getAttributeImplsIterator();
+ it = src.getAttributeImplsIterator();
Token tok = (Token) it.next();
assertFalse("There should be only one attribute implementation instance", it.hasNext());