You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2011/04/23 01:07:50 UTC
svn commit: r1096078 - in /lucene/dev/branches/branch_3x: ./ lucene/
lucene/CHANGES.txt lucene/backwards/
lucene/src/java/org/apache/lucene/util/AttributeSource.java
lucene/src/test/org/apache/lucene/util/TestAttributeSource.java solr/
Author: uschindler
Date: Fri Apr 22 23:07:49 2011
New Revision: 1096078
URL: http://svn.apache.org/viewvc?rev=1096078&view=rev
Log:
LUCENE-3042: When a filter or consumer added Attributes to a TokenStream chain after it was already (partly) consumed [or clearAttributes(), captureState(), cloneAttributes(),... was called by the Tokenizer], the Tokenizer calling clearAttributes() or capturing state after addition may not do this on the newly added Attribute
Modified:
lucene/dev/branches/branch_3x/ (props changed)
lucene/dev/branches/branch_3x/lucene/ (props changed)
lucene/dev/branches/branch_3x/lucene/CHANGES.txt
lucene/dev/branches/branch_3x/lucene/backwards/ (props changed)
lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeSource.java
lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java
lucene/dev/branches/branch_3x/solr/ (props changed)
Modified: lucene/dev/branches/branch_3x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/CHANGES.txt?rev=1096078&r1=1096077&r2=1096078&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_3x/lucene/CHANGES.txt Fri Apr 22 23:07:49 2011
@@ -1,10 +1,5 @@
Lucene Change Log
-Build
-
-* LUCENE-3006: Building javadocs will fail on warnings by default. Override with -Dfailonjavadocwarning=false (sarowe, gsingers)
-
-
======================= Lucene 3.x (not yet released) =======================
Changes in backwards compatibility policy
@@ -29,6 +24,14 @@ Bug fixes
seeking TermEnum (eg used by Solr's faceting) (Tom Burton-West, Mike
McCandless)
+* LUCENE-3042: When a filter or consumer added Attributes to a TokenStream
+ chain after it was already (partly) consumed [or clearAttributes(),
+ captureState(), cloneAttributes(),... was called by the Tokenizer],
+ the Tokenizer calling clearAttributes() or capturing state after addition
+ may not do this on the newly added Attribute. This bug affected only
+ very special use cases of the TokenStream-API, most users would not
+ have recognized it. (Uwe Schindler, Robert Muir)
+
Test Cases
* LUCENE-3002: added 'tests.iter.min' to control 'tests.iter' by allowing to
Modified: lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeSource.java?rev=1096078&r1=1096077&r2=1096078&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeSource.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/java/org/apache/lucene/util/AttributeSource.java Fri Apr 22 23:07:49 2011
@@ -101,10 +101,33 @@ public class AttributeSource {
}
}
+ /**
+ * This class holds the state of an AttributeSource.
+ * @see #captureState
+ * @see #restoreState
+ */
+ public static final class State implements Cloneable {
+ AttributeImpl attribute;
+ State next;
+
+ @Override
+ public Object clone() {
+ State clone = new State();
+ clone.attribute = (AttributeImpl) attribute.clone();
+
+ if (next != null) {
+ clone.next = (State) next.clone();
+ }
+
+ return clone;
+ }
+ }
+
// These two maps must always be in sync!!!
// So they are private, final and read-only from the outside (read-only iterators)
private final Map<Class<? extends Attribute>, AttributeImpl> attributes;
private final Map<Class<? extends AttributeImpl>, AttributeImpl> attributeImpls;
+ private final State[] currentState;
private AttributeFactory factory;
@@ -124,6 +147,7 @@ public class AttributeSource {
}
this.attributes = input.attributes;
this.attributeImpls = input.attributeImpls;
+ this.currentState = input.currentState;
this.factory = input.factory;
}
@@ -133,6 +157,7 @@ public class AttributeSource {
public AttributeSource(AttributeFactory factory) {
this.attributes = new LinkedHashMap<Class<? extends Attribute>, AttributeImpl>();
this.attributeImpls = new LinkedHashMap<Class<? extends AttributeImpl>, AttributeImpl>();
+ this.currentState = new State[1];
this.factory = factory;
}
@@ -155,11 +180,8 @@ public class AttributeSource {
* if one instance implements more than one Attribute interface.
*/
public Iterator<AttributeImpl> getAttributeImplsIterator() {
- if (hasAttributes()) {
- if (currentState == null) {
- computeCurrentState();
- }
- final State initState = currentState;
+ final State initState = getCurrentState();
+ if (initState != null) {
return new Iterator<AttributeImpl>() {
private State state = initState;
@@ -233,7 +255,7 @@ public class AttributeSource {
// Attribute is a superclass of this interface
if (!attributes.containsKey(curInterface)) {
// invalidate state to force recomputation in captureState()
- this.currentState = null;
+ this.currentState[0] = null;
attributes.put(curInterface, att);
attributeImpls.put(clazz, att);
}
@@ -291,41 +313,21 @@ public class AttributeSource {
}
return attClass.cast(attImpl);
}
-
- /**
- * This class holds the state of an AttributeSource.
- * @see #captureState
- * @see #restoreState
- */
- public static final class State implements Cloneable {
- AttributeImpl attribute;
- State next;
- @Override
- public Object clone() {
- State clone = new State();
- clone.attribute = (AttributeImpl) attribute.clone();
-
- if (next != null) {
- clone.next = (State) next.clone();
- }
-
- return clone;
+ private State getCurrentState() {
+ State s = currentState[0];
+ if (s != null || !hasAttributes()) {
+ return s;
}
- }
-
- private State currentState = null;
-
- private void computeCurrentState() {
- currentState = new State();
- State c = currentState;
+ State c = s = currentState[0] = new State();
final Iterator<AttributeImpl> it = attributeImpls.values().iterator();
c.attribute = it.next();
while (it.hasNext()) {
c.next = new State();
c = c.next;
c.attribute = it.next();
- }
+ }
+ return s;
}
/**
@@ -333,13 +335,8 @@ public class AttributeSource {
* {@link AttributeImpl#clear()} on each Attribute implementation.
*/
public void clearAttributes() {
- if (hasAttributes()) {
- if (currentState == null) {
- computeCurrentState();
- }
- for (State state = currentState; state != null; state = state.next) {
- state.attribute.clear();
- }
+ for (State state = getCurrentState(); state != null; state = state.next) {
+ state.attribute.clear();
}
}
@@ -348,14 +345,8 @@ public class AttributeSource {
* {@link #restoreState} to restore the state of this or another AttributeSource.
*/
public State captureState() {
- if (!hasAttributes()) {
- return null;
- }
-
- if (currentState == null) {
- computeCurrentState();
- }
- return (State) this.currentState.clone();
+ final State state = this.getCurrentState();
+ return (state == null) ? null : (State) state.clone();
}
/**
@@ -390,15 +381,9 @@ public class AttributeSource {
@Override
public int hashCode() {
int code = 0;
- if (hasAttributes()) {
- if (currentState == null) {
- computeCurrentState();
- }
- for (State state = currentState; state != null; state = state.next) {
- code = code * 31 + state.attribute.hashCode();
- }
+ for (State state = getCurrentState(); state != null; state = state.next) {
+ code = code * 31 + state.attribute.hashCode();
}
-
return code;
}
@@ -421,14 +406,8 @@ public class AttributeSource {
}
// it is only equal if all attribute impls are the same in the same order
- if (this.currentState == null) {
- this.computeCurrentState();
- }
- State thisState = this.currentState;
- if (other.currentState == null) {
- other.computeCurrentState();
- }
- State otherState = other.currentState;
+ State thisState = this.getCurrentState();
+ State otherState = other.getCurrentState();
while (thisState != null && otherState != null) {
if (otherState.attribute.getClass() != thisState.attribute.getClass() || !otherState.attribute.equals(thisState.attribute)) {
return false;
@@ -460,11 +439,8 @@ public class AttributeSource {
public String toString() {
final StringBuilder sb = new StringBuilder().append('(');
if (hasAttributes()) {
- if (currentState == null) {
- computeCurrentState();
- }
- for (State state = currentState; state != null; state = state.next) {
- if (state != currentState) sb.append(',');
+ for (State state = getCurrentState(); state != null; state = state.next) {
+ if (sb.length() > 1) sb.append(',');
sb.append(state.attribute.toString());
}
}
@@ -508,13 +484,8 @@ public class AttributeSource {
* @see AttributeImpl#reflectWith
*/
public final void reflectWith(AttributeReflector reflector) {
- if (hasAttributes()) {
- if (currentState == null) {
- computeCurrentState();
- }
- for (State state = currentState; state != null; state = state.next) {
- state.attribute.reflectWith(reflector);
- }
+ for (State state = getCurrentState(); state != null; state = state.next) {
+ state.attribute.reflectWith(reflector);
}
}
@@ -530,10 +501,7 @@ public class AttributeSource {
if (hasAttributes()) {
// first clone the impls
- if (currentState == null) {
- computeCurrentState();
- }
- for (State state = currentState; state != null; state = state.next) {
+ for (State state = getCurrentState(); state != null; state = state.next) {
clone.attributeImpls.put(state.attribute.getClass(), (AttributeImpl) state.attribute.clone());
}
@@ -555,18 +523,13 @@ public class AttributeSource {
* {@link #cloneAttributes} instead of {@link #captureState}.
*/
public final void copyTo(AttributeSource target) {
- if (hasAttributes()) {
- if (currentState == null) {
- computeCurrentState();
- }
- for (State state = currentState; state != null; state = state.next) {
- final AttributeImpl targetImpl = target.attributeImpls.get(state.attribute.getClass());
- if (targetImpl == null) {
- throw new IllegalArgumentException("This AttributeSource contains AttributeImpl of type " +
- state.attribute.getClass().getName() + " that is not in the target");
- }
- state.attribute.copyTo(targetImpl);
+ for (State state = getCurrentState(); state != null; state = state.next) {
+ final AttributeImpl targetImpl = target.attributeImpls.get(state.attribute.getClass());
+ if (targetImpl == null) {
+ throw new IllegalArgumentException("This AttributeSource contains AttributeImpl of type " +
+ state.attribute.getClass().getName() + " that is not in the target");
}
+ state.attribute.copyTo(targetImpl);
}
}
Modified: lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java?rev=1096078&r1=1096077&r2=1096078&view=diff
==============================================================================
--- lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java (original)
+++ lucene/dev/branches/branch_3x/lucene/src/test/org/apache/lucene/util/TestAttributeSource.java Fri Apr 22 23:07:49 2011
@@ -178,6 +178,16 @@ public class TestAttributeSource extends
} catch (IllegalArgumentException iae) {}
}
+ public void testLUCENE_3042() throws Exception {
+ final AttributeSource src1 = new AttributeSource();
+ src1.addAttribute(CharTermAttribute.class).append("foo");
+ int hash1 = src1.hashCode(); // this triggers a cached state
+ final AttributeSource src2 = new AttributeSource(src1);
+ src2.addAttribute(TypeAttribute.class).setType("bar");
+ assertTrue("The hashCode is identical, so the captured state was preserved.", hash1 != src1.hashCode());
+ assertEquals(src2.hashCode(), src1.hashCode());
+ }
+
// this class is included in external class check, so no assertion errors occur
@Deprecated
static class TestAttributeImpl extends AttributeImpl implements FlagsAttribute {