You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2016/08/19 15:26:21 UTC
lucene-solr:branch_6x: LUCENE-7419: Don't lookup
PositionIncrementAttribute every time in TokenStream.end()
Repository: lucene-solr
Updated Branches:
refs/heads/branch_6x 082f36e01 -> 326db813f
LUCENE-7419: Don't lookup PositionIncrementAttribute every time in TokenStream.end()
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/326db813
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/326db813
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/326db813
Branch: refs/heads/branch_6x
Commit: 326db813fa1791a3ada5d5dafebfb47268f03632
Parents: 082f36e
Author: Robert Muir <rm...@apache.org>
Authored: Fri Aug 19 11:14:08 2016 -0400
Committer: Robert Muir <rm...@apache.org>
Committed: Fri Aug 19 11:15:21 2016 -0400
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../src/java/org/apache/lucene/analysis/TokenStream.java | 7 +------
.../tokenattributes/PackedTokenAttributeImpl.java | 11 +++++++++++
.../tokenattributes/PositionIncrementAttributeImpl.java | 5 +++++
.../src/java/org/apache/lucene/util/AttributeImpl.java | 11 +++++++++++
.../src/java/org/apache/lucene/util/AttributeSource.java | 10 ++++++++++
6 files changed, 41 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/326db813/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4cd8f3c..4e4078d 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -74,6 +74,9 @@ Bug Fixes
* SOLR-9413: Fix analysis/kuromoji's CSVUtil.quoteEscape logic, add TestCSVUtil test.
(AppChecker, Christine Poerschke)
+* LUCENE-7419: Fix performance bug with TokenStream.end(), where it would lookup
+ PositionIncrementAttribute every time. (Mike McCandless, Robert Muir)
+
Improvements
* LUCENE-7323: Compound file writing now verifies the incoming
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/326db813/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java b/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java
index 6a78e1c..af1e7bd 100644
--- a/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java
+++ b/lucene/core/src/java/org/apache/lucene/analysis/TokenStream.java
@@ -22,7 +22,6 @@ import java.io.Closeable;
import java.lang.reflect.Modifier;
import org.apache.lucene.analysis.tokenattributes.PackedTokenAttributeImpl;
-import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
@@ -176,11 +175,7 @@ public abstract class TokenStream extends AttributeSource implements Closeable {
* @throws IOException If an I/O error occurs
*/
public void end() throws IOException {
- clearAttributes(); // LUCENE-3849: don't consume dirty atts
- PositionIncrementAttribute posIncAtt = getAttribute(PositionIncrementAttribute.class);
- if (posIncAtt != null) {
- posIncAtt.setPositionIncrement(0);
- }
+ endAttributes(); // LUCENE-3849: don't consume dirty atts
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/326db813/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java
index a84d7b7..aaa3316 100644
--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java
+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PackedTokenAttributeImpl.java
@@ -138,6 +138,17 @@ public class PackedTokenAttributeImpl extends CharTermAttributeImpl
startOffset = endOffset = 0;
type = DEFAULT_TYPE;
}
+
+ /** Resets the attributes at end
+ */
+ @Override
+ public void end() {
+ super.end();
+ positionIncrement = 0;
+ positionLength = 1;
+ startOffset = endOffset = 0;
+ type = DEFAULT_TYPE;
+ }
@Override
public PackedTokenAttributeImpl clone() {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/326db813/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java
index 283f481..4d63d6f 100644
--- a/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java
+++ b/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttributeImpl.java
@@ -47,6 +47,11 @@ public class PositionIncrementAttributeImpl extends AttributeImpl implements Pos
}
@Override
+ public void end() {
+ this.positionIncrement = 0;
+ }
+
+ @Override
public boolean equals(Object other) {
if (other == this) {
return true;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/326db813/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java b/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java
index b722661..8a7c542 100644
--- a/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java
+++ b/lucene/core/src/java/org/apache/lucene/util/AttributeImpl.java
@@ -33,6 +33,17 @@ public abstract class AttributeImpl implements Cloneable, Attribute {
public abstract void clear();
/**
+ * Clears the values in this AttributeImpl and resets it to its value
+ * at the end of the field. If this implementation implements more than one Attribute interface
+ * it clears all.
+ * <p>
+ * The default implementation simply calls {@link #clear()}
+ */
+ public void end() {
+ clear();
+ }
+
+ /**
* This method returns the current attribute values as a string in the following format
* by calling the {@link #reflectWith(AttributeReflector)} method:
*
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/326db813/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java b/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
index 6c39025..e962fed 100644
--- a/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
+++ b/lucene/core/src/java/org/apache/lucene/util/AttributeSource.java
@@ -270,6 +270,16 @@ public class AttributeSource {
state.attribute.clear();
}
}
+
+ /**
+ * Resets all Attributes in this AttributeSource by calling
+ * {@link AttributeImpl#end()} on each Attribute implementation.
+ */
+ public final void endAttributes() {
+ for (State state = getCurrentState(); state != null; state = state.next) {
+ state.attribute.end();
+ }
+ }
/**
* Removes all attributes and their implementations from this AttributeSource.