You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by us...@apache.org on 2010/10/29 15:44:30 UTC
svn commit: r1028739 - in /lucene/java/branches/lucene_3_0: ./
src/java/org/apache/lucene/analysis/standard/
Author: uschindler
Date: Fri Oct 29 13:44:30 2010
New Revision: 1028739
URL: http://svn.apache.org/viewvc?rev=1028739&view=rev
Log:
LUCENE-2384: Reset zzBuffer in StandardTokenizerImpl when lexer is reset
Modified:
lucene/java/branches/lucene_3_0/CHANGES.txt
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
Modified: lucene/java/branches/lucene_3_0/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/CHANGES.txt?rev=1028739&r1=1028738&r2=1028739&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/CHANGES.txt (original)
+++ lucene/java/branches/lucene_3_0/CHANGES.txt Fri Oct 29 13:44:30 2010
@@ -48,6 +48,9 @@ Bug fixes
* LUCENE-2650: Added extra safety to MMapIndexInput clones to prevent accessing
an unmapped buffer if the input is closed (Mike McCandless, Uwe Schindler, Robert Muir)
+* LUCENE-2384: Reset zzBuffer in StandardTokenizerImpl when lexer is reset.
+ (Ruben Laguna via Uwe Schindler, sub-issue of LUCENE-2074)
+
Documentation
* LUCENE-2239: Documented limitations in NIOFSDirectory and MMapDirectory due
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=1028739&r1=1028738&r2=1028739&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Fri Oct 29 13:44:30 2010
@@ -203,21 +203,10 @@ public final class StandardTokenizer ext
offsetAtt.setOffset(finalOffset, finalOffset);
}
- /*
- * (non-Javadoc)
- *
- * @see org.apache.lucene.analysis.TokenStream#reset()
- */
- @Override
- public void reset() throws IOException {
- super.reset();
- scanner.yyreset(input);
- }
-
@Override
public void reset(Reader reader) throws IOException {
super.reset(reader);
- reset();
+ scanner.reset(reader);
}
/**
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=1028739&r1=1028738&r2=1028739&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java Fri Oct 29 13:44:30 2010
@@ -364,6 +364,17 @@ public final int yychar()
}
/**
+ * Resets the Tokenizer to a new Reader.
+ */
+final void reset(java.io.Reader r) {
+ // reset to default buffer size, if buffer has grown
+ if (zzBuffer.length > ZZ_BUFFERSIZE) {
+ zzBuffer = new char[ZZ_BUFFERSIZE];
+ }
+ yyreset(r);
+}
+
+/**
* Fills Lucene token with the current token text.
*/
final void getText(Token t) {
Modified: lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex?rev=1028739&r1=1028738&r2=1028739&view=diff
==============================================================================
--- lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex (original)
+++ lucene/java/branches/lucene_3_0/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex Fri Oct 29 13:44:30 2010
@@ -65,6 +65,17 @@ public final int yychar()
}
/**
+ * Resets the Tokenizer to a new Reader.
+ */
+final void reset(java.io.Reader r) {
+ // reset to default buffer size, if buffer has grown
+ if (zzBuffer.length > ZZ_BUFFERSIZE) {
+ zzBuffer = new char[ZZ_BUFFERSIZE];
+ }
+ yyreset(r);
+}
+
+/**
* Fills Lucene token with the current token text.
*/
final void getText(Token t) {