You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2010/05/17 15:13:11 UTC
svn commit: r945130 - in /lucene/dev/trunk:
lucene/src/java/org/apache/lucene/analysis/standard/
modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/
Author: uschindler
Date: Mon May 17 13:13:10 2010
New Revision: 945130
URL: http://svn.apache.org/viewvc?rev=945130&view=rev
Log:
LUCENE-2384: Remove hack, as JFlex trunk now has the zzBuffer bug fixed
Modified:
lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt
lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex
lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.java
lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex
lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerInterface.java
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt?rev=945130&r1=945129&r2=945130&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt Mon May 17 13:13:10 2010
@@ -17,4 +17,5 @@
WARNING: if you change StandardTokenizerImpl*.jflex and need to regenerate
- the tokenizer, only use the trunk version of JFlex 1.5 at the moment!
+ the tokenizer, only use the trunk version of JFlex 1.5 (with a minimum
+ SVN revision 591) at the moment!
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=945130&r1=945129&r2=945130&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Mon May 17 13:13:10 2010
@@ -201,7 +201,7 @@ public final class StandardTokenizer ext
@Override
public void reset(Reader reader) throws IOException {
super.reset(reader);
- scanner.reset(reader);
+ scanner.yyreset(reader);
}
/**
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.java?rev=945130&r1=945129&r2=945130&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.java Mon May 17 13:13:10 2010
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 10.04.10 13:07 */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 17.05.10 14:50 */
package org.apache.lucene.analysis.standard;
@@ -33,8 +33,8 @@ import org.apache.lucene.analysis.tokena
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 10.04.10 13:07 from the specification file
- * <tt>C:/Users/Uwe Schindler/Projects/lucene/trunk-full1/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex</tt>
+ * on 17.05.10 14:50 from the specification file
+ * <tt>C:/Users/Uwe Schindler/Projects/lucene/newtrunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex</tt>
*/
class StandardTokenizerImpl31 implements StandardTokenizerInterface {
@@ -379,17 +379,6 @@ public final void getText(CharTermAttrib
t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
}
-/**
- * Resets the Tokenizer to a new Reader.
- */
-public final void reset(Reader r) {
- // reset to default buffer size, if buffer has grown
- if (zzBuffer.length > ZZ_BUFFERSIZE) {
- zzBuffer = new char[ZZ_BUFFERSIZE];
- }
- yyreset(r);
-}
-
/**
@@ -505,6 +494,8 @@ public final void reset(Reader r) {
* <b>cannot</b> be reused (internal buffer is discarded and lost).
* Lexical state is set to <tt>ZZ_INITIAL</tt>.
*
+ * Internal scan buffer is resized down to its initial length, if it has grown.
+ *
* @param reader the new input stream
*/
public final void yyreset(java.io.Reader reader) {
@@ -516,6 +507,8 @@ public final void reset(Reader r) {
zzCurrentPos = zzMarkedPos = 0;
yyline = yychar = yycolumn = 0;
zzLexicalState = YYINITIAL;
+ if (zzBuffer.length > ZZ_BUFFERSIZE)
+ zzBuffer = new char[ZZ_BUFFERSIZE];
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex?rev=945130&r1=945129&r2=945130&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl31.jflex Mon May 17 13:13:10 2010
@@ -67,17 +67,6 @@ public final void getText(CharTermAttrib
t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
}
-/**
- * Resets the Tokenizer to a new Reader.
- */
-public final void reset(Reader r) {
- // reset to default buffer size, if buffer has grown
- if (zzBuffer.length > ZZ_BUFFERSIZE) {
- zzBuffer = new char[ZZ_BUFFERSIZE];
- }
- yyreset(r);
-}
-
%}
THAI = [\u0E00-\u0E59]
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.java?rev=945130&r1=945129&r2=945130&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.java Mon May 17 13:13:10 2010
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 10.04.10 13:07 */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 17.05.10 14:50 */
package org.apache.lucene.analysis.standard;
@@ -33,8 +33,8 @@ import org.apache.lucene.analysis.tokena
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 10.04.10 13:07 from the specification file
- * <tt>C:/Users/Uwe Schindler/Projects/lucene/trunk-full1/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex</tt>
+ * on 17.05.10 14:50 from the specification file
+ * <tt>C:/Users/Uwe Schindler/Projects/lucene/newtrunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex</tt>
*/
class StandardTokenizerImplOrig implements StandardTokenizerInterface {
@@ -375,17 +375,6 @@ public final void getText(CharTermAttrib
t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
}
-/**
- * Resets the Tokenizer to a new Reader.
- */
-public final void reset(Reader r) {
- // reset to default buffer size, if buffer has grown
- if (zzBuffer.length > ZZ_BUFFERSIZE) {
- zzBuffer = new char[ZZ_BUFFERSIZE];
- }
- yyreset(r);
-}
-
/**
@@ -501,6 +490,8 @@ public final void reset(Reader r) {
* <b>cannot</b> be reused (internal buffer is discarded and lost).
* Lexical state is set to <tt>ZZ_INITIAL</tt>.
*
+ * Internal scan buffer is resized down to its initial length, if it has grown.
+ *
* @param reader the new input stream
*/
public final void yyreset(java.io.Reader reader) {
@@ -512,6 +503,8 @@ public final void reset(Reader r) {
zzCurrentPos = zzMarkedPos = 0;
yyline = yychar = yycolumn = 0;
zzLexicalState = YYINITIAL;
+ if (zzBuffer.length > ZZ_BUFFERSIZE)
+ zzBuffer = new char[ZZ_BUFFERSIZE];
}
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex?rev=945130&r1=945129&r2=945130&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImplOrig.jflex Mon May 17 13:13:10 2010
@@ -67,17 +67,6 @@ public final void getText(CharTermAttrib
t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
}
-/**
- * Resets the Tokenizer to a new Reader.
- */
-public final void reset(Reader r) {
- // reset to default buffer size, if buffer has grown
- if (zzBuffer.length > ZZ_BUFFERSIZE) {
- zzBuffer = new char[ZZ_BUFFERSIZE];
- }
- yyreset(r);
-}
-
%}
THAI = [\u0E00-\u0E59]
Modified: lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerInterface.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerInterface.java?rev=945130&r1=945129&r2=945130&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerInterface.java (original)
+++ lucene/dev/trunk/lucene/src/java/org/apache/lucene/analysis/standard/StandardTokenizerInterface.java Mon May 17 13:13:10 2010
@@ -47,7 +47,7 @@ interface StandardTokenizerInterface {
*
* @param reader the new input stream
*/
- void reset(Reader reader);
+ void yyreset(Reader reader);
/**
* Returns the length of the matched text region.
Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java?rev=945130&r1=945129&r2=945130&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java Mon May 17 13:13:10 2010
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 10.04.10 12:56 */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 17.05.10 14:51 */
package org.apache.lucene.analysis.wikipedia;
@@ -25,8 +25,8 @@ import org.apache.lucene.analysis.tokena
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 10.04.10 12:56 from the specification file
- * <tt>C:/Users/Uwe Schindler/Projects/lucene/trunk-full1/contrib/analyzers/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
+ * on 17.05.10 14:51 from the specification file
+ * <tt>C:/Users/Uwe Schindler/Projects/lucene/newtrunk/modules/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
*/
class WikipediaTokenizerImpl {
@@ -613,6 +613,8 @@ final int setText(StringBuilder buffer){
* <b>cannot</b> be reused (internal buffer is discarded and lost).
* Lexical state is set to <tt>ZZ_INITIAL</tt>.
*
+ * Internal scan buffer is resized down to its initial length, if it has grown.
+ *
* @param reader the new input stream
*/
public final void yyreset(java.io.Reader reader) {
@@ -624,6 +626,8 @@ final int setText(StringBuilder buffer){
zzCurrentPos = zzMarkedPos = 0;
yyline = yychar = yycolumn = 0;
zzLexicalState = YYINITIAL;
+ if (zzBuffer.length > ZZ_BUFFERSIZE)
+ zzBuffer = new char[ZZ_BUFFERSIZE];
}