You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/08/06 19:16:48 UTC
svn commit: r1369883 - in /lucene/dev/trunk/lucene: ./
analysis/common/src/java/org/apache/lucene/analysis/charfilter/
analysis/common/src/java/org/apache/lucene/analysis/standard/
analysis/common/src/java/org/apache/lucene/analysis/wikipedia/
Author: rmuir
Date: Mon Aug 6 17:16:47 2012
New Revision: 1369883
URL: http://svn.apache.org/viewvc?rev=1369883&view=rev
Log:
LUCENE-4291: reduce jflex buffer sizes
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1369883&r1=1369882&r2=1369883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Mon Aug 6 17:16:47 2012
@@ -120,6 +120,10 @@ Optimizations
making them substantially more lightweight. Behavior is unchanged.
(Robert Muir)
+* LUCENE-4291: Reduced internal buffer size for Jflex-based tokenizers
+ such as StandardTokenizer from 32kb to 8kb.
+ (Raintung Li, Steven Rowe, Robert Muir)
+
Bug Fixes
* LUCENE-4109: BooleanQueries are not parsed correctly with the
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java?rev=1369883&r1=1369882&r2=1369883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java Mon Aug 6 17:16:47 2012
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 7/26/12 6:22 PM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 8/6/12 11:57 AM */
package org.apache.lucene.analysis.charfilter;
@@ -40,8 +40,8 @@ import org.apache.lucene.analysis.util.O
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 7/26/12 6:22 PM from the specification file
- * <tt>C:/svn/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex</tt>
+ * on 8/6/12 11:57 AM from the specification file
+ * <tt>/home/rmuir/workspace/lucene-trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex</tt>
*/
public final class HTMLStripCharFilter extends BaseCharFilter {
@@ -31255,88 +31255,56 @@ public final class HTMLStripCharFilter e
{ yybegin(STYLE);
}
case 55: break;
- case 51:
- { // Handle paired UTF-16 surrogates.
- String surrogatePair = yytext();
- char highSurrogate = '\u0000';
- char lowSurrogate = '\u0000';
- try {
- highSurrogate = (char)Integer.parseInt(surrogatePair.substring(2, 6), 16);
- } catch(Exception e) { // should never happen
- assert false: "Exception parsing high surrogate '"
- + surrogatePair.substring(2, 6) + "'";
- }
- try { // Low surrogates are in decimal range [56320, 57343]
- lowSurrogate = (char)Integer.parseInt(surrogatePair.substring(9, 14));
- } catch(Exception e) { // should never happen
- assert false: "Exception parsing low surrogate '"
- + surrogatePair.substring(9, 14) + "'";
- }
- if (Character.isLowSurrogate(lowSurrogate)) {
- outputSegment = entitySegment;
- outputSegment.clear();
- outputSegment.unsafeWrite(lowSurrogate);
- // add (previously matched input length) + (this match length) - (substitution length)
- cumulativeDiff += inputSegment.length() + yylength() - 2;
- // position the correction at (already output length) + (substitution length)
- addOffCorrectMap(outputCharCount + 2, cumulativeDiff);
- inputSegment.clear();
- yybegin(YYINITIAL);
- return highSurrogate;
- }
- yypushback(surrogatePair.length() - 1); // Consume only '#'
- inputSegment.append('#');
- yybegin(NUMERIC_CHARACTER);
+ case 27:
+ { // add (previously matched input length) + (this match length) - (substitution length)
+ cumulativeDiff += inputSegment.length() + yylength() - 1;
+ // position the correction at (already output length) + (substitution length)
+ addOffCorrectMap(outputCharCount + 1, cumulativeDiff);
+ inputSegment.clear();
+ yybegin(YYINITIAL);
+ return BLOCK_LEVEL_START_TAG_REPLACEMENT;
}
case 56: break;
- case 21:
- { previousRestoreState = restoreState;
- restoreState = SERVER_SIDE_INCLUDE;
- yybegin(SINGLE_QUOTED_STRING);
+ case 30:
+ { int length = yylength();
+ inputSegment.write(zzBuffer, zzStartRead, length);
+ entitySegment.clear();
+ char ch = entityValues.get(zzBuffer, zzStartRead, length).charValue();
+ entitySegment.append(ch);
+ outputSegment = entitySegment;
+ yybegin(CHARACTER_REFERENCE_TAIL);
}
case 57: break;
- case 31:
- { int matchLength = yylength();
- inputSegment.write(zzBuffer, zzStartRead, matchLength);
- if (matchLength <= 6) { // 10FFFF: max 6 hex chars
- String hexCharRef
- = new String(zzBuffer, zzStartRead + 1, matchLength - 1);
- int codePoint = 0;
- try {
- codePoint = Integer.parseInt(hexCharRef, 16);
- } catch(Exception e) {
- assert false: "Exception parsing hex code point '" + hexCharRef + "'";
- }
- if (codePoint <= 0x10FFFF) {
- outputSegment = entitySegment;
- outputSegment.clear();
- if (codePoint >= Character.MIN_SURROGATE
- && codePoint <= Character.MAX_SURROGATE) {
- outputSegment.unsafeWrite(REPLACEMENT_CHARACTER);
- } else {
- outputSegment.setLength
- (Character.toChars(codePoint, outputSegment.getArray(), 0));
- }
- yybegin(CHARACTER_REFERENCE_TAIL);
- } else {
- outputSegment = inputSegment;
- yybegin(YYINITIAL);
- return outputSegment.nextChar();
- }
- } else {
+ case 48:
+ { inputSegment.clear();
+ yybegin(YYINITIAL);
+ // add (previously matched input length) -- current match and substitution handled below
+ cumulativeDiff += yychar - inputStart;
+ // position the offset correction at (already output length) -- substitution handled below
+ int offsetCorrectionPos = outputCharCount;
+ int returnValue;
+ if (escapeSTYLE) {
+ inputSegment.write(zzBuffer, zzStartRead, yylength());
outputSegment = inputSegment;
- yybegin(YYINITIAL);
- return outputSegment.nextChar();
+ returnValue = outputSegment.nextChar();
+ } else {
+ // add (this match length) - (substitution length)
+ cumulativeDiff += yylength() - 1;
+ // add (substitution length)
+ ++offsetCorrectionPos;
+ returnValue = STYLE_REPLACEMENT;
}
+ addOffCorrectMap(offsetCorrectionPos, cumulativeDiff);
+ return returnValue;
}
case 58: break;
- case 19:
+ case 8:
{ inputSegment.write(zzBuffer, zzStartRead, yylength());
if (null != escapedTags
&& escapedTags.contains(zzBuffer, zzStartRead, yylength())) {
- yybegin(END_TAG_TAIL_INCLUDE);
+ yybegin(START_TAG_TAIL_INCLUDE);
} else {
- yybegin(END_TAG_TAIL_EXCLUDE);
+ yybegin(START_TAG_TAIL_SUBSTITUTE);
}
}
case 59: break;
@@ -31347,113 +31315,79 @@ public final class HTMLStripCharFilter e
yybegin(LEFT_ANGLE_BRACKET);
}
case 60: break;
- case 27:
- { // add (previously matched input length) + (this match length) - (substitution length)
- cumulativeDiff += inputSegment.length() + yylength() - 1;
- // position the correction at (already output length) + (substitution length)
- addOffCorrectMap(outputCharCount + 1, cumulativeDiff);
- inputSegment.clear();
- yybegin(YYINITIAL);
- return BLOCK_LEVEL_START_TAG_REPLACEMENT;
- }
- case 61: break;
case 44:
{ restoreState = STYLE_COMMENT; yybegin(SERVER_SIDE_INCLUDE);
}
+ case 61: break;
+ case 21:
+ { previousRestoreState = restoreState;
+ restoreState = SERVER_SIDE_INCLUDE;
+ yybegin(SINGLE_QUOTED_STRING);
+ }
case 62: break;
+ case 11:
+ { inputSegment.write(zzBuffer, zzStartRead, yylength());
+ yybegin(LEFT_ANGLE_BRACKET_SPACE);
+ }
+ case 63: break;
case 35:
{ yybegin(SCRIPT);
}
- case 63: break;
+ case 64: break;
case 42:
{ restoreState = COMMENT; yybegin(SERVER_SIDE_INCLUDE);
}
- case 64: break;
+ case 65: break;
case 10:
{ inputSegment.append('!'); yybegin(BANG);
}
- case 65: break;
- case 33:
- { yybegin(YYINITIAL);
- if (escapeBR) {
- inputSegment.write(zzBuffer, zzStartRead, yylength());
- outputSegment = inputSegment;
- return outputSegment.nextChar();
- } else {
- // add (previously matched input length) + (this match length) - (substitution length)
- cumulativeDiff += inputSegment.length() + yylength() - 1;
- // position the correction at (already output length) + (substitution length)
- addOffCorrectMap(outputCharCount + 1, cumulativeDiff);
- inputSegment.reset();
- return BR_START_TAG_REPLACEMENT;
- }
- }
case 66: break;
- case 53:
+ case 51:
{ // Handle paired UTF-16 surrogates.
String surrogatePair = yytext();
char highSurrogate = '\u0000';
- try { // High surrogates are in decimal range [55296, 56319]
- highSurrogate = (char)Integer.parseInt(surrogatePair.substring(1, 6));
+ char lowSurrogate = '\u0000';
+ try {
+ highSurrogate = (char)Integer.parseInt(surrogatePair.substring(2, 6), 16);
} catch(Exception e) { // should never happen
assert false: "Exception parsing high surrogate '"
- + surrogatePair.substring(1, 6) + "'";
+ + surrogatePair.substring(2, 6) + "'";
}
- if (Character.isHighSurrogate(highSurrogate)) {
- char lowSurrogate = '\u0000';
- try { // Low surrogates are in decimal range [56320, 57343]
- lowSurrogate = (char)Integer.parseInt(surrogatePair.substring(9, 14));
- } catch(Exception e) { // should never happen
- assert false: "Exception parsing low surrogate '"
- + surrogatePair.substring(9, 14) + "'";
- }
- if (Character.isLowSurrogate(lowSurrogate)) {
- outputSegment = entitySegment;
- outputSegment.clear();
- outputSegment.unsafeWrite(lowSurrogate);
- // add (previously matched input length) + (this match length) - (substitution length)
- cumulativeDiff += inputSegment.length() + yylength() - 2;
- // position the correction at (already output length) + (substitution length)
- addOffCorrectMap(outputCharCount + 2, cumulativeDiff);
- inputSegment.clear();
- yybegin(YYINITIAL);
- return highSurrogate;
- }
+ try { // Low surrogates are in decimal range [56320, 57343]
+ lowSurrogate = (char)Integer.parseInt(surrogatePair.substring(9, 14));
+ } catch(Exception e) { // should never happen
+ assert false: "Exception parsing low surrogate '"
+ + surrogatePair.substring(9, 14) + "'";
+ }
+ if (Character.isLowSurrogate(lowSurrogate)) {
+ outputSegment = entitySegment;
+ outputSegment.clear();
+ outputSegment.unsafeWrite(lowSurrogate);
+ // add (previously matched input length) + (this match length) - (substitution length)
+ cumulativeDiff += inputSegment.length() + yylength() - 2;
+ // position the correction at (already output length) + (substitution length)
+ addOffCorrectMap(outputCharCount + 2, cumulativeDiff);
+ inputSegment.clear();
+ yybegin(YYINITIAL);
+ return highSurrogate;
}
yypushback(surrogatePair.length() - 1); // Consume only '#'
inputSegment.append('#');
yybegin(NUMERIC_CHARACTER);
}
case 67: break;
- case 43:
- { restoreState = SCRIPT_COMMENT; yybegin(SERVER_SIDE_INCLUDE);
+ case 4:
+ { yypushback(1);
+ outputSegment = inputSegment;
+ outputSegment.restart();
+ yybegin(YYINITIAL);
+ return outputSegment.nextChar();
}
case 68: break;
- case 30:
- { int length = yylength();
- inputSegment.write(zzBuffer, zzStartRead, length);
- entitySegment.clear();
- char ch = entityValues.get(zzBuffer, zzStartRead, length).charValue();
- entitySegment.append(ch);
- outputSegment = entitySegment;
- yybegin(CHARACTER_REFERENCE_TAIL);
+ case 43:
+ { restoreState = SCRIPT_COMMENT; yybegin(SERVER_SIDE_INCLUDE);
}
case 69: break;
- case 28:
- { restoreState = STYLE_COMMENT; yybegin(SINGLE_QUOTED_STRING);
- }
- case 70: break;
- case 3:
- { inputStart = yychar;
- inputSegment.clear();
- inputSegment.append('&');
- yybegin(AMPERSAND);
- }
- case 71: break;
- case 16:
- { restoreState = SCRIPT_COMMENT; yybegin(SINGLE_QUOTED_STRING);
- }
- case 72: break;
case 52:
{ // Handle paired UTF-16 surrogates.
String surrogatePair = yytext();
@@ -31486,174 +31420,11 @@ public final class HTMLStripCharFilter e
inputSegment.append('#');
yybegin(NUMERIC_CHARACTER);
}
- case 73: break;
- case 6:
- { int matchLength = yylength();
- inputSegment.write(zzBuffer, zzStartRead, matchLength);
- if (matchLength <= 7) { // 0x10FFFF = 1114111: max 7 decimal chars
- String decimalCharRef = yytext();
- int codePoint = 0;
- try {
- codePoint = Integer.parseInt(decimalCharRef);
- } catch(Exception e) {
- assert false: "Exception parsing code point '" + decimalCharRef + "'";
- }
- if (codePoint <= 0x10FFFF) {
- outputSegment = entitySegment;
- outputSegment.clear();
- if (codePoint >= Character.MIN_SURROGATE
- && codePoint <= Character.MAX_SURROGATE) {
- outputSegment.unsafeWrite(REPLACEMENT_CHARACTER);
- } else {
- outputSegment.setLength
- (Character.toChars(codePoint, outputSegment.getArray(), 0));
- }
- yybegin(CHARACTER_REFERENCE_TAIL);
- } else {
- outputSegment = inputSegment;
- yybegin(YYINITIAL);
- return outputSegment.nextChar();
- }
- } else {
- outputSegment = inputSegment;
- yybegin(YYINITIAL);
- return outputSegment.nextChar();
- }
- }
- case 74: break;
- case 37:
- { // add (this match length) [ - (substitution length) = 0 ]
- cumulativeDiff += yylength();
- // position the correction at (already output length) [ + (substitution length) = 0 ]
- addOffCorrectMap(outputCharCount, cumulativeDiff);
- yybegin(YYINITIAL);
- }
- case 75: break;
- case 8:
- { inputSegment.write(zzBuffer, zzStartRead, yylength());
- if (null != escapedTags
- && escapedTags.contains(zzBuffer, zzStartRead, yylength())) {
- yybegin(START_TAG_TAIL_INCLUDE);
- } else {
- yybegin(START_TAG_TAIL_SUBSTITUTE);
- }
- }
- case 76: break;
- case 46:
- { yybegin(SCRIPT);
- if (escapeSCRIPT) {
- inputSegment.write(zzBuffer, zzStartRead, yylength());
- outputSegment = inputSegment;
- inputStart += 1 + yylength();
- return outputSegment.nextChar();
- }
- }
- case 77: break;
- case 11:
- { inputSegment.write(zzBuffer, zzStartRead, yylength());
- yybegin(LEFT_ANGLE_BRACKET_SPACE);
- }
- case 78: break;
- case 20:
- { inputSegment.write(zzBuffer, zzStartRead, yylength());
- }
- case 79: break;
- case 34:
- { // add (previously matched input length) + (this match length) [ - (substitution length) = 0]
- cumulativeDiff += yychar - inputStart + yylength();
- // position the correction at (already output length) [ + (substitution length) = 0]
- addOffCorrectMap(outputCharCount, cumulativeDiff);
- inputSegment.clear();
- yybegin(YYINITIAL);
- }
- case 80: break;
- case 23:
- { yybegin(restoreState); restoreState = previousRestoreState;
- }
- case 81: break;
- case 32:
- { yybegin(COMMENT);
- }
- case 82: break;
- case 14:
- { // add (previously matched input length) + (this match length) [ - (substitution length) = 0 ]
- cumulativeDiff += inputSegment.length() + yylength();
- // position the correction at (already output length) [ + (substitution length) = 0 ]
- addOffCorrectMap(outputCharCount, cumulativeDiff);
- inputSegment.clear();
- yybegin(YYINITIAL);
- }
- case 83: break;
- case 18:
- { inputSegment.write(zzBuffer, zzStartRead, yylength());
- if (null != escapedTags
- && escapedTags.contains(zzBuffer, zzStartRead, yylength())) {
- yybegin(END_TAG_TAIL_INCLUDE);
- } else {
- yybegin(END_TAG_TAIL_SUBSTITUTE);
- }
- }
- case 84: break;
- case 25:
- { // add (previously matched input length) + (this match length) - (substitution length)
- cumulativeDiff += inputSegment.length() + yylength() - 1;
- // position the correction at (already output length) + (substitution length)
- addOffCorrectMap(outputCharCount + 1, cumulativeDiff);
- inputSegment.clear();
- yybegin(YYINITIAL);
- return BLOCK_LEVEL_END_TAG_REPLACEMENT;
- }
- case 85: break;
- case 7:
- { // add (previously matched input length) + (this match length) - (substitution length)
- cumulativeDiff += inputSegment.length() + yylength() - outputSegment.length();
- // position the correction at (already output length) + (substitution length)
- addOffCorrectMap(outputCharCount + outputSegment.length(), cumulativeDiff);
- yybegin(YYINITIAL);
- return outputSegment.nextChar();
- }
- case 86: break;
- case 48:
- { inputSegment.clear();
- yybegin(YYINITIAL);
- // add (previously matched input length) -- current match and substitution handled below
- cumulativeDiff += yychar - inputStart;
- // position the offset correction at (already output length) -- substitution handled below
- int offsetCorrectionPos = outputCharCount;
- int returnValue;
- if (escapeSTYLE) {
- inputSegment.write(zzBuffer, zzStartRead, yylength());
- outputSegment = inputSegment;
- returnValue = outputSegment.nextChar();
- } else {
- // add (this match length) - (substitution length)
- cumulativeDiff += yylength() - 1;
- // add (substitution length)
- ++offsetCorrectionPos;
- returnValue = STYLE_REPLACEMENT;
- }
- addOffCorrectMap(offsetCorrectionPos, cumulativeDiff);
- return returnValue;
- }
- case 87: break;
- case 5:
- { inputSegment.append('#'); yybegin(NUMERIC_CHARACTER);
- }
- case 88: break;
- case 26:
- { // add (previously matched input length) + (this match length) [ - (substitution length) = 0 ]
- cumulativeDiff += inputSegment.length() + yylength();
- // position the correction at (already output length) [ + (substitution length) = 0 ]
- addOffCorrectMap(outputCharCount, cumulativeDiff);
- inputSegment.clear();
- outputSegment = inputSegment;
- yybegin(YYINITIAL);
- }
- case 89: break;
- case 13:
- { inputSegment.append(zzBuffer[zzStartRead]);
+ case 70: break;
+ case 28:
+ { restoreState = STYLE_COMMENT; yybegin(SINGLE_QUOTED_STRING);
}
- case 90: break;
+ case 71: break;
case 50:
{ // Handle paired UTF-16 surrogates.
outputSegment = entitySegment;
@@ -31681,32 +31452,41 @@ public final class HTMLStripCharFilter e
yybegin(YYINITIAL);
return highSurrogate;
}
- case 91: break;
- case 40:
- { yybegin(SCRIPT_COMMENT);
- }
- case 92: break;
- case 45:
- { yybegin(STYLE);
- if (escapeSTYLE) {
- inputSegment.write(zzBuffer, zzStartRead, yylength());
- outputSegment = inputSegment;
- inputStart += 1 + yylength();
- return outputSegment.nextChar();
- }
+ case 72: break;
+ case 16:
+ { restoreState = SCRIPT_COMMENT; yybegin(SINGLE_QUOTED_STRING);
}
- case 93: break;
+ case 73: break;
case 22:
{ previousRestoreState = restoreState;
restoreState = SERVER_SIDE_INCLUDE;
yybegin(DOUBLE_QUOTED_STRING);
}
- case 94: break;
- case 12:
- { inputSegment.append('/'); yybegin(LEFT_ANGLE_BRACKET_SLASH);
+ case 74: break;
+ case 26:
+ { // add (previously matched input length) + (this match length) [ - (substitution length) = 0 ]
+ cumulativeDiff += inputSegment.length() + yylength();
+ // position the correction at (already output length) [ + (substitution length) = 0 ]
+ addOffCorrectMap(outputCharCount, cumulativeDiff);
+ inputSegment.clear();
+ outputSegment = inputSegment;
+ yybegin(YYINITIAL);
}
- case 95: break;
- case 36:
+ case 75: break;
+ case 20:
+ { inputSegment.write(zzBuffer, zzStartRead, yylength());
+ }
+ case 76: break;
+ case 47:
+ { // add (previously matched input length) + (this match length) [ - (substitution length) = 0 ]
+ cumulativeDiff += inputSegment.length() + yylength();
+ // position the correction at (already output length) [ + (substitution length) = 0 ]
+ addOffCorrectMap(outputCharCount, cumulativeDiff);
+ inputSegment.clear();
+ yybegin(CDATA);
+ }
+ case 77: break;
+ case 33:
{ yybegin(YYINITIAL);
if (escapeBR) {
inputSegment.write(zzBuffer, zzStartRead, yylength());
@@ -31718,34 +31498,128 @@ public final class HTMLStripCharFilter e
// position the correction at (already output length) + (substitution length)
addOffCorrectMap(outputCharCount + 1, cumulativeDiff);
inputSegment.reset();
- return BR_END_TAG_REPLACEMENT;
+ return BR_START_TAG_REPLACEMENT;
}
}
- case 96: break;
+ case 78: break;
+ case 23:
+ { yybegin(restoreState); restoreState = previousRestoreState;
+ }
+ case 79: break;
+ case 32:
+ { yybegin(COMMENT);
+ }
+ case 80: break;
case 24:
{ inputSegment.write(zzBuffer, zzStartRead, yylength());
outputSegment = inputSegment;
yybegin(YYINITIAL);
return outputSegment.nextChar();
}
- case 97: break;
- case 47:
+ case 81: break;
+ case 3:
+ { inputStart = yychar;
+ inputSegment.clear();
+ inputSegment.append('&');
+ yybegin(AMPERSAND);
+ }
+ case 82: break;
+ case 46:
+ { yybegin(SCRIPT);
+ if (escapeSCRIPT) {
+ inputSegment.write(zzBuffer, zzStartRead, yylength());
+ outputSegment = inputSegment;
+ inputStart += 1 + yylength();
+ return outputSegment.nextChar();
+ }
+ }
+ case 83: break;
+ case 14:
{ // add (previously matched input length) + (this match length) [ - (substitution length) = 0 ]
cumulativeDiff += inputSegment.length() + yylength();
// position the correction at (already output length) [ + (substitution length) = 0 ]
addOffCorrectMap(outputCharCount, cumulativeDiff);
inputSegment.clear();
- yybegin(CDATA);
+ yybegin(YYINITIAL);
}
- case 98: break;
- case 29:
- { restoreState = STYLE_COMMENT; yybegin(DOUBLE_QUOTED_STRING);
+ case 84: break;
+ case 6:
+ { int matchLength = yylength();
+ inputSegment.write(zzBuffer, zzStartRead, matchLength);
+ if (matchLength <= 7) { // 0x10FFFF = 1114111: max 7 decimal chars
+ String decimalCharRef = yytext();
+ int codePoint = 0;
+ try {
+ codePoint = Integer.parseInt(decimalCharRef);
+ } catch(Exception e) {
+ assert false: "Exception parsing code point '" + decimalCharRef + "'";
+ }
+ if (codePoint <= 0x10FFFF) {
+ outputSegment = entitySegment;
+ outputSegment.clear();
+ if (codePoint >= Character.MIN_SURROGATE
+ && codePoint <= Character.MAX_SURROGATE) {
+ outputSegment.unsafeWrite(REPLACEMENT_CHARACTER);
+ } else {
+ outputSegment.setLength
+ (Character.toChars(codePoint, outputSegment.getArray(), 0));
+ }
+ yybegin(CHARACTER_REFERENCE_TAIL);
+ } else {
+ outputSegment = inputSegment;
+ yybegin(YYINITIAL);
+ return outputSegment.nextChar();
+ }
+ } else {
+ outputSegment = inputSegment;
+ yybegin(YYINITIAL);
+ return outputSegment.nextChar();
+ }
}
- case 99: break;
- case 17:
- { restoreState = SCRIPT_COMMENT; yybegin(DOUBLE_QUOTED_STRING);
+ case 85: break;
+ case 34:
+ { // add (previously matched input length) + (this match length) [ - (substitution length) = 0]
+ cumulativeDiff += yychar - inputStart + yylength();
+ // position the correction at (already output length) [ + (substitution length) = 0]
+ addOffCorrectMap(outputCharCount, cumulativeDiff);
+ inputSegment.clear();
+ yybegin(YYINITIAL);
}
- case 100: break;
+ case 86: break;
+ case 5:
+ { inputSegment.append('#'); yybegin(NUMERIC_CHARACTER);
+ }
+ case 87: break;
+ case 13:
+ { inputSegment.append(zzBuffer[zzStartRead]);
+ }
+ case 88: break;
+ case 18:
+ { inputSegment.write(zzBuffer, zzStartRead, yylength());
+ if (null != escapedTags
+ && escapedTags.contains(zzBuffer, zzStartRead, yylength())) {
+ yybegin(END_TAG_TAIL_INCLUDE);
+ } else {
+ yybegin(END_TAG_TAIL_SUBSTITUTE);
+ }
+ }
+ case 89: break;
+ case 40:
+ { yybegin(SCRIPT_COMMENT);
+ }
+ case 90: break;
+ case 37:
+ { // add (this match length) [ - (substitution length) = 0 ]
+ cumulativeDiff += yylength();
+ // position the correction at (already output length) [ + (substitution length) = 0 ]
+ addOffCorrectMap(outputCharCount, cumulativeDiff);
+ yybegin(YYINITIAL);
+ }
+ case 91: break;
+ case 12:
+ { inputSegment.append('/'); yybegin(LEFT_ANGLE_BRACKET_SLASH);
+ }
+ case 92: break;
case 9:
{ inputSegment.write(zzBuffer, zzStartRead, yylength());
if (null != escapedTags
@@ -31755,7 +31629,7 @@ public final class HTMLStripCharFilter e
yybegin(START_TAG_TAIL_EXCLUDE);
}
}
- case 101: break;
+ case 93: break;
case 49:
{ inputSegment.clear();
yybegin(YYINITIAL);
@@ -31778,26 +31652,152 @@ public final class HTMLStripCharFilter e
addOffCorrectMap(offsetCorrectionPos, cumulativeDiff);
return returnValue;
}
+ case 94: break;
+ case 29:
+ { restoreState = STYLE_COMMENT; yybegin(DOUBLE_QUOTED_STRING);
+ }
+ case 95: break;
+ case 17:
+ { restoreState = SCRIPT_COMMENT; yybegin(DOUBLE_QUOTED_STRING);
+ }
+ case 96: break;
+ case 45:
+ { yybegin(STYLE);
+ if (escapeSTYLE) {
+ inputSegment.write(zzBuffer, zzStartRead, yylength());
+ outputSegment = inputSegment;
+ inputStart += 1 + yylength();
+ return outputSegment.nextChar();
+ }
+ }
+ case 97: break;
+ case 7:
+ { // add (previously matched input length) + (this match length) - (substitution length)
+ cumulativeDiff += inputSegment.length() + yylength() - outputSegment.length();
+ // position the correction at (already output length) + (substitution length)
+ addOffCorrectMap(outputCharCount + outputSegment.length(), cumulativeDiff);
+ yybegin(YYINITIAL);
+ return outputSegment.nextChar();
+ }
+ case 98: break;
+ case 19:
+ { inputSegment.write(zzBuffer, zzStartRead, yylength());
+ if (null != escapedTags
+ && escapedTags.contains(zzBuffer, zzStartRead, yylength())) {
+ yybegin(END_TAG_TAIL_INCLUDE);
+ } else {
+ yybegin(END_TAG_TAIL_EXCLUDE);
+ }
+ }
+ case 99: break;
+ case 25:
+ { // add (previously matched input length) + (this match length) - (substitution length)
+ cumulativeDiff += inputSegment.length() + yylength() - 1;
+ // position the correction at (already output length) + (substitution length)
+ addOffCorrectMap(outputCharCount + 1, cumulativeDiff);
+ inputSegment.clear();
+ yybegin(YYINITIAL);
+ return BLOCK_LEVEL_END_TAG_REPLACEMENT;
+ }
+ case 100: break;
+ case 31:
+ { int matchLength = yylength();
+ inputSegment.write(zzBuffer, zzStartRead, matchLength);
+ if (matchLength <= 6) { // 10FFFF: max 6 hex chars
+ String hexCharRef
+ = new String(zzBuffer, zzStartRead + 1, matchLength - 1);
+ int codePoint = 0;
+ try {
+ codePoint = Integer.parseInt(hexCharRef, 16);
+ } catch(Exception e) {
+ assert false: "Exception parsing hex code point '" + hexCharRef + "'";
+ }
+ if (codePoint <= 0x10FFFF) {
+ outputSegment = entitySegment;
+ outputSegment.clear();
+ if (codePoint >= Character.MIN_SURROGATE
+ && codePoint <= Character.MAX_SURROGATE) {
+ outputSegment.unsafeWrite(REPLACEMENT_CHARACTER);
+ } else {
+ outputSegment.setLength
+ (Character.toChars(codePoint, outputSegment.getArray(), 0));
+ }
+ yybegin(CHARACTER_REFERENCE_TAIL);
+ } else {
+ outputSegment = inputSegment;
+ yybegin(YYINITIAL);
+ return outputSegment.nextChar();
+ }
+ } else {
+ outputSegment = inputSegment;
+ yybegin(YYINITIAL);
+ return outputSegment.nextChar();
+ }
+ }
+ case 101: break;
+ case 53:
+ { // Handle paired UTF-16 surrogates.
+ String surrogatePair = yytext();
+ char highSurrogate = '\u0000';
+ try { // High surrogates are in decimal range [55296, 56319]
+ highSurrogate = (char)Integer.parseInt(surrogatePair.substring(1, 6));
+ } catch(Exception e) { // should never happen
+ assert false: "Exception parsing high surrogate '"
+ + surrogatePair.substring(1, 6) + "'";
+ }
+ if (Character.isHighSurrogate(highSurrogate)) {
+ char lowSurrogate = '\u0000';
+ try { // Low surrogates are in decimal range [56320, 57343]
+ lowSurrogate = (char)Integer.parseInt(surrogatePair.substring(9, 14));
+ } catch(Exception e) { // should never happen
+ assert false: "Exception parsing low surrogate '"
+ + surrogatePair.substring(9, 14) + "'";
+ }
+ if (Character.isLowSurrogate(lowSurrogate)) {
+ outputSegment = entitySegment;
+ outputSegment.clear();
+ outputSegment.unsafeWrite(lowSurrogate);
+ // add (previously matched input length) + (this match length) - (substitution length)
+ cumulativeDiff += inputSegment.length() + yylength() - 2;
+ // position the correction at (already output length) + (substitution length)
+ addOffCorrectMap(outputCharCount + 2, cumulativeDiff);
+ inputSegment.clear();
+ yybegin(YYINITIAL);
+ return highSurrogate;
+ }
+ }
+ yypushback(surrogatePair.length() - 1); // Consume only '#'
+ inputSegment.append('#');
+ yybegin(NUMERIC_CHARACTER);
+ }
case 102: break;
+ case 36:
+ { yybegin(YYINITIAL);
+ if (escapeBR) {
+ inputSegment.write(zzBuffer, zzStartRead, yylength());
+ outputSegment = inputSegment;
+ return outputSegment.nextChar();
+ } else {
+ // add (previously matched input length) + (this match length) - (substitution length)
+ cumulativeDiff += inputSegment.length() + yylength() - 1;
+ // position the correction at (already output length) + (substitution length)
+ addOffCorrectMap(outputCharCount + 1, cumulativeDiff);
+ inputSegment.reset();
+ return BR_END_TAG_REPLACEMENT;
+ }
+ }
+ case 103: break;
case 38:
{ yybegin(restoreState);
}
- case 103: break;
+ case 104: break;
case 41:
{ yybegin(STYLE_COMMENT);
}
- case 104: break;
+ case 105: break;
case 1:
{ return zzBuffer[zzStartRead];
}
- case 105: break;
- case 4:
- { yypushback(1);
- outputSegment = inputSegment;
- outputSegment.restart();
- yybegin(YYINITIAL);
- return outputSegment.nextChar();
- }
case 106: break;
default:
if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex?rev=1369883&r1=1369882&r2=1369883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex Mon Aug 6 17:16:47 2012
@@ -141,9 +141,9 @@ InlineElment = ( [aAbBiIqQsSuU]
[vV][aA][rR] )
-%include src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex
+%include HTMLCharacterEntities.jflex
-%include src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.SUPPLEMENTARY.jflex-macro
+%include HTMLStripCharFilter.SUPPLEMENTARY.jflex-macro
%{
private static final int INITIAL_INPUT_SEGMENT_SIZE = 1024;
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java?rev=1369883&r1=1369882&r2=1369883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java Mon Aug 6 17:16:47 2012
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 7/15/12 1:57 AM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 8/6/12 11:57 AM */
package org.apache.lucene.analysis.standard;
@@ -33,8 +33,8 @@ import org.apache.lucene.analysis.tokena
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 7/15/12 1:57 AM from the specification file
- * <tt>C:/cygwin/home/s/svn/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex</tt>
+ * on 8/6/12 11:57 AM from the specification file
+ * <tt>/home/rmuir/workspace/lucene-trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex</tt>
*/
class ClassicTokenizerImpl implements StandardTokenizerInterface {
@@ -42,7 +42,7 @@ class ClassicTokenizerImpl implements St
public static final int YYEOF = -1;
/** initial size of the lookahead buffer */
- private static final int ZZ_BUFFERSIZE = 16384;
+ private static final int ZZ_BUFFERSIZE = 4096;
/** lexical states */
public static final int YYINITIAL = 0;
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex?rev=1369883&r1=1369882&r2=1369883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex Mon Aug 6 17:16:47 2012
@@ -36,6 +36,7 @@ import org.apache.lucene.analysis.tokena
%function getNextToken
%pack
%char
+%buffer 4096
%{
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro?rev=1369883&r1=1369882&r2=1369883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro Mon Aug 6 17:16:47 2012
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-// Generated using ICU4J 49.1.0.0 on Thursday, July 26, 2012 10:22:01 PM UTC
+// Generated using ICU4J 49.1.0.0 on Monday, August 6, 2012 3:57:23 PM UTC
// by org.apache.lucene.analysis.icu.GenerateJFlexSupplementaryMacros
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=1369883&r1=1369882&r2=1369883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java Mon Aug 6 17:16:47 2012
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 7/26/12 6:22 PM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 8/6/12 11:57 AM */
package org.apache.lucene.analysis.standard;
@@ -43,7 +43,7 @@ public final class StandardTokenizerImpl
public static final int YYEOF = -1;
/** initial size of the lookahead buffer */
- private static final int ZZ_BUFFERSIZE = 16384;
+ private static final int ZZ_BUFFERSIZE = 4096;
/** lexical states */
public static final int YYINITIAL = 0;
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex?rev=1369883&r1=1369882&r2=1369883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex Mon Aug 6 17:16:47 2012
@@ -44,8 +44,9 @@ import org.apache.lucene.analysis.tokena
%implements StandardTokenizerInterface
%function getNextToken
%char
+%buffer 4096
-%include src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
+%include SUPPLEMENTARY.jflex-macro
ALetter = ([\p{WB:ALetter}] | {ALetterSupp})
Format = ([\p{WB:Format}] | {FormatSupp})
Numeric = ([\p{WB:Numeric}] | {NumericSupp})
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java?rev=1369883&r1=1369882&r2=1369883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java Mon Aug 6 17:16:47 2012
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 7/26/12 6:22 PM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 8/6/12 11:57 AM */
package org.apache.lucene.analysis.standard;
@@ -46,7 +46,7 @@ public final class UAX29URLEmailTokenize
public static final int YYEOF = -1;
/** initial size of the lookahead buffer */
- private static final int ZZ_BUFFERSIZE = 16384;
+ private static final int ZZ_BUFFERSIZE = 4096;
/** lexical states */
public static final int YYINITIAL = 0;
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex?rev=1369883&r1=1369882&r2=1369883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex Mon Aug 6 17:16:47 2012
@@ -47,8 +47,9 @@ import org.apache.lucene.analysis.tokena
%implements StandardTokenizerInterface
%function getNextToken
%char
+%buffer 4096
-%include src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
+%include SUPPLEMENTARY.jflex-macro
ALetter = ([\p{WB:ALetter}] | {ALetterSupp})
Format = ([\p{WB:Format}] | {FormatSupp})
Numeric = ([\p{WB:Numeric}] | {NumericSupp})
@@ -88,7 +89,7 @@ HiraganaEx = {Hiragana} ({Format} | {Ext
// RFC-5321: Simple Mail Transfer Protocol
// RFC-5322: Internet Message Format
-%include src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro
+%include ASCIITLD.jflex-macro
DomainLabel = [A-Za-z0-9] ([-A-Za-z0-9]* [A-Za-z0-9])?
DomainNameStrict = {DomainLabel} ("." {DomainLabel})* {ASCIITLD}
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java?rev=1369883&r1=1369882&r2=1369883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java Mon Aug 6 17:16:47 2012
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 7/15/12 1:57 AM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 8/6/12 11:57 AM */
package org.apache.lucene.analysis.wikipedia;
@@ -25,8 +25,8 @@ import org.apache.lucene.analysis.tokena
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 7/15/12 1:57 AM from the specification file
- * <tt>C:/cygwin/home/s/svn/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
+ * on 8/6/12 11:57 AM from the specification file
+ * <tt>/home/rmuir/workspace/lucene-trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
*/
class WikipediaTokenizerImpl {
@@ -34,7 +34,7 @@ class WikipediaTokenizerImpl {
public static final int YYEOF = -1;
/** initial size of the lookahead buffer */
- private static final int ZZ_BUFFERSIZE = 16384;
+ private static final int ZZ_BUFFERSIZE = 4096;
/** lexical states */
public static final int THREE_SINGLE_QUOTES_STATE = 10;
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex?rev=1369883&r1=1369882&r2=1369883&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex Mon Aug 6 17:16:47 2012
@@ -27,6 +27,7 @@ import org.apache.lucene.analysis.tokena
%function getNextToken
%pack
%char
+%buffer 4096
%{