You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/08/09 12:21:07 UTC
svn commit: r1371142 [3/32] - in /lucene/dev/branches/lucene3312: ./
dev-tools/ dev-tools/eclipse/ dev-tools/maven/ dev-tools/maven/lucene/
dev-tools/maven/lucene/analysis/common/
dev-tools/maven/lucene/analysis/icu/ dev-tools/maven/lucene/analysis/kur...
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java Thu Aug 9 10:20:53 2012
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 7/26/12 6:22 PM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 8/6/12 11:57 AM */
package org.apache.lucene.analysis.charfilter;
@@ -40,8 +40,8 @@ import org.apache.lucene.analysis.util.O
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 7/26/12 6:22 PM from the specification file
- * <tt>C:/svn/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex</tt>
+ * on 8/6/12 11:57 AM from the specification file
+ * <tt>/home/rmuir/workspace/lucene-trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex</tt>
*/
public final class HTMLStripCharFilter extends BaseCharFilter {
@@ -31255,88 +31255,56 @@ public final class HTMLStripCharFilter e
{ yybegin(STYLE);
}
case 55: break;
- case 51:
- { // Handle paired UTF-16 surrogates.
- String surrogatePair = yytext();
- char highSurrogate = '\u0000';
- char lowSurrogate = '\u0000';
- try {
- highSurrogate = (char)Integer.parseInt(surrogatePair.substring(2, 6), 16);
- } catch(Exception e) { // should never happen
- assert false: "Exception parsing high surrogate '"
- + surrogatePair.substring(2, 6) + "'";
- }
- try { // Low surrogates are in decimal range [56320, 57343]
- lowSurrogate = (char)Integer.parseInt(surrogatePair.substring(9, 14));
- } catch(Exception e) { // should never happen
- assert false: "Exception parsing low surrogate '"
- + surrogatePair.substring(9, 14) + "'";
- }
- if (Character.isLowSurrogate(lowSurrogate)) {
- outputSegment = entitySegment;
- outputSegment.clear();
- outputSegment.unsafeWrite(lowSurrogate);
- // add (previously matched input length) + (this match length) - (substitution length)
- cumulativeDiff += inputSegment.length() + yylength() - 2;
- // position the correction at (already output length) + (substitution length)
- addOffCorrectMap(outputCharCount + 2, cumulativeDiff);
- inputSegment.clear();
- yybegin(YYINITIAL);
- return highSurrogate;
- }
- yypushback(surrogatePair.length() - 1); // Consume only '#'
- inputSegment.append('#');
- yybegin(NUMERIC_CHARACTER);
+ case 27:
+ { // add (previously matched input length) + (this match length) - (substitution length)
+ cumulativeDiff += inputSegment.length() + yylength() - 1;
+ // position the correction at (already output length) + (substitution length)
+ addOffCorrectMap(outputCharCount + 1, cumulativeDiff);
+ inputSegment.clear();
+ yybegin(YYINITIAL);
+ return BLOCK_LEVEL_START_TAG_REPLACEMENT;
}
case 56: break;
- case 21:
- { previousRestoreState = restoreState;
- restoreState = SERVER_SIDE_INCLUDE;
- yybegin(SINGLE_QUOTED_STRING);
+ case 30:
+ { int length = yylength();
+ inputSegment.write(zzBuffer, zzStartRead, length);
+ entitySegment.clear();
+ char ch = entityValues.get(zzBuffer, zzStartRead, length).charValue();
+ entitySegment.append(ch);
+ outputSegment = entitySegment;
+ yybegin(CHARACTER_REFERENCE_TAIL);
}
case 57: break;
- case 31:
- { int matchLength = yylength();
- inputSegment.write(zzBuffer, zzStartRead, matchLength);
- if (matchLength <= 6) { // 10FFFF: max 6 hex chars
- String hexCharRef
- = new String(zzBuffer, zzStartRead + 1, matchLength - 1);
- int codePoint = 0;
- try {
- codePoint = Integer.parseInt(hexCharRef, 16);
- } catch(Exception e) {
- assert false: "Exception parsing hex code point '" + hexCharRef + "'";
- }
- if (codePoint <= 0x10FFFF) {
- outputSegment = entitySegment;
- outputSegment.clear();
- if (codePoint >= Character.MIN_SURROGATE
- && codePoint <= Character.MAX_SURROGATE) {
- outputSegment.unsafeWrite(REPLACEMENT_CHARACTER);
- } else {
- outputSegment.setLength
- (Character.toChars(codePoint, outputSegment.getArray(), 0));
- }
- yybegin(CHARACTER_REFERENCE_TAIL);
- } else {
- outputSegment = inputSegment;
- yybegin(YYINITIAL);
- return outputSegment.nextChar();
- }
- } else {
+ case 48:
+ { inputSegment.clear();
+ yybegin(YYINITIAL);
+ // add (previously matched input length) -- current match and substitution handled below
+ cumulativeDiff += yychar - inputStart;
+ // position the offset correction at (already output length) -- substitution handled below
+ int offsetCorrectionPos = outputCharCount;
+ int returnValue;
+ if (escapeSTYLE) {
+ inputSegment.write(zzBuffer, zzStartRead, yylength());
outputSegment = inputSegment;
- yybegin(YYINITIAL);
- return outputSegment.nextChar();
+ returnValue = outputSegment.nextChar();
+ } else {
+ // add (this match length) - (substitution length)
+ cumulativeDiff += yylength() - 1;
+ // add (substitution length)
+ ++offsetCorrectionPos;
+ returnValue = STYLE_REPLACEMENT;
}
+ addOffCorrectMap(offsetCorrectionPos, cumulativeDiff);
+ return returnValue;
}
case 58: break;
- case 19:
+ case 8:
{ inputSegment.write(zzBuffer, zzStartRead, yylength());
if (null != escapedTags
&& escapedTags.contains(zzBuffer, zzStartRead, yylength())) {
- yybegin(END_TAG_TAIL_INCLUDE);
+ yybegin(START_TAG_TAIL_INCLUDE);
} else {
- yybegin(END_TAG_TAIL_EXCLUDE);
+ yybegin(START_TAG_TAIL_SUBSTITUTE);
}
}
case 59: break;
@@ -31347,113 +31315,79 @@ public final class HTMLStripCharFilter e
yybegin(LEFT_ANGLE_BRACKET);
}
case 60: break;
- case 27:
- { // add (previously matched input length) + (this match length) - (substitution length)
- cumulativeDiff += inputSegment.length() + yylength() - 1;
- // position the correction at (already output length) + (substitution length)
- addOffCorrectMap(outputCharCount + 1, cumulativeDiff);
- inputSegment.clear();
- yybegin(YYINITIAL);
- return BLOCK_LEVEL_START_TAG_REPLACEMENT;
- }
- case 61: break;
case 44:
{ restoreState = STYLE_COMMENT; yybegin(SERVER_SIDE_INCLUDE);
}
+ case 61: break;
+ case 21:
+ { previousRestoreState = restoreState;
+ restoreState = SERVER_SIDE_INCLUDE;
+ yybegin(SINGLE_QUOTED_STRING);
+ }
case 62: break;
+ case 11:
+ { inputSegment.write(zzBuffer, zzStartRead, yylength());
+ yybegin(LEFT_ANGLE_BRACKET_SPACE);
+ }
+ case 63: break;
case 35:
{ yybegin(SCRIPT);
}
- case 63: break;
+ case 64: break;
case 42:
{ restoreState = COMMENT; yybegin(SERVER_SIDE_INCLUDE);
}
- case 64: break;
+ case 65: break;
case 10:
{ inputSegment.append('!'); yybegin(BANG);
}
- case 65: break;
- case 33:
- { yybegin(YYINITIAL);
- if (escapeBR) {
- inputSegment.write(zzBuffer, zzStartRead, yylength());
- outputSegment = inputSegment;
- return outputSegment.nextChar();
- } else {
- // add (previously matched input length) + (this match length) - (substitution length)
- cumulativeDiff += inputSegment.length() + yylength() - 1;
- // position the correction at (already output length) + (substitution length)
- addOffCorrectMap(outputCharCount + 1, cumulativeDiff);
- inputSegment.reset();
- return BR_START_TAG_REPLACEMENT;
- }
- }
case 66: break;
- case 53:
+ case 51:
{ // Handle paired UTF-16 surrogates.
String surrogatePair = yytext();
char highSurrogate = '\u0000';
- try { // High surrogates are in decimal range [55296, 56319]
- highSurrogate = (char)Integer.parseInt(surrogatePair.substring(1, 6));
+ char lowSurrogate = '\u0000';
+ try {
+ highSurrogate = (char)Integer.parseInt(surrogatePair.substring(2, 6), 16);
} catch(Exception e) { // should never happen
assert false: "Exception parsing high surrogate '"
- + surrogatePair.substring(1, 6) + "'";
+ + surrogatePair.substring(2, 6) + "'";
}
- if (Character.isHighSurrogate(highSurrogate)) {
- char lowSurrogate = '\u0000';
- try { // Low surrogates are in decimal range [56320, 57343]
- lowSurrogate = (char)Integer.parseInt(surrogatePair.substring(9, 14));
- } catch(Exception e) { // should never happen
- assert false: "Exception parsing low surrogate '"
- + surrogatePair.substring(9, 14) + "'";
- }
- if (Character.isLowSurrogate(lowSurrogate)) {
- outputSegment = entitySegment;
- outputSegment.clear();
- outputSegment.unsafeWrite(lowSurrogate);
- // add (previously matched input length) + (this match length) - (substitution length)
- cumulativeDiff += inputSegment.length() + yylength() - 2;
- // position the correction at (already output length) + (substitution length)
- addOffCorrectMap(outputCharCount + 2, cumulativeDiff);
- inputSegment.clear();
- yybegin(YYINITIAL);
- return highSurrogate;
- }
+ try { // Low surrogates are in decimal range [56320, 57343]
+ lowSurrogate = (char)Integer.parseInt(surrogatePair.substring(9, 14));
+ } catch(Exception e) { // should never happen
+ assert false: "Exception parsing low surrogate '"
+ + surrogatePair.substring(9, 14) + "'";
+ }
+ if (Character.isLowSurrogate(lowSurrogate)) {
+ outputSegment = entitySegment;
+ outputSegment.clear();
+ outputSegment.unsafeWrite(lowSurrogate);
+ // add (previously matched input length) + (this match length) - (substitution length)
+ cumulativeDiff += inputSegment.length() + yylength() - 2;
+ // position the correction at (already output length) + (substitution length)
+ addOffCorrectMap(outputCharCount + 2, cumulativeDiff);
+ inputSegment.clear();
+ yybegin(YYINITIAL);
+ return highSurrogate;
}
yypushback(surrogatePair.length() - 1); // Consume only '#'
inputSegment.append('#');
yybegin(NUMERIC_CHARACTER);
}
case 67: break;
- case 43:
- { restoreState = SCRIPT_COMMENT; yybegin(SERVER_SIDE_INCLUDE);
+ case 4:
+ { yypushback(1);
+ outputSegment = inputSegment;
+ outputSegment.restart();
+ yybegin(YYINITIAL);
+ return outputSegment.nextChar();
}
case 68: break;
- case 30:
- { int length = yylength();
- inputSegment.write(zzBuffer, zzStartRead, length);
- entitySegment.clear();
- char ch = entityValues.get(zzBuffer, zzStartRead, length).charValue();
- entitySegment.append(ch);
- outputSegment = entitySegment;
- yybegin(CHARACTER_REFERENCE_TAIL);
+ case 43:
+ { restoreState = SCRIPT_COMMENT; yybegin(SERVER_SIDE_INCLUDE);
}
case 69: break;
- case 28:
- { restoreState = STYLE_COMMENT; yybegin(SINGLE_QUOTED_STRING);
- }
- case 70: break;
- case 3:
- { inputStart = yychar;
- inputSegment.clear();
- inputSegment.append('&');
- yybegin(AMPERSAND);
- }
- case 71: break;
- case 16:
- { restoreState = SCRIPT_COMMENT; yybegin(SINGLE_QUOTED_STRING);
- }
- case 72: break;
case 52:
{ // Handle paired UTF-16 surrogates.
String surrogatePair = yytext();
@@ -31486,174 +31420,11 @@ public final class HTMLStripCharFilter e
inputSegment.append('#');
yybegin(NUMERIC_CHARACTER);
}
- case 73: break;
- case 6:
- { int matchLength = yylength();
- inputSegment.write(zzBuffer, zzStartRead, matchLength);
- if (matchLength <= 7) { // 0x10FFFF = 1114111: max 7 decimal chars
- String decimalCharRef = yytext();
- int codePoint = 0;
- try {
- codePoint = Integer.parseInt(decimalCharRef);
- } catch(Exception e) {
- assert false: "Exception parsing code point '" + decimalCharRef + "'";
- }
- if (codePoint <= 0x10FFFF) {
- outputSegment = entitySegment;
- outputSegment.clear();
- if (codePoint >= Character.MIN_SURROGATE
- && codePoint <= Character.MAX_SURROGATE) {
- outputSegment.unsafeWrite(REPLACEMENT_CHARACTER);
- } else {
- outputSegment.setLength
- (Character.toChars(codePoint, outputSegment.getArray(), 0));
- }
- yybegin(CHARACTER_REFERENCE_TAIL);
- } else {
- outputSegment = inputSegment;
- yybegin(YYINITIAL);
- return outputSegment.nextChar();
- }
- } else {
- outputSegment = inputSegment;
- yybegin(YYINITIAL);
- return outputSegment.nextChar();
- }
- }
- case 74: break;
- case 37:
- { // add (this match length) [ - (substitution length) = 0 ]
- cumulativeDiff += yylength();
- // position the correction at (already output length) [ + (substitution length) = 0 ]
- addOffCorrectMap(outputCharCount, cumulativeDiff);
- yybegin(YYINITIAL);
- }
- case 75: break;
- case 8:
- { inputSegment.write(zzBuffer, zzStartRead, yylength());
- if (null != escapedTags
- && escapedTags.contains(zzBuffer, zzStartRead, yylength())) {
- yybegin(START_TAG_TAIL_INCLUDE);
- } else {
- yybegin(START_TAG_TAIL_SUBSTITUTE);
- }
- }
- case 76: break;
- case 46:
- { yybegin(SCRIPT);
- if (escapeSCRIPT) {
- inputSegment.write(zzBuffer, zzStartRead, yylength());
- outputSegment = inputSegment;
- inputStart += 1 + yylength();
- return outputSegment.nextChar();
- }
- }
- case 77: break;
- case 11:
- { inputSegment.write(zzBuffer, zzStartRead, yylength());
- yybegin(LEFT_ANGLE_BRACKET_SPACE);
- }
- case 78: break;
- case 20:
- { inputSegment.write(zzBuffer, zzStartRead, yylength());
- }
- case 79: break;
- case 34:
- { // add (previously matched input length) + (this match length) [ - (substitution length) = 0]
- cumulativeDiff += yychar - inputStart + yylength();
- // position the correction at (already output length) [ + (substitution length) = 0]
- addOffCorrectMap(outputCharCount, cumulativeDiff);
- inputSegment.clear();
- yybegin(YYINITIAL);
- }
- case 80: break;
- case 23:
- { yybegin(restoreState); restoreState = previousRestoreState;
- }
- case 81: break;
- case 32:
- { yybegin(COMMENT);
- }
- case 82: break;
- case 14:
- { // add (previously matched input length) + (this match length) [ - (substitution length) = 0 ]
- cumulativeDiff += inputSegment.length() + yylength();
- // position the correction at (already output length) [ + (substitution length) = 0 ]
- addOffCorrectMap(outputCharCount, cumulativeDiff);
- inputSegment.clear();
- yybegin(YYINITIAL);
- }
- case 83: break;
- case 18:
- { inputSegment.write(zzBuffer, zzStartRead, yylength());
- if (null != escapedTags
- && escapedTags.contains(zzBuffer, zzStartRead, yylength())) {
- yybegin(END_TAG_TAIL_INCLUDE);
- } else {
- yybegin(END_TAG_TAIL_SUBSTITUTE);
- }
- }
- case 84: break;
- case 25:
- { // add (previously matched input length) + (this match length) - (substitution length)
- cumulativeDiff += inputSegment.length() + yylength() - 1;
- // position the correction at (already output length) + (substitution length)
- addOffCorrectMap(outputCharCount + 1, cumulativeDiff);
- inputSegment.clear();
- yybegin(YYINITIAL);
- return BLOCK_LEVEL_END_TAG_REPLACEMENT;
- }
- case 85: break;
- case 7:
- { // add (previously matched input length) + (this match length) - (substitution length)
- cumulativeDiff += inputSegment.length() + yylength() - outputSegment.length();
- // position the correction at (already output length) + (substitution length)
- addOffCorrectMap(outputCharCount + outputSegment.length(), cumulativeDiff);
- yybegin(YYINITIAL);
- return outputSegment.nextChar();
- }
- case 86: break;
- case 48:
- { inputSegment.clear();
- yybegin(YYINITIAL);
- // add (previously matched input length) -- current match and substitution handled below
- cumulativeDiff += yychar - inputStart;
- // position the offset correction at (already output length) -- substitution handled below
- int offsetCorrectionPos = outputCharCount;
- int returnValue;
- if (escapeSTYLE) {
- inputSegment.write(zzBuffer, zzStartRead, yylength());
- outputSegment = inputSegment;
- returnValue = outputSegment.nextChar();
- } else {
- // add (this match length) - (substitution length)
- cumulativeDiff += yylength() - 1;
- // add (substitution length)
- ++offsetCorrectionPos;
- returnValue = STYLE_REPLACEMENT;
- }
- addOffCorrectMap(offsetCorrectionPos, cumulativeDiff);
- return returnValue;
- }
- case 87: break;
- case 5:
- { inputSegment.append('#'); yybegin(NUMERIC_CHARACTER);
- }
- case 88: break;
- case 26:
- { // add (previously matched input length) + (this match length) [ - (substitution length) = 0 ]
- cumulativeDiff += inputSegment.length() + yylength();
- // position the correction at (already output length) [ + (substitution length) = 0 ]
- addOffCorrectMap(outputCharCount, cumulativeDiff);
- inputSegment.clear();
- outputSegment = inputSegment;
- yybegin(YYINITIAL);
- }
- case 89: break;
- case 13:
- { inputSegment.append(zzBuffer[zzStartRead]);
+ case 70: break;
+ case 28:
+ { restoreState = STYLE_COMMENT; yybegin(SINGLE_QUOTED_STRING);
}
- case 90: break;
+ case 71: break;
case 50:
{ // Handle paired UTF-16 surrogates.
outputSegment = entitySegment;
@@ -31681,32 +31452,41 @@ public final class HTMLStripCharFilter e
yybegin(YYINITIAL);
return highSurrogate;
}
- case 91: break;
- case 40:
- { yybegin(SCRIPT_COMMENT);
- }
- case 92: break;
- case 45:
- { yybegin(STYLE);
- if (escapeSTYLE) {
- inputSegment.write(zzBuffer, zzStartRead, yylength());
- outputSegment = inputSegment;
- inputStart += 1 + yylength();
- return outputSegment.nextChar();
- }
+ case 72: break;
+ case 16:
+ { restoreState = SCRIPT_COMMENT; yybegin(SINGLE_QUOTED_STRING);
}
- case 93: break;
+ case 73: break;
case 22:
{ previousRestoreState = restoreState;
restoreState = SERVER_SIDE_INCLUDE;
yybegin(DOUBLE_QUOTED_STRING);
}
- case 94: break;
- case 12:
- { inputSegment.append('/'); yybegin(LEFT_ANGLE_BRACKET_SLASH);
+ case 74: break;
+ case 26:
+ { // add (previously matched input length) + (this match length) [ - (substitution length) = 0 ]
+ cumulativeDiff += inputSegment.length() + yylength();
+ // position the correction at (already output length) [ + (substitution length) = 0 ]
+ addOffCorrectMap(outputCharCount, cumulativeDiff);
+ inputSegment.clear();
+ outputSegment = inputSegment;
+ yybegin(YYINITIAL);
}
- case 95: break;
- case 36:
+ case 75: break;
+ case 20:
+ { inputSegment.write(zzBuffer, zzStartRead, yylength());
+ }
+ case 76: break;
+ case 47:
+ { // add (previously matched input length) + (this match length) [ - (substitution length) = 0 ]
+ cumulativeDiff += inputSegment.length() + yylength();
+ // position the correction at (already output length) [ + (substitution length) = 0 ]
+ addOffCorrectMap(outputCharCount, cumulativeDiff);
+ inputSegment.clear();
+ yybegin(CDATA);
+ }
+ case 77: break;
+ case 33:
{ yybegin(YYINITIAL);
if (escapeBR) {
inputSegment.write(zzBuffer, zzStartRead, yylength());
@@ -31718,34 +31498,128 @@ public final class HTMLStripCharFilter e
// position the correction at (already output length) + (substitution length)
addOffCorrectMap(outputCharCount + 1, cumulativeDiff);
inputSegment.reset();
- return BR_END_TAG_REPLACEMENT;
+ return BR_START_TAG_REPLACEMENT;
}
}
- case 96: break;
+ case 78: break;
+ case 23:
+ { yybegin(restoreState); restoreState = previousRestoreState;
+ }
+ case 79: break;
+ case 32:
+ { yybegin(COMMENT);
+ }
+ case 80: break;
case 24:
{ inputSegment.write(zzBuffer, zzStartRead, yylength());
outputSegment = inputSegment;
yybegin(YYINITIAL);
return outputSegment.nextChar();
}
- case 97: break;
- case 47:
+ case 81: break;
+ case 3:
+ { inputStart = yychar;
+ inputSegment.clear();
+ inputSegment.append('&');
+ yybegin(AMPERSAND);
+ }
+ case 82: break;
+ case 46:
+ { yybegin(SCRIPT);
+ if (escapeSCRIPT) {
+ inputSegment.write(zzBuffer, zzStartRead, yylength());
+ outputSegment = inputSegment;
+ inputStart += 1 + yylength();
+ return outputSegment.nextChar();
+ }
+ }
+ case 83: break;
+ case 14:
{ // add (previously matched input length) + (this match length) [ - (substitution length) = 0 ]
cumulativeDiff += inputSegment.length() + yylength();
// position the correction at (already output length) [ + (substitution length) = 0 ]
addOffCorrectMap(outputCharCount, cumulativeDiff);
inputSegment.clear();
- yybegin(CDATA);
+ yybegin(YYINITIAL);
}
- case 98: break;
- case 29:
- { restoreState = STYLE_COMMENT; yybegin(DOUBLE_QUOTED_STRING);
+ case 84: break;
+ case 6:
+ { int matchLength = yylength();
+ inputSegment.write(zzBuffer, zzStartRead, matchLength);
+ if (matchLength <= 7) { // 0x10FFFF = 1114111: max 7 decimal chars
+ String decimalCharRef = yytext();
+ int codePoint = 0;
+ try {
+ codePoint = Integer.parseInt(decimalCharRef);
+ } catch(Exception e) {
+ assert false: "Exception parsing code point '" + decimalCharRef + "'";
+ }
+ if (codePoint <= 0x10FFFF) {
+ outputSegment = entitySegment;
+ outputSegment.clear();
+ if (codePoint >= Character.MIN_SURROGATE
+ && codePoint <= Character.MAX_SURROGATE) {
+ outputSegment.unsafeWrite(REPLACEMENT_CHARACTER);
+ } else {
+ outputSegment.setLength
+ (Character.toChars(codePoint, outputSegment.getArray(), 0));
+ }
+ yybegin(CHARACTER_REFERENCE_TAIL);
+ } else {
+ outputSegment = inputSegment;
+ yybegin(YYINITIAL);
+ return outputSegment.nextChar();
+ }
+ } else {
+ outputSegment = inputSegment;
+ yybegin(YYINITIAL);
+ return outputSegment.nextChar();
+ }
}
- case 99: break;
- case 17:
- { restoreState = SCRIPT_COMMENT; yybegin(DOUBLE_QUOTED_STRING);
+ case 85: break;
+ case 34:
+ { // add (previously matched input length) + (this match length) [ - (substitution length) = 0]
+ cumulativeDiff += yychar - inputStart + yylength();
+ // position the correction at (already output length) [ + (substitution length) = 0]
+ addOffCorrectMap(outputCharCount, cumulativeDiff);
+ inputSegment.clear();
+ yybegin(YYINITIAL);
}
- case 100: break;
+ case 86: break;
+ case 5:
+ { inputSegment.append('#'); yybegin(NUMERIC_CHARACTER);
+ }
+ case 87: break;
+ case 13:
+ { inputSegment.append(zzBuffer[zzStartRead]);
+ }
+ case 88: break;
+ case 18:
+ { inputSegment.write(zzBuffer, zzStartRead, yylength());
+ if (null != escapedTags
+ && escapedTags.contains(zzBuffer, zzStartRead, yylength())) {
+ yybegin(END_TAG_TAIL_INCLUDE);
+ } else {
+ yybegin(END_TAG_TAIL_SUBSTITUTE);
+ }
+ }
+ case 89: break;
+ case 40:
+ { yybegin(SCRIPT_COMMENT);
+ }
+ case 90: break;
+ case 37:
+ { // add (this match length) [ - (substitution length) = 0 ]
+ cumulativeDiff += yylength();
+ // position the correction at (already output length) [ + (substitution length) = 0 ]
+ addOffCorrectMap(outputCharCount, cumulativeDiff);
+ yybegin(YYINITIAL);
+ }
+ case 91: break;
+ case 12:
+ { inputSegment.append('/'); yybegin(LEFT_ANGLE_BRACKET_SLASH);
+ }
+ case 92: break;
case 9:
{ inputSegment.write(zzBuffer, zzStartRead, yylength());
if (null != escapedTags
@@ -31755,7 +31629,7 @@ public final class HTMLStripCharFilter e
yybegin(START_TAG_TAIL_EXCLUDE);
}
}
- case 101: break;
+ case 93: break;
case 49:
{ inputSegment.clear();
yybegin(YYINITIAL);
@@ -31778,26 +31652,152 @@ public final class HTMLStripCharFilter e
addOffCorrectMap(offsetCorrectionPos, cumulativeDiff);
return returnValue;
}
+ case 94: break;
+ case 29:
+ { restoreState = STYLE_COMMENT; yybegin(DOUBLE_QUOTED_STRING);
+ }
+ case 95: break;
+ case 17:
+ { restoreState = SCRIPT_COMMENT; yybegin(DOUBLE_QUOTED_STRING);
+ }
+ case 96: break;
+ case 45:
+ { yybegin(STYLE);
+ if (escapeSTYLE) {
+ inputSegment.write(zzBuffer, zzStartRead, yylength());
+ outputSegment = inputSegment;
+ inputStart += 1 + yylength();
+ return outputSegment.nextChar();
+ }
+ }
+ case 97: break;
+ case 7:
+ { // add (previously matched input length) + (this match length) - (substitution length)
+ cumulativeDiff += inputSegment.length() + yylength() - outputSegment.length();
+ // position the correction at (already output length) + (substitution length)
+ addOffCorrectMap(outputCharCount + outputSegment.length(), cumulativeDiff);
+ yybegin(YYINITIAL);
+ return outputSegment.nextChar();
+ }
+ case 98: break;
+ case 19:
+ { inputSegment.write(zzBuffer, zzStartRead, yylength());
+ if (null != escapedTags
+ && escapedTags.contains(zzBuffer, zzStartRead, yylength())) {
+ yybegin(END_TAG_TAIL_INCLUDE);
+ } else {
+ yybegin(END_TAG_TAIL_EXCLUDE);
+ }
+ }
+ case 99: break;
+ case 25:
+ { // add (previously matched input length) + (this match length) - (substitution length)
+ cumulativeDiff += inputSegment.length() + yylength() - 1;
+ // position the correction at (already output length) + (substitution length)
+ addOffCorrectMap(outputCharCount + 1, cumulativeDiff);
+ inputSegment.clear();
+ yybegin(YYINITIAL);
+ return BLOCK_LEVEL_END_TAG_REPLACEMENT;
+ }
+ case 100: break;
+ case 31:
+ { int matchLength = yylength();
+ inputSegment.write(zzBuffer, zzStartRead, matchLength);
+ if (matchLength <= 6) { // 10FFFF: max 6 hex chars
+ String hexCharRef
+ = new String(zzBuffer, zzStartRead + 1, matchLength - 1);
+ int codePoint = 0;
+ try {
+ codePoint = Integer.parseInt(hexCharRef, 16);
+ } catch(Exception e) {
+ assert false: "Exception parsing hex code point '" + hexCharRef + "'";
+ }
+ if (codePoint <= 0x10FFFF) {
+ outputSegment = entitySegment;
+ outputSegment.clear();
+ if (codePoint >= Character.MIN_SURROGATE
+ && codePoint <= Character.MAX_SURROGATE) {
+ outputSegment.unsafeWrite(REPLACEMENT_CHARACTER);
+ } else {
+ outputSegment.setLength
+ (Character.toChars(codePoint, outputSegment.getArray(), 0));
+ }
+ yybegin(CHARACTER_REFERENCE_TAIL);
+ } else {
+ outputSegment = inputSegment;
+ yybegin(YYINITIAL);
+ return outputSegment.nextChar();
+ }
+ } else {
+ outputSegment = inputSegment;
+ yybegin(YYINITIAL);
+ return outputSegment.nextChar();
+ }
+ }
+ case 101: break;
+ case 53:
+ { // Handle paired UTF-16 surrogates.
+ String surrogatePair = yytext();
+ char highSurrogate = '\u0000';
+ try { // High surrogates are in decimal range [55296, 56319]
+ highSurrogate = (char)Integer.parseInt(surrogatePair.substring(1, 6));
+ } catch(Exception e) { // should never happen
+ assert false: "Exception parsing high surrogate '"
+ + surrogatePair.substring(1, 6) + "'";
+ }
+ if (Character.isHighSurrogate(highSurrogate)) {
+ char lowSurrogate = '\u0000';
+ try { // Low surrogates are in decimal range [56320, 57343]
+ lowSurrogate = (char)Integer.parseInt(surrogatePair.substring(9, 14));
+ } catch(Exception e) { // should never happen
+ assert false: "Exception parsing low surrogate '"
+ + surrogatePair.substring(9, 14) + "'";
+ }
+ if (Character.isLowSurrogate(lowSurrogate)) {
+ outputSegment = entitySegment;
+ outputSegment.clear();
+ outputSegment.unsafeWrite(lowSurrogate);
+ // add (previously matched input length) + (this match length) - (substitution length)
+ cumulativeDiff += inputSegment.length() + yylength() - 2;
+ // position the correction at (already output length) + (substitution length)
+ addOffCorrectMap(outputCharCount + 2, cumulativeDiff);
+ inputSegment.clear();
+ yybegin(YYINITIAL);
+ return highSurrogate;
+ }
+ }
+ yypushback(surrogatePair.length() - 1); // Consume only '#'
+ inputSegment.append('#');
+ yybegin(NUMERIC_CHARACTER);
+ }
case 102: break;
+ case 36:
+ { yybegin(YYINITIAL);
+ if (escapeBR) {
+ inputSegment.write(zzBuffer, zzStartRead, yylength());
+ outputSegment = inputSegment;
+ return outputSegment.nextChar();
+ } else {
+ // add (previously matched input length) + (this match length) - (substitution length)
+ cumulativeDiff += inputSegment.length() + yylength() - 1;
+ // position the correction at (already output length) + (substitution length)
+ addOffCorrectMap(outputCharCount + 1, cumulativeDiff);
+ inputSegment.reset();
+ return BR_END_TAG_REPLACEMENT;
+ }
+ }
+ case 103: break;
case 38:
{ yybegin(restoreState);
}
- case 103: break;
+ case 104: break;
case 41:
{ yybegin(STYLE_COMMENT);
}
- case 104: break;
+ case 105: break;
case 1:
{ return zzBuffer[zzStartRead];
}
- case 105: break;
- case 4:
- { yypushback(1);
- outputSegment = inputSegment;
- outputSegment.restart();
- yybegin(YYINITIAL);
- return outputSegment.nextChar();
- }
case 106: break;
default:
if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex Thu Aug 9 10:20:53 2012
@@ -141,9 +141,9 @@ InlineElment = ( [aAbBiIqQsSuU]
[vV][aA][rR] )
-%include src/java/org/apache/lucene/analysis/charfilter/HTMLCharacterEntities.jflex
+%include HTMLCharacterEntities.jflex
-%include src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.SUPPLEMENTARY.jflex-macro
+%include HTMLStripCharFilter.SUPPLEMENTARY.jflex-macro
%{
private static final int INITIAL_INPUT_SEGMENT_SIZE = 1024;
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilter.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilter.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilter.java Thu Aug 9 10:20:53 2012
@@ -24,6 +24,8 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
import org.apache.lucene.util.ArrayUtil;
@@ -35,6 +37,12 @@ import org.apache.lucene.util.ArrayUtil;
* {@link #CJKBigramFilter(TokenStream, int)} to explicitly control which
* of the CJK scripts are turned into bigrams.
* <p>
+ * By default, when a CJK character has no adjacent characters to form
+ * a bigram, it is output in unigram form. If you want to always output
+ * both unigrams and bigrams, set the <code>outputUnigrams</code>
+ * flag in {@link CJKBigramFilter#CJKBigramFilter(TokenStream, int, boolean)}.
+ * This can be used for a combined unigram+bigram approach.
+ * <p>
* In all cases, all non-CJK input is passed thru unmodified.
*/
public final class CJKBigramFilter extends TokenFilter {
@@ -67,10 +75,16 @@ public final class CJKBigramFilter exten
private final Object doHiragana;
private final Object doKatakana;
private final Object doHangul;
+
+ // true if we should output unigram tokens always
+ private final boolean outputUnigrams;
+ private boolean ngramState; // false = output unigram, true = output bigram
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+ private final PositionIncrementAttribute posIncAtt = addAttribute(PositionIncrementAttribute.class);
+ private final PositionLengthAttribute posLengthAtt = addAttribute(PositionLengthAttribute.class);
// buffers containing codepoint and offsets in parallel
int buffer[] = new int[8];
@@ -88,23 +102,36 @@ public final class CJKBigramFilter exten
/**
* Calls {@link CJKBigramFilter#CJKBigramFilter(TokenStream, int)
- * CJKBigramFilter(HAN | HIRAGANA | KATAKANA | HANGUL)}
+ * CJKBigramFilter(in, HAN | HIRAGANA | KATAKANA | HANGUL)}
*/
public CJKBigramFilter(TokenStream in) {
this(in, HAN | HIRAGANA | KATAKANA | HANGUL);
}
/**
- * Create a new CJKBigramFilter, specifying which writing systems should be bigrammed.
+ * Calls {@link CJKBigramFilter#CJKBigramFilter(TokenStream, int, boolean)
+ * CJKBigramFilter(in, flags, false)}
+ */
+ public CJKBigramFilter(TokenStream in, int flags) {
+ this(in, flags, false);
+ }
+
+ /**
+ * Create a new CJKBigramFilter, specifying which writing systems should be bigrammed,
+ * and whether or not unigrams should also be output.
* @param flags OR'ed set from {@link CJKBigramFilter#HAN}, {@link CJKBigramFilter#HIRAGANA},
* {@link CJKBigramFilter#KATAKANA}, {@link CJKBigramFilter#HANGUL}
+ * @param outputUnigrams true if unigrams for the selected writing systems should also be output.
+ * when this is false, this is only done when there are no adjacent characters to form
+ * a bigram.
*/
- public CJKBigramFilter(TokenStream in, int flags) {
+ public CJKBigramFilter(TokenStream in, int flags, boolean outputUnigrams) {
super(in);
doHan = (flags & HAN) == 0 ? NO : HAN_TYPE;
doHiragana = (flags & HIRAGANA) == 0 ? NO : HIRAGANA_TYPE;
doKatakana = (flags & KATAKANA) == 0 ? NO : KATAKANA_TYPE;
doHangul = (flags & HANGUL) == 0 ? NO : HANGUL_TYPE;
+ this.outputUnigrams = outputUnigrams;
}
/*
@@ -120,7 +147,24 @@ public final class CJKBigramFilter exten
// case 1: we have multiple remaining codepoints buffered,
// so we can emit a bigram here.
- flushBigram();
+ if (outputUnigrams) {
+
+ // when also outputting unigrams, we output the unigram first,
+ // then rewind back to revisit the bigram.
+ // so an input of ABC is A + (rewind)AB + B + (rewind)BC + C
+ // the logic in hasBufferedUnigram ensures we output the C,
+ // even though it did actually have adjacent CJK characters.
+
+ if (ngramState) {
+ flushBigram();
+ } else {
+ flushUnigram();
+ index--;
+ }
+ ngramState = !ngramState;
+ } else {
+ flushBigram();
+ }
return true;
} else if (doNext()) {
@@ -260,6 +304,11 @@ public final class CJKBigramFilter exten
termAtt.setLength(len2);
offsetAtt.setOffset(startOffset[index], endOffset[index+1]);
typeAtt.setType(DOUBLE_TYPE);
+ // when outputting unigrams, all bigrams are synonyms that span two unigrams
+ if (outputUnigrams) {
+ posIncAtt.setPositionIncrement(0);
+ posLengthAtt.setPositionLength(2);
+ }
index++;
}
@@ -292,7 +341,13 @@ public final class CJKBigramFilter exten
* inputs.
*/
private boolean hasBufferedUnigram() {
- return bufferLen == 1 && index == 0;
+ if (outputUnigrams) {
+ // when outputting unigrams always
+ return bufferLen - index == 1;
+ } else {
+ // otherwise its only when we have a lone CJK character
+ return bufferLen == 1 && index == 0;
+ }
}
@Override
@@ -303,5 +358,6 @@ public final class CJKBigramFilter exten
lastEndOffset = 0;
loneState = null;
exhausted = false;
+ ngramState = false;
}
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilterFactory.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilterFactory.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/cjk/CJKBigramFilterFactory.java Thu Aug 9 10:20:53 2012
@@ -33,12 +33,13 @@ import org.apache.lucene.analysis.util.T
* <filter class="solr.LowerCaseFilterFactory"/>
* <filter class="solr.CJKBigramFilterFactory"
* han="true" hiragana="true"
- * katakana="true" hangul="true" />
+ * katakana="true" hangul="true" outputUnigrams="false" />
* </analyzer>
* </fieldType></pre>
*/
public class CJKBigramFilterFactory extends TokenFilterFactory {
int flags;
+ boolean outputUnigrams;
@Override
public void init(Map<String,String> args) {
@@ -56,10 +57,11 @@ public class CJKBigramFilterFactory exte
if (getBoolean("hangul", true)) {
flags |= CJKBigramFilter.HANGUL;
}
+ outputUnigrams = getBoolean("outputUnigrams", false);
}
@Override
public TokenStream create(TokenStream input) {
- return new CJKBigramFilter(input, flags);
+ return new CJKBigramFilter(input, flags, outputUnigrams);
}
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/fr/FrenchAnalyzer.java Thu Aug 9 10:20:53 2012
@@ -28,6 +28,7 @@ import org.apache.lucene.analysis.standa
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.standard.StandardAnalyzer; // for javadoc
import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.ElisionFilter;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.IOUtils;
@@ -35,6 +36,7 @@ import org.apache.lucene.util.Version;
import java.io.IOException;
import java.io.Reader;
+import java.util.Arrays;
/**
* {@link Analyzer} for French language.
@@ -54,6 +56,11 @@ public final class FrenchAnalyzer extend
/** File containing default French stopwords. */
public final static String DEFAULT_STOPWORD_FILE = "french_stop.txt";
+ /** Default set of articles for ElisionFilter */
+ public static final CharArraySet DEFAULT_ARTICLES = CharArraySet.unmodifiableSet(
+ new CharArraySet(Version.LUCENE_CURRENT, Arrays.asList(
+ "l", "m", "t", "qu", "n", "s", "j"), true));
+
/**
* Contains words that should be indexed but not stemmed.
*/
@@ -134,7 +141,7 @@ public final class FrenchAnalyzer extend
Reader reader) {
final Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
- result = new ElisionFilter(matchVersion, result);
+ result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
if(!excltable.isEmpty())
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishAnalyzer.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishAnalyzer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/ga/IrishAnalyzer.java Thu Aug 9 10:20:53 2012
@@ -23,7 +23,6 @@ import java.util.Arrays;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.StopFilter;
-import org.apache.lucene.analysis.fr.ElisionFilter;
import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
@@ -31,6 +30,7 @@ import org.apache.lucene.analysis.snowba
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.ElisionFilter;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;
import org.tartarus.snowball.ext.IrishStemmer;
@@ -140,7 +140,7 @@ public final class IrishAnalyzer extends
StopFilter s = new StopFilter(matchVersion, result, HYPHENATIONS);
s.setEnablePositionIncrements(false);
result = s;
- result = new ElisionFilter(matchVersion, result, DEFAULT_ARTICLES);
+ result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new IrishLowerCaseFilter(result);
result = new StopFilter(matchVersion, result, stopwords);
if(!stemExclusionSet.isEmpty())
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/it/ItalianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/it/ItalianAnalyzer.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/it/ItalianAnalyzer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/it/ItalianAnalyzer.java Thu Aug 9 10:20:53 2012
@@ -24,7 +24,6 @@ import java.util.Arrays;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopFilter;
-import org.apache.lucene.analysis.fr.ElisionFilter;
import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
@@ -32,6 +31,7 @@ import org.apache.lucene.analysis.snowba
import org.apache.lucene.analysis.standard.StandardFilter;
import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.ElisionFilter;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
import org.apache.lucene.util.IOUtils;
@@ -129,7 +129,7 @@ public final class ItalianAnalyzer exten
Reader reader) {
final Tokenizer source = new StandardTokenizer(matchVersion, reader);
TokenStream result = new StandardFilter(matchVersion, source);
- result = new ElisionFilter(matchVersion, result, DEFAULT_ARTICLES);
+ result = new ElisionFilter(result, DEFAULT_ARTICLES);
result = new LowerCaseFilter(matchVersion, result);
result = new StopFilter(matchVersion, result, stopwords);
if(!stemExclusionSet.isEmpty())
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/path/PathHierarchyTokenizerFactory.java Thu Aug 9 10:20:53 2012
@@ -27,13 +27,47 @@ import org.apache.lucene.analysis.util.T
/**
* Factory for {@link PathHierarchyTokenizer}.
+ * <p>
+ * This factory is typically configured for use only in the <code>index</code>
+ * Analyzer (or only in the <code>query</code> Analyzer, but never both).
+ * </p>
+ * <p>
+ * For example, in the configuration below a query for
+ * <code>Books/NonFic</code> will match documents indexed with values like
+ * <code>Books/NonFic</code>, <code>Books/NonFic/Law</code>,
+ * <code>Books/NonFic/Science/Physics</code>, etc. But it will not match
+ * documents indexed with values like <code>Books</code>, or
+ * <code>Books/Fic</code>...
+ * </p>
+ *
* <pre class="prettyprint" >
- * <fieldType name="text_path" class="solr.TextField" positionIncrementGap="100">
- * <analyzer>
- * <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="\" replace="/"/>
+ * <fieldType name="descendent_path" class="solr.TextField">
+ * <analyzer type="index">
+ * <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
* </analyzer>
- * </fieldType></pre>
- *
+ * <analyzer type="query">
+ * <tokenizer class="solr.KeywordTokenizerFactory" />
+ * </analyzer>
+ * </fieldType>
+ * </pre>
+ * <p>
+ * In this example however we see the oposite configuration, so that a query
+ * for <code>Books/NonFic/Science/Physics</code> would match documents
+ * containing <code>Books/NonFic</code>, <code>Books/NonFic/Science</code>,
+ * or <code>Books/NonFic/Science/Physics</code>, but not
+ * <code>Books/NonFic/Science/Physics/Theory</code> or
+ * <code>Books/NonFic/Law</code>.
+ * </p>
+ * <pre class="prettyprint" >
+ * <fieldType name="descendent_path" class="solr.TextField">
+ * <analyzer type="index">
+ * <tokenizer class="solr.KeywordTokenizerFactory" />
+ * </analyzer>
+ * <analyzer type="query">
+ * <tokenizer class="solr.PathHierarchyTokenizerFactory" delimiter="/" />
+ * </analyzer>
+ * </fieldType>
+ * </pre>
*/
public class PathHierarchyTokenizerFactory extends TokenizerFactory {
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java Thu Aug 9 10:20:53 2012
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 7/15/12 1:57 AM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 8/6/12 11:57 AM */
package org.apache.lucene.analysis.standard;
@@ -33,8 +33,8 @@ import org.apache.lucene.analysis.tokena
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 7/15/12 1:57 AM from the specification file
- * <tt>C:/cygwin/home/s/svn/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex</tt>
+ * on 8/6/12 11:57 AM from the specification file
+ * <tt>/home/rmuir/workspace/lucene-trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex</tt>
*/
class ClassicTokenizerImpl implements StandardTokenizerInterface {
@@ -42,7 +42,7 @@ class ClassicTokenizerImpl implements St
public static final int YYEOF = -1;
/** initial size of the lookahead buffer */
- private static final int ZZ_BUFFERSIZE = 16384;
+ private static final int ZZ_BUFFERSIZE = 4096;
/** lexical states */
public static final int YYINITIAL = 0;
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex Thu Aug 9 10:20:53 2012
@@ -36,6 +36,7 @@ import org.apache.lucene.analysis.tokena
%function getNextToken
%pack
%char
+%buffer 4096
%{
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro Thu Aug 9 10:20:53 2012
@@ -14,7 +14,7 @@
* limitations under the License.
*/
-// Generated using ICU4J 49.1.0.0 on Thursday, July 26, 2012 10:22:01 PM UTC
+// Generated using ICU4J 49.1.0.0 on Monday, August 6, 2012 3:57:23 PM UTC
// by org.apache.lucene.analysis.icu.GenerateJFlexSupplementaryMacros
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java Thu Aug 9 10:20:53 2012
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 7/26/12 6:22 PM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 8/6/12 11:57 AM */
package org.apache.lucene.analysis.standard;
@@ -43,7 +43,7 @@ public final class StandardTokenizerImpl
public static final int YYEOF = -1;
/** initial size of the lookahead buffer */
- private static final int ZZ_BUFFERSIZE = 16384;
+ private static final int ZZ_BUFFERSIZE = 4096;
/** lexical states */
public static final int YYINITIAL = 0;
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex Thu Aug 9 10:20:53 2012
@@ -44,8 +44,9 @@ import org.apache.lucene.analysis.tokena
%implements StandardTokenizerInterface
%function getNextToken
%char
+%buffer 4096
-%include src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
+%include SUPPLEMENTARY.jflex-macro
ALetter = ([\p{WB:ALetter}] | {ALetterSupp})
Format = ([\p{WB:Format}] | {FormatSupp})
Numeric = ([\p{WB:Numeric}] | {NumericSupp})
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java Thu Aug 9 10:20:53 2012
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 7/26/12 6:22 PM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 8/6/12 11:57 AM */
package org.apache.lucene.analysis.standard;
@@ -46,7 +46,7 @@ public final class UAX29URLEmailTokenize
public static final int YYEOF = -1;
/** initial size of the lookahead buffer */
- private static final int ZZ_BUFFERSIZE = 16384;
+ private static final int ZZ_BUFFERSIZE = 4096;
/** lexical states */
public static final int YYINITIAL = 0;
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex Thu Aug 9 10:20:53 2012
@@ -47,8 +47,9 @@ import org.apache.lucene.analysis.tokena
%implements StandardTokenizerInterface
%function getNextToken
%char
+%buffer 4096
-%include src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
+%include SUPPLEMENTARY.jflex-macro
ALetter = ([\p{WB:ALetter}] | {ALetterSupp})
Format = ([\p{WB:Format}] | {FormatSupp})
Numeric = ([\p{WB:Numeric}] | {NumericSupp})
@@ -88,7 +89,7 @@ HiraganaEx = {Hiragana} ({Format} | {Ext
// RFC-5321: Simple Mail Transfer Protocol
// RFC-5322: Internet Message Format
-%include src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro
+%include ASCIITLD.jflex-macro
DomainLabel = [A-Za-z0-9] ([-A-Za-z0-9]* [A-Za-z0-9])?
DomainNameStrict = {DomainLabel} ("." {DomainLabel})* {ASCIITLD}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java Thu Aug 9 10:20:53 2012
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 7/15/12 1:57 AM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 8/6/12 11:57 AM */
package org.apache.lucene.analysis.wikipedia;
@@ -25,8 +25,8 @@ import org.apache.lucene.analysis.tokena
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 7/15/12 1:57 AM from the specification file
- * <tt>C:/cygwin/home/s/svn/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
+ * on 8/6/12 11:57 AM from the specification file
+ * <tt>/home/rmuir/workspace/lucene-trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex</tt>
*/
class WikipediaTokenizerImpl {
@@ -34,7 +34,7 @@ class WikipediaTokenizerImpl {
public static final int YYEOF = -1;
/** initial size of the lookahead buffer */
- private static final int ZZ_BUFFERSIZE = 16384;
+ private static final int ZZ_BUFFERSIZE = 4096;
/** lexical states */
public static final int THREE_SINGLE_QUOTES_STATE = 10;
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex?rev=1371142&r1=1371141&r2=1371142&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex Thu Aug 9 10:20:53 2012
@@ -27,6 +27,7 @@ import org.apache.lucene.analysis.tokena
%function getNextToken
%pack
%char
+%buffer 4096
%{