You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rj...@apache.org on 2014/10/09 08:13:30 UTC
svn commit: r1630295 - in /lucene/dev/branches/branch_5x: ./ lucene/
lucene/analysis/
lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/
Author: rjernst
Date: Thu Oct 9 06:13:30 2014
New Revision: 1630295
URL: http://svn.apache.org/r1630295
Log:
LUCENE-6000: Removed StandardTokenizerInterface
Removed:
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerInterface.java
Modified:
lucene/dev/branches/branch_5x/ (props changed)
lucene/dev/branches/branch_5x/lucene/ (props changed)
lucene/dev/branches/branch_5x/lucene/CHANGES.txt (contents, props changed)
lucene/dev/branches/branch_5x/lucene/analysis/ (props changed)
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex
Modified: lucene/dev/branches/branch_5x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/CHANGES.txt?rev=1630295&r1=1630294&r2=1630295&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_5x/lucene/CHANGES.txt Thu Oct 9 06:13:30 2014
@@ -139,6 +139,10 @@ API Changes
* LUCENE-5998: Simplify Field/SegmentInfoFormat to read+write methods.
(Robert Muir)
+* LUCENE-6000: Removed StandardTokenizerInterface. Tokenizers now use
+ their jflex impl directly.
+ (Ryan Ernst)
+
Bug Fixes
* LUCENE-5650: Enforce read-only access to any path outside the temporary
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java?rev=1630295&r1=1630294&r2=1630295&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java Thu Oct 9 06:13:30 2014
@@ -49,7 +49,7 @@ import org.apache.lucene.util.AttributeF
public final class ClassicTokenizer extends Tokenizer {
/** A private instance of the JFlex-constructed scanner */
- private StandardTokenizerInterface scanner;
+ private ClassicTokenizerImpl scanner;
public static final int ALPHANUM = 0;
public static final int APOSTROPHE = 1;
@@ -135,7 +135,7 @@ public final class ClassicTokenizer exte
while(true) {
int tokenType = scanner.getNextToken();
- if (tokenType == StandardTokenizerInterface.YYEOF) {
+ if (tokenType == ClassicTokenizerImpl.YYEOF) {
return false;
}
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java?rev=1630295&r1=1630294&r2=1630295&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java Thu Oct 9 06:13:30 2014
@@ -33,7 +33,7 @@ import org.apache.lucene.analysis.tokena
* This class implements the classic lucene StandardTokenizer up until 3.0
*/
-class ClassicTokenizerImpl implements StandardTokenizerInterface {
+class ClassicTokenizerImpl {
/** This character denotes the end of file */
public static final int YYEOF = -1;
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex?rev=1630295&r1=1630294&r2=1630295&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex Thu Oct 9 06:13:30 2014
@@ -33,7 +33,6 @@ import org.apache.lucene.analysis.tokena
%%
%class ClassicTokenizerImpl
-%implements StandardTokenizerInterface
%unicode 3.0
%integer
%function getNextToken
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=1630295&r1=1630294&r2=1630295&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Thu Oct 9 06:13:30 2014
@@ -39,7 +39,7 @@ import org.apache.lucene.util.AttributeF
public final class StandardTokenizer extends Tokenizer {
/** A private instance of the JFlex-constructed scanner */
- private StandardTokenizerInterface scanner;
+ private StandardTokenizerImpl scanner;
// TODO: how can we remove these old types?!
public static final int ALPHANUM = 0;
@@ -150,7 +150,7 @@ public final class StandardTokenizer ext
while(true) {
int tokenType = scanner.getNextToken();
- if (tokenType == StandardTokenizerInterface.YYEOF) {
+ if (tokenType == StandardTokenizerImpl.YYEOF) {
return false;
}
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java?rev=1630295&r1=1630294&r2=1630295&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java Thu Oct 9 06:13:30 2014
@@ -39,7 +39,7 @@ import org.apache.lucene.analysis.tokena
* </ul>
*/
-public final class StandardTokenizerImpl implements StandardTokenizerInterface {
+public final class StandardTokenizerImpl {
/** This character denotes the end of file */
public static final int YYEOF = -1;
@@ -804,7 +804,7 @@ public final class StandardTokenizerImpl
if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
zzAtEOF = true;
{
- return StandardTokenizerInterface.YYEOF;
+ return YYEOF;
}
}
else {
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex?rev=1630295&r1=1630294&r2=1630295&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex Thu Oct 9 06:13:30 2014
@@ -43,7 +43,6 @@ import org.apache.lucene.analysis.tokena
%final
%public
%class StandardTokenizerImpl
-%implements StandardTokenizerInterface
%function getNextToken
%char
%buffer 255
@@ -118,7 +117,7 @@ ComplexContextEx = \p{LB:Complex_Cont
// UAX#29 WB1. sot ÷
// WB2. ÷ eot
//
-<<EOF>> { return StandardTokenizerInterface.YYEOF; }
+<<EOF>> { return YYEOF; }
// UAX#29 WB8. Numeric à Numeric
// WB11. Numeric (MidNum | MidNumLet | Single_Quote) Ã Numeric
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java?rev=1630295&r1=1630294&r2=1630295&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java Thu Oct 9 06:13:30 2014
@@ -47,7 +47,7 @@ import org.apache.lucene.util.AttributeF
public final class UAX29URLEmailTokenizer extends Tokenizer {
/** A private instance of the JFlex-constructed scanner */
- private final StandardTokenizerInterface scanner;
+ private final UAX29URLEmailTokenizerImpl scanner;
public static final int ALPHANUM = 0;
public static final int NUM = 1;
@@ -108,7 +108,7 @@ public final class UAX29URLEmailTokenize
this.scanner = getScanner();
}
- private StandardTokenizerInterface getScanner() {
+ private UAX29URLEmailTokenizerImpl getScanner() {
return new UAX29URLEmailTokenizerImpl(input);
}
@@ -127,7 +127,7 @@ public final class UAX29URLEmailTokenize
while(true) {
int tokenType = scanner.getNextToken();
- if (tokenType == StandardTokenizerInterface.YYEOF) {
+ if (tokenType == UAX29URLEmailTokenizerImpl.YYEOF) {
return false;
}
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java?rev=1630295&r1=1630294&r2=1630295&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java Thu Oct 9 06:13:30 2014
@@ -42,7 +42,7 @@ import org.apache.lucene.analysis.tokena
* </ul>
*/
-public final class UAX29URLEmailTokenizerImpl implements StandardTokenizerInterface {
+public final class UAX29URLEmailTokenizerImpl {
/** This character denotes the end of file */
public static final int YYEOF = -1;
@@ -7204,11 +7204,11 @@ public final class UAX29URLEmailTokenize
zzAtEOF = true;
switch (zzLexicalState) {
case YYINITIAL: {
- return StandardTokenizerInterface.YYEOF;
+ return YYEOF;
}
case 2910: break;
case AVOID_BAD_URL: {
- return StandardTokenizerInterface.YYEOF;
+ return YYEOF;
}
case 2911: break;
default:
Modified: lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex?rev=1630295&r1=1630294&r2=1630295&view=diff
==============================================================================
--- lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex (original)
+++ lucene/dev/branches/branch_5x/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex Thu Oct 9 06:13:30 2014
@@ -46,7 +46,6 @@ import org.apache.lucene.analysis.tokena
%final
%public
%class UAX29URLEmailTokenizerImpl
-%implements StandardTokenizerInterface
%function getNextToken
%char
%xstate AVOID_BAD_URL
@@ -208,7 +207,7 @@ EMAIL = {EMAILlocalPart} "@" ({DomainNam
// UAX#29 WB1. sot ÷
// WB2. ÷ eot
//
- <<EOF>> { return StandardTokenizerInterface.YYEOF; }
+ <<EOF>> { return YYEOF; }
{URL} { yybegin(YYINITIAL); return URL_TYPE; }