You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by si...@apache.org on 2010/12/08 20:00:19 UTC
svn commit: r1043605 [1/3] - in /lucene/dev/branches/docvalues: ./
modules/analysis/ modules/analysis/common/
modules/analysis/common/src/java/org/apache/lucene/analysis/ar/
modules/analysis/common/src/java/org/apache/lucene/analysis/fa/
modules/analys...
Author: simonw
Date: Wed Dec 8 19:00:16 2010
New Revision: 1043605
URL: http://svn.apache.org/viewvc?rev=1043605&view=rev
Log:
catch up with trunk
Added:
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java
- copied unchanged from r1043147, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java
- copied unchanged from r1043147, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.java
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.jflex
- copied unchanged from r1043147, lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.jflex
lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java
- copied unchanged from r1043147, lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java
lucene/dev/branches/docvalues/modules/analysis/icu/lib/icu4j-4_6.jar
- copied unchanged from r1043147, lucene/dev/trunk/modules/analysis/icu/lib/icu4j-4_6.jar
lucene/dev/branches/docvalues/solr/contrib/analysis-extras/lib/icu4j-4_6.jar
- copied unchanged from r1043147, lucene/dev/trunk/solr/contrib/analysis-extras/lib/icu4j-4_6.jar
lucene/dev/branches/docvalues/solr/contrib/extraction/lib/icu4j-4_6.jar
- copied unchanged from r1043147, lucene/dev/trunk/solr/contrib/extraction/lib/icu4j-4_6.jar
lucene/dev/branches/docvalues/solr/example/lib/jetty-6.1.26.jar
- copied unchanged from r1043147, lucene/dev/trunk/solr/example/lib/jetty-6.1.26.jar
lucene/dev/branches/docvalues/solr/example/lib/jetty-util-6.1.26.jar
- copied unchanged from r1043147, lucene/dev/trunk/solr/example/lib/jetty-util-6.1.26.jar
lucene/dev/branches/docvalues/solr/example/lib/jsp-2.1/jsp-2.1-glassfish-2.1.v20091210.jar
- copied unchanged from r1043147, lucene/dev/trunk/solr/example/lib/jsp-2.1/jsp-2.1-glassfish-2.1.v20091210.jar
lucene/dev/branches/docvalues/solr/example/lib/jsp-2.1/jsp-2.1-jetty-6.1.26.jar
- copied unchanged from r1043147, lucene/dev/trunk/solr/example/lib/jsp-2.1/jsp-2.1-jetty-6.1.26.jar
lucene/dev/branches/docvalues/solr/example/lib/jsp-2.1/jsp-api-2.1-glassfish-2.1.v20091210.jar
- copied unchanged from r1043147, lucene/dev/trunk/solr/example/lib/jsp-2.1/jsp-api-2.1-glassfish-2.1.v20091210.jar
lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/PersianCharFilterFactory.java
- copied unchanged from r1043147, lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/PersianCharFilterFactory.java
lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java
- copied unchanged from r1043147, lucene/dev/trunk/solr/src/java/org/apache/solr/analysis/UAX29URLEmailTokenizerFactory.java
lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/analysis/TestUAX29URLEmailTokenizerFactory.java
- copied unchanged from r1043147, lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestUAX29URLEmailTokenizerFactory.java
Removed:
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29Tokenizer.java
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29Tokenizer.jflex
lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29Tokenizer.java
lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/analysis/in/TestIndicTokenizer.java
lucene/dev/branches/docvalues/modules/analysis/icu/lib/icu4j-4_4_2.jar
lucene/dev/branches/docvalues/solr/contrib/analysis-extras/lib/icu4j-4_4_2.jar
lucene/dev/branches/docvalues/solr/contrib/extraction/lib/icu4j-4_4_2.jar
lucene/dev/branches/docvalues/solr/example/lib/jetty-6.1.22.jar
lucene/dev/branches/docvalues/solr/example/lib/jetty-util-6.1.22.jar
lucene/dev/branches/docvalues/solr/example/lib/jsp-2.1/jsp-2.1-glassfish-9.1.1.B60.25.p2.jar
lucene/dev/branches/docvalues/solr/example/lib/jsp-2.1/jsp-2.1-jetty-6.1.22.jar
lucene/dev/branches/docvalues/solr/example/lib/jsp-2.1/jsp-api-2.1-glassfish-9.1.1.B60.25.p2.jar
lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/IndicTokenizerFactory.java
lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/UAX29TokenizerFactory.java
lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/analysis/TestUAX29TokenizerFactory.java
Modified:
lucene/dev/branches/docvalues/ (props changed)
lucene/dev/branches/docvalues/modules/analysis/CHANGES.txt
lucene/dev/branches/docvalues/modules/analysis/README.txt
lucene/dev/branches/docvalues/modules/analysis/common/build.xml
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicLetterTokenizer.java
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/package.html
lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/util/ReusableAnalyzerBase.java
lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/analysis/ar/TestArabicLetterTokenizer.java
lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/analysis/core/TestStandardAnalyzer.java
lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/collation/CollationTestBase.java
lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyAnalyzer.java
lucene/dev/branches/docvalues/modules/analysis/common/src/test/org/apache/lucene/collation/TestCollationKeyFilter.java
lucene/dev/branches/docvalues/modules/analysis/icu/src/data/utr30/DiacriticFolding.txt
lucene/dev/branches/docvalues/modules/analysis/icu/src/data/utr30/NativeDigitFolding.txt
lucene/dev/branches/docvalues/modules/analysis/icu/src/data/utr30/nfkc.txt
lucene/dev/branches/docvalues/modules/analysis/icu/src/data/utr30/nfkc_cf.txt
lucene/dev/branches/docvalues/modules/analysis/icu/src/java/overview.html
lucene/dev/branches/docvalues/modules/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Hebrew.brk
lucene/dev/branches/docvalues/modules/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Khmer.brk
lucene/dev/branches/docvalues/modules/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Lao.brk
lucene/dev/branches/docvalues/modules/analysis/icu/src/resources/org/apache/lucene/analysis/icu/segmentation/Myanmar.brk
lucene/dev/branches/docvalues/modules/analysis/icu/src/resources/org/apache/lucene/analysis/icu/utr30.nrm
lucene/dev/branches/docvalues/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyAnalyzer.java
lucene/dev/branches/docvalues/modules/analysis/icu/src/test/org/apache/lucene/collation/TestICUCollationKeyFilter.java
lucene/dev/branches/docvalues/solr/ (props changed)
lucene/dev/branches/docvalues/solr/CHANGES.txt (contents, props changed)
lucene/dev/branches/docvalues/solr/KEYS (props changed)
lucene/dev/branches/docvalues/solr/LICENSE.txt (props changed)
lucene/dev/branches/docvalues/solr/NOTICE.txt (props changed)
lucene/dev/branches/docvalues/solr/README.txt (props changed)
lucene/dev/branches/docvalues/solr/build.xml (props changed)
lucene/dev/branches/docvalues/solr/client/ (props changed)
lucene/dev/branches/docvalues/solr/common-build.xml (props changed)
lucene/dev/branches/docvalues/solr/contrib/ (props changed)
lucene/dev/branches/docvalues/solr/contrib/dataimporthandler/src/main/java/org/apache/solr/handler/dataimport/FieldStreamDataSource.java
lucene/dev/branches/docvalues/solr/example/ (props changed)
lucene/dev/branches/docvalues/solr/example/etc/jetty.xml
lucene/dev/branches/docvalues/solr/example/etc/webdefault.xml
lucene/dev/branches/docvalues/solr/lib/ (props changed)
lucene/dev/branches/docvalues/solr/site/ (props changed)
lucene/dev/branches/docvalues/solr/src/ (props changed)
lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/analysis/ArabicLetterTokenizerFactory.java
lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/handler/CSVRequestHandler.java
lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/schema/LatLonType.java
lucene/dev/branches/docvalues/solr/src/java/org/apache/solr/search/MutableValueInt.java
lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/analysis/TestArabicFilters.java
lucene/dev/branches/docvalues/solr/src/test/org/apache/solr/analysis/TestHindiFilters.java
lucene/dev/branches/docvalues/solr/testlogging.properties (props changed)
Modified: lucene/dev/branches/docvalues/modules/analysis/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/CHANGES.txt?rev=1043605&r1=1043604&r2=1043605&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/CHANGES.txt (original)
+++ lucene/dev/branches/docvalues/modules/analysis/CHANGES.txt Wed Dec 8 19:00:16 2010
@@ -9,15 +9,17 @@ API Changes
* LUCENE-2413: Removed the AnalyzerUtil in common/miscellaneous. (Robert Muir)
- * LUCENE-2167: StandardTokenizer/Analyzer in common/standard/ now implement
- the Word Break rules from the Unicode Text Segmentation algorithm (UAX#29),
- as well as tokenizing URLs and email addresses according to the relevant
- RFCs. ClassicTokenizer/Analyzer retains the old StandardTokenizer/Analyzer
- behavior. (Steven Rowe, Robert Muir, Uwe Schindler)
-
- * LUCENE-2699: Update StandardTokenizer and UAX29Tokenizer to Unicode 6.0.0.
- (Steven Rowe)
+ * LUCENE-2167,LUCENE-2699,LUCENE-2763: StandardTokenizer/Analyzer in
+ common/standard/ now implement the Word Break rules from the Unicode 6.0.0
+ Text Segmentation algorithm (UAX#29).
+ ClassicTokenizer/Analyzer retains the old StandardTokenizer/Analyzer
+ implementation and behavior.
+
+ UAX29URLEmailTokenizer tokenizes URLs and E-mail addresses according to the
+ relevant RFCs, in addition to implementing the UAX#29 Word Break rules.
+ (Steven Rowe, Robert Muir, Uwe Schindler)
+
* LUCENE-1370: Added ShingleFilter option to output unigrams if no shingles
can be generated. (Chris Harris via Steven Rowe)
Modified: lucene/dev/branches/docvalues/modules/analysis/README.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/README.txt?rev=1043605&r1=1043604&r2=1043605&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/README.txt (original)
+++ lucene/dev/branches/docvalues/modules/analysis/README.txt Wed Dec 8 19:00:16 2010
@@ -20,7 +20,7 @@ lucene-analyzers-common-XX.jar
lucene-analyzers-icu-XX.jar
An add-on analysis library that provides improved Unicode support via
International Components for Unicode (ICU). Note: this module depends on
- the ICU4j jar file (version >= 4.4.0)
+ the ICU4j jar file (version >= 4.6.0)
lucene-analyzers-phonetic-XX.jar
An add-on analysis library that provides phonetic encoders via Apache
Modified: lucene/dev/branches/docvalues/modules/analysis/common/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/common/build.xml?rev=1043605&r1=1043604&r2=1043605&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/common/build.xml (original)
+++ lucene/dev/branches/docvalues/modules/analysis/common/build.xml Wed Dec 8 19:00:16 2010
@@ -38,7 +38,7 @@
<target name="compile-core" depends="jflex-notice, common.compile-core"/>
- <target name="jflex" depends="jflex-check,clean-jflex,jflex-StandardAnalyzer,jflex-UAX29Tokenizer,jflex-wiki-tokenizer"/>
+ <target name="jflex" depends="jflex-check,clean-jflex,jflex-StandardAnalyzer,jflex-UAX29URLEmailTokenizer,jflex-wiki-tokenizer"/>
<target name="jflex-wiki-tokenizer" depends="init,jflex-check" if="jflex.present">
<taskdef classname="jflex.anttask.JFlexTask" name="jflex">
@@ -62,11 +62,11 @@
nobak="on" />
</target>
- <target name="jflex-UAX29Tokenizer" depends="jflex-check" if="jflex.present">
+ <target name="jflex-UAX29URLEmailTokenizer" depends="jflex-check" if="jflex.present">
<taskdef classname="jflex.anttask.JFlexTask" name="jflex">
<classpath refid="jflex.classpath"/>
</taskdef>
- <jflex file="src/java/org/apache/lucene/analysis/standard/UAX29Tokenizer.jflex"
+ <jflex file="src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizer.jflex"
outdir="src/java/org/apache/lucene/analysis/standard"
nobak="on" />
</target>
Modified: lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java?rev=1043605&r1=1043604&r2=1043605&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java (original)
+++ lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java Wed Dec 8 19:00:16 2010
@@ -27,6 +27,7 @@ import org.apache.lucene.analysis.Analyz
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopFilter;
import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.analysis.util.WordlistLoader;
@@ -132,7 +133,7 @@ public final class ArabicAnalyzer extend
* used to tokenize all the text in the provided {@link Reader}.
*
* @return {@link org.apache.lucene.analysis.util.ReusableAnalyzerBase.TokenStreamComponents}
- * built from an {@link ArabicLetterTokenizer} filtered with
+ * built from an {@link StandardTokenizer} filtered with
* {@link LowerCaseFilter}, {@link StopFilter},
* {@link ArabicNormalizationFilter}, {@link KeywordMarkerFilter}
* if a stem exclusion set is provided and {@link ArabicStemFilter}.
@@ -140,7 +141,8 @@ public final class ArabicAnalyzer extend
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
- final Tokenizer source = new ArabicLetterTokenizer(matchVersion, reader);
+ final Tokenizer source = matchVersion.onOrAfter(Version.LUCENE_31) ?
+ new StandardTokenizer(matchVersion, reader) : new ArabicLetterTokenizer(matchVersion, reader);
TokenStream result = new LowerCaseFilter(matchVersion, source);
// the order here is important: the stopword list is not normalized!
result = new StopFilter( matchVersion, result, stopwords);
Modified: lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicLetterTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicLetterTokenizer.java?rev=1043605&r1=1043604&r2=1043605&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicLetterTokenizer.java (original)
+++ lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/ar/ArabicLetterTokenizer.java Wed Dec 8 19:00:16 2010
@@ -20,6 +20,7 @@ import java.io.Reader;
import org.apache.lucene.analysis.CharTokenizer;
import org.apache.lucene.analysis.core.LetterTokenizer;
+import org.apache.lucene.analysis.standard.StandardTokenizer; // javadoc @link
import org.apache.lucene.util.AttributeSource;
import org.apache.lucene.util.Version;
@@ -38,7 +39,9 @@ import org.apache.lucene.util.Version;
* detect token characters. See {@link #isTokenChar(int)} and
* {@link #normalize(int)} for details.</li>
* </ul>
+ * @deprecated (3.1) Use {@link StandardTokenizer} instead.
*/
+@Deprecated
public class ArabicLetterTokenizer extends LetterTokenizer {
/**
* Construct a new ArabicLetterTokenizer.
Modified: lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java?rev=1043605&r1=1043604&r2=1043605&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java (original)
+++ lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java Wed Dec 8 19:00:16 2010
@@ -22,12 +22,14 @@ import java.io.Reader;
import java.util.Set;
import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.ar.ArabicLetterTokenizer;
import org.apache.lucene.analysis.ar.ArabicNormalizationFilter;
import org.apache.lucene.analysis.core.LowerCaseFilter;
import org.apache.lucene.analysis.core.StopFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
import org.apache.lucene.util.Version;
@@ -109,14 +111,19 @@ public final class PersianAnalyzer exten
* used to tokenize all the text in the provided {@link Reader}.
*
* @return {@link org.apache.lucene.analysis.util.ReusableAnalyzerBase.TokenStreamComponents}
- * built from a {@link ArabicLetterTokenizer} filtered with
+ * built from a {@link StandardTokenizer} filtered with
* {@link LowerCaseFilter}, {@link ArabicNormalizationFilter},
* {@link PersianNormalizationFilter} and Persian Stop words
*/
@Override
protected TokenStreamComponents createComponents(String fieldName,
Reader reader) {
- final Tokenizer source = new ArabicLetterTokenizer(matchVersion, reader);
+ final Tokenizer source;
+ if (matchVersion.onOrAfter(Version.LUCENE_31)) {
+ source = new StandardTokenizer(matchVersion, reader);
+ } else {
+ source = new ArabicLetterTokenizer(matchVersion, reader);
+ }
TokenStream result = new LowerCaseFilter(matchVersion, source);
result = new ArabicNormalizationFilter(result);
/* additional persian-specific normalization */
@@ -127,4 +134,14 @@ public final class PersianAnalyzer exten
*/
return new TokenStreamComponents(source, new StopFilter(matchVersion, result, stopwords));
}
+
+ /**
+ * Wraps the Reader with {@link PersianCharFilter}
+ */
+ @Override
+ protected Reader initReader(Reader reader) {
+ return matchVersion.onOrAfter(Version.LUCENE_31) ?
+ new PersianCharFilter(CharReader.get(reader)) :
+ reader;
+ }
}
Modified: lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro?rev=1043605&r1=1043604&r2=1043605&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro (original)
+++ lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro Wed Dec 8 19:00:16 2010
@@ -15,8 +15,8 @@
*/
// Generated from IANA Root Zone Database <http://www.internic.net/zones/root.zone>
-// file version from Tuesday, October 12, 2010 11:34:09 AM UTC
-// generated on Wednesday, October 13, 2010 4:12:27 AM UTC
+// file version from Saturday, December 4, 2010 12:34:19 PM UTC
+// generated on Sunday, December 5, 2010 12:24:12 AM UTC
// by org.apache.lucene.analysis.standard.GenerateJflexTLDMacros
ASCIITLD = "." (
Modified: lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java?rev=1043605&r1=1043604&r2=1043605&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java (original)
+++ lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java Wed Dec 8 19:00:16 2010
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 10/3/10 9:07 AM */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 12/4/10 7:24 PM */
package org.apache.lucene.analysis.standard;
@@ -33,8 +33,8 @@ import org.apache.lucene.analysis.tokena
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 10/3/10 9:07 AM from the specification file
- * <tt>C:/Users/rmuir/workspace/lucene-clean/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex</tt>
+ * on 12/4/10 7:24 PM from the specification file
+ * <tt>C:/cygwin/home/us/svn/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex</tt>
*/
class ClassicTokenizerImpl implements StandardTokenizerInterface {
@@ -630,6 +630,12 @@ public final void getText(CharTermAttrib
zzState = ZZ_LEXSTATE[zzLexicalState];
+ // set up zzAction for empty match case:
+ int zzAttributes = zzAttrL[zzState];
+ if ( (zzAttributes & 1) == 1 ) {
+ zzAction = zzState;
+ }
+
zzForAction: {
while (true) {
@@ -662,7 +668,7 @@ public final void getText(CharTermAttrib
if (zzNext == -1) break zzForAction;
zzState = zzNext;
- int zzAttributes = zzAttrL[zzState];
+ zzAttributes = zzAttrL[zzState];
if ( (zzAttributes & 1) == 1 ) {
zzAction = zzState;
zzMarkedPosL = zzCurrentPosL;
@@ -676,44 +682,44 @@ public final void getText(CharTermAttrib
zzMarkedPos = zzMarkedPosL;
switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
- case 10:
- { return EMAIL;
+ case 5:
+ { return NUM;
}
case 11: break;
- case 2:
- { return ALPHANUM;
+ case 9:
+ { return ACRONYM;
}
case 12: break;
- case 4:
- { return HOST;
+ case 7:
+ { return COMPANY;
}
case 13: break;
- case 1:
- { /* ignore */
+ case 10:
+ { return EMAIL;
}
case 14: break;
- case 8:
- { return ACRONYM_DEP;
+ case 1:
+ { /* ignore */
}
case 15: break;
- case 5:
- { return NUM;
+ case 6:
+ { return APOSTROPHE;
}
case 16: break;
- case 9:
- { return ACRONYM;
+ case 3:
+ { return CJ;
}
case 17: break;
- case 7:
- { return COMPANY;
+ case 8:
+ { return ACRONYM_DEP;
}
case 18: break;
- case 6:
- { return APOSTROPHE;
+ case 2:
+ { return ALPHANUM;
}
case 19: break;
- case 3:
- { return CJ;
+ case 4:
+ { return HOST;
}
case 20: break;
default:
Modified: lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt?rev=1043605&r1=1043604&r2=1043605&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt (original)
+++ lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt Wed Dec 8 19:00:16 2010
@@ -16,6 +16,6 @@
*/
-WARNING: if you change StandardTokenizerImpl*.jflex and need to regenerate
- the tokenizer, only use the trunk version of JFlex 1.5 (with a minimum
- SVN revision 597) at the moment!
+WARNING: if you change StandardTokenizerImpl*.jflex or UAX29URLEmailTokenizer
+ and need to regenerate the tokenizer, only use the trunk version
+ of JFlex 1.5 (with a minimum SVN revision 597) at the moment!
Modified: lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=1043605&r1=1043604&r2=1043605&view=diff
==============================================================================
--- lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/dev/branches/docvalues/modules/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Wed Dec 8 19:00:16 2010
@@ -83,10 +83,9 @@ public final class StandardTokenizer ext
@Deprecated
public static final int ACRONYM_DEP = 8;
- public static final int URL = 9;
- public static final int SOUTHEAST_ASIAN = 10;
- public static final int IDEOGRAPHIC = 11;
- public static final int HIRAGANA = 12;
+ public static final int SOUTHEAST_ASIAN = 9;
+ public static final int IDEOGRAPHIC = 10;
+ public static final int HIRAGANA = 11;
/** String token types that correspond to token type int constants */
public static final String [] TOKEN_TYPES = new String [] {
@@ -99,7 +98,6 @@ public final class StandardTokenizer ext
"<NUM>",
"<CJ>",
"<ACRONYM_DEP>",
- "<URL>",
"<SOUTHEAST_ASIAN>",
"<IDEOGRAPHIC>",
"<HIRAGANA>"