You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2013/12/06 17:51:45 UTC
svn commit: r1548595 [1/6] - in /lucene/dev/trunk/lucene: ./ analysis/common/
analysis/common/src/java/org/apache/lucene/analysis/charfilter/
analysis/common/src/java/org/apache/lucene/analysis/standard/
analysis/common/src/java/org/apache/lucene/analy...
Author: sarowe
Date: Fri Dec 6 16:51:43 2013
New Revision: 1548595
URL: http://svn.apache.org/r1548595
Log:
LUCENE-5357: Upgrade StandardTokenizer and UAX29URLEmailTokenizer to Unicode 6.3; update UAX29URLEmailTokenizer's recognized top level domains in URLs and Emails from the IANA Root Zone Database.
Added:
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/WordBreakTestUnicode_6_3_0.java (with props)
Removed:
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/WordBreakTestUnicode_6_1_0.java
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/analysis/common/build.xml
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizerImpl.jflex
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/UAX29URLEmailTokenizerImpl.jflex
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.java
lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/wikipedia/WikipediaTokenizerImpl.jflex
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestStandardAnalyzer.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/TestUAX29URLEmailTokenizer.java
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/email.addresses.from.random.text.with.email.addresses.txt
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/generateJavaUnicodeWordBreakTest.pl
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/random.text.with.email.addresses.txt
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/random.text.with.urls.txt
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/core/urls.from.random.text.with.urls.txt
lucene/dev/trunk/lucene/analysis/common/src/test/org/apache/lucene/analysis/standard/TestUAX29URLEmailTokenizerFactory.java
lucene/dev/trunk/lucene/analysis/common/src/tools/java/org/apache/lucene/analysis/standard/GenerateJflexTLDMacros.java
lucene/dev/trunk/lucene/analysis/icu/src/tools/java/org/apache/lucene/analysis/icu/GenerateJFlexSupplementaryMacros.java
lucene/dev/trunk/lucene/common-build.xml
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1548595&r1=1548594&r2=1548595&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Fri Dec 6 16:51:43 2013
@@ -91,6 +91,11 @@ Build
* LUCENE-4381: Upgrade analysis/icu to 52.1. (Robert Muir)
+* LUCENE-5357: Upgrade StandardTokenizer and UAX29URLEmailTokenizer to
+ Unicode 6.3; update UAX29URLEmailTokenizer's recognized top level
+ domains in URLs and Emails from the IANA Root Zone Database.
+ (Steve Rowe)
+
Bug fixes
* LUCENE-5285: Improved highlighting of multi-valued fields with
Modified: lucene/dev/trunk/lucene/analysis/common/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/build.xml?rev=1548595&r1=1548594&r2=1548595&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/build.xml (original)
+++ lucene/dev/trunk/lucene/analysis/common/build.xml Fri Dec 6 16:51:43 2013
@@ -45,17 +45,13 @@
<taskdef classname="jflex.anttask.JFlexTask" name="jflex">
<classpath refid="jflex.classpath"/>
</taskdef>
- <!-- this logic below looks duplicated with run-jflex, but its not, the regexp is different! -->
<jflex file="src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex"
outdir="src/java/org/apache/lucene/analysis/charfilter"
- nobak="on"/>
- <!-- Remove the inappropriate JFlex-generated constructors -->
+ nobak="on" inputstreamctor="false"/>
+ <!-- Remove the inappropriate JFlex-generated constructor -->
<replaceregexp file="src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java"
- match="/\*\*\s*\*\s*Creates a new scanner.*this\(new java\.io\.InputStreamReader\(in\)\);\s*\}"
- replace="" flags="sg"/>
- <replaceregexp file="src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java"
- match="\/\*\s*The following code was generated by JFlex.*"
- replace="\/\* The following code was generated by JFlex. \*\/" flags=""/>
+ match="/\*\*\s*\*\s*Creates a new scanner\s*\*\s*\*\s*@param\s*in\s*the java.io.Reader to read input from\.\s*\*/\s*public HTMLStripCharFilter\(java\.io\.Reader in\)\s*\{\s*this.zzReader = in;\s*\}"
+ replace="" flags="s"/>
</target>
<target name="generate-jflex-html-char-entities">
@@ -96,15 +92,7 @@
<attribute name="dir"/>
<attribute name="name"/>
<sequential>
- <jflex file="@{dir}/@{name}.jflex"
- outdir="@{dir}"
- nobak="on" />
- <replaceregexp file="@{dir}/@{name}.java"
- match="/\*\*\s*\*\s*Creates a new scanner\..*this\(new java\.io\.InputStreamReader\(in\)\);\s*\}"
- replace="" flags="sg"/>
- <replaceregexp file="@{dir}/@{name}.java"
- match="\/\*\s*The following code was generated by JFlex.*"
- replace="\/\* The following code was generated by JFlex. \*\/" flags=""/>
+ <jflex file="@{dir}/@{name}.jflex" outdir="@{dir}" nobak="on" inputstreamctor="false"/>
</sequential>
</macrodef>
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java?rev=1548595&r1=1548594&r2=1548595&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.java Fri Dec 6 16:51:43 2013
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex. */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT */
package org.apache.lucene.analysis.charfilter;
@@ -152,77 +152,77 @@ public final class HTMLStripCharFilter e
"\21\1\1\41\32\1\5\0\113\1\3\0\3\1\17\0\15\1\1\0"+
"\4\1\3\2\13\0\22\1\3\2\13\0\22\1\2\2\14\0\15\1"+
"\1\0\3\1\1\0\2\2\14\0\64\1\40\2\3\0\1\1\4\0"+
- "\1\1\1\2\2\0\12\274\41\0\3\2\1\41\1\0\12\274\6\0"+
- "\130\1\10\0\51\1\1\2\1\1\5\0\106\1\12\0\35\1\3\0"+
- "\14\2\4\0\14\2\12\0\12\274\36\1\2\0\5\1\13\0\54\1"+
- "\4\0\21\2\7\1\2\2\6\0\12\274\1\2\45\0\27\1\5\2"+
- "\4\0\65\1\12\2\1\0\35\2\2\0\1\2\12\274\6\0\12\274"+
- "\15\0\1\1\130\0\5\2\57\1\21\2\7\1\4\0\12\274\21\0"+
- "\11\2\14\0\3\2\36\1\15\2\2\1\12\274\54\1\16\2\14\0"+
- "\44\1\24\2\10\0\12\274\3\0\3\1\12\274\44\1\122\0\3\2"+
- "\1\0\25\2\4\1\1\2\4\1\3\2\2\1\11\0\300\1\47\2"+
- "\25\0\4\2\u0116\1\2\0\6\1\2\0\46\1\2\0\6\1\2\0"+
- "\10\1\1\0\1\1\1\0\1\1\1\0\1\1\1\0\37\1\2\0"+
- "\65\1\1\0\7\1\1\0\1\1\3\0\3\1\1\0\7\1\3\0"+
- "\4\1\2\0\6\1\4\0\15\1\5\0\3\1\1\0\7\1\3\0"+
- "\13\41\35\0\2\41\5\0\1\41\17\0\2\2\23\0\1\2\12\0"+
- "\1\41\21\0\1\1\15\0\1\1\20\0\15\1\63\0\15\2\4\0"+
- "\1\2\3\0\14\2\21\0\1\1\4\0\1\1\2\0\12\1\1\0"+
- "\1\1\2\0\6\1\6\0\1\1\1\0\1\1\1\0\1\1\1\0"+
- "\20\1\2\0\4\1\5\0\5\1\4\0\1\1\21\0\51\1\u0a77\0"+
- "\57\1\1\0\57\1\1\0\205\1\6\0\4\1\3\2\2\1\14\0"+
- "\46\1\1\0\1\1\5\0\1\1\2\0\70\1\7\0\1\1\17\0"+
- "\1\2\27\1\11\0\7\1\1\0\7\1\1\0\7\1\1\0\7\1"+
- "\1\0\7\1\1\0\7\1\1\0\7\1\1\0\7\1\1\0\40\2"+
- "\u0200\0\1\41\4\0\3\1\31\0\11\1\6\2\1\0\5\1\2\0"+
- "\5\1\4\0\126\1\2\0\2\2\5\1\1\0\132\1\1\0\4\1"+
- "\5\0\51\1\3\0\136\1\21\0\33\1\65\0\20\1\u0200\0\u19b6\1"+
- "\112\0\u51cd\1\63\0\u048d\1\103\0\56\1\2\0\u010d\1\3\0\20\1"+
- "\12\274\2\1\24\0\57\1\1\2\4\0\12\2\1\0\31\1\7\0"+
- "\1\2\120\1\2\2\45\0\11\1\2\0\147\1\2\0\4\1\1\0"+
- "\4\1\14\0\13\1\115\0\12\1\1\2\3\1\1\2\4\1\1\2"+
- "\27\1\5\2\30\0\64\1\14\0\2\2\62\1\21\2\13\0\12\274"+
- "\6\0\22\2\6\1\3\0\1\1\4\0\12\274\34\1\10\2\2\0"+
- "\27\1\15\2\14\0\35\1\3\0\4\2\57\1\16\2\16\0\1\1"+
- "\12\274\46\0\51\1\16\2\11\0\3\1\1\2\10\1\2\2\2\0"+
- "\12\274\6\0\27\1\3\0\1\1\1\2\4\0\60\1\1\2\1\1"+
- "\3\2\2\1\2\2\5\1\2\2\1\1\1\2\1\1\30\0\3\1"+
- "\2\0\13\1\5\2\2\0\3\1\2\2\12\0\6\1\2\0\6\1"+
- "\2\0\6\1\11\0\7\1\1\0\7\1\221\0\43\1\10\2\1\0"+
- "\2\2\2\0\12\274\6\0\u2ba4\1\14\0\27\1\4\0\61\1\4\0"+
- "\1\170\1\223\1\103\1\165\1\136\1\214\2\0\1\160\1\153\2\0"+
- "\1\120\1\210\14\0\1\105\1\127\20\0\1\122\7\0\1\256\1\112"+
- "\5\0\1\143\4\0\51\120\1\110\3\120\1\124\1\220\17\0\1\133"+
- "\u02c1\0\1\252\277\0\2\123\1\212\3\222\2\211\1\222\1\211\2\222"+
- "\1\221\21\222\11\213\1\157\7\213\7\204\1\156\1\204\1\246\2\207"+
- "\1\166\1\246\1\207\1\166\10\246\2\167\5\203\2\155\5\203\1\107"+
- "\10\202\5\154\3\224\12\251\20\224\3\225\32\227\1\226\2\200\2\234"+
- "\1\235\2\234\2\235\2\234\1\235\3\200\1\177\2\200\12\250\1\247"+
- "\1\176\1\171\7\176\1\171\13\176\31\200\7\176\12\250\1\176\5\134"+
- "\3\245\3\142\1\140\4\142\2\140\10\142\1\140\7\141\1\137\2\141"+
- "\7\142\16\245\1\135\4\245\1\106\4\244\1\106\5\255\1\254\1\255"+
- "\3\254\7\255\1\254\23\255\5\264\3\255\6\264\2\255\6\253\5\263"+
- "\3\262\2\142\7\257\36\142\4\257\5\142\5\245\6\244\2\245\1\244"+
- "\4\141\13\253\12\244\26\253\15\134\1\243\2\134\1\152\3\237\1\134"+
- "\2\237\5\151\4\237\4\152\1\151\3\152\1\151\5\152\2\147\1\116"+
- "\2\147\1\116\1\147\2\116\1\147\1\116\12\147\1\116\4\146\1\115"+
- "\1\236\1\240\1\150\3\164\1\240\2\164\1\260\2\261\2\164\1\150"+
- "\1\164\1\150\1\164\1\150\1\164\3\150\1\164\2\150\1\164\1\150"+
- "\2\164\1\150\1\164\1\150\1\164\1\150\1\164\1\150\1\164\1\150"+
- "\1\162\2\145\1\162\1\145\2\162\4\145\1\162\7\145\1\162\4\145"+
- "\1\162\4\145\1\164\1\150\1\164\12\216\1\217\21\216\1\217\3\215"+
- "\1\217\3\216\1\217\1\216\2\144\2\216\1\217\15\241\4\201\4\206"+
- "\1\242\1\161\10\242\7\206\6\164\4\113\1\121\37\113\1\121\4\113"+
- "\25\174\1\131\11\174\21\130\5\174\1\104\12\117\5\174\6\205\4\162"+
- "\1\163\1\130\5\231\12\232\17\231\1\125\3\114\14\230\1\126\11\173"+
- "\1\172\5\173\4\233\13\175\2\132\11\173\1\172\31\173\1\172\4\126"+
- "\4\173\2\172\2\265\1\111\5\265\52\111\u1900\0\u016e\1\2\0\152\1"+
- "\46\0\7\1\14\0\5\1\5\0\1\1\1\2\12\1\1\0\15\1"+
- "\1\0\5\1\1\0\1\1\1\0\2\1\1\0\2\1\1\0\154\1"+
- "\41\0\u016b\1\22\0\100\1\2\0\66\1\50\0\14\1\4\0\20\2"+
- "\20\0\7\2\14\0\2\2\30\0\3\2\40\0\5\1\1\0\207\1"+
- "\23\0\12\274\7\0\32\1\4\0\1\2\1\0\32\1\13\0\131\1"+
- "\3\0\6\1\2\0\6\1\2\0\6\1\2\0\3\1\43\0";
+ "\1\1\1\2\2\0\12\274\41\0\3\2\2\0\12\274\6\0\130\1"+
+ "\10\0\51\1\1\2\1\1\5\0\106\1\12\0\35\1\3\0\14\2"+
+ "\4\0\14\2\12\0\12\274\36\1\2\0\5\1\13\0\54\1\4\0"+
+ "\21\2\7\1\2\2\6\0\12\274\1\2\45\0\27\1\5\2\4\0"+
+ "\65\1\12\2\1\0\35\2\2\0\1\2\12\274\6\0\12\274\15\0"+
+ "\1\1\130\0\5\2\57\1\21\2\7\1\4\0\12\274\21\0\11\2"+
+ "\14\0\3\2\36\1\15\2\2\1\12\274\54\1\16\2\14\0\44\1"+
+ "\24\2\10\0\12\274\3\0\3\1\12\274\44\1\122\0\3\2\1\0"+
+ "\25\2\4\1\1\2\4\1\3\2\2\1\11\0\300\1\47\2\25\0"+
+ "\4\2\u0116\1\2\0\6\1\2\0\46\1\2\0\6\1\2\0\10\1"+
+ "\1\0\1\1\1\0\1\1\1\0\1\1\1\0\37\1\2\0\65\1"+
+ "\1\0\7\1\1\0\1\1\3\0\3\1\1\0\7\1\3\0\4\1"+
+ "\2\0\6\1\4\0\15\1\5\0\3\1\1\0\7\1\3\0\13\41"+
+ "\35\0\2\41\5\0\1\41\17\0\2\2\23\0\1\2\12\0\1\41"+
+ "\21\0\1\1\15\0\1\1\20\0\15\1\63\0\15\2\4\0\1\2"+
+ "\3\0\14\2\21\0\1\1\4\0\1\1\2\0\12\1\1\0\1\1"+
+ "\2\0\6\1\6\0\1\1\1\0\1\1\1\0\1\1\1\0\20\1"+
+ "\2\0\4\1\5\0\5\1\4\0\1\1\21\0\51\1\u0a77\0\57\1"+
+ "\1\0\57\1\1\0\205\1\6\0\4\1\3\2\2\1\14\0\46\1"+
+ "\1\0\1\1\5\0\1\1\2\0\70\1\7\0\1\1\17\0\1\2"+
+ "\27\1\11\0\7\1\1\0\7\1\1\0\7\1\1\0\7\1\1\0"+
+ "\7\1\1\0\7\1\1\0\7\1\1\0\7\1\1\0\40\2\u0200\0"+
+ "\1\41\4\0\3\1\31\0\11\1\6\2\1\0\5\1\2\0\5\1"+
+ "\4\0\126\1\2\0\2\2\5\1\1\0\132\1\1\0\4\1\5\0"+
+ "\51\1\3\0\136\1\21\0\33\1\65\0\20\1\u0200\0\u19b6\1\112\0"+
+ "\u51cd\1\63\0\u048d\1\103\0\56\1\2\0\u010d\1\3\0\20\1\12\274"+
+ "\2\1\24\0\57\1\1\2\4\0\12\2\1\0\31\1\7\0\1\2"+
+ "\120\1\2\2\45\0\11\1\2\0\147\1\2\0\4\1\1\0\4\1"+
+ "\14\0\13\1\115\0\12\1\1\2\3\1\1\2\4\1\1\2\27\1"+
+ "\5\2\30\0\64\1\14\0\2\2\62\1\21\2\13\0\12\274\6\0"+
+ "\22\2\6\1\3\0\1\1\4\0\12\274\34\1\10\2\2\0\27\1"+
+ "\15\2\14\0\35\1\3\0\4\2\57\1\16\2\16\0\1\1\12\274"+
+ "\46\0\51\1\16\2\11\0\3\1\1\2\10\1\2\2\2\0\12\274"+
+ "\6\0\27\1\3\0\1\1\1\2\4\0\60\1\1\2\1\1\3\2"+
+ "\2\1\2\2\5\1\2\2\1\1\1\2\1\1\30\0\3\1\2\0"+
+ "\13\1\5\2\2\0\3\1\2\2\12\0\6\1\2\0\6\1\2\0"+
+ "\6\1\11\0\7\1\1\0\7\1\221\0\43\1\10\2\1\0\2\2"+
+ "\2\0\12\274\6\0\u2ba4\1\14\0\27\1\4\0\61\1\4\0\1\170"+
+ "\1\223\1\103\1\165\1\136\1\214\2\0\1\160\1\153\2\0\1\120"+
+ "\1\210\14\0\1\105\1\127\20\0\1\122\7\0\1\256\1\112\5\0"+
+ "\1\143\4\0\51\120\1\110\3\120\1\124\1\220\17\0\1\133\u02c1\0"+
+ "\1\252\277\0\2\123\1\212\3\222\2\211\1\222\1\211\2\222\1\221"+
+ "\21\222\11\213\1\157\7\213\7\204\1\156\1\204\1\246\2\207\1\166"+
+ "\1\246\1\207\1\166\10\246\2\167\5\203\2\155\5\203\1\107\10\202"+
+ "\5\154\3\224\12\251\20\224\3\225\32\227\1\226\2\200\2\234\1\235"+
+ "\2\234\2\235\2\234\1\235\3\200\1\177\2\200\12\250\1\247\1\176"+
+ "\1\171\7\176\1\171\13\176\31\200\7\176\12\250\1\176\5\134\3\245"+
+ "\3\142\1\140\4\142\2\140\10\142\1\140\7\141\1\137\2\141\7\142"+
+ "\16\245\1\135\4\245\1\106\4\244\1\106\5\255\1\254\1\255\3\254"+
+ "\7\255\1\254\23\255\5\264\3\255\6\264\2\255\6\253\5\263\3\262"+
+ "\2\142\7\257\36\142\4\257\5\142\5\245\6\244\2\245\1\244\4\141"+
+ "\13\253\12\244\26\253\15\134\1\243\2\134\1\152\3\237\1\134\2\237"+
+ "\5\151\4\237\4\152\1\151\3\152\1\151\5\152\2\147\1\116\2\147"+
+ "\1\116\1\147\2\116\1\147\1\116\12\147\1\116\4\146\1\115\1\236"+
+ "\1\240\1\150\3\164\1\240\2\164\1\260\2\261\2\164\1\150\1\164"+
+ "\1\150\1\164\1\150\1\164\3\150\1\164\2\150\1\164\1\150\2\164"+
+ "\1\150\1\164\1\150\1\164\1\150\1\164\1\150\1\164\1\150\1\162"+
+ "\2\145\1\162\1\145\2\162\4\145\1\162\7\145\1\162\4\145\1\162"+
+ "\4\145\1\164\1\150\1\164\12\216\1\217\21\216\1\217\3\215\1\217"+
+ "\3\216\1\217\1\216\2\144\2\216\1\217\15\241\4\201\4\206\1\242"+
+ "\1\161\10\242\7\206\6\164\4\113\1\121\37\113\1\121\4\113\25\174"+
+ "\1\131\11\174\21\130\5\174\1\104\12\117\5\174\6\205\4\162\1\163"+
+ "\1\130\5\231\12\232\17\231\1\125\3\114\14\230\1\126\11\173\1\172"+
+ "\5\173\4\233\13\175\2\132\11\173\1\172\31\173\1\172\4\126\4\173"+
+ "\2\172\2\265\1\111\5\265\52\111\u1900\0\u016e\1\2\0\152\1\46\0"+
+ "\7\1\14\0\5\1\5\0\1\1\1\2\12\1\1\0\15\1\1\0"+
+ "\5\1\1\0\1\1\1\0\2\1\1\0\2\1\1\0\154\1\41\0"+
+ "\u016b\1\22\0\100\1\2\0\66\1\50\0\14\1\4\0\20\2\20\0"+
+ "\7\2\14\0\2\2\30\0\3\2\40\0\5\1\1\0\207\1\23\0"+
+ "\12\274\7\0\32\1\4\0\1\2\1\0\32\1\13\0\131\1\3\0"+
+ "\6\1\2\0\6\1\2\0\6\1\2\0\3\1\43\0";
/**
* Translates characters to character classes
@@ -30895,6 +30895,7 @@ public final class HTMLStripCharFilter e
+
/**
* Unpacks the compressed character translation table.
*
@@ -30905,7 +30906,7 @@ public final class HTMLStripCharFilter e
char [] map = new char[0x10000];
int i = 0; /* index in packed string */
int j = 0; /* index in unpacked array */
- while (i < 2778) {
+ while (i < 2776) {
int count = packed.charAt(i++);
char value = packed.charAt(i++);
do map[j++] = value; while (--count > 0);
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex?rev=1548595&r1=1548594&r2=1548595&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex Fri Dec 6 16:51:43 2013
@@ -34,7 +34,7 @@ import org.apache.lucene.analysis.util.O
*/
%%
-%unicode 6.1
+%unicode 6.3
%apiprivate
%type int
%final
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro?rev=1548595&r1=1548594&r2=1548595&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro Fri Dec 6 16:51:43 2013
@@ -1,11 +1,12 @@
/*
- * Copyright 2001-2005 The Apache Software Foundation.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,10 +14,9 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
// Generated from IANA Root Zone Database <http://www.internic.net/zones/root.zone>
-// file version from Saturday, July 14, 2012 4:34:14 AM UTC
-// generated on Sunday, July 15, 2012 12:59:44 AM UTC
+// file version from Friday, December 6, 2013 4:34:10 AM UTC
+// generated on Friday, December 6, 2013 3:21:59 PM UTC
// by org.apache.lucene.analysis.standard.GenerateJflexTLDMacros
ASCIITLD = "." (
@@ -49,6 +49,7 @@ ASCIITLD = "." (
| [bB][gG]
| [bB][hH]
| [bB][iI]
+ | [bB][iI][kK][eE]
| [bB][iI][zZ]
| [bB][jJ]
| [bB][mM]
@@ -62,6 +63,7 @@ ASCIITLD = "." (
| [bB][yY]
| [bB][zZ]
| [cC][aA]
+ | [cC][aA][mM][eE][rR][aA]
| [cC][aA][tT]
| [cC][cC]
| [cC][dD]
@@ -71,10 +73,13 @@ ASCIITLD = "." (
| [cC][iI]
| [cC][kK]
| [cC][lL]
+ | [cC][lL][oO][tT][hH][iI][nN][gG]
| [cC][mM]
| [cC][nN]
| [cC][oO]
| [cC][oO][mM]
+ | [cC][oO][nN][sS][tT][rR][uU][cC][tT][iI][oO][nN]
+ | [cC][oO][nN][tT][rR][aA][cC][tT][oO][rR][sS]
| [cC][oO][oO][pP]
| [cC][rR]
| [cC][uU]
@@ -84,6 +89,8 @@ ASCIITLD = "." (
| [cC][yY]
| [cC][zZ]
| [dD][eE]
+ | [dD][iI][aA][mM][oO][nN][dD][sS]
+ | [dD][iI][rR][eE][cC][tT][oO][rR][yY]
| [dD][jJ]
| [dD][kK]
| [dD][mM]
@@ -93,8 +100,11 @@ ASCIITLD = "." (
| [eE][dD][uU]
| [eE][eE]
| [eE][gG]
+ | [eE][nN][tT][eE][rR][pP][rR][iI][sS][eE][sS]
+ | [eE][qQ][uU][iI][pP][mM][eE][nN][tT]
| [eE][rR]
| [eE][sS]
+ | [eE][sS][tT][aA][tT][eE]
| [eE][tT]
| [eE][uU]
| [fF][iI]
@@ -104,6 +114,7 @@ ASCIITLD = "." (
| [fF][oO]
| [fF][rR]
| [gG][aA]
+ | [gG][aA][lL][lL][eE][rR][yY]
| [gG][bB]
| [gG][dD]
| [gG][eE]
@@ -118,14 +129,17 @@ ASCIITLD = "." (
| [gG][pP]
| [gG][qQ]
| [gG][rR]
+ | [gG][rR][aA][pP][hH][iI][cC][sS]
| [gG][sS]
| [gG][tT]
| [gG][uU]
+ | [gG][uU][rR][uU]
| [gG][wW]
| [gG][yY]
| [hH][kK]
| [hH][mM]
| [hH][nN]
+ | [hH][oO][lL][dD][iI][nN][gG][sS]
| [hH][rR]
| [hH][tT]
| [hH][uU]
@@ -150,6 +164,7 @@ ASCIITLD = "." (
| [kK][gG]
| [kK][hH]
| [kK][iI]
+ | [kK][iI][tT][cC][hH][eE][nN]
| [kK][mM]
| [kK][nN]
| [kK][pP]
@@ -158,9 +173,11 @@ ASCIITLD = "." (
| [kK][yY]
| [kK][zZ]
| [lL][aA]
+ | [lL][aA][nN][dD]
| [lL][bB]
| [lL][cC]
| [lL][iI]
+ | [lL][iI][gG][hH][tT][iI][nN][gG]
| [lL][kK]
| [lL][rR]
| [lL][sS]
@@ -172,6 +189,7 @@ ASCIITLD = "." (
| [mM][cC]
| [mM][dD]
| [mM][eE]
+ | [mM][eE][nN][uU]
| [mM][gG]
| [mM][hH]
| [mM][iI][lL]
@@ -214,10 +232,13 @@ ASCIITLD = "." (
| [pP][fF]
| [pP][gG]
| [pP][hH]
+ | [pP][hH][oO][tT][oO][gG][rR][aA][pP][hH][yY]
| [pP][kK]
| [pP][lL]
+ | [pP][lL][uU][mM][bB][iI][nN][gG]
| [pP][mM]
| [pP][nN]
+ | [pP][oO][sS][tT]
| [pP][rR]
| [pP][rR][oO]
| [pP][sS]
@@ -235,9 +256,11 @@ ASCIITLD = "." (
| [sS][cC]
| [sS][dD]
| [sS][eE]
+ | [sS][eE][xX][yY]
| [sS][gG]
| [sS][hH]
| [sS][iI]
+ | [sS][iI][nN][gG][lL][eE][sS]
| [sS][jJ]
| [sS][kK]
| [sS][lL]
@@ -251,18 +274,22 @@ ASCIITLD = "." (
| [sS][xX]
| [sS][yY]
| [sS][zZ]
+ | [tT][aA][tT][tT][oO][oO]
| [tT][cC]
| [tT][dD]
+ | [tT][eE][cC][hH][nN][oO][lL][oO][gG][yY]
| [tT][eE][lL]
| [tT][fF]
| [tT][gG]
| [tT][hH]
+ | [tT][iI][pP][sS]
| [tT][jJ]
| [tT][kK]
| [tT][lL]
| [tT][mM]
| [tT][nN]
| [tT][oO]
+ | [tT][oO][dD][aA][yY]
| [tT][pP]
| [tT][rR]
| [tT][rR][aA][vV][eE][lL]
@@ -273,61 +300,62 @@ ASCIITLD = "." (
| [uU][aA]
| [uU][gG]
| [uU][kK]
+ | [uU][nN][oO]
| [uU][sS]
| [uU][yY]
| [uU][zZ]
| [vV][aA]
| [vV][cC]
| [vV][eE]
+ | [vV][eE][nN][tT][uU][rR][eE][sS]
| [vV][gG]
| [vV][iI]
| [vV][nN]
+ | [vV][oO][yY][aA][gG][eE]
| [vV][uU]
| [wW][fF]
| [wW][sS]
- | [xX][nN]--0[zZ][wW][mM]56[dD]
- | [xX][nN]--11[bB]5[bB][sS]3[aA]9[aA][jJ]6[gG]
| [xX][nN]--3[eE]0[bB]707[eE]
| [xX][nN]--45[bB][rR][jJ]9[cC]
- | [xX][nN]--80[aA][kK][hH][bB][yY][kK][nN][jJ]4[fF]
| [xX][nN]--80[aA][oO]21[aA]
+ | [xX][nN]--80[aA][sS][eE][hH][dD][bB]
+ | [xX][nN]--80[aA][sS][wW][gG]
| [xX][nN]--90[aA]3[aA][cC]
- | [xX][nN]--9[tT]4[bB]11[yY][iI]5[aA]
| [xX][nN]--[cC][lL][cC][hH][cC]0[eE][aA]0[bB]2[gG]2[aA]9[gG][cC][dD]
- | [xX][nN]--[dD][eE][bB][aA]0[aA][dD]
| [xX][nN]--[fF][iI][qQ][sS]8[sS]
| [xX][nN]--[fF][iI][qQ][zZ]9[sS]
| [xX][nN]--[fF][pP][cC][rR][jJ]9[cC]3[dD]
| [xX][nN]--[fF][zZ][cC]2[cC]9[eE]2[cC]
- | [xX][nN]--[gG]6[wW]251[dD]
| [xX][nN]--[gG][eE][cC][rR][jJ]9[cC]
| [xX][nN]--[hH]2[bB][rR][jJ]9[cC]
- | [xX][nN]--[hH][gG][bB][kK]6[aA][jJ]7[fF]53[bB][bB][aA]
- | [xX][nN]--[hH][lL][cC][jJ]6[aA][yY][aA]9[eE][sS][cC]7[aA]
+ | [xX][nN]--[jJ]1[aA][mM][hH]
| [xX][nN]--[jJ]6[wW]193[gG]
- | [xX][nN]--[jJ][xX][aA][lL][pP][dD][lL][pP]
- | [xX][nN]--[kK][gG][bB][eE][cC][hH][tT][vV]
| [xX][nN]--[kK][pP][rR][wW]13[dD]
| [xX][nN]--[kK][pP][rR][yY]57[dD]
+ | [xX][nN]--[lL]1[aA][cC][cC]
| [xX][nN]--[lL][gG][bB][bB][aA][tT]1[aA][dD]8[jJ]
| [xX][nN]--[mM][gG][bB]9[aA][wW][bB][fF]
+ | [xX][nN]--[mM][gG][bB][aA]3[aA]4[fF]16[aA]
| [xX][nN]--[mM][gG][bB][aA][aA][mM]7[aA]8[hH]
| [xX][nN]--[mM][gG][bB][aA][yY][hH]7[gG][pP][aA]
| [xX][nN]--[mM][gG][bB][bB][hH]1[aA]71[eE]
| [xX][nN]--[mM][gG][bB][cC]0[aA]9[aA][zZ][cC][gG]
| [xX][nN]--[mM][gG][bB][eE][rR][pP]4[aA]5[dD]4[aA][rR]
+ | [xX][nN]--[mM][gG][bB][xX]4[cC][dD]0[aA][bB]
+ | [xX][nN]--[nN][gG][bB][cC]5[aA][zZ][dD]
| [xX][nN]--[oO]3[cC][wW]4[hH]
| [xX][nN]--[oO][gG][bB][pP][fF]8[fF][lL]
| [xX][nN]--[pP]1[aA][iI]
| [xX][nN]--[pP][gG][bB][sS]0[dD][hH]
+ | [xX][nN]--[qQ]9[jJ][yY][bB]4[cC]
| [xX][nN]--[sS]9[bB][rR][jJ]9[cC]
+ | [xX][nN]--[uU][nN][uU][pP]4[yY]
| [xX][nN]--[wW][gG][bB][hH]1[cC]
| [xX][nN]--[wW][gG][bB][lL]6[aA]
| [xX][nN]--[xX][kK][cC]2[aA][lL]3[hH][yY][eE]2[aA]
| [xX][nN]--[xX][kK][cC]2[dD][lL]3[aA]5[eE][eE]0[hH]
| [xX][nN]--[yY][fF][rR][oO]4[iI]67[oO]
| [xX][nN]--[yY][gG][bB][iI]2[aA][mM][mM][xX]
- | [xX][nN]--[zZ][cC][kK][zZ][aA][hH]
| [xX][xX][xX]
| [yY][eE]
| [yY][tT]
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java?rev=1548595&r1=1548594&r2=1548595&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java Fri Dec 6 16:51:43 2013
@@ -1,4 +1,4 @@
-/* The following code was generated by JFlex. */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT */
package org.apache.lucene.analysis.standard;
@@ -58,64 +58,63 @@ class ClassicTokenizerImpl implements St
* Translates characters to character classes
*/
private static final String ZZ_CMAP_PACKED =
- "\11\0\1\0\1\15\1\0\1\0\1\14\22\0\1\0\5\0\1\5"+
- "\1\3\4\0\1\11\1\7\1\4\1\11\12\2\6\0\1\6\32\12"+
- "\4\0\1\10\1\0\32\12\57\0\1\12\12\0\1\12\4\0\1\12"+
- "\5\0\27\12\1\0\37\12\1\0\u0128\12\2\0\22\12\34\0\136\12"+
- "\2\0\11\12\2\0\7\12\16\0\2\12\16\0\5\12\11\0\1\12"+
- "\213\0\1\12\13\0\1\12\1\0\3\12\1\0\1\12\1\0\24\12"+
- "\1\0\54\12\1\0\10\12\2\0\32\12\14\0\202\12\12\0\71\12"+
- "\2\0\2\12\2\0\2\12\3\0\46\12\2\0\2\12\67\0\46\12"+
- "\2\0\1\12\7\0\47\12\110\0\33\12\5\0\3\12\56\0\32\12"+
- "\5\0\13\12\25\0\12\2\7\0\143\12\1\0\1\12\17\0\2\12"+
- "\11\0\12\2\3\12\23\0\1\12\1\0\33\12\123\0\46\12\u015f\0"+
- "\65\12\3\0\1\12\22\0\1\12\7\0\12\12\4\0\12\2\25\0"+
- "\10\12\2\0\2\12\2\0\26\12\1\0\7\12\1\0\1\12\3\0"+
- "\4\12\42\0\2\12\1\0\3\12\4\0\12\2\2\12\23\0\6\12"+
- "\4\0\2\12\2\0\26\12\1\0\7\12\1\0\2\12\1\0\2\12"+
- "\1\0\2\12\37\0\4\12\1\0\1\12\7\0\12\2\2\0\3\12"+
- "\20\0\7\12\1\0\1\12\1\0\3\12\1\0\26\12\1\0\7\12"+
- "\1\0\2\12\1\0\5\12\3\0\1\12\22\0\1\12\17\0\1\12"+
- "\5\0\12\2\25\0\10\12\2\0\2\12\2\0\26\12\1\0\7\12"+
- "\1\0\2\12\2\0\4\12\3\0\1\12\36\0\2\12\1\0\3\12"+
- "\4\0\12\2\25\0\6\12\3\0\3\12\1\0\4\12\3\0\2\12"+
- "\1\0\1\12\1\0\2\12\3\0\2\12\3\0\3\12\3\0\10\12"+
- "\1\0\3\12\55\0\11\2\25\0\10\12\1\0\3\12\1\0\27\12"+
- "\1\0\12\12\1\0\5\12\46\0\2\12\4\0\12\2\25\0\10\12"+
- "\1\0\3\12\1\0\27\12\1\0\12\12\1\0\5\12\44\0\1\12"+
- "\1\0\2\12\4\0\12\2\25\0\10\12\1\0\3\12\1\0\27\12"+
- "\1\0\20\12\46\0\2\12\4\0\12\2\25\0\22\12\3\0\30\12"+
- "\1\0\11\12\1\0\1\12\2\0\7\12\71\0\1\1\60\12\1\1"+
- "\2\12\14\1\7\12\11\1\12\2\47\0\2\12\1\0\1\12\2\0"+
- "\2\12\1\0\1\12\2\0\1\12\6\0\4\12\1\0\7\12\1\0"+
- "\3\12\1\0\1\12\1\0\1\12\2\0\2\12\1\0\4\12\1\0"+
- "\2\12\11\0\1\12\2\0\5\12\1\0\1\12\11\0\12\2\2\0"+
- "\2\12\42\0\1\12\37\0\12\2\26\0\10\12\1\0\42\12\35\0"+
- "\4\12\164\0\42\12\1\0\5\12\1\0\2\12\25\0\12\2\6\0"+
- "\6\12\112\0\46\12\12\0\47\12\11\0\132\12\5\0\104\12\5\0"+
- "\122\12\6\0\7\12\1\0\77\12\1\0\1\12\1\0\4\12\2\0"+
- "\7\12\1\0\1\12\1\0\4\12\2\0\47\12\1\0\1\12\1\0"+
- "\4\12\2\0\37\12\1\0\1\12\1\0\4\12\2\0\7\12\1\0"+
- "\1\12\1\0\4\12\2\0\7\12\1\0\7\12\1\0\27\12\1\0"+
- "\37\12\1\0\1\12\1\0\4\12\2\0\7\12\1\0\47\12\1\0"+
- "\23\12\16\0\11\2\56\0\125\12\14\0\u026c\12\2\0\10\12\12\0"+
- "\32\12\5\0\113\12\225\0\64\12\54\0\12\2\46\0\12\2\6\0"+
- "\130\12\10\0\51\12\u0557\0\234\12\4\0\132\12\6\0\26\12\2\0"+
- "\6\12\2\0\46\12\2\0\6\12\2\0\10\12\1\0\1\12\1\0"+
- "\1\12\1\0\1\12\1\0\37\12\2\0\65\12\1\0\7\12\1\0"+
- "\1\12\3\0\3\12\1\0\7\12\3\0\4\12\2\0\6\12\4\0"+
- "\15\12\5\0\3\12\1\0\7\12\202\0\1\12\202\0\1\12\4\0"+
- "\1\12\2\0\12\12\1\0\1\12\3\0\5\12\6\0\1\12\1\0"+
- "\1\12\1\0\1\12\1\0\4\12\1\0\3\12\1\0\7\12\u0ecb\0"+
- "\2\12\52\0\5\12\12\0\1\13\124\13\10\13\2\13\2\13\132\13"+
- "\1\13\3\13\6\13\50\13\3\13\1\0\136\12\21\0\30\12\70\0"+
- "\20\13\u0100\0\200\13\200\0\u19b6\13\12\13\100\0\u51a6\13\132\13\u048d\12"+
- "\u0773\0\u2ba4\12\u215c\0\u012e\13\322\13\7\12\14\0\5\12\5\0\1\12"+
- "\1\0\12\12\1\0\15\12\1\0\5\12\1\0\1\12\1\0\2\12"+
- "\1\0\2\12\1\0\154\12\41\0\u016b\12\22\0\100\12\2\0\66\12"+
- "\50\0\14\12\164\0\3\12\1\0\1\12\1\0\207\12\23\0\12\2"+
- "\7\0\32\12\6\0\32\12\12\0\1\13\72\13\37\12\3\0\6\12"+
- "\2\0\6\12\2\0\6\12\2\0\3\12\43\0";
+ "\46\0\1\5\1\3\4\0\1\11\1\7\1\4\1\11\12\2\6\0"+
+ "\1\6\32\12\4\0\1\10\1\0\32\12\57\0\1\12\12\0\1\12"+
+ "\4\0\1\12\5\0\27\12\1\0\37\12\1\0\u0128\12\2\0\22\12"+
+ "\34\0\136\12\2\0\11\12\2\0\7\12\16\0\2\12\16\0\5\12"+
+ "\11\0\1\12\213\0\1\12\13\0\1\12\1\0\3\12\1\0\1\12"+
+ "\1\0\24\12\1\0\54\12\1\0\10\12\2\0\32\12\14\0\202\12"+
+ "\12\0\71\12\2\0\2\12\2\0\2\12\3\0\46\12\2\0\2\12"+
+ "\67\0\46\12\2\0\1\12\7\0\47\12\110\0\33\12\5\0\3\12"+
+ "\56\0\32\12\5\0\13\12\25\0\12\2\7\0\143\12\1\0\1\12"+
+ "\17\0\2\12\11\0\12\2\3\12\23\0\1\12\1\0\33\12\123\0"+
+ "\46\12\u015f\0\65\12\3\0\1\12\22\0\1\12\7\0\12\12\4\0"+
+ "\12\2\25\0\10\12\2\0\2\12\2\0\26\12\1\0\7\12\1\0"+
+ "\1\12\3\0\4\12\42\0\2\12\1\0\3\12\4\0\12\2\2\12"+
+ "\23\0\6\12\4\0\2\12\2\0\26\12\1\0\7\12\1\0\2\12"+
+ "\1\0\2\12\1\0\2\12\37\0\4\12\1\0\1\12\7\0\12\2"+
+ "\2\0\3\12\20\0\7\12\1\0\1\12\1\0\3\12\1\0\26\12"+
+ "\1\0\7\12\1\0\2\12\1\0\5\12\3\0\1\12\22\0\1\12"+
+ "\17\0\1\12\5\0\12\2\25\0\10\12\2\0\2\12\2\0\26\12"+
+ "\1\0\7\12\1\0\2\12\2\0\4\12\3\0\1\12\36\0\2\12"+
+ "\1\0\3\12\4\0\12\2\25\0\6\12\3\0\3\12\1\0\4\12"+
+ "\3\0\2\12\1\0\1\12\1\0\2\12\3\0\2\12\3\0\3\12"+
+ "\3\0\10\12\1\0\3\12\55\0\11\2\25\0\10\12\1\0\3\12"+
+ "\1\0\27\12\1\0\12\12\1\0\5\12\46\0\2\12\4\0\12\2"+
+ "\25\0\10\12\1\0\3\12\1\0\27\12\1\0\12\12\1\0\5\12"+
+ "\44\0\1\12\1\0\2\12\4\0\12\2\25\0\10\12\1\0\3\12"+
+ "\1\0\27\12\1\0\20\12\46\0\2\12\4\0\12\2\25\0\22\12"+
+ "\3\0\30\12\1\0\11\12\1\0\1\12\2\0\7\12\71\0\1\1"+
+ "\60\12\1\1\2\12\14\1\7\12\11\1\12\2\47\0\2\12\1\0"+
+ "\1\12\2\0\2\12\1\0\1\12\2\0\1\12\6\0\4\12\1\0"+
+ "\7\12\1\0\3\12\1\0\1\12\1\0\1\12\2\0\2\12\1\0"+
+ "\4\12\1\0\2\12\11\0\1\12\2\0\5\12\1\0\1\12\11\0"+
+ "\12\2\2\0\2\12\42\0\1\12\37\0\12\2\26\0\10\12\1\0"+
+ "\42\12\35\0\4\12\164\0\42\12\1\0\5\12\1\0\2\12\25\0"+
+ "\12\2\6\0\6\12\112\0\46\12\12\0\47\12\11\0\132\12\5\0"+
+ "\104\12\5\0\122\12\6\0\7\12\1\0\77\12\1\0\1\12\1\0"+
+ "\4\12\2\0\7\12\1\0\1\12\1\0\4\12\2\0\47\12\1\0"+
+ "\1\12\1\0\4\12\2\0\37\12\1\0\1\12\1\0\4\12\2\0"+
+ "\7\12\1\0\1\12\1\0\4\12\2\0\7\12\1\0\7\12\1\0"+
+ "\27\12\1\0\37\12\1\0\1\12\1\0\4\12\2\0\7\12\1\0"+
+ "\47\12\1\0\23\12\16\0\11\2\56\0\125\12\14\0\u026c\12\2\0"+
+ "\10\12\12\0\32\12\5\0\113\12\225\0\64\12\54\0\12\2\46\0"+
+ "\12\2\6\0\130\12\10\0\51\12\u0557\0\234\12\4\0\132\12\6\0"+
+ "\26\12\2\0\6\12\2\0\46\12\2\0\6\12\2\0\10\12\1\0"+
+ "\1\12\1\0\1\12\1\0\1\12\1\0\37\12\2\0\65\12\1\0"+
+ "\7\12\1\0\1\12\3\0\3\12\1\0\7\12\3\0\4\12\2\0"+
+ "\6\12\4\0\15\12\5\0\3\12\1\0\7\12\202\0\1\12\202\0"+
+ "\1\12\4\0\1\12\2\0\12\12\1\0\1\12\3\0\5\12\6\0"+
+ "\1\12\1\0\1\12\1\0\1\12\1\0\4\12\1\0\3\12\1\0"+
+ "\7\12\u0ecb\0\2\12\52\0\5\12\12\0\1\13\124\13\10\13\2\13"+
+ "\2\13\132\13\1\13\3\13\6\13\50\13\3\13\1\0\136\12\21\0"+
+ "\30\12\70\0\20\13\u0100\0\200\13\200\0\u19b6\13\12\13\100\0\u51a6\13"+
+ "\132\13\u048d\12\u0773\0\u2ba4\12\u215c\0\u012e\13\322\13\7\12\14\0\5\12"+
+ "\5\0\1\12\1\0\12\12\1\0\15\12\1\0\5\12\1\0\1\12"+
+ "\1\0\2\12\1\0\2\12\1\0\154\12\41\0\u016b\12\22\0\100\12"+
+ "\2\0\66\12\50\0\14\12\164\0\3\12\1\0\1\12\1\0\207\12"+
+ "\23\0\12\2\7\0\32\12\6\0\32\12\12\0\1\13\72\13\37\12"+
+ "\3\0\6\12\2\0\6\12\2\0\6\12\2\0\3\12\43\0";
/**
* Translates characters to character classes
@@ -128,13 +127,12 @@ class ClassicTokenizerImpl implements St
private static final int [] ZZ_ACTION = zzUnpackAction();
private static final String ZZ_ACTION_PACKED_0 =
- "\1\0\1\1\3\2\1\3\1\1\13\0\1\2\3\4"+
- "\2\0\1\5\1\0\1\5\3\4\6\5\1\6\1\4"+
- "\2\7\1\10\1\0\1\10\3\0\2\10\1\11\1\12"+
- "\1\4";
+ "\1\0\1\1\3\2\1\3\13\0\1\2\3\4\2\0"+
+ "\1\5\1\0\1\5\3\4\6\5\1\6\1\4\2\7"+
+ "\1\10\1\0\1\10\3\0\2\10\1\11\1\12\1\4";
private static int [] zzUnpackAction() {
- int [] result = new int[51];
+ int [] result = new int[50];
int offset = 0;
offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
return result;
@@ -159,16 +157,16 @@ class ClassicTokenizerImpl implements St
private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
private static final String ZZ_ROWMAP_PACKED_0 =
- "\0\0\0\16\0\34\0\52\0\70\0\16\0\106\0\124"+
- "\0\142\0\160\0\176\0\214\0\232\0\250\0\266\0\304"+
- "\0\322\0\340\0\356\0\374\0\u010a\0\u0118\0\u0126\0\u0134"+
- "\0\u0142\0\u0150\0\u015e\0\u016c\0\u017a\0\u0188\0\u0196\0\u01a4"+
- "\0\u01b2\0\u01c0\0\u01ce\0\u01dc\0\u01ea\0\u01f8\0\322\0\u0206"+
- "\0\u0214\0\u0222\0\u0230\0\u023e\0\u024c\0\u025a\0\124\0\214"+
- "\0\u0268\0\u0276\0\u0284";
+ "\0\0\0\14\0\30\0\44\0\60\0\14\0\74\0\110"+
+ "\0\124\0\140\0\154\0\170\0\204\0\220\0\234\0\250"+
+ "\0\264\0\300\0\314\0\330\0\344\0\360\0\374\0\u0108"+
+ "\0\u0114\0\u0120\0\u012c\0\u0138\0\u0144\0\u0150\0\u015c\0\u0168"+
+ "\0\u0174\0\u0180\0\u018c\0\u0198\0\u01a4\0\250\0\u01b0\0\u01bc"+
+ "\0\u01c8\0\u01d4\0\u01e0\0\u01ec\0\u01f8\0\74\0\154\0\u0204"+
+ "\0\u0210\0\u021c";
private static int [] zzUnpackRowMap() {
- int [] result = new int[51];
+ int [] result = new int[50];
int offset = 0;
offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
return result;
@@ -191,49 +189,49 @@ class ClassicTokenizerImpl implements St
private static final int [] ZZ_TRANS = zzUnpackTrans();
private static final String ZZ_TRANS_PACKED_0 =
- "\1\2\1\3\1\4\7\2\1\5\1\6\1\7\1\2"+
- "\17\0\2\3\1\0\1\10\1\0\1\11\2\12\1\13"+
- "\1\3\4\0\1\3\1\4\1\0\1\14\1\0\1\11"+
- "\2\15\1\16\1\4\4\0\1\3\1\4\1\17\1\20"+
- "\1\21\1\22\2\12\1\13\1\23\20\0\1\2\1\0"+
- "\1\24\1\25\7\0\1\26\4\0\2\27\7\0\1\27"+
- "\4\0\1\30\1\31\7\0\1\32\5\0\1\33\7\0"+
- "\1\13\4\0\1\34\1\35\7\0\1\36\4\0\1\37"+
- "\1\40\7\0\1\41\4\0\1\42\1\43\7\0\1\44"+
- "\15\0\1\45\4\0\1\24\1\25\7\0\1\46\15\0"+
- "\1\47\4\0\2\27\7\0\1\50\4\0\1\3\1\4"+
- "\1\17\1\10\1\21\1\22\2\12\1\13\1\23\4\0"+
- "\2\24\1\0\1\51\1\0\1\11\2\52\1\0\1\24"+
- "\4\0\1\24\1\25\1\0\1\53\1\0\1\11\2\54"+
- "\1\55\1\25\4\0\1\24\1\25\1\0\1\51\1\0"+
- "\1\11\2\52\1\0\1\26\4\0\2\27\1\0\1\56"+
- "\2\0\1\56\2\0\1\27\4\0\2\30\1\0\1\52"+
- "\1\0\1\11\2\52\1\0\1\30\4\0\1\30\1\31"+
- "\1\0\1\54\1\0\1\11\2\54\1\55\1\31\4\0"+
- "\1\30\1\31\1\0\1\52\1\0\1\11\2\52\1\0"+
- "\1\32\5\0\1\33\1\0\1\55\2\0\3\55\1\33"+
- "\4\0\2\34\1\0\1\57\1\0\1\11\2\12\1\13"+
- "\1\34\4\0\1\34\1\35\1\0\1\60\1\0\1\11"+
- "\2\15\1\16\1\35\4\0\1\34\1\35\1\0\1\57"+
- "\1\0\1\11\2\12\1\13\1\36\4\0\2\37\1\0"+
- "\1\12\1\0\1\11\2\12\1\13\1\37\4\0\1\37"+
- "\1\40\1\0\1\15\1\0\1\11\2\15\1\16\1\40"+
- "\4\0\1\37\1\40\1\0\1\12\1\0\1\11\2\12"+
- "\1\13\1\41\4\0\2\42\1\0\1\13\2\0\3\13"+
- "\1\42\4\0\1\42\1\43\1\0\1\16\2\0\3\16"+
- "\1\43\4\0\1\42\1\43\1\0\1\13\2\0\3\13"+
- "\1\44\6\0\1\17\6\0\1\45\4\0\1\24\1\25"+
- "\1\0\1\61\1\0\1\11\2\52\1\0\1\26\4\0"+
- "\2\27\1\0\1\56\2\0\1\56\2\0\1\50\4\0"+
- "\2\24\7\0\1\24\4\0\2\30\7\0\1\30\4\0"+
- "\2\34\7\0\1\34\4\0\2\37\7\0\1\37\4\0"+
- "\2\42\7\0\1\42\4\0\2\62\7\0\1\62\4\0"+
- "\2\24\7\0\1\63\4\0\2\62\1\0\1\56\2\0"+
- "\1\56\2\0\1\62\4\0\2\24\1\0\1\61\1\0"+
- "\1\11\2\52\1\0\1\24\3\0";
+ "\1\2\1\3\1\4\7\2\1\5\1\6\15\0\2\3"+
+ "\1\0\1\7\1\0\1\10\2\11\1\12\1\3\2\0"+
+ "\1\3\1\4\1\0\1\13\1\0\1\10\2\14\1\15"+
+ "\1\4\2\0\1\3\1\4\1\16\1\17\1\20\1\21"+
+ "\2\11\1\12\1\22\2\0\1\23\1\24\7\0\1\25"+
+ "\2\0\2\26\7\0\1\26\2\0\1\27\1\30\7\0"+
+ "\1\31\3\0\1\32\7\0\1\12\2\0\1\33\1\34"+
+ "\7\0\1\35\2\0\1\36\1\37\7\0\1\40\2\0"+
+ "\1\41\1\42\7\0\1\43\13\0\1\44\2\0\1\23"+
+ "\1\24\7\0\1\45\13\0\1\46\2\0\2\26\7\0"+
+ "\1\47\2\0\1\3\1\4\1\16\1\7\1\20\1\21"+
+ "\2\11\1\12\1\22\2\0\2\23\1\0\1\50\1\0"+
+ "\1\10\2\51\1\0\1\23\2\0\1\23\1\24\1\0"+
+ "\1\52\1\0\1\10\2\53\1\54\1\24\2\0\1\23"+
+ "\1\24\1\0\1\50\1\0\1\10\2\51\1\0\1\25"+
+ "\2\0\2\26\1\0\1\55\2\0\1\55\2\0\1\26"+
+ "\2\0\2\27\1\0\1\51\1\0\1\10\2\51\1\0"+
+ "\1\27\2\0\1\27\1\30\1\0\1\53\1\0\1\10"+
+ "\2\53\1\54\1\30\2\0\1\27\1\30\1\0\1\51"+
+ "\1\0\1\10\2\51\1\0\1\31\3\0\1\32\1\0"+
+ "\1\54\2\0\3\54\1\32\2\0\2\33\1\0\1\56"+
+ "\1\0\1\10\2\11\1\12\1\33\2\0\1\33\1\34"+
+ "\1\0\1\57\1\0\1\10\2\14\1\15\1\34\2\0"+
+ "\1\33\1\34\1\0\1\56\1\0\1\10\2\11\1\12"+
+ "\1\35\2\0\2\36\1\0\1\11\1\0\1\10\2\11"+
+ "\1\12\1\36\2\0\1\36\1\37\1\0\1\14\1\0"+
+ "\1\10\2\14\1\15\1\37\2\0\1\36\1\37\1\0"+
+ "\1\11\1\0\1\10\2\11\1\12\1\40\2\0\2\41"+
+ "\1\0\1\12\2\0\3\12\1\41\2\0\1\41\1\42"+
+ "\1\0\1\15\2\0\3\15\1\42\2\0\1\41\1\42"+
+ "\1\0\1\12\2\0\3\12\1\43\4\0\1\16\6\0"+
+ "\1\44\2\0\1\23\1\24\1\0\1\60\1\0\1\10"+
+ "\2\51\1\0\1\25\2\0\2\26\1\0\1\55\2\0"+
+ "\1\55\2\0\1\47\2\0\2\23\7\0\1\23\2\0"+
+ "\2\27\7\0\1\27\2\0\2\33\7\0\1\33\2\0"+
+ "\2\36\7\0\1\36\2\0\2\41\7\0\1\41\2\0"+
+ "\2\61\7\0\1\61\2\0\2\23\7\0\1\62\2\0"+
+ "\2\61\1\0\1\55\2\0\1\55\2\0\1\61\2\0"+
+ "\2\23\1\0\1\60\1\0\1\10\2\51\1\0\1\23"+
+ "\1\0";
private static int [] zzUnpackTrans() {
- int [] result = new int[658];
+ int [] result = new int[552];
int offset = 0;
offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
return result;
@@ -271,11 +269,11 @@ class ClassicTokenizerImpl implements St
private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
private static final String ZZ_ATTRIBUTE_PACKED_0 =
- "\1\0\1\11\3\1\1\11\1\1\13\0\4\1\2\0"+
- "\1\1\1\0\17\1\1\0\1\1\3\0\5\1";
+ "\1\0\1\11\3\1\1\11\13\0\4\1\2\0\1\1"+
+ "\1\0\17\1\1\0\1\1\3\0\5\1";
private static int [] zzUnpackAttribute() {
- int [] result = new int[51];
+ int [] result = new int[50];
int offset = 0;
offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
return result;
@@ -372,7 +370,6 @@ public final void getText(CharTermAttrib
/**
* Creates a new scanner
- * There is also a java.io.InputStream version of this constructor.
*
* @param in the java.io.Reader to read input from.
*/
@@ -380,7 +377,6 @@ public final void getText(CharTermAttrib
this.zzReader = in;
}
-
/**
* Unpacks the compressed character translation table.
@@ -392,7 +388,7 @@ public final void getText(CharTermAttrib
char [] map = new char[0x10000];
int i = 0; /* index in packed string */
int j = 0; /* index in unpacked array */
- while (i < 1154) {
+ while (i < 1138) {
int count = packed.charAt(i++);
char value = packed.charAt(i++);
do map[j++] = value; while (--count > 0);
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex?rev=1548595&r1=1548594&r2=1548595&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex Fri Dec 6 16:51:43 2013
@@ -116,8 +116,6 @@ LETTER = !(![:letter:]|{CJ})
// Chinese and Japanese (but NOT Korean, which is included in [:letter:])
CJ = [\u3100-\u312f\u3040-\u309F\u30A0-\u30FF\u31F0-\u31FF\u3300-\u337f\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uff65-\uff9f]
-WHITESPACE = \r\n | [ \r\n\t\f]
-
%%
{ALPHANUM} { return ALPHANUM; }
@@ -131,4 +129,4 @@ WHITESPACE = \r\n | [ \r\n\t\f]
{ACRONYM_DEP} { return ACRONYM_DEP; }
/** Ignore the rest */
-. | {WHITESPACE} { /* Break so we don't hit fall-through warning: */ break;/* ignore */ }
+[^] { /* Break so we don't hit fall-through warning: */ break;/* ignore */ }
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt?rev=1548595&r1=1548594&r2=1548595&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt Fri Dec 6 16:51:43 2013
@@ -18,4 +18,4 @@
WARNING: if you change StandardTokenizerImpl*.jflex or UAX29URLEmailTokenizer
and need to regenerate the tokenizer, only use the trunk version
- of JFlex 1.5 (with a minimum SVN revision 607) at the moment!
+ of JFlex 1.5 (with a minimum SVN revision 722) at the moment!
Modified: lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro?rev=1548595&r1=1548594&r2=1548595&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro (original)
+++ lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro Fri Dec 6 16:51:43 2013
@@ -1,11 +1,12 @@
/*
- * Copyright 2010 The Apache Software Foundation.
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
*
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
@@ -13,7 +14,6 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
// Generated using ICU4J 52.1.0.0
// by org.apache.lucene.analysis.icu.GenerateJFlexSupplementaryMacros
@@ -39,6 +39,12 @@ FormatSupp = (
| ([\ud834][\uDD73-\uDD7A])
| ([\udb40][\uDC01\uDC20-\uDC7F])
)
+NumericSupp = (
+ ([\ud805][\uDEC0-\uDEC9])
+ | ([\ud804][\uDC66-\uDC6F\uDCF0-\uDCF9\uDD36-\uDD3F\uDDD0-\uDDD9])
+ | ([\ud835][\uDFCE-\uDFFF])
+ | ([\ud801][\uDCA0-\uDCA9])
+)
ExtendSupp = (
([\ud81b][\uDF51-\uDF7E\uDF8F-\uDF92])
| ([\ud805][\uDEAB-\uDEB7])
@@ -48,12 +54,6 @@ ExtendSupp = (
| ([\udb40][\uDD00-\uDDEF])
| ([\ud802][\uDE01-\uDE03\uDE05\uDE06\uDE0C-\uDE0F\uDE38-\uDE3A\uDE3F])
)
-NumericSupp = (
- ([\ud805][\uDEC0-\uDEC9])
- | ([\ud804][\uDC66-\uDC6F\uDCF0-\uDCF9\uDD36-\uDD3F\uDDD0-\uDDD9])
- | ([\ud835][\uDFCE-\uDFFF])
- | ([\ud801][\uDCA0-\uDCA9])
-)
KatakanaSupp = (
([\ud82c][\uDC00])
)
@@ -129,3 +129,15 @@ HiraganaSupp = (
([\ud83c][\uDE00])
| ([\ud82c][\uDC01])
)
+SingleQuoteSupp = (
+ []
+)
+DoubleQuoteSupp = (
+ []
+)
+HebrewLetterSupp = (
+ []
+)
+RegionalIndicatorSupp = (
+ ([\ud83c][\uDDE6-\uDDFF])
+)