You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/07/28 13:28:27 UTC
svn commit: r1366643 [4/19] - in /lucene/dev/branches/lucene3312: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/copyright/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/ dev-tools/maven/
dev-tools/maven/lucene/benchmark/ dev-tools/maven...
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/HTMLStripCharFilter.jflex Sat Jul 28 11:27:51 2012
@@ -18,13 +18,13 @@ package org.apache.lucene.analysis.charf
*/
import java.io.IOException;
+import java.io.Reader;
import java.util.Arrays;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.apache.lucene.util.Version;
-import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.util.CharArrayMap;
import org.apache.lucene.analysis.util.CharArraySet;
import org.apache.lucene.analysis.util.OpenStringBuilder;
@@ -173,7 +173,7 @@ InlineElment = ( [aAbBiIqQsSuU]
/**
* @param source
*/
- public HTMLStripCharFilter(CharStream source) {
+ public HTMLStripCharFilter(Reader source) {
super(source);
this.zzReader = source;
}
@@ -183,7 +183,7 @@ InlineElment = ( [aAbBiIqQsSuU]
* @param escapedTags Tags in this set (both start and end tags)
* will not be filtered out.
*/
- public HTMLStripCharFilter(CharStream source, Set<String> escapedTags) {
+ public HTMLStripCharFilter(Reader source, Set<String> escapedTags) {
super(source);
this.zzReader = source;
if (null != escapedTags) {
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/charfilter/MappingCharFilter.java Sat Jul 28 11:27:51 2012
@@ -21,8 +21,7 @@ import java.io.IOException;
import java.io.Reader;
import java.util.Map;
-import org.apache.lucene.analysis.CharReader;
-import org.apache.lucene.analysis.CharStream;
+import org.apache.lucene.analysis.CharFilter; // javadocs
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.RollingCharBuffer;
import org.apache.lucene.util.fst.CharSequenceOutputs;
@@ -51,8 +50,8 @@ public class MappingCharFilter extends B
private int replacementPointer;
private int inputOff;
- /** Default constructor that takes a {@link CharStream}. */
- public MappingCharFilter(NormalizeCharMap normMap, CharStream in) {
+ /** Default constructor that takes a {@link Reader}. */
+ public MappingCharFilter(NormalizeCharMap normMap, Reader in) {
super(in);
buffer.reset(in);
@@ -66,15 +65,10 @@ public class MappingCharFilter extends B
}
}
- /** Easy-use constructor that takes a {@link Reader}. */
- public MappingCharFilter(NormalizeCharMap normMap, Reader in) {
- this(normMap, CharReader.get(in));
- }
-
@Override
public void reset() throws IOException {
super.reset();
- buffer.reset(input);
+ buffer.reset(in);
replacement = null;
inputOff = 0;
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/DictionaryCompoundWordTokenFilter.java Sat Jul 28 11:27:51 2012
@@ -57,6 +57,9 @@ public class DictionaryCompoundWordToken
*/
public DictionaryCompoundWordTokenFilter(Version matchVersion, TokenStream input, CharArraySet dictionary) {
super(matchVersion, input, dictionary);
+ if (dictionary == null) {
+ throw new IllegalArgumentException("dictionary cannot be null");
+ }
}
/**
@@ -83,6 +86,9 @@ public class DictionaryCompoundWordToken
public DictionaryCompoundWordTokenFilter(Version matchVersion, TokenStream input, CharArraySet dictionary,
int minWordSize, int minSubwordSize, int maxSubwordSize, boolean onlyLongestMatch) {
super(matchVersion, input, dictionary, minWordSize, minSubwordSize, maxSubwordSize, onlyLongestMatch);
+ if (dictionary == null) {
+ throw new IllegalArgumentException("dictionary cannot be null");
+ }
}
@Override
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java Sat Jul 28 11:27:51 2012
@@ -18,6 +18,7 @@ package org.apache.lucene.analysis.compo
*/
import java.io.File;
+import java.io.IOException;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
@@ -131,10 +132,10 @@ public class HyphenationCompoundWordToke
*
* @param hyphenationFilename the filename of the XML grammar to load
* @return An object representing the hyphenation patterns
- * @throws Exception
+ * @throws IOException
*/
public static HyphenationTree getHyphenationTree(String hyphenationFilename)
- throws Exception {
+ throws IOException {
return getHyphenationTree(new InputSource(hyphenationFilename));
}
@@ -143,10 +144,10 @@ public class HyphenationCompoundWordToke
*
* @param hyphenationFile the file of the XML grammar to load
* @return An object representing the hyphenation patterns
- * @throws Exception
+ * @throws IOException
*/
public static HyphenationTree getHyphenationTree(File hyphenationFile)
- throws Exception {
+ throws IOException {
return getHyphenationTree(new InputSource(hyphenationFile.toURL().toExternalForm()));
}
@@ -155,10 +156,10 @@ public class HyphenationCompoundWordToke
*
* @param hyphenationSource the InputSource pointing to the XML grammar
* @return An object representing the hyphenation patterns
- * @throws Exception
+ * @throws IOException
*/
public static HyphenationTree getHyphenationTree(InputSource hyphenationSource)
- throws Exception {
+ throws IOException {
HyphenationTree tree = new HyphenationTree();
tree.loadPatterns(hyphenationSource);
return tree;
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/HyphenationTree.java Sat Jul 28 11:27:51 2012
@@ -18,7 +18,8 @@
package org.apache.lucene.analysis.compound.hyphenation;
import java.io.File;
-import java.net.MalformedURLException;
+import java.io.IOException;
+import java.io.PrintStream;
import java.util.ArrayList;
import java.util.HashMap;
@@ -107,25 +108,20 @@ public class HyphenationTree extends Ter
* Read hyphenation patterns from an XML file.
*
* @param f the filename
- * @throws HyphenationException In case the parsing fails
+ * @throws IOException In case the parsing fails
*/
- public void loadPatterns(File f) throws HyphenationException {
- try {
- InputSource src = new InputSource(f.toURL().toExternalForm());
- loadPatterns(src);
- } catch (MalformedURLException e) {
- throw new HyphenationException("Error converting the File '" + f
- + "' to a URL: " + e.getMessage());
- }
+ public void loadPatterns(File f) throws IOException {
+ InputSource src = new InputSource(f.toURL().toExternalForm());
+ loadPatterns(src);
}
/**
* Read hyphenation patterns from an XML file.
*
* @param source the InputSource for the file
- * @throws HyphenationException In case the parsing fails
+ * @throws IOException In case the parsing fails
*/
- public void loadPatterns(InputSource source) throws HyphenationException {
+ public void loadPatterns(InputSource source) throws IOException {
PatternParser pp = new PatternParser(this);
ivalues = new TernaryTree();
@@ -463,10 +459,10 @@ public class HyphenationTree extends Ter
}
@Override
- public void printStats() {
- System.out.println("Value space size = "
+ public void printStats(PrintStream out) {
+ out.println("Value space size = "
+ Integer.toString(vspace.length()));
- super.printStats();
+ super.printStats(out);
}
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/PatternParser.java Sat Jul 28 11:27:51 2012
@@ -27,9 +27,7 @@ import org.xml.sax.Attributes;
// Java
import java.io.File;
-import java.io.FileNotFoundException;
import java.io.IOException;
-import java.net.MalformedURLException;
import java.util.ArrayList;
import javax.xml.parsers.SAXParserFactory;
@@ -40,7 +38,7 @@ import javax.xml.parsers.SAXParserFactor
*
* This class has been taken from the Apache FOP project (http://xmlgraphics.apache.org/fop/). They have been slightly modified.
*/
-public class PatternParser extends DefaultHandler implements PatternConsumer {
+public class PatternParser extends DefaultHandler {
XMLReader parser;
@@ -87,9 +85,9 @@ public class PatternParser extends Defau
* Parses a hyphenation pattern file.
*
* @param filename the filename
- * @throws HyphenationException In case of an exception while parsing
+ * @throws IOException In case of an exception while parsing
*/
- public void parse(String filename) throws HyphenationException {
+ public void parse(String filename) throws IOException {
parse(new InputSource(filename));
}
@@ -97,33 +95,24 @@ public class PatternParser extends Defau
* Parses a hyphenation pattern file.
*
* @param file the pattern file
- * @throws HyphenationException In case of an exception while parsing
+ * @throws IOException In case of an exception while parsing
*/
- public void parse(File file) throws HyphenationException {
- try {
- InputSource src = new InputSource(file.toURL().toExternalForm());
- parse(src);
- } catch (MalformedURLException e) {
- throw new HyphenationException("Error converting the File '" + file
- + "' to a URL: " + e.getMessage());
- }
+ public void parse(File file) throws IOException {
+ InputSource src = new InputSource(file.toURL().toExternalForm());
+ parse(src);
}
/**
* Parses a hyphenation pattern file.
*
* @param source the InputSource for the file
- * @throws HyphenationException In case of an exception while parsing
+ * @throws IOException In case of an exception while parsing
*/
- public void parse(InputSource source) throws HyphenationException {
+ public void parse(InputSource source) throws IOException {
try {
parser.parse(source);
- } catch (FileNotFoundException fnfe) {
- throw new HyphenationException("File not found: " + fnfe.getMessage());
- } catch (IOException ioe) {
- throw new HyphenationException(ioe.getMessage());
} catch (SAXException e) {
- throw new HyphenationException(errMsg);
+ throw new IOException(e);
}
}
@@ -402,25 +391,4 @@ public class PatternParser extends Defau
return str.toString();
} // getLocationString(SAXParseException):String
-
- // PatternConsumer implementation for testing purposes
- public void addClass(String c) {
- System.out.println("class: " + c);
- }
-
- public void addException(String w, ArrayList<Object> e) {
- System.out.println("exception: " + w + " : " + e.toString());
- }
-
- public void addPattern(String p, String v) {
- System.out.println("pattern: " + p + " : " + v);
- }
-
- public static void main(String[] args) throws Exception {
- if (args.length > 0) {
- PatternParser pp = new PatternParser();
- pp.setConsumer(pp);
- pp.parse(args[0]);
- }
- }
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/hyphenation/TernaryTree.java Sat Jul 28 11:27:51 2012
@@ -17,6 +17,7 @@
package org.apache.lucene.analysis.compound.hyphenation;
+import java.io.PrintStream;
import java.util.Enumeration;
import java.util.Stack;
@@ -633,11 +634,11 @@ public class TernaryTree implements Clon
}
- public void printStats() {
- System.out.println("Number of keys = " + Integer.toString(length));
- System.out.println("Node count = " + Integer.toString(freenode));
+ public void printStats(PrintStream out) {
+ out.println("Number of keys = " + Integer.toString(length));
+ out.println("Node count = " + Integer.toString(freenode));
// System.out.println("Array length = " + Integer.toString(eq.length));
- System.out.println("Key Array length = " + Integer.toString(kv.length()));
+ out.println("Key Array length = " + Integer.toString(kv.length()));
/*
* for(int i=0; i<kv.length(); i++) if ( kv.get(i) != 0 )
@@ -647,7 +648,7 @@ public class TernaryTree implements Clon
*/
}
-
+/*
public static void main(String[] args) {
TernaryTree tt = new TernaryTree();
tt.insert("Carlos", 'C');
@@ -658,7 +659,8 @@ public class TernaryTree implements Clon
System.out.println((char) tt.find("Car"));
System.out.println((char) tt.find("Carlos"));
System.out.println((char) tt.find("alto"));
- tt.printStats();
+ tt.printStats(System.out);
}
+ */
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/core/KeywordTokenizer.java Sat Jul 28 11:27:51 2012
@@ -94,8 +94,8 @@ public final class KeywordTokenizer exte
}
@Override
- public void reset(Reader input) throws IOException {
- super.reset(input);
+ public void setReader(Reader input) throws IOException {
+ super.setReader(input);
this.done = false;
}
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/KStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/KStemmer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/KStemmer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/KStemmer.java Sat Jul 28 11:27:51 2012
@@ -289,7 +289,7 @@ public class KStemmer {
entry = new DictEntry(exceptionWords[i], true);
d.put(exceptionWords[i], entry);
} else {
- System.out.println("Warning: Entry [" + exceptionWords[i]
+ throw new RuntimeException("Warning: Entry [" + exceptionWords[i]
+ "] already in dictionary 1");
}
}
@@ -299,7 +299,7 @@ public class KStemmer {
entry = new DictEntry(directConflations[i][1], false);
d.put(directConflations[i][0], entry);
} else {
- System.out.println("Warning: Entry [" + directConflations[i][0]
+ throw new RuntimeException("Warning: Entry [" + directConflations[i][0]
+ "] already in dictionary 2");
}
}
@@ -309,7 +309,7 @@ public class KStemmer {
entry = new DictEntry(countryNationality[i][1], false);
d.put(countryNationality[i][0], entry);
} else {
- System.out.println("Warning: Entry [" + countryNationality[i][0]
+ throw new RuntimeException("Warning: Entry [" + countryNationality[i][0]
+ "] already in dictionary 3");
}
}
@@ -323,7 +323,7 @@ public class KStemmer {
if (!d.containsKey(array[i])) {
d.put(array[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + array[i]
+ throw new RuntimeException("Warning: Entry [" + array[i]
+ "] already in dictionary 4");
}
}
@@ -333,7 +333,7 @@ public class KStemmer {
if (!d.containsKey(array[i])) {
d.put(array[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + array[i]
+ throw new RuntimeException("Warning: Entry [" + array[i]
+ "] already in dictionary 4");
}
}
@@ -343,7 +343,7 @@ public class KStemmer {
if (!d.containsKey(array[i])) {
d.put(array[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + array[i]
+ throw new RuntimeException("Warning: Entry [" + array[i]
+ "] already in dictionary 4");
}
}
@@ -353,7 +353,7 @@ public class KStemmer {
if (!d.containsKey(array[i])) {
d.put(array[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + array[i]
+ throw new RuntimeException("Warning: Entry [" + array[i]
+ "] already in dictionary 4");
}
}
@@ -363,7 +363,7 @@ public class KStemmer {
if (!d.containsKey(array[i])) {
d.put(array[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + array[i]
+ throw new RuntimeException("Warning: Entry [" + array[i]
+ "] already in dictionary 4");
}
}
@@ -373,7 +373,7 @@ public class KStemmer {
if (!d.containsKey(array[i])) {
d.put(array[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + array[i]
+ throw new RuntimeException("Warning: Entry [" + array[i]
+ "] already in dictionary 4");
}
}
@@ -383,7 +383,7 @@ public class KStemmer {
if (!d.containsKey(array[i])) {
d.put(array[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + array[i]
+ throw new RuntimeException("Warning: Entry [" + array[i]
+ "] already in dictionary 4");
}
}
@@ -392,7 +392,7 @@ public class KStemmer {
if (!d.containsKey(KStemData8.data[i])) {
d.put(KStemData8.data[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + KStemData8.data[i]
+ throw new RuntimeException("Warning: Entry [" + KStemData8.data[i]
+ "] already in dictionary 4");
}
}
@@ -401,7 +401,7 @@ public class KStemmer {
if (!d.containsKey(supplementDict[i])) {
d.put(supplementDict[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + supplementDict[i]
+ throw new RuntimeException("Warning: Entry [" + supplementDict[i]
+ "] already in dictionary 5");
}
}
@@ -410,7 +410,7 @@ public class KStemmer {
if (!d.containsKey(properNouns[i])) {
d.put(properNouns[i], defaultEntry);
} else {
- System.out.println("Warning: Entry [" + properNouns[i]
+ throw new RuntimeException("Warning: Entry [" + properNouns[i]
+ "] already in dictionary 6");
}
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/PorterStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/PorterStemmer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/PorterStemmer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/en/PorterStemmer.java Sat Jul 28 11:27:51 2012
@@ -492,10 +492,9 @@ class PorterStemmer
return dirty;
}
- /** Test program for demonstrating the Stemmer. It reads a file and
+ /* Test program for demonstrating the Stemmer. It reads a file and
* stems each word, writing the result to standard out.
* Usage: Stemmer file-name
- */
public static void main(String[] args) {
PorterStemmer s = new PorterStemmer();
@@ -542,6 +541,6 @@ class PorterStemmer
System.out.println("error reading " + args[i]);
}
}
- }
+ }*/
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianAnalyzer.java Sat Jul 28 11:27:51 2012
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.io.Reader;
import org.apache.lucene.analysis.Analyzer;
-import org.apache.lucene.analysis.CharReader;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.ar.ArabicNormalizationFilter;
@@ -134,6 +133,6 @@ public final class PersianAnalyzer exten
*/
@Override
protected Reader initReader(String fieldName, Reader reader) {
- return new PersianCharFilter(CharReader.get(reader));
+ return new PersianCharFilter(reader);
}
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/fa/PersianCharFilter.java Sat Jul 28 11:27:51 2012
@@ -18,9 +18,9 @@ package org.apache.lucene.analysis.fa;
*/
import java.io.IOException;
+import java.io.Reader;
-import org.apache.lucene.analysis.CharStream;
-import org.apache.lucene.analysis.charfilter.CharFilter;
+import org.apache.lucene.analysis.CharFilter;
/**
* CharFilter that replaces instances of Zero-width non-joiner with an
@@ -28,7 +28,7 @@ import org.apache.lucene.analysis.charfi
*/
public class PersianCharFilter extends CharFilter {
- public PersianCharFilter(CharStream in) {
+ public PersianCharFilter(Reader in) {
super(in);
}
@@ -45,4 +45,9 @@ public class PersianCharFilter extends C
}
return charsRead;
}
+
+ @Override
+ protected int correct(int currentOff) {
+ return currentOff; // we don't change the length of the string
+ }
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellDictionary.java Sat Jul 28 11:27:51 2012
@@ -66,10 +66,11 @@ public class HunspellDictionary {
/**
* Creates a new HunspellDictionary containing the information read from the provided InputStreams to hunspell affix
- * and dictionary files
+ * and dictionary files.
+ * You have to close the provided InputStreams yourself.
*
- * @param affix InputStream for reading the hunspell affix file
- * @param dictionary InputStream for reading the hunspell dictionary file
+ * @param affix InputStream for reading the hunspell affix file (won't be closed).
+ * @param dictionary InputStream for reading the hunspell dictionary file (won't be closed).
* @param version Lucene Version
* @throws IOException Can be thrown while reading from the InputStreams
* @throws ParseException Can be thrown if the content of the files does not meet expected formats
@@ -80,10 +81,11 @@ public class HunspellDictionary {
/**
* Creates a new HunspellDictionary containing the information read from the provided InputStreams to hunspell affix
- * and dictionary files
+ * and dictionary files.
+ * You have to close the provided InputStreams yourself.
*
- * @param affix InputStream for reading the hunspell affix file
- * @param dictionary InputStream for reading the hunspell dictionary file
+ * @param affix InputStream for reading the hunspell affix file (won't be closed).
+ * @param dictionary InputStream for reading the hunspell dictionary file (won't be closed).
* @param version Lucene Version
* @param ignoreCase If true, dictionary matching will be case insensitive
* @throws IOException Can be thrown while reading from the InputStreams
@@ -95,10 +97,11 @@ public class HunspellDictionary {
/**
* Creates a new HunspellDictionary containing the information read from the provided InputStreams to hunspell affix
- * and dictionary files
+ * and dictionary files.
+ * You have to close the provided InputStreams yourself.
*
- * @param affix InputStream for reading the hunspell affix file
- * @param dictionaries InputStreams for reading the hunspell dictionary file
+ * @param affix InputStream for reading the hunspell affix file (won't be closed).
+ * @param dictionaries InputStreams for reading the hunspell dictionary file (won't be closed).
* @param version Lucene Version
* @param ignoreCase If true, dictionary matching will be case insensitive
* @throws IOException Can be thrown while reading from the InputStreams
@@ -110,10 +113,11 @@ public class HunspellDictionary {
/**
* Creates a new HunspellDictionary containing the information read from the provided InputStreams to hunspell affix
- * and dictionary files
+ * and dictionary files.
+ * You have to close the provided InputStreams yourself.
*
- * @param affix InputStream for reading the hunspell affix file
- * @param dictionaries InputStreams for reading the hunspell dictionary file
+ * @param affix InputStream for reading the hunspell affix file (won't be closed).
+ * @param dictionaries InputStreams for reading the hunspell dictionary file (won't be closed).
* @param version Lucene Version
* @param ignoreCase If true, dictionary matching will be case insensitive
* @param strictAffixParsing Affix strict parsing enabled or not (an error while reading a rule causes exception or is ignored)
@@ -194,7 +198,6 @@ public class HunspellDictionary {
flagParsingStrategy = getFlagParsingStrategy(line);
}
}
- reader.close();
}
/**
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/HunspellStemmer.java Sat Jul 28 11:27:51 2012
@@ -299,13 +299,12 @@ public class HunspellStemmer {
// ================================================= Entry Point ===================================================
- /**
+ /*
* HunspellStemmer entry point. Accepts two arguments: location of affix file and location of dic file
*
* @param args Program arguments. Should contain location of affix file and location of dic file
* @throws IOException Can be thrown while reading from the files
* @throws ParseException Can be thrown while parsing the files
- */
public static void main(String[] args) throws IOException, ParseException {
boolean ignoreCase = false;
int offset = 0;
@@ -347,12 +346,10 @@ public class HunspellStemmer {
}
}
- /**
* Prints the results of the stemming of a word
*
* @param originalWord Word that has been stemmed
* @param stems Stems of the word
- */
private static void printStemResults(String originalWord, List<Stem> stems) {
StringBuilder builder = new StringBuilder().append("stem(").append(originalWord).append(")").append("\n");
@@ -382,13 +379,12 @@ public class HunspellStemmer {
System.out.println(builder);
}
- /**
* Simple utility to check if the given String has any text
*
* @param str String to check if it has any text
* @return {@code true} if the String has text, {@code false} otherwise
- */
private static boolean hasText(String str) {
return str != null && str.length() > 0;
}
+ */
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceCharFilter.java Sat Jul 28 11:27:51 2012
@@ -23,7 +23,6 @@ import java.io.StringReader;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.charfilter.BaseCharFilter;
/**
@@ -54,7 +53,7 @@ public class PatternReplaceCharFilter ex
private final String replacement;
private Reader transformedInput;
- public PatternReplaceCharFilter(Pattern pattern, String replacement, CharStream in) {
+ public PatternReplaceCharFilter(Pattern pattern, String replacement, Reader in) {
super(in);
this.pattern = pattern;
this.replacement = replacement;
@@ -64,16 +63,29 @@ public class PatternReplaceCharFilter ex
public int read(char[] cbuf, int off, int len) throws IOException {
// Buffer all input on the first call.
if (transformedInput == null) {
- StringBuilder buffered = new StringBuilder();
- char [] temp = new char [1024];
- for (int cnt = input.read(temp); cnt > 0; cnt = input.read(temp)) {
- buffered.append(temp, 0, cnt);
- }
- transformedInput = new StringReader(processPattern(buffered).toString());
+ fill();
}
return transformedInput.read(cbuf, off, len);
}
+
+ private void fill() throws IOException {
+ StringBuilder buffered = new StringBuilder();
+ char [] temp = new char [1024];
+ for (int cnt = in.read(temp); cnt > 0; cnt = in.read(temp)) {
+ buffered.append(temp, 0, cnt);
+ }
+ transformedInput = new StringReader(processPattern(buffered).toString());
+ }
+
+ @Override
+ public int read() throws IOException {
+ if (transformedInput == null) {
+ fill();
+ }
+
+ return transformedInput.read();
+ }
@Override
protected int correct(int currentOff) {
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternTokenizer.java Sat Jul 28 11:27:51 2012
@@ -136,8 +136,8 @@ public final class PatternTokenizer exte
}
@Override
- public void reset(Reader input) throws IOException {
- super.reset(input);
+ public void setReader(Reader input) throws IOException {
+ super.setReader(input);
fillBuffer(str, input);
matcher.reset(str);
index = 0;
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/payloads/NumericPayloadTokenFilter.java Sat Jul 28 11:27:51 2012
@@ -40,6 +40,9 @@ public class NumericPayloadTokenFilter e
public NumericPayloadTokenFilter(TokenStream input, float payload, String typeMatch) {
super(input);
+ if (typeMatch == null) {
+ throw new IllegalArgumentException("typeMatch cannot be null");
+ }
//Need to encode the payload
thePayload = new BytesRef(PayloadHelper.encodeFloat(payload));
this.typeMatch = typeMatch;
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/pt/RSLPStemmerBase.java Sat Jul 28 11:27:51 2012
@@ -132,7 +132,7 @@ public abstract class RSLPStemmerBase {
super(suffix, min, replacement);
for (int i = 0; i < exceptions.length; i++) {
if (!exceptions[i].endsWith(suffix))
- System.err.println("warning: useless exception '" + exceptions[i] + "' does not end with '" + suffix + "'");
+ throw new RuntimeException("useless exception '" + exceptions[i] + "' does not end with '" + suffix + "'");
}
this.exceptions = new CharArraySet(Version.LUCENE_50,
Arrays.asList(exceptions), false);
@@ -156,7 +156,7 @@ public abstract class RSLPStemmerBase {
super(suffix, min, replacement);
for (int i = 0; i < exceptions.length; i++) {
if (!exceptions[i].endsWith(suffix))
- System.err.println("warning: useless exception '" + exceptions[i] + "' does not end with '" + suffix + "'");
+ throw new RuntimeException("warning: useless exception '" + exceptions[i] + "' does not end with '" + suffix + "'");
}
this.exceptions = new char[exceptions.length][];
for (int i = 0; i < exceptions.length; i++)
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ASCIITLD.jflex-macro Sat Jul 28 11:27:51 2012
@@ -15,8 +15,8 @@
*/
// Generated from IANA Root Zone Database <http://www.internic.net/zones/root.zone>
-// file version from Sunday, March 18, 2012 4:34:02 AM UTC
-// generated on Sunday, March 18, 2012 4:02:55 PM UTC
+// file version from Saturday, July 14, 2012 4:34:14 AM UTC
+// generated on Sunday, July 15, 2012 12:59:44 AM UTC
// by org.apache.lucene.analysis.standard.GenerateJflexTLDMacros
ASCIITLD = "." (
@@ -310,6 +310,7 @@ ASCIITLD = "." (
| [xX][nN]--[kK][pP][rR][wW]13[dD]
| [xX][nN]--[kK][pP][rR][yY]57[dD]
| [xX][nN]--[lL][gG][bB][bB][aA][tT]1[aA][dD]8[jJ]
+ | [xX][nN]--[mM][gG][bB]9[aA][wW][bB][fF]
| [xX][nN]--[mM][gG][bB][aA][aA][mM]7[aA]8[hH]
| [xX][nN]--[mM][gG][bB][aA][yY][hH]7[gG][pP][aA]
| [xX][nN]--[mM][gG][bB][bB][hH]1[aA]71[eE]
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicAnalyzer.java Sat Jul 28 11:27:51 2012
@@ -114,9 +114,9 @@ public final class ClassicAnalyzer exten
tok = new StopFilter(matchVersion, tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
- protected void reset(final Reader reader) throws IOException {
+ protected void setReader(final Reader reader) throws IOException {
src.setMaxTokenLength(ClassicAnalyzer.this.maxTokenLength);
- super.reset(reader);
+ super.setReader(reader);
}
};
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizer.java Sat Jul 28 11:27:51 2012
@@ -175,8 +175,8 @@ public final class ClassicTokenizer exte
}
@Override
- public void reset(Reader reader) throws IOException {
- super.reset(reader);
+ public void setReader(Reader reader) throws IOException {
+ super.setReader(reader);
scanner.yyreset(reader);
}
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.java Sat Jul 28 11:27:51 2012
@@ -1,8 +1,8 @@
-/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 08.07.12 16:59 */
+/* The following code was generated by JFlex 1.5.0-SNAPSHOT on 7/15/12 1:57 AM */
package org.apache.lucene.analysis.standard;
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
@@ -33,8 +33,8 @@ import org.apache.lucene.analysis.tokena
/**
* This class is a scanner generated by
* <a href="http://www.jflex.de/">JFlex</a> 1.5.0-SNAPSHOT
- * on 08.07.12 16:59 from the specification file
- * <tt>C:/Users/Uwe Schindler/Projects/lucene/lucene4199/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex</tt>
+ * on 7/15/12 1:57 AM from the specification file
+ * <tt>C:/cygwin/home/s/svn/lucene/dev/trunk/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex</tt>
*/
class ClassicTokenizerImpl implements StandardTokenizerInterface {
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/ClassicTokenizerImpl.jflex Sat Jul 28 11:27:51 2012
@@ -1,6 +1,6 @@
package org.apache.lucene.analysis.standard;
-/**
+/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/READ_BEFORE_REGENERATING.txt Sat Jul 28 11:27:51 2012
@@ -18,4 +18,4 @@
WARNING: if you change StandardTokenizerImpl*.jflex or UAX29URLEmailTokenizer
and need to regenerate the tokenizer, only use the trunk version
- of JFlex 1.5 (with a minimum SVN revision 597) at the moment!
+ of JFlex 1.5 (with a minimum SVN revision 607) at the moment!
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/SUPPLEMENTARY.jflex-macro Sat Jul 28 11:27:51 2012
@@ -14,22 +14,25 @@
* limitations under the License.
*/
-// Generated using ICU4J 4.8.1.1 on Sunday, July 8, 2012 2:59:49 PM UTC
+// Generated using ICU4J 49.1.0.0 on Thursday, July 26, 2012 10:22:01 PM UTC
// by org.apache.lucene.analysis.icu.GenerateJFlexSupplementaryMacros
ALetterSupp = (
- ([\ud80d][\uDC00-\uDC2E])
+ ([\ud83b][\uDE00-\uDE03\uDE05-\uDE1F\uDE21\uDE22\uDE24\uDE27\uDE29-\uDE32\uDE34-\uDE37\uDE39\uDE3B\uDE42\uDE47\uDE49\uDE4B\uDE4D-\uDE4F\uDE51\uDE52\uDE54\uDE57\uDE59\uDE5B\uDE5D\uDE5F\uDE61\uDE62\uDE64\uDE67-\uDE6A\uDE6C-\uDE72\uDE74-\uDE77\uDE79-\uDE7C\uDE7E\uDE80-\uDE89\uDE8B-\uDE9B\uDEA1-\uDEA3\uDEA5-\uDEA9\uDEAB-\uDEBB])
+ | ([\ud81a][\uDC00-\uDE38])
+ | ([\ud81b][\uDF00-\uDF44\uDF50\uDF93-\uDF9F])
+ | ([\ud835][\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDEC0\uDEC2-\uDEDA\uDEDC-\uDEFA\uDEFC-\uDF14\uDF16-\uDF34\uDF36-\uDF4E\uDF50-\uDF6E\uDF70-\uDF88\uDF8A-\uDFA8\uDFAA-\uDFC2\uDFC4-\uDFCB])
+ | ([\ud80d][\uDC00-\uDC2E])
| ([\ud80c][\uDC00-\uDFFF])
| ([\ud809][\uDC00-\uDC62])
| ([\ud808][\uDC00-\uDF6E])
- | ([\ud81a][\uDC00-\uDE38])
- | ([\ud804][\uDC03-\uDC37\uDC83-\uDCAF])
- | ([\ud835][\uDC00-\uDC54\uDC56-\uDC9C\uDC9E\uDC9F\uDCA2\uDCA5\uDCA6\uDCA9-\uDCAC\uDCAE-\uDCB9\uDCBB\uDCBD-\uDCC3\uDCC5-\uDD05\uDD07-\uDD0A\uDD0D-\uDD14\uDD16-\uDD1C\uDD1E-\uDD39\uDD3B-\uDD3E\uDD40-\uDD44\uDD46\uDD4A-\uDD50\uDD52-\uDEA5\uDEA8-\uDEC0\uDEC2-\uDEDA\uDEDC-\uDEFA\uDEFC-\uDF14\uDF16-\uDF34\uDF36-\uDF4E\uDF50-\uDF6E\uDF70-\uDF88\uDF8A-\uDFA8\uDFAA-\uDFC2\uDFC4-\uDFCB])
+ | ([\ud805][\uDE80-\uDEAA])
+ | ([\ud804][\uDC03-\uDC37\uDC83-\uDCAF\uDCD0-\uDCE8\uDD03-\uDD26\uDD83-\uDDB2\uDDC1-\uDDC4])
| ([\ud801][\uDC00-\uDC9D])
| ([\ud800][\uDC00-\uDC0B\uDC0D-\uDC26\uDC28-\uDC3A\uDC3C\uDC3D\uDC3F-\uDC4D\uDC50-\uDC5D\uDC80-\uDCFA\uDD40-\uDD74\uDE80-\uDE9C\uDEA0-\uDED0\uDF00-\uDF1E\uDF30-\uDF4A\uDF80-\uDF9D\uDFA0-\uDFC3\uDFC8-\uDFCF\uDFD1-\uDFD5])
| ([\ud803][\uDC00-\uDC48])
- | ([\ud802][\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F-\uDC55\uDD00-\uDD15\uDD20-\uDD39\uDE00\uDE10-\uDE13\uDE15-\uDE17\uDE19-\uDE33\uDE60-\uDE7C\uDF00-\uDF35\uDF40-\uDF55\uDF60-\uDF72])
+ | ([\ud802][\uDC00-\uDC05\uDC08\uDC0A-\uDC35\uDC37\uDC38\uDC3C\uDC3F-\uDC55\uDD00-\uDD15\uDD20-\uDD39\uDD80-\uDDB7\uDDBE\uDDBF\uDE00\uDE10-\uDE13\uDE15-\uDE17\uDE19-\uDE33\uDE60-\uDE7C\uDF00-\uDF35\uDF40-\uDF55\uDF60-\uDF72])
)
FormatSupp = (
([\ud804][\uDCBD])
@@ -37,14 +40,17 @@ FormatSupp = (
| ([\udb40][\uDC01\uDC20-\uDC7F])
)
ExtendSupp = (
- ([\ud804][\uDC00-\uDC02\uDC38-\uDC46\uDC80-\uDC82\uDCB0-\uDCBA])
+ ([\ud81b][\uDF51-\uDF7E\uDF8F-\uDF92])
+ | ([\ud805][\uDEAB-\uDEB7])
+ | ([\ud804][\uDC00-\uDC02\uDC38-\uDC46\uDC80-\uDC82\uDCB0-\uDCBA\uDD00-\uDD02\uDD27-\uDD34\uDD80-\uDD82\uDDB3-\uDDC0])
| ([\ud834][\uDD65-\uDD69\uDD6D-\uDD72\uDD7B-\uDD82\uDD85-\uDD8B\uDDAA-\uDDAD\uDE42-\uDE44])
| ([\ud800][\uDDFD])
| ([\udb40][\uDD00-\uDDEF])
| ([\ud802][\uDE01-\uDE03\uDE05\uDE06\uDE0C-\uDE0F\uDE38-\uDE3A\uDE3F])
)
NumericSupp = (
- ([\ud804][\uDC66-\uDC6F])
+ ([\ud805][\uDEC0-\uDEC9])
+ | ([\ud804][\uDC66-\uDC6F\uDCF0-\uDCF9\uDD36-\uDD3F\uDDD0-\uDDD9])
| ([\ud835][\uDFCE-\uDFFF])
| ([\ud801][\uDCA0-\uDCA9])
)
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardAnalyzer.java Sat Jul 28 11:27:51 2012
@@ -115,9 +115,9 @@ public final class StandardAnalyzer exte
tok = new StopFilter(matchVersion, tok, stopwords);
return new TokenStreamComponents(src, tok) {
@Override
- protected void reset(final Reader reader) throws IOException {
+ protected void setReader(final Reader reader) throws IOException {
src.setMaxTokenLength(StandardAnalyzer.this.maxTokenLength);
- super.reset(reader);
+ super.setReader(reader);
}
};
}
Modified: lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java?rev=1366643&r1=1366642&r2=1366643&view=diff
==============================================================================
--- lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java (original)
+++ lucene/dev/branches/lucene3312/lucene/analysis/common/src/java/org/apache/lucene/analysis/standard/StandardTokenizer.java Sat Jul 28 11:27:51 2012
@@ -183,8 +183,8 @@ public final class StandardTokenizer ext
}
@Override
- public void reset(Reader reader) throws IOException {
- super.reset(reader);
+ public void setReader(Reader reader) throws IOException {
+ super.setReader(reader);
scanner.yyreset(reader);
}
}