You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2010/09/19 17:40:06 UTC
svn commit: r998684 - in /lucene/dev/trunk: lucene/contrib/
modules/analysis/common/src/java/org/apache/lucene/analysis/th/
modules/analysis/common/src/test/org/apache/lucene/analysis/th/
solr/src/test/org/apache/solr/analysis/
Author: rmuir
Date: Sun Sep 19 15:40:06 2010
New Revision: 998684
URL: http://svn.apache.org/viewvc?rev=998684&view=rev
Log:
LUCENE-2653: ThaiAnalyzer assumes things about your jre
Modified:
lucene/dev/trunk/lucene/contrib/CHANGES.txt
lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java
lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java
Modified: lucene/dev/trunk/lucene/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/contrib/CHANGES.txt?rev=998684&r1=998683&r2=998684&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/contrib/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/contrib/CHANGES.txt Sun Sep 19 15:40:06 2010
@@ -122,6 +122,10 @@ Bug fixes
you to customize its normalization/folding, by editing the source data files in src/data
and regenerating a new .nrm with 'ant gennorm2'. (David Bowen via Robert Muir)
+* LUCENE-2653: ThaiWordFilter depends on the JRE having a Thai dictionary, which is not
+ always the case. If the dictionary is unavailable, the filter will now throw
+ UnsupportedOperationException in the constructor. (Robert Muir)
+
API Changes
Modified: lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java?rev=998684&r1=998683&r2=998684&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/java/org/apache/lucene/analysis/th/ThaiWordFilter.java Sun Sep 19 15:40:06 2010
@@ -38,10 +38,24 @@ import org.apache.lucene.util.Version;
* {@link ThaiAnalyzer} will insert a {@link LowerCaseFilter} before this filter
* so the behaviour of the Analyzer does not change. With version 3.1, the filter handles
* position increments correctly.
+ * <p>WARNING: this filter may not be supported by all JREs.
+ * It is known to work with Sun/Oracle and Harmony JREs.
+ * If your application needs to be fully portable, consider using ICUTokenizer instead,
+ * which uses an ICU Thai BreakIterator that will always be available.
*/
public final class ThaiWordFilter extends TokenFilter {
-
- private final BreakIterator breaker = BreakIterator.getWordInstance(new Locale("th"));
+ /**
+ * True if the JRE supports a working dictionary-based breakiterator for Thai.
+ * If this is false, this filter will not work at all!
+ */
+ public static final boolean DBBI_AVAILABLE;
+ private static final BreakIterator proto = BreakIterator.getWordInstance(new Locale("th"));
+ static {
+ // check that we have a working dictionary-based break iterator for thai
+ proto.setText("ภาษาà¹à¸à¸¢");
+ DBBI_AVAILABLE = proto.isBoundary(4);
+ }
+ private final BreakIterator breaker = (BreakIterator) proto.clone();
private final Segment charIterator = new Segment();
private final boolean handlePosIncr;
@@ -67,6 +81,8 @@ public final class ThaiWordFilter extend
public ThaiWordFilter(Version matchVersion, TokenStream input) {
super(matchVersion.onOrAfter(Version.LUCENE_31) ?
input : new LowerCaseFilter(matchVersion, input));
+ if (!DBBI_AVAILABLE)
+ throw new UnsupportedOperationException("This JRE does not have support for Thai segmentation");
handlePosIncr = matchVersion.onOrAfter(Version.LUCENE_31);
}
Modified: lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java?rev=998684&r1=998683&r2=998684&view=diff
==============================================================================
--- lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java (original)
+++ lucene/dev/trunk/modules/analysis/common/src/test/org/apache/lucene/analysis/th/TestThaiAnalyzer.java Sun Sep 19 15:40:06 2010
@@ -18,6 +18,7 @@ package org.apache.lucene.analysis.th;
*/
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.junit.Assume;
/**
* Test case for ThaiAnalyzer, modified from TestFrenchAnalyzer
@@ -31,6 +32,7 @@ public class TestThaiAnalyzer extends Ba
* testcase for offsets
*/
public void testOffsets() throws Exception {
+ Assume.assumeTrue(ThaiWordFilter.DBBI_AVAILABLE);
assertAnalyzesTo(new ThaiAnalyzer(TEST_VERSION_CURRENT), "à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ",
new String[] { "à¸à¸²à¸£", "à¸à¸µà¹", "à¹à¸à¹", "à¸à¹à¸à¸", "à¹à¸ªà¸à¸", "วà¹à¸²", "à¸à¸²à¸", "à¸à¸µ" },
new int[] { 0, 3, 6, 9, 13, 17, 20, 23 },
@@ -49,6 +51,7 @@ public class TestThaiAnalyzer extends Ba
* Instead, allow the definition of alphanum to include relevant categories like nonspacing marks!
*/
public void testBuggyTokenType() throws Exception {
+ Assume.assumeTrue(ThaiWordFilter.DBBI_AVAILABLE);
assertAnalyzesTo(new ThaiAnalyzer(TEST_VERSION_CURRENT), "à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ à¹à¹à¹",
new String[] { "à¸à¸²à¸£", "à¸à¸µà¹", "à¹à¸à¹", "à¸à¹à¸à¸", "à¹à¸ªà¸à¸", "วà¹à¸²", "à¸à¸²à¸", "à¸à¸µ", "à¹à¹à¹" },
new String[] { "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>", "<ALPHANUM>",
@@ -65,6 +68,7 @@ public class TestThaiAnalyzer extends Ba
*/
public void testAnalyzer() throws Exception {
+ Assume.assumeTrue(ThaiWordFilter.DBBI_AVAILABLE);
ThaiAnalyzer analyzer = new ThaiAnalyzer(TEST_VERSION_CURRENT);
assertAnalyzesTo(analyzer, "", new String[] {});
@@ -90,6 +94,7 @@ public class TestThaiAnalyzer extends Ba
* Test that position increments are adjusted correctly for stopwords.
*/
public void testPositionIncrements() throws Exception {
+ Assume.assumeTrue(ThaiWordFilter.DBBI_AVAILABLE);
ThaiAnalyzer analyzer = new ThaiAnalyzer(TEST_VERSION_CURRENT);
assertAnalyzesTo(new ThaiAnalyzer(TEST_VERSION_CURRENT), "à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸ the à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ",
@@ -107,6 +112,7 @@ public class TestThaiAnalyzer extends Ba
}
public void testReusableTokenStream() throws Exception {
+ Assume.assumeTrue(ThaiWordFilter.DBBI_AVAILABLE);
ThaiAnalyzer analyzer = new ThaiAnalyzer(TEST_VERSION_CURRENT);
assertAnalyzesToReuse(analyzer, "", new String[] {});
Modified: lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java?rev=998684&r1=998683&r2=998684&view=diff
==============================================================================
--- lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java (original)
+++ lucene/dev/trunk/solr/src/test/org/apache/solr/analysis/TestThaiWordFilterFactory.java Sun Sep 19 15:40:06 2010
@@ -23,6 +23,8 @@ import java.io.StringReader;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.WhitespaceTokenizer;
+import org.apache.lucene.analysis.th.ThaiWordFilter;
+import org.junit.Assume;
/**
* Simple tests to ensure the Thai word filter factory is working.
@@ -32,6 +34,7 @@ public class TestThaiWordFilterFactory e
* Ensure the filter actually decomposes text.
*/
public void testWordBreak() throws Exception {
+ Assume.assumeTrue(ThaiWordFilter.DBBI_AVAILABLE);
Reader reader = new StringReader("à¸à¸²à¸£à¸à¸µà¹à¹à¸à¹à¸à¹à¸à¸à¹à¸ªà¸à¸à¸§à¹à¸²à¸à¸²à¸à¸à¸µ");
Tokenizer tokenizer = new WhitespaceTokenizer(DEFAULT_VERSION, reader);
ThaiWordFilterFactory factory = new ThaiWordFilterFactory();