You are viewing a plain text version of this content. The canonical link for it is here.
Posted to java-commits@lucene.apache.org by rm...@apache.org on 2009/08/05 20:22:22 UTC
svn commit: r801348 - in /lucene/java/trunk/contrib: ./ analyzers/common/src/java/org/apache/lucene/analysis/ar/ analyzers/common/src/resources/org/apache/lucene/analysis/ar/ analyzers/common/src/test/org/apache/lucene/analysis/ar/

Author: rmuir
Date: Wed Aug  5 18:22:22 2009
New Revision: 801348

URL: http://svn.apache.org/viewvc?rev=801348&view=rev
Log:
LUCENE-1758: Update ArabicAnalyzer to light10 stemming, stopwords improvements, lowercase non-arabic text

Modified:
    lucene/java/trunk/contrib/CHANGES.txt
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java
    lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicStemmer.java
    lucene/java/trunk/contrib/analyzers/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt
    lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java
    lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java

Modified: lucene/java/trunk/contrib/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/CHANGES.txt?rev=801348&r1=801347&r2=801348&view=diff
==============================================================================
--- lucene/java/trunk/contrib/CHANGES.txt (original)
+++ lucene/java/trunk/contrib/CHANGES.txt Wed Aug  5 18:22:22 2009
@@ -8,6 +8,12 @@
     number conversion.  You'll need to fully re-index any previously created indexes.
     This isn't a break in back-compatibility because local Lucene has not yet
     been released.  (Mike McCandless)
+ 
+ 2. LUCENE-1758: ArabicAnalyzer now uses the light10 algorithm, has a refined
+    default stopword list, and lowercases non-Arabic text.  
+    You'll need to fully re-index any previously created indexes. This isn't a 
+    break in back-compatibility because ArabicAnalyzer has not yet been 
+    released.  (Robert Muir)
 
 
 API Changes

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java?rev=801348&r1=801347&r2=801348&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicAnalyzer.java Wed Aug  5 18:22:22 2009
@@ -27,6 +27,7 @@
 import java.util.Set;
 
 import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.LowerCaseFilter;
 import org.apache.lucene.analysis.StopFilter;
 import org.apache.lucene.analysis.TokenStream;
 import org.apache.lucene.analysis.WordlistLoader;
@@ -36,10 +37,9 @@
  * <p>
  * This analyzer implements light-stemming as specified by:
  * <i>
- * Improving Stemming for Arabic Information Retrieval: 
- *      Light Stemming and Co-occurrence Analysis
+ * Light Stemming for Arabic Information Retrieval
  * </i>    
- * http://ciir.cs.umass.edu/pubfiles/ir-249.pdf
+ * http://www.mtholyoke.edu/~lballest/Pubs/arab_stem05.pdf
  * <p>
  * The analysis package contains three primary components:
  * <ul>
@@ -109,12 +109,13 @@
   /**
    * Creates a TokenStream which tokenizes all the text in the provided Reader.
    *
-   * @return  A TokenStream build from a StandardTokenizer filtered with
-   * 			StandardFilter, StopFilter, ArabicNormalizationFilter and ArabicStemFilter.
+   * @return  A TokenStream build from an ArabicTokenizer filtered with
+   * 			StopFilter, LowerCaseFilter, ArabicNormalizationFilter and ArabicStemFilter.
    */
   public final TokenStream tokenStream(String fieldName, Reader reader) {
     TokenStream result = new ArabicLetterTokenizer( reader );
     result = new StopFilter( result, stoptable );
+    result = new LowerCaseFilter(result);
     result = new ArabicNormalizationFilter( result );
     result = new ArabicStemFilter( result );
 

Modified: lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicStemmer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicStemmer.java?rev=801348&r1=801347&r2=801348&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicStemmer.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/java/org/apache/lucene/analysis/ar/ArabicStemmer.java Wed Aug  5 18:22:22 2009
@@ -47,6 +47,7 @@
       ("" + BEH + ALEF + LAM).toCharArray(),
       ("" + KAF + ALEF + LAM).toCharArray(),
       ("" + FEH + ALEF + LAM).toCharArray(),
+      ("" + LAM + LAM).toCharArray(),
       ("" + WAW).toCharArray(),
   };
   

Modified: lucene/java/trunk/contrib/analyzers/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt?rev=801348&r1=801347&r2=801348&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/resources/org/apache/lucene/analysis/ar/stopwords.txt Wed Aug  5 18:22:22 2009
@@ -5,75 +5,39 @@
 Ø§
 Ø£
 Ø
-Ø¹Ø´Ø±
 Ø¹Ø¨Ø¯
-Ø¹Ø¯Ø¯
-Ø¹Ø¯Ø©
-Ø¹Ø´Ø±Ø©
 Ø¹Ø¯Ù
 Ø¹Ø§Ù
 Ø¹Ø§ÙØ§
-Ø¹Ø±ÙØ§Øª
 Ø¹Ù
 Ø¹ÙØ¯
-Ø¹ÙØ§Ù
 Ø¹ÙØ¯ÙØ§
 Ø¹ÙÙ
-Ø¹ÙÙ
 Ø¹ÙÙÙ
 Ø¹ÙÙÙØ§
-Ø¹ÙÙÙØ©
-Ø²ÙØ§Ø±Ø©
-Ø³Ø¨ØªÙØ¨Ø±
-Ø³Ø§Ø±Ø§ÙÙÙÙ
-Ø³ÙØ©
-Ø³ÙØ±ÙØ§
-Ø³ÙÙØ§Øª
-ØªØ´Ø±ÙÙ
 ØªÙ
-ØªÙÙØ²
 Ø¶Ø¯
 Ø¨Ø¹Ø¯
 Ø¨Ø¹Ø¶
 Ø§Ø¹Ø§Ø¯Ø©
 Ø§Ø¹ÙÙ
 Ø§Ø¹ÙÙØª
-ØØ²Ø¨
-ØØ²ÙØ±Ø§Ù
 Ø¨Ø³Ø¨Ø¨
-Ø§Ø³Ø±Ø§Ø¦ÙÙ
-ØØ³ÙÙ
 ØØªÙ
 Ø§ØªÙØ§Ù
-ØµØ±Ø¨
 Ø§Ø°Ø§
 Ø§ØØ¯
 Ø§Ø«Ø±
-ØºØ²Ø©
-Ø¨Ø±Ø³
-Ø¨Ø§Ø³Ù
 Ø§Ø¬ØªÙØ§Ø¹
-ØºØ¯Ø§
-Ø´Ø®ØµØ§
-ØµØ¨Ø§Ø
 Ø§Ø·Ø§Ø±
 Ø§Ø±Ø¨Ø¹Ø©
-Ø¨ØºØ¯Ø§Ø¯
 Ø§Ø®Ø±Ù
-Ø¨Ø§Ø±ÙØ³
-Ø±Ø§Ø¨ÙÙ
-Ø´Ø±Ù
 Ø¨Ø§Ù
 Ø§Ø¨Ù
 Ø§Ø¬Ù
 ØºÙØ±
-ØØ±ÙØ©
-Ø±Ø¦ÙØ³
-Ø¬Ø¯ÙØ¯Ø©
 Ø§Ø·ÙØ§Ù
 Ø¨Ø´ÙÙ
-Ø¨Ø·ÙÙØ©
-ØµØÙÙØ©
 ØØ§ÙÙØ§
 Ø¨Ù
 Ø¨Ù
@@ -84,166 +48,40 @@
 Ø§Ù
 Ø¨ÙØ§
 Ø¬ÙØ©
-ØµÙØ±
 ØÙØ«
 Ø§ÙØ¯
 Ø§ÙØ§
 Ø§ÙØ§
-Ø§ÙØ¹Ø³ÙØ±ÙØ©
-Ø§ÙØ¹Ø±Ø§Ù
-Ø§ÙØ¹Ø§ØµÙØ©
-Ø§ÙØ¹Ø±Ø¨ÙØ©
-Ø§ÙØ¹Ø±Ø§ÙÙ
-Ø§ÙØ¹Ø±Ø§ÙÙØ©
 Ø§ÙØ¹Ø§Ù
-Ø§ÙØ¹Ø§ÙÙ
-Ø§ÙØ¹ÙØ§ÙØ§Øª
-Ø§ÙØ¹ÙÙ
-Ø§ÙØ³
-Ø§ÙØ³Ø¹ÙØ¯ÙØ©
-Ø§ÙØ³Ø§Ø¹Ø©
-Ø§ÙØ³Ø¨Øª
 Ø§ÙØ³Ø§Ø¨Ù
-Ø±ÙØ³ÙØ§
-Ø§ÙØ³ÙØ·Ø©
-Ø§ÙØ³ÙØ·Ø§Øª
 Ø§ÙØ³ÙØ§Ù
 Ø§ÙØªØ¹Ø§ÙÙ
-Ø§ÙØªØØ±ÙØ±
 Ø§ÙØªÙ
 Ø§ÙØªÙ
-Ø§ÙØªÙØ¨Ø±
-Ø¯ÙØ±Ø©
 Ø§ÙØ«Ø±
-Ø§ÙØ§Ø±
 Ø§ÙØ¶Ø§
-Ø§ÙØ¬Ø²Ø§Ø¦Ø±
-ØÙØ§Ø³
-Ø§ÙØ§Ø³Ø±Ø§Ø¦ÙÙÙ
-Ø§ÙØ§Ø³Ø±Ø§Ø¦ÙÙÙØ©
-Ø§ÙØ§Ø³Ø¨ÙØ¹
-Ø§ÙØ§Ø³ÙØØ©
-Ø§ÙØ§Ø³ÙØ§ÙÙØ©
-Ø°ÙØ±Øª
-Ø§ÙØ§ØªØØ§Ø¯
-Ø§ÙØ§ØªÙØ§Ù
-Ø«ÙØ§Ø«Ø©
-Ø§ÙØØ±Ø¨
-Ø§ÙØ§ØØ¯
 Ø§ÙØ°Ø§ØªÙ
-Ø§ÙØ´Ø±Ø·Ø©
-Ø§ÙØ§Ø±Ø¨Ø¹Ø§Ø¡
-Ø§ÙØºØ±Ø¨ÙØ©
-Ø§ÙØ®Ø§Ø±Ø¬ÙØ©
-Ø§ÙØ§Ø±Ø¯Ù
-Ø§ÙØ´Ø±Ù
-Ø§ÙØ±Ø§Ù
-Ø§ÙØØ¯ÙØ¯
-Ø§ÙØ±Ø¦ÙØ³
 Ø§ÙØ§Ø®ÙØ±Ø©
 Ø§ÙØ«Ø§ÙÙ
 Ø§ÙØ«Ø§ÙÙØ©
-Ø§ÙØ§Ø«ÙÙÙ
-Ø´ÙØ§Ù
-Ø¨ÙØ§Ù
-Ø¯ÙØ´Ù
 Ø§ÙØ°Ù
 Ø§ÙØ°Ù
 Ø§ÙØ§Ù
-Ø§ÙØ§Ù
-Ø§ÙØ§Ù
 Ø®ÙØ§Ù
-Ø§ÙØ´ÙØ®
-Ø§ÙØ¬ÙØ´
-Ø§ÙØ¯ÙØ±
-Ø§ÙØ¶ÙØ©
-Ø§ÙØ¬ÙØ¹Ø©
-Ø¨ÙØ±ÙØ²
-Ø§ÙØ§ÙØ³Ø·
-Ø§ÙØ±ÙØ³Ù
-Ø§ÙØ¨ÙØ³ÙØ©
-Ø§ÙØ±ÙØ³ÙØ©
-Ø¨ÙØ±ÙØª
-Ø§ÙØ§ÙØªØ®Ø§Ø¨Ø§Øª
-Ø§ÙØ¨ÙØ§Ø¯
-Ø§ÙØ¯ÙØ§Ø¹
-Ø§ÙØ«ÙØ«Ø§Ø¡
-Ø§ÙØ§ÙØ¨Ø§Ø¡
-Ø§ÙØ«ÙØ§Ø«Ø§Ø¡
-Ø§ÙØ§ÙØ±ÙØ¨Ù
 ØÙØ§ÙÙ
 Ø§ÙØ°ÙÙ
-Ø§ÙØ¯ÙÙ
 Ø§ÙØÙÙ
-Ø§ÙØ§ÙÙ
-Ø§ÙØ§ÙÙ
 Ø§ÙØ§ÙÙ
-Ø§ÙØ¯ÙÙØ©
-Ø§ÙØ®ÙÙØ¬
-Ø§ÙØ®ÙÙØ³
-Ø§ÙØ§ÙÙØ±ÙÙ
-Ø§ÙØ§ÙÙØ±ÙÙØ©
-Ø§ÙØ¯ÙÙÙ
 Ø§ÙØ§ÙÙÙ
-Ø§ÙØ¯ÙÙÙØ©
-Ø§ÙØÙÙÙØ©
 Ø¨ÙÙ
 Ø°ÙÙ
-Ø¯ÙÙ
-Ø¯ÙÙ
-ØÙÙ
-ØÙÙ
-Ø§ÙÙ
 Ø§ÙÙ
 Ø§ÙÙ
-Ø§ÙÙ
 Ø¶ÙÙ
-Ø¬ÙÙØ¨
-Ø¯ÙÙØ©
 Ø§ÙÙØ§
 Ø¬ÙÙØ¹
-Ø§ÙÙØ²Ø±Ø§Ø¡
-Ø§ÙÙØªØØ¯Ø«
-Ø§ÙÙØªØØ¯Ø©
-Ø¯ÙÙØ§Ø±
-Ø§ÙÙØ§Ø±
-Ø§ÙÙØ¶Ø¹
-Ø§ÙÙØ¯Ø³
-Ø§ÙÙØØªÙØ©
-Ø§ÙÙØµØ¯Ø±
-Ø§ÙÙØ¨Ø§Ø±Ø§Ø©
-Ø§ÙÙØµØ±Ù
 Ø§ÙÙØ§Ø¶Ù
-Ø§ÙÙØµØ±ÙØ©
-Ø§ÙÙØ±ØÙØ©
-Ø§ÙÙØ¯Ù
-Ø§ÙÙØ¬ÙØ©
-Ø§ÙÙØ¬ÙØ³
-Ø§ÙÙØ±ÙØ³Ù
-Ø§ÙÙØ±ÙØ³ÙØ©
-Ø§ÙÙØ§ÙØ±Ø©
-Ø§ÙÙØ¯ÙÙØ©
-Ø§ÙÙØ§ÙÙØ§
-Ø§ÙÙØ·ÙÙØ©
-Ø§ÙÙØ¬ÙÙØ¹Ø©
-Ø§ÙÙÙ
-Ø§ÙÙÙØ³Ø·ÙÙÙ
-Ø§ÙÙÙØ³Ø·ÙÙÙØ©
-Ø§ÙÙÙØ³Ø·ÙÙÙÙÙ
-Ø§ÙÙÙØª
-Ø§ÙÙÙØ±Ø±
-Ø§ÙÙÙØ§Øª
-Ø§ÙÙÙØ§Ø¦Ù
 Ø§ÙÙÙØ¨Ù
-Ø§ÙÙÙØ·ÙØ©
-Ø§ÙÙÙØ§ÙØ§Øª
-Ø§ÙÙÙØ§ÙØ¶Ø§Øª
-Ø§ÙÙÙÙ
-Ø§ÙÙÙÙ
-Ø§ÙÙÙÙ
-Ø§ÙÙÙÙ
-Ø§ÙÙÙÙØª
-Ù
 Ù
 Ù
 Ù6
@@ -251,48 +89,19 @@
 ÙØ§
 ÙØ§
 ÙØ¹
-ÙØ²Ø§Ø±Ø©
-ÙØ²ÙØ±
-ÙØ³Ø§Ø¡
-ÙØªÙ
-ÙØ±Ø©
-ÙØµØ±
 ÙØ°Ø§
-ÙØ§Ø²
-ÙØ£Ø³
-ÙØ§Ø³Ø±
-ÙØ±Ø§Ø±
-ÙØµØ¯Ø±
-ÙØ§ØØ¯
-ÙØ·Ø§Ø¹
-ÙØµØ§Ø¯Ø±
-ÙØ¨Ø§Ø±Ø§Ø©
-ÙØ¨Ø§Ø±Ù
 ÙØ§Ø¶Ø§Ù
 ÙØ§Ø¶Ø§ÙØª
-ÙØ±Ø§ÙØ³
-ÙØ§Ø´ÙØ·Ù
 ÙØ§Ù
 ÙØ¨Ù
-ÙØ§Ù
 ÙØ§Ù
 ÙØ¯Ù
 ÙØÙ
 ÙØ°Ù
 ÙØ§Ù
-ÙØÙØ¯
-ÙØ§ÙØ¯
 ÙØ°ÙØ±
-ÙØ¬ÙØ³
-ÙØ±ÙØ³Ø§
-ÙØ±ÙØ³ØªÙÙØ±
 ÙØ§ÙØª
 ÙØ§ÙØ¶Ø
-ÙØ¨ÙØ§Ù
-ÙØ§ÙÙ
-ÙØ¯ÙÙØ©
-ÙØ¬ÙÙØ¹Ø©
-ÙØ§ÙÙÙ
 ÙÙ
 ÙÙ
 ÙÙ
@@ -302,49 +111,51 @@
 ÙÙ
 ÙÙ
 ÙÙ
-ÙÙØ©
 ÙÙØ§
 ÙÙØ§
 ÙÙØ°
-ÙÙØ¯
 ÙÙØ§
-ÙÙØ³Ù
-ÙÙØ³ÙÙ
-ÙÙØªÙ
-ÙÙØ§Ø¡
-ÙÙØ±Ø©
-ÙÙØ·Ø©
-ÙÙØ§Øª
 ÙÙØ§Ø¨Ù
-ÙÙØ¯Ù
 ÙÙØ§Ù
-ÙÙØ§Ù
 ÙÙØ§Ù
-ÙÙØ·ÙØ©
-ÙÙØ¸ÙØ©
-ÙÙØ§ÙØ©
-ÙÙØ§ÙØ©
-ÙÙØ§ÙØª
 ÙÙØ§ÙØª
-ÙÙØ§ÙÙ
 ÙÙÙ
-ÙÙÙ
 ÙÙÙ
 ÙÙÙ
-ÙÙÙ
 ÙÙÙ
 ÙÙÙ
 ÙÙÙ
 ÙÙÙ
-ÙÙÙ
 ÙÙÙØ§
 ÙÙÙØ§
-ÙÙÙØ§Ø±
-ÙÙÙØ§ÙØ©
 ÙÙÙÙ
-ÙÙÙÙ
-ÙÙÙÙØªÙÙ
-ÙÙÙÙÙ
-ÙÙÙÙÙ
-ÙÙÙÙÙ
-ÙÙÙÙÙØ±Ù
+Ø£Ø®Ø±Ù
+Ø¥Ø°Ø§
+Ø£Ø±Ø¨Ø¹Ø©
+Ø¥Ø·Ø§Ø±
+Ø¥Ø¹Ø§Ø¯Ø©
+Ø£Ø¹ÙÙ
+Ø£Ø¹ÙÙØª
+Ø£Ù
+Ø£ÙØ«Ø±
+Ø£ÙØ¯
+Ø¥ÙØ§
+Ø§ÙØ£Ø®ÙØ±Ø©
+Ø§ÙØ¢Ù
+Ø§ÙØ£ÙÙ
+Ø§ÙØ£ÙÙÙ
+Ø¥ÙÙ
+Ø£ÙØ§
+Ø£Ù
+Ø¥Ù
+Ø¥ÙÙ
+Ø£ÙÙ
+Ø£ÙÙØ§
+Ø¥ÙÙØ§
+Ø£Ù
+Ø§Ù
+Ø£Ù
+Ø£Ù
+Ø£ÙØ¶Ø§
+Ø¨Ø£Ù
+ÙØ¥Ù

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java?rev=801348&r1=801347&r2=801348&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicAnalyzer.java Wed Aug  5 18:22:22 2009
@@ -17,6 +17,12 @@
  * limitations under the License.
  */
 
+import java.io.StringReader;
+
+import org.apache.lucene.analysis.Analyzer;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.TermAttribute;
+
 import junit.framework.TestCase;
 
 /**
@@ -31,6 +37,48 @@
     new ArabicAnalyzer();
   }
   
-  /* TODO: more tests */
+  /**
+   * Some simple tests showing some features of the analyzer, how some regular forms will conflate
+   */
+  public void testBasicFeatures() throws Exception {
+    ArabicAnalyzer a = new ArabicAnalyzer();
+    assertAnalyzesTo(a, "ÙØ¨ÙØ±", new String[] { "ÙØ¨ÙØ±" });
+    assertAnalyzesTo(a, "ÙØ¨ÙØ±Ø©", new String[] { "ÙØ¨ÙØ±" }); // feminine marker
+    
+    assertAnalyzesTo(a, "ÙØ´Ø±ÙØ¨", new String[] { "ÙØ´Ø±ÙØ¨" });
+    assertAnalyzesTo(a, "ÙØ´Ø±ÙØ¨Ø§Øª", new String[] { "ÙØ´Ø±ÙØ¨" }); // plural -at
+    
+    assertAnalyzesTo(a, "Ø£ÙØ±ÙÙÙÙÙ", new String[] { "Ø§ÙØ±ÙÙ" }); // plural -in
+    assertAnalyzesTo(a, "Ø§ÙØ±ÙÙÙ", new String[] { "Ø§ÙØ±ÙÙ" }); // singular with bare alif
+    
+    assertAnalyzesTo(a, "ÙØªØ§Ø¨", new String[] { "ÙØªØ§Ø¨" }); 
+    assertAnalyzesTo(a, "Ø§ÙÙØªØ§Ø¨", new String[] { "ÙØªØ§Ø¨" }); // definite article
+    
+    assertAnalyzesTo(a, "ÙØ§ ÙÙÙØª Ø£ÙÙØ§ÙÙÙ", new String[] { "ÙÙÙØª", "Ø§ÙÙØ§ÙÙÙ"});
+    assertAnalyzesTo(a, "Ø§ÙØ°ÙÙ ÙÙÙØª Ø£ÙÙØ§ÙÙÙ", new String[] { "ÙÙÙØª", "Ø§ÙÙØ§ÙÙÙ" }); // stopwords
+  }
+
+  /**
+   * Non-arabic text gets treated in a similar way as SimpleAnalyzer.
+   */
+  public void testEnglishInput() throws Exception {
+    assertAnalyzesTo(new ArabicAnalyzer(), "English text.", new String[] {
+        "english", "text" });
+  }
+  
+  private void assertAnalyzesTo(Analyzer a, String input, String[] output)
+      throws Exception {
+    TokenStream ts = a.tokenStream("dummy", new StringReader(input));
+    TermAttribute termAtt = (TermAttribute) ts
+        .getAttribute(TermAttribute.class);
+
+    for (int i = 0; i < output.length; i++) {
+      assertTrue(ts.incrementToken());
+      assertEquals(output[i], termAtt.term());
+    }
+
+    assertFalse(ts.incrementToken());
+    ts.close();
+  }
 
 }

Modified: lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java
URL: http://svn.apache.org/viewvc/lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java?rev=801348&r1=801347&r2=801348&view=diff
==============================================================================
--- lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java (original)
+++ lucene/java/trunk/contrib/analyzers/common/src/test/org/apache/lucene/analysis/ar/TestArabicStemFilter.java Wed Aug  5 18:22:22 2009
@@ -50,6 +50,10 @@
     check("ÙØ§ÙØØ³Ù", "ØØ³Ù");
   }    
 
+  public void testLlPrefix() throws IOException {
+    check("ÙÙØ§Ø®Ø±", "Ø§Ø®Ø±"); 
+  }
+  
   public void testWaPrefix() throws IOException {
     check("ÙØØ³Ù", "ØØ³Ù");
   }