You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sa...@apache.org on 2012/12/03 19:32:23 UTC

svn commit: r1416629 - in /lucene/dev/branches/branch_4x: ./ dev-tools/ dev-tools/eclipse/ dev-tools/idea/lucene/analysis/icu/ lucene/ lucene/analysis/ lucene/analysis/icu/ lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ lucen...

Author: sarowe
Date: Mon Dec  3 18:32:17 2012
New Revision: 1416629

URL: http://svn.apache.org/viewvc?rev=1416629&view=rev
Log:
SOLR-4123: Add per-script customizability to ICUTokenizerFactory via rule files in the ICU RuleBasedBreakIterator format. (merged trunk r1416617)

Added:
    lucene/dev/branches/branch_4x/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/KeywordTokenizer.rbbi
      - copied unchanged from r1416617, lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/KeywordTokenizer.rbbi
    lucene/dev/branches/branch_4x/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/Latin-break-only-on-whitespace.rbbi
      - copied unchanged from r1416617, lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/Latin-break-only-on-whitespace.rbbi
    lucene/dev/branches/branch_4x/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/Latin-dont-break-on-hyphens.rbbi
      - copied unchanged from r1416617, lucene/dev/trunk/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/Latin-dont-break-on-hyphens.rbbi
Modified:
    lucene/dev/branches/branch_4x/   (props changed)
    lucene/dev/branches/branch_4x/dev-tools/   (props changed)
    lucene/dev/branches/branch_4x/dev-tools/eclipse/dot.classpath
    lucene/dev/branches/branch_4x/dev-tools/idea/lucene/analysis/icu/icu.iml
    lucene/dev/branches/branch_4x/lucene/   (props changed)
    lucene/dev/branches/branch_4x/lucene/CHANGES.txt   (contents, props changed)
    lucene/dev/branches/branch_4x/lucene/analysis/   (props changed)
    lucene/dev/branches/branch_4x/lucene/analysis/icu/build.xml
    lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java
    lucene/dev/branches/branch_4x/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java

Modified: lucene/dev/branches/branch_4x/dev-tools/eclipse/dot.classpath
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/dev-tools/eclipse/dot.classpath?rev=1416629&r1=1416628&r2=1416629&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/dev-tools/eclipse/dot.classpath (original)
+++ lucene/dev/branches/branch_4x/dev-tools/eclipse/dot.classpath Mon Dec  3 18:32:17 2012
@@ -25,6 +25,7 @@
   <classpathentry kind="src" path="lucene/analysis/icu/src/java"/>
   <classpathentry kind="src" output="eclipse-build/analysis-icu" path="lucene/analysis/icu/src/resources"/>
   <classpathentry kind="src" path="lucene/analysis/icu/src/test"/>
+  <classpathentry kind="src" path="lucene/analysis/icu/src/test-files"/>
   <classpathentry kind="src" path="lucene/analysis/kuromoji/src/java"/>
   <classpathentry kind="src" output="eclipse-build/analysis-kuromoji" path="lucene/analysis/kuromoji/src/resources"/>
   <classpathentry kind="src" path="lucene/analysis/kuromoji/src/test"/>

Modified: lucene/dev/branches/branch_4x/dev-tools/idea/lucene/analysis/icu/icu.iml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/dev-tools/idea/lucene/analysis/icu/icu.iml?rev=1416629&r1=1416628&r2=1416629&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/dev-tools/idea/lucene/analysis/icu/icu.iml (original)
+++ lucene/dev/branches/branch_4x/dev-tools/idea/lucene/analysis/icu/icu.iml Mon Dec  3 18:32:17 2012
@@ -9,6 +9,7 @@
       <sourceFolder url="file://$MODULE_DIR$/src/tools/java" isTestSource="false" />
       <sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
       <sourceFolder url="file://$MODULE_DIR$/src/resources" isTestSource="false" />
+      <sourceFolder url="file://$MODULE_DIR$/src/test-files" isTestSource="true" />
     </content>
     <orderEntry type="inheritedJdk" />
     <orderEntry type="sourceFolder" forTests="false" />

Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1416629&r1=1416628&r2=1416629&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Mon Dec  3 18:32:17 2012
@@ -100,6 +100,10 @@ New Features
   be notified whenever a new searcher was opened. (selckin via Shai
   Erera, Mike McCandless)
 
+* SOLR-4123: Add per-script customizability to ICUTokenizerFactory via
+  rule files in the ICU RuleBasedBreakIterator format.
+  (Shawn Heisey, Robert Muir, Steve Rowe)
+
 API Changes
 
 * LUCENE-4399: Deprecated AppendingCodec. Lucene's term dictionaries

Modified: lucene/dev/branches/branch_4x/lucene/analysis/icu/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/icu/build.xml?rev=1416629&r1=1416628&r2=1416629&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/icu/build.xml (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/icu/build.xml Mon Dec  3 18:32:17 2012
@@ -35,6 +35,11 @@
     <path refid="base.classpath"/>
   </path>
 
+  <path id="test.classpath">
+    <path refid="test.base.classpath" />
+    <pathelement path="src/test-files" />
+  </path>
+
   <target name="compile-core" depends="jar-analyzers-common, common.compile-core" />
 
   <property name="utr30.data.dir" location="src/data/utr30"/>

Modified: lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java?rev=1416629&r1=1416628&r2=1416629&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/icu/src/java/org/apache/lucene/analysis/icu/segmentation/ICUTokenizerFactory.java Mon Dec  3 18:32:17 2012
@@ -17,22 +17,135 @@ package org.apache.lucene.analysis.icu.s
  * limitations under the License.
  */
 
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
 import java.io.Reader;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
 
 import org.apache.lucene.analysis.Tokenizer;
-import org.apache.lucene.analysis.icu.segmentation.ICUTokenizer;
 import org.apache.lucene.analysis.util.AbstractAnalysisFactory; // javadocs
+import org.apache.lucene.analysis.util.ResourceLoader;
+import org.apache.lucene.analysis.util.ResourceLoaderAware;
 import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.IOUtils;
 
-/** Factory for {@link ICUTokenizer} */
-public class ICUTokenizerFactory extends TokenizerFactory {
+import com.ibm.icu.lang.UCharacter;
+import com.ibm.icu.lang.UProperty;
+import com.ibm.icu.lang.UScript;
+import com.ibm.icu.text.BreakIterator;
+import com.ibm.icu.text.RuleBasedBreakIterator;
+
+/**
+ * Factory for {@link ICUTokenizer}.
+ * Words are broken across script boundaries, then segmented according to
+ * the BreakIterator and typing provided by the {@link DefaultICUTokenizerConfig}.
+ *
+ * <p/>
+ *
+ * To use the default set of per-script rules:
+ *
+ * <pre class="prettyprint" >
+ * &lt;fieldType name="text_icu" class="solr.TextField" positionIncrementGap="100"&gt;
+ *   &lt;analyzer&gt;
+ *     &lt;tokenizer class="solr.ICUTokenizerFactory"/&gt;
+ *   &lt;/analyzer&gt;
+ * &lt;/fieldType&gt;</pre>
+ *
+ * <p/>
+ *
+ * You can customize this tokenizer's behavior by specifying per-script rule files,
+ * which are compiled by the ICU RuleBasedBreakIterator.  See the
+ * <a href="http://userguide.icu-project.org/boundaryanalysis#TOC-RBBI-Rules"
+ * >ICU RuleBasedBreakIterator syntax reference</a>.
+ *
+ * To add per-script rules, add a "rulefiles" argument, which should contain a
+ * comma-separated list of <tt>code:rulefile</tt> pairs in the following format:
+ * <a href="http://unicode.org/iso15924/iso15924-codes.html"
+ * >four-letter ISO 15924 script code</a>, followed by a colon, then a resource
+ * path.  E.g. to specify rules for Latin (script code "Latn") and Cyrillic
+ * (script code "Cyrl"):
+ *
+ * <pre class="prettyprint" >
+ * &lt;fieldType name="text_icu_custom" class="solr.TextField" positionIncrementGap="100"&gt;
+ *   &lt;analyzer&gt;
+ *     &lt;tokenizer class="solr.ICUTokenizerFactory"
+ *                rulefiles="Latn:my.Latin.rules.rbbi,Cyrl:my.Cyrillic.rules.rbbi"/&gt;
+ *   &lt;/analyzer&gt;
+ * &lt;/fieldType&gt;</pre>
+ *
+ */
+public class ICUTokenizerFactory extends TokenizerFactory implements ResourceLoaderAware {
+  static final String RULEFILES = "rulefiles";
+  private Map<Integer,String> tailored;
+  private ICUTokenizerConfig config;
   
   /** Sole constructor. See {@link AbstractAnalysisFactory} for initialization lifecycle. */
   public ICUTokenizerFactory() {}
 
-  // TODO: add support for custom configs
+  @Override
+  public void init(Map<String,String> args) {
+    super.init(args);
+    tailored = new HashMap<Integer,String>();
+    String rulefilesArg = args.get(RULEFILES);
+    if (rulefilesArg != null) {
+      List<String> scriptAndResourcePaths = splitFileNames(rulefilesArg);
+      for (String scriptAndResourcePath : scriptAndResourcePaths) {
+        int colonPos = scriptAndResourcePath.indexOf(":");
+        String scriptCode = scriptAndResourcePath.substring(0, colonPos).trim();
+        String resourcePath = scriptAndResourcePath.substring(colonPos+1).trim();
+        tailored.put(UCharacter.getPropertyValueEnum(UProperty.SCRIPT, scriptCode), resourcePath);
+      }
+    }
+  }
+
+  @Override
+  public void inform(ResourceLoader loader) throws IOException {
+    assert tailored != null : "init must be called first!";
+    if (tailored.isEmpty()) {
+      config = new DefaultICUTokenizerConfig();
+    } else {
+      final BreakIterator breakers[] = new BreakIterator[UScript.CODE_LIMIT];
+      for (Map.Entry<Integer,String> entry : tailored.entrySet()) {
+        int code = entry.getKey();
+        String resourcePath = entry.getValue();
+        breakers[code] = parseRules(resourcePath, loader);
+      }
+      config = new DefaultICUTokenizerConfig() {
+        
+        @Override
+        public BreakIterator getBreakIterator(int script) {
+          if (breakers[script] != null) {
+            return (BreakIterator) breakers[script].clone();
+          } else {
+            return super.getBreakIterator(script);
+          }
+        }
+        // TODO: we could also allow codes->types mapping
+      };
+    }
+  }
+  
+  private BreakIterator parseRules(String filename, ResourceLoader loader) throws IOException {
+    StringBuilder rules = new StringBuilder();
+    InputStream rulesStream = loader.openResource(filename);
+    BufferedReader reader = new BufferedReader
+        (IOUtils.getDecodingReader(rulesStream, IOUtils.CHARSET_UTF_8));
+    String line = null;
+    while ((line = reader.readLine()) != null) {
+      if ( ! line.startsWith("#"))
+        rules.append(line);
+      rules.append('\n');
+    }
+    reader.close();
+    return new RuleBasedBreakIterator(rules.toString());
+  }
+
   @Override
   public Tokenizer create(Reader input) {
-    return new ICUTokenizer(input);
+    assert config != null : "inform must be called first!";
+    return new ICUTokenizer(input, config);
   }
 }

Modified: lucene/dev/branches/branch_4x/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java?rev=1416629&r1=1416628&r2=1416629&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java (original)
+++ lucene/dev/branches/branch_4x/lucene/analysis/icu/src/test/org/apache/lucene/analysis/icu/segmentation/TestICUTokenizerFactory.java Mon Dec  3 18:32:17 2012
@@ -19,18 +19,73 @@ package org.apache.lucene.analysis.icu.s
 
 import java.io.Reader;
 import java.io.StringReader;
+import java.util.HashMap;
+import java.util.Map;
 
 import org.apache.lucene.analysis.BaseTokenStreamTestCase;
 import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.util.ClasspathResourceLoader;
 
 /** basic tests for {@link ICUTokenizerFactory} **/
 public class TestICUTokenizerFactory extends BaseTokenStreamTestCase {
   public void testMixedText() throws Exception {
     Reader reader = new StringReader("การที่ได้ต้องแสดงว่างานดี  This is a test ກວ່າດອກ");
     ICUTokenizerFactory factory = new ICUTokenizerFactory();
+    factory.init(new HashMap<String,String>());
+    factory.inform(new ClasspathResourceLoader(getClass()));
     TokenStream stream = factory.create(reader);
     assertTokenStreamContents(stream,
         new String[] { "การ", "ที่", "ได้", "ต้อง", "แสดง", "ว่า", "งาน", "ดี",
         "This", "is", "a", "test", "ກວ່າ", "ດອກ"});
   }
+
+  public void testTokenizeLatinOnWhitespaceOnly() throws Exception {
+    // “ U+201C LEFT DOUBLE QUOTATION MARK; ” U+201D RIGHT DOUBLE QUOTATION MARK
+    Reader reader = new StringReader
+        ("  Don't,break.at?/(punct)!  \u201Cnice\u201D\r\n\r\n85_At:all; `really\" +2=3$5,&813 !@#%$^)(*@#$   ");
+    ICUTokenizerFactory factory = new ICUTokenizerFactory();
+    final Map<String,String> args = new HashMap<String,String>();
+    args.put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-break-only-on-whitespace.rbbi");
+    factory.init(args);
+    factory.inform(new ClasspathResourceLoader(this.getClass()));
+    TokenStream stream = factory.create(reader);
+    assertTokenStreamContents(stream,
+        new String[] { "Don't,break.at?/(punct)!", "\u201Cnice\u201D", "85_At:all;", "`really\"",  "+2=3$5,&813", "!@#%$^)(*@#$" },
+        new String[] { "<ALPHANUM>",               "<ALPHANUM>",       "<ALPHANUM>", "<ALPHANUM>", "<NUM>",       "<OTHER>" });
+  }
+
+  public void testTokenizeLatinDontBreakOnHyphens() throws Exception {
+    Reader reader = new StringReader
+        ("One-two punch.  Brang-, not brung-it.  This one--not that one--is the right one, -ish.");
+    ICUTokenizerFactory factory = new ICUTokenizerFactory();
+    final Map<String,String> args = new HashMap<String,String>();
+    args.put(ICUTokenizerFactory.RULEFILES, "Latn:Latin-dont-break-on-hyphens.rbbi");
+    factory.init(args);
+    factory.inform(new ClasspathResourceLoader(getClass()));
+    TokenStream stream = factory.create(reader);
+    assertTokenStreamContents(stream,
+        new String[] { "One-two", "punch",
+            "Brang", "not", "brung-it",
+            "This", "one", "not", "that", "one", "is", "the", "right", "one", "ish" });
+  }
+
+  /**
+   * Specify more than one script/rule file pair.
+   * Override default DefaultICUTokenizerConfig Thai script tokenization.
+   * Use the same rule file for both scripts.
+   */
+  public void testKeywordTokenizeCyrillicAndThai() throws Exception {
+    Reader reader = new StringReader
+        ("Some English.  Немного русский.  ข้อความภาษาไทยเล็ก ๆ น้อย ๆ  More English.");
+    ICUTokenizerFactory factory = new ICUTokenizerFactory();
+    final Map<String,String> args = new HashMap<String,String>();
+    args.put(ICUTokenizerFactory.RULEFILES, "Cyrl:KeywordTokenizer.rbbi,Thai:KeywordTokenizer.rbbi");
+    factory.init(args);
+    factory.inform(new ClasspathResourceLoader(getClass()));
+    TokenStream stream = factory.create(reader);
+    assertTokenStreamContents(stream, new String[] { "Some", "English",
+        "Немного русский.  ",
+        "ข้อความภาษาไทยเล็ก ๆ น้อย ๆ  ",
+        "More", "English" });
+  }
 }