You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/04/16 17:21:04 UTC
[24/46] lucene-solr:jira/solr-11833: LUCENE-8231: Add a new analysis
module (nori) similar to Kuromoji but to handle Korean
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e851b89c/lucene/analysis/nori/src/tools/test/org/apache/lucene/analysis/ko/dict/UnknownDictionaryTest.java
----------------------------------------------------------------------
diff --git a/lucene/analysis/nori/src/tools/test/org/apache/lucene/analysis/ko/dict/UnknownDictionaryTest.java b/lucene/analysis/nori/src/tools/test/org/apache/lucene/analysis/ko/dict/UnknownDictionaryTest.java
new file mode 100644
index 0000000..2bba714
--- /dev/null
+++ b/lucene/analysis/nori/src/tools/test/org/apache/lucene/analysis/ko/dict/UnknownDictionaryTest.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.ko.dict;
+
+
+import org.apache.lucene.analysis.ko.util.CSVUtil;
+import org.apache.lucene.analysis.ko.util.UnknownDictionaryWriter;
+import org.apache.lucene.util.LuceneTestCase;
+import org.junit.Test;
+
+public class UnknownDictionaryTest extends LuceneTestCase {
+
+ @Test
+ public void testPutCharacterCategory() {
+ UnknownDictionaryWriter unkDic = new UnknownDictionaryWriter(10 * 1024 * 1024);
+
+ try{
+ unkDic.putCharacterCategory(0, "DUMMY_NAME");
+ fail();
+ } catch(Exception e) {
+
+ }
+
+ try{
+ unkDic.putCharacterCategory(-1, "HANGUL");
+ fail();
+ } catch(Exception e) {
+
+ }
+
+ unkDic.putCharacterCategory(0, "DEFAULT");
+ unkDic.putCharacterCategory(1, "GREEK");
+ unkDic.putCharacterCategory(2, "HANJA");
+ unkDic.putCharacterCategory(3, "HANGUL");
+ unkDic.putCharacterCategory(4, "KANJI");
+ }
+
+ @Test
+ public void testPut() {
+ UnknownDictionaryWriter unkDic = new UnknownDictionaryWriter(10 * 1024 * 1024);
+ try{
+ unkDic.put(CSVUtil.parse("HANGUL,1800,3562,UNKNOWN,*,*,*,*,*,*,*"));
+ fail();
+ } catch(Exception e){
+
+ }
+
+ String entry1 = "ALPHA,1793,3533,795,SL,*,*,*,*,*,*,*";
+ String entry2 = "HANGUL,1800,3562,10247,UNKNOWN,*,*,*,*,*,*,*";
+ String entry3 = "HANJA,1792,3554,-821,SH,*,*,*,*,*,*,*";
+
+ unkDic.putCharacterCategory(0, "ALPHA");
+ unkDic.putCharacterCategory(1, "HANGUL");
+ unkDic.putCharacterCategory(2, "HANJA");
+
+ unkDic.put(CSVUtil.parse(entry1));
+ unkDic.put(CSVUtil.parse(entry2));
+ unkDic.put(CSVUtil.parse(entry3));
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e851b89c/lucene/ivy-versions.properties
----------------------------------------------------------------------
diff --git a/lucene/ivy-versions.properties b/lucene/ivy-versions.properties
index f19e33d..14e7194 100644
--- a/lucene/ivy-versions.properties
+++ b/lucene/ivy-versions.properties
@@ -85,6 +85,7 @@ io.prometheus.version = 0.2.0
/junit/junit = 4.10
/mecab/mecab-ipadic = 2.7.0-20070801
+/mecab/mecab-ko-dic = 2.0.3-20170922
/mecab/mecab-naist-jdic = 0.6.3b-20111013
/net.arnx/jsonic = 1.2.7
/net.bytebuddy/byte-buddy = 1.6.2
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/e851b89c/lucene/module-build.xml
----------------------------------------------------------------------
diff --git a/lucene/module-build.xml b/lucene/module-build.xml
index 92ca60e..9f17c1c 100644
--- a/lucene/module-build.xml
+++ b/lucene/module-build.xml
@@ -451,6 +451,28 @@
<property name="analyzers-morfologik-javadocs.uptodate" value="true"/>
</target>
+ <property name="analyzers-nori.jar" value="${common.dir}/build/analysis/nori/lucene-analyzers-nori-${version}.jar"/>
+ <target name="check-analyzers-nori-uptodate" unless="analyzers-nori.uptodate">
+ <module-uptodate name="analysis/nori" jarfile="${analyzers-nori.jar}" property="analyzers-nori.uptodate"/>
+ </target>
+ <target name="jar-analyzers-nori" unless="analyzers-nori.uptodate" depends="check-analyzers-nori-uptodate">
+ <ant dir="${common.dir}/analysis/nori" target="jar-core" inheritAll="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ </ant>
+ <property name="analyzers-nori.uptodate" value="true"/>
+ </target>
+
+ <property name="analyzers-nori-javadoc.jar" value="${common.dir}/build/analysis/nori/lucene-analyzers-nori-${version}-javadoc.jar"/>
+ <target name="check-analyzers-nori-javadocs-uptodate" unless="analyzers-nori-javadocs.uptodate">
+ <module-uptodate name="analysis/nori" jarfile="${analyzers-nori-javadoc.jar}" property="analyzers-nori-javadocs.uptodate"/>
+ </target>
+ <target name="javadocs-analyzers-nori" unless="analyzers-nori-javadocs.uptodate" depends="check-analyzers-nori-javadocs-uptodate">
+ <ant dir="${common.dir}/analysis/nori" target="javadocs" inheritAll="false">
+ <propertyset refid="uptodate.and.compiled.properties"/>
+ </ant>
+ <property name="analyzers-nori-javadocs.uptodate" value="true"/>
+ </target>
+
<property name="codecs.jar" value="${common.dir}/build/codecs/lucene-codecs-${version}.jar"/>
<target name="check-codecs-uptodate" unless="codecs.uptodate">
<module-uptodate name="codecs" jarfile="${codecs.jar}" property="codecs.uptodate"/>