You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cm...@apache.org on 2013/05/05 05:40:06 UTC
svn commit: r1479234 [4/15] - in /lucene/dev/branches/lucene4956:
dev-tools/idea/.idea/ dev-tools/idea/lucene/analysis/arirang/
lucene/analysis/ lucene/analysis/arirang/ lucene/analysis/arirang/src/
lucene/analysis/arirang/src/java/ lucene/analysis/ari...
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/FileUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/FileUtil.java?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/FileUtil.java (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/FileUtil.java Sun May 5 03:39:51 2013
@@ -0,0 +1,316 @@
+package org.apache.lucene.analysis.kr.utils;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.io.UnsupportedEncodingException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.lucene.analysis.kr.morph.MorphException;
+
+/**
+ * file utility class
+ * @author S.M.Lee
+ *
+ */
+public class FileUtil {
+
+
+ /**
+ * Given a file name for a file that is located somewhere in the application
+ * classpath, return a File object representing the file.
+ *
+ * @param filename The name of the file (relative to the classpath) that is
+ * to be retrieved.
+ * @return A file object representing the requested filename
+ * @throws Exception Thrown if the classloader can not be found or if
+ * the file can not be found in the classpath.
+ */
+ public static File getClassLoaderFile(String filename) throws MorphException {
+ // note that this method is used when initializing logging, so it must
+ // not attempt to log anything.
+ File file = null;
+ ClassLoader loader = FileUtil.class.getClassLoader();
+ URL url = loader.getResource(filename);
+ if (url == null) {
+ url = ClassLoader.getSystemResource(filename);
+ if (url == null) {
+ throw new MorphException("Unable to find " + filename);
+ }
+ file = toFile(url);
+ } else {
+ file = toFile(url);
+ }
+ if (file==null||!file.exists()) {
+ return null;
+ }
+ return file;
+ }
+
+ /**
+ * Reads the contents of a file line by line to a List of Strings.
+ * The file is always closed.
+ *
+ * @param file the file to read, must not be <code>null</code>
+ * @param encoding the encoding to use, <code>null</code> means platform default
+ * @return the list of Strings representing each line in the file, never <code>null</code>
+ * @throws IOException in case of an I/O error
+ * @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM
+ * @since Commons IO 1.1
+ */
+ public static List readLines(File file, String encoding) throws IOException {
+ InputStream in = null;
+ try {
+ in = openInputStream(file);
+ return readLines(in, encoding);
+ } finally {
+ closeQuietly(in);
+ }
+ }
+
+ /**
+ * Reads the contents of a file line by line to a List of Strings.
+ * The file is always closed.
+ *
+ * @param file the file to read, must not be <code>null</code>
+ * @param encoding the encoding to use, <code>null</code> means platform default
+ * @return the list of Strings representing each line in the file, never <code>null</code>
+ * @throws MorphException
+ * @throws IOException
+ * @throws Exception
+ * @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM
+ * @since Commons IO 1.1
+ */
+ public static List readLines(String fName, String encoding) throws MorphException, IOException {
+ InputStream in = null;
+ try {
+
+ File file = getClassLoaderFile(fName);
+ if(file!=null&&file.exists()) {
+ in = openInputStream(file);
+ } else {
+ in = new ByteArrayInputStream(readByteFromCurrentJar(fName));
+ }
+
+ return readLines(in, encoding);
+ } finally {
+ closeQuietly(in);
+ }
+ }
+
+ //-----------------------------------------------------------------------
+ /**
+ * Opens a {@link FileInputStream} for the specified file, providing better
+ * error messages than simply calling <code>new FileInputStream(file)</code>.
+ * <p>
+ * At the end of the method either the stream will be successfully opened,
+ * or an exception will have been thrown.
+ * <p>
+ * An exception is thrown if the file does not exist.
+ * An exception is thrown if the file object exists but is a directory.
+ * An exception is thrown if the file exists but cannot be read.
+ *
+ * @param file the file to open for input, must not be <code>null</code>
+ * @return a new {@link FileInputStream} for the specified file
+ * @throws FileNotFoundException if the file does not exist
+ * @throws IOException if the file object is a directory
+ * @throws IOException if the file cannot be read
+ * @since Commons IO 1.3
+ */
+ public static FileInputStream openInputStream(File file) throws IOException {
+ if (file.exists()) {
+ if (file.isDirectory()) {
+ throw new IOException("File '" + file + "' exists but is a directory");
+ }
+ if (file.canRead() == false) {
+ throw new IOException("File '" + file + "' cannot be read");
+ }
+ } else {
+ throw new FileNotFoundException("File '" + file + "' does not exist");
+ }
+ return new FileInputStream(file);
+ }
+
+ // readLines
+ //-----------------------------------------------------------------------
+ /**
+ * Get the contents of an <code>InputStream</code> as a list of Strings,
+ * one entry per line, using the default character encoding of the platform.
+ * <p>
+ * This method buffers the input internally, so there is no need to use a
+ * <code>BufferedInputStream</code>.
+ *
+ * @param input the <code>InputStream</code> to read from, not null
+ * @return the list of Strings, never null
+ * @throws NullPointerException if the input is null
+ * @throws IOException if an I/O error occurs
+ * @since Commons IO 1.1
+ */
+ public static List readLines(InputStream input) throws IOException {
+ InputStreamReader reader = new InputStreamReader(input);
+ return readLines(reader);
+ }
+
+ /**
+ * Get the contents of an <code>InputStream</code> as a list of Strings,
+ * one entry per line, using the specified character encoding.
+ * <p>
+ * Character encoding names can be found at
+ * <a href="http://www.iana.org/assignments/character-sets">IANA</a>.
+ * <p>
+ * This method buffers the input internally, so there is no need to use a
+ * <code>BufferedInputStream</code>.
+ *
+ * @param input the <code>InputStream</code> to read from, not null
+ * @param encoding the encoding to use, null means platform default
+ * @return the list of Strings, never null
+ * @throws NullPointerException if the input is null
+ * @throws IOException if an I/O error occurs
+ * @since Commons IO 1.1
+ */
+ public static List readLines(InputStream input, String encoding) throws IOException {
+ if (encoding == null) {
+ return readLines(input);
+ } else {
+ InputStreamReader reader = new InputStreamReader(input, encoding);
+ return readLines(reader);
+ }
+ }
+
+ /**
+ * Get the contents of a <code>Reader</code> as a list of Strings,
+ * one entry per line.
+ * <p>
+ * This method buffers the input internally, so there is no need to use a
+ * <code>BufferedReader</code>.
+ *
+ * @param input the <code>Reader</code> to read from, not null
+ * @return the list of Strings, never null
+ * @throws NullPointerException if the input is null
+ * @throws IOException if an I/O error occurs
+ * @since Commons IO 1.1
+ */
+ public static List readLines(Reader input) throws IOException {
+ BufferedReader reader = new BufferedReader(input);
+ List list = new ArrayList();
+ String line = reader.readLine();
+ while (line != null) {
+ list.add(line);
+ line = reader.readLine();
+ }
+ return list;
+ }
+
+ /**
+ * Unconditionally close an <code>InputStream</code>.
+ * <p>
+ * Equivalent to {@link InputStream#close()}, except any exceptions will be ignored.
+ * This is typically used in finally blocks.
+ *
+ * @param input the InputStream to close, may be null or already closed
+ */
+ public static void closeQuietly(InputStream input) {
+ try {
+ if (input != null) {
+ input.close();
+ }
+ } catch (IOException ioe) {
+ // ignore
+ }
+ }
+
+
+
+ //-----------------------------------------------------------------------
+ /**
+ * Convert from a <code>URL</code> to a <code>File</code>.
+ * <p>
+ * From version 1.1 this method will decode the URL.
+ * Syntax such as <code>file:///my%20docs/file.txt</code> will be
+ * correctly decoded to <code>/my docs/file.txt</code>.
+ *
+ * @param url the file URL to convert, <code>null</code> returns <code>null</code>
+ * @return the equivalent <code>File</code> object, or <code>null</code>
+ * if the URL's protocol is not <code>file</code>
+ * @throws IllegalArgumentException if the file is incorrectly encoded
+ */
+ public static File toFile(URL url) {
+ if (url == null || !url.getProtocol().equals("file")) {
+ return null;
+ } else {
+ String filename = url.getFile().replace('/', File.separatorChar);
+ int pos =0;
+ while ((pos = filename.indexOf('%', pos)) >= 0) {
+ if (pos + 2 < filename.length()) {
+ String hexStr = filename.substring(pos + 1, pos + 3);
+ char ch = (char) Integer.parseInt(hexStr, 16);
+ filename = filename.substring(0, pos) + ch + filename.substring(pos + 3);
+ }
+ }
+ return new File(filename);
+ }
+ }
+
+
+ //-----------------------------------------------------------------------
+ /**
+ * Reads the contents of a file into a String.
+ * The file is always closed.
+ *
+ * @param file the file to read, must not be <code>null</code>
+ * @param encoding the encoding to use, <code>null</code> means platform default
+ * @return the file contents, never <code>null</code>
+ * @throws IOException in case of an I/O error
+ * @throws java.io.UnsupportedEncodingException if the encoding is not supported by the VM
+ */
+ public static String readFileToString(File file, String encoding) throws IOException {
+ InputStream in = null;
+ try {
+ in = openInputStream(file);
+ return StringUtil.toString(in, encoding);
+ } finally {
+ closeQuietly(in);
+ }
+ }
+
+
+ public static byte[] readByteFromCurrentJar(String resource) throws MorphException {
+
+ String jarPath = FileUtil.class.getProtectionDomain().getCodeSource().getLocation().getPath();
+
+ JarResources jar = new JarResources(jarPath);
+ try {
+ return jar.getResource(resource);
+ } catch (Exception e) {
+ throw new MorphException(e.getMessage(),e);
+ }
+ }
+
+}
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/HanjaUtils.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/HanjaUtils.java?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/HanjaUtils.java (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/HanjaUtils.java Sun May 5 03:39:51 2013
@@ -0,0 +1,74 @@
+package org.apache.lucene.analysis.kr.utils;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.lucene.analysis.kr.morph.MorphException;
+
+public class HanjaUtils {
+
+ private static Map<String, char[]> mapHanja;
+
+ public synchronized static void loadDictionary() throws MorphException {
+ try {
+ List<String> strList = FileUtil.readLines("org/apache/lucene/analysis/kr/dic/mapHanja.dic","UTF-8");
+ mapHanja = new HashMap();
+
+ for(int i=0;i<strList.size();i++) {
+
+ if(strList.get(i).length()<1||
+ strList.get(i).indexOf(",")==-1) continue;
+
+ String[] hanInfos = StringUtil.split(strList.get(i),",");
+
+ if(hanInfos.length!=2) continue;
+
+ String hanja = StringEscapeUtil.unescapeJava(hanInfos[0]);
+
+ mapHanja.put(hanja, hanInfos[1].toCharArray());
+ }
+ } catch (IOException e) {
+ throw new MorphException(e);
+ }
+ }
+
+ /**
+ * íìì ëìíë íê¸ì ì°¾ìì ë°ííë¤.
+ * íëì íìë ì¬ë¬ ìì¼ë¡ ì½ì¼ ì ìì¼ë¯ë¡ ê°ë¥í 모ë ìì íê¸ë¡ ë°ííë¤.
+ * @param hanja
+ * @return
+ * @throws MorphException
+ */
+ public static char[] convertToHangul(char hanja) throws MorphException {
+
+ if(mapHanja==null) loadDictionary();
+
+// if(hanja>0x9FFF||hanja<0x3400) return new char[]{hanja};
+
+ char[] result = mapHanja.get(new String(new char[]{hanja}));
+ if(result==null) return new char[]{hanja};
+
+ return result;
+ }
+}
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/IrregularUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/IrregularUtil.java?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/IrregularUtil.java (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/IrregularUtil.java Sun May 5 03:39:51 2013
@@ -0,0 +1,372 @@
+package org.apache.lucene.analysis.kr.utils;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.kr.morph.MorphException;
+import org.apache.lucene.analysis.kr.morph.WordEntry;
+
+/**
+ *
+ * ëì¬ì ë¶ê·ì¹ ë³íì ì²ë¦¬íë Utility Class
+ */
+public class IrregularUtil {
+
+ // ã
ë¶ê·ì¹
+ public static final char IRR_TYPE_BIUP = 'B';
+
+ // ã
ë¶ê·ì¹
+ public static final char IRR_TYPE_HIOOT = 'H';
+
+ // ã¹ ë¶ê·ì¹
+ public static final char IRR_TYPE_LIUL = 'U';
+
+ // 르 ë¶ê·ì¹
+ public static final char IRR_TYPE_LOO = 'L';
+
+ // ã
ë¶ê·ì¹
+ public static final char IRR_TYPE_SIUT = 'S';
+
+ // ã· ë¶ê·ì¹
+ public static final char IRR_TYPE_DI = 'D';
+
+ // ë¬ ë¶ê·ì¹
+ public static final char IRR_TYPE_RU = 'R';
+
+ // ì¼ íë½
+ public static final char IRR_TYPE_UI = 'X';
+
+ // ê·ì¹í
+ public static final char IRR_TYPE_REGULAR = 'X';
+
+ public static String[] restoreIrregularVerb(String start, String end) throws MorphException {
+
+ if(end==null) end="";
+ char[] jasos = new char[0];
+
+ if(end.length()>0) jasos = MorphUtil.decompose(end.charAt(0));
+
+ if(end.startsWith("ã´")) {
+ String[] irrs = restoreBIrregular(start,end);
+ if(irrs!=null) return irrs;
+ irrs = restoreHIrregular(start,end);
+ if(irrs!=null) return irrs;
+ irrs = restoreELIrregular(start,end);
+ if(irrs!=null) return irrs;
+ }else if(end.startsWith("ã¹")) {
+ String[] irrs = restoreBIrregular(start,end);
+ if(irrs!=null) return irrs;
+ irrs = restoreHIrregular(start,end);
+ if(irrs!=null) return irrs;
+ irrs = restoreELIrregular(start,end);
+ if(irrs!=null) return irrs;
+ }else if(end.startsWith("ã
")) {
+ String[] irrs = restoreBIrregular(start,end);
+ if(irrs!=null) return irrs;
+ irrs = restoreHIrregular(start,end);
+ if(irrs!=null) return irrs;
+ }else if(end.startsWith("ã
")) {
+ String[] irrs = restoreBIrregular(start,end);
+ if(irrs!=null) return irrs;
+ irrs = restoreHIrregular(start,end);
+ if(irrs!=null) return irrs;
+ irrs = restoreELIrregular(start,end);
+ if(irrs!=null) return irrs;
+ }else if(start.endsWith("ì°")||start.endsWith("ì¤")) {
+ String[] irrs = restoreBIrregular(start,end);
+ if(irrs!=null) return irrs;
+ }else if(end.startsWith("ì¤")) {
+ String[] irrs = restoreBIrregular(start,end);
+ if(irrs!=null) return irrs;
+ }else if(end.startsWith("ì")) {
+ String[] irrs = restoreBIrregular(start,end);
+ if(irrs!=null) return irrs;
+ irrs = restoreELIrregular(start,end);
+ if(irrs!=null) return irrs;
+ }else if(end.startsWith("ì¼")) {
+ String[] irrs = restoreBIrregular(start,end);
+ if(irrs!=null) return irrs;
+ }else if(jasos.length>1&&jasos[0]=='ã
'&&(jasos[1]=='ã
'||jasos[1]=='ã
')) {
+ String[] irrs = restoreDIrregular(start,end);
+ if(irrs!=null) return irrs;
+ irrs = restoreSIrregular(start,end);
+ if(irrs!=null) return irrs;
+ irrs = restoreLIrregular(start,end);
+ if(irrs!=null) return irrs;
+ irrs = restoreHIrregular(start,end);
+ if(irrs!=null) return irrs;
+ irrs = restoreUIrregular(start,end);
+ if(irrs!=null) return irrs;
+ irrs = restoreRUIrregular(start,end);
+ if(irrs!=null) return irrs;
+ }else if(jasos.length>1&&jasos[0]=='ã
'&&jasos[1]=='ã
¡') {
+ String[] irrs = restoreDIrregular(start,end);
+ if(irrs!=null) return irrs;
+ irrs = restoreSIrregular(start,end);
+ if(irrs!=null) return irrs;
+ }else if(("ê°".equals(start)&&"ê±°ë¼".equals(end))||
+ ("ì¤".equals(start)&&"ëë¼".equals(end))) {
+ return new String[]{start,end};
+ }
+
+ return null;
+
+ }
+
+ /**
+ * ã
ë¶ê·ì¹ ìíì ë³µìíë¤. (ëë¤, ê³±ë¤)
+ * @param start
+ * @param end
+ * @return
+ * @throws MorphException
+ */
+ private static String[] restoreBIrregular(String start, String end) throws MorphException {
+
+ if(start==null||"".equals(start)||end==null) return null;
+
+ if(start.length()<2) return null;
+
+ if(!(start.endsWith("ì¤")||start.endsWith("ì°"))) return null;
+
+ char convEnd = MorphUtil.makeChar(end.charAt(0), 0);
+ if("ã
".equals(end)||"ã´".equals(end)||"ã¹".equals(end)||
+ convEnd=='ì'||convEnd=='ì´') { // ëì°(ë), ê³ ì¤(ê³±), ì¤ë¬ì°(ì¤ë½) ë±ì¼ë¡ ë³íëë¯ë¡ ë°ëì 2ì ì´ìì
+
+ char ch = start.charAt(start.length()-2);
+ ch = MorphUtil.makeChar(ch, 17);
+
+ if(start.length()>2)
+ start = Utilities.arrayToString(new String[]{start.substring(0,start.length()-2),Character.toString(ch)});
+ else
+ start = Character.toString(ch);
+
+ WordEntry entry = DictionaryUtil.getVerb(start);
+ if(entry!=null&&entry.getFeature(WordEntry.IDX_REGURA)==IRR_TYPE_BIUP)
+ return new String[]{start,end};
+ }
+
+
+ return null;
+ }
+
+ /**
+ * ã· ë¶ê·ì¹ ìíì ë³µìíë¤. (깨ë«ë¤, 묻ë¤)
+ * @param start
+ * @param end
+ * @return
+ * @throws MorphException
+ */
+ private static String[] restoreDIrregular(String start, String end) throws MorphException {
+ if(start==null||"".equals(start)) return null;
+
+ char ch = start.charAt(start.length()-1);
+ char[] jasos = MorphUtil.decompose(ch);
+ if(jasos.length!=3||jasos[2]!='ã¹') return null;
+
+ ch = MorphUtil.makeChar(ch, 7);
+ if(start.length()>1)
+ start = Utilities.arrayToString(new String[]{start.substring(0,start.length()-1),Character.toString(ch)});
+ else
+ start = Character.toString(ch);
+
+ WordEntry entry = DictionaryUtil.getVerb(start);
+ if(entry!=null&&entry.getFeature(WordEntry.IDX_REGURA)==IRR_TYPE_DI)
+ return new String[]{start,end};
+
+ return null;
+ }
+
+ /**
+ * ã
ë¶ê·ì¹ ìíì ë³µìíë¤. (ê¸ë¤--ê·¸ì´)
+ * @param start
+ * @param end
+ * @return
+ * @throws MorphException
+ */
+ private static String[] restoreSIrregular(String start, String end) throws MorphException {
+ if(start==null||"".equals(start)) return null;
+
+ char ch = start.charAt(start.length()-1);
+ char[] jasos = MorphUtil.decompose(ch);
+ if(jasos.length!=2) return null;
+
+ ch = MorphUtil.makeChar(ch, 19);
+ if(start.length()>1)
+ start = start.substring(0,start.length()-1)+ch;
+ else
+ start = Character.toString(ch);
+
+ WordEntry entry = DictionaryUtil.getVerb(start);
+ if(entry!=null&&entry.getFeature(WordEntry.IDX_REGURA)==IRR_TYPE_SIUT)
+ return new String[]{start,end};
+
+ return null;
+ }
+
+ /**
+ * 르 ë¶ê·ì¹ ìíì ë³µìíë¤. (í르ë¤-->íë¬)
+ * "ë°ë¥´ë¤"ë ã¹ë¶ê·ì¹ì´ ìëì§ë§.. ì¸ ê²ì²ë¼ ì²ë¦¬íë¤.
+ * @param start
+ * @param end
+ * @return
+ * @throws MorphException
+ */
+ private static String[] restoreLIrregular(String start, String end) throws MorphException {
+
+ if(start.length()<2) return null;
+
+ char ch1 = start.charAt(start.length()-2);
+ char ch2 = start.charAt(start.length()-1);
+
+ char[] jasos1 = MorphUtil.decompose(ch1);
+
+
+ if(((jasos1.length==3&&jasos1[2]=='ã¹')||jasos1.length==2)&&(ch2=='ë¬'||ch2=='ë¼')) {
+
+ StringBuffer sb = new StringBuffer();
+
+ ch1 = MorphUtil.makeChar(ch1, 0);
+ if(start.length()>2)
+ sb.append(start.substring(0,start.length()-2)).append(ch1).append("르");
+ else
+ sb.append(Character.toString(ch1)).append("르");
+
+ WordEntry entry = DictionaryUtil.getVerb(sb.toString());
+ if(entry!=null&&entry.getFeature(WordEntry.IDX_REGURA)==IRR_TYPE_LOO)
+ return new String[]{sb.toString(),end};
+
+ }
+
+ return null;
+ }
+
+ /**
+ * ã¹ë¶ê·ì¹ ìíì ë³µìíë¤. (길ë¤-->긴, ìë¤-->ì)
+ * ì´ê°ì ëìë¦¬ì¸ âã¹âì´ âã´â, âã¹â, âã
â, âì¤â, âìâ ììì íë½íë íì©ì íì
+ * @param start
+ * @param end
+ * @return
+ * @throws MorphException
+ */
+ private static String[] restoreELIrregular(String start, String end) throws MorphException {
+
+ if(start==null || start.length()==0 || end==null||end.length()==0) return null;
+
+ if(!(end.charAt(0)=='ã´'||end.charAt(0)=='ã¹'||end.charAt(0)=='ã
'||end.charAt(0)=='ì¤'||end.charAt(0)=='ì')) return null;
+
+ char convEnd = MorphUtil.makeChar(start.charAt(start.length()-1), 8);
+ start = start.substring(0,start.length()-1)+convEnd;
+
+ WordEntry entry = DictionaryUtil.getVerb(start);
+ if(entry!=null)
+ return new String[]{start,end};
+
+ return null;
+ }
+
+ /**
+ * ë¬ ë¶ê·ì¹ ìíì ë³µìíë¤. (ì´ë¥´ë¤->ì´ë¥´ë¬, í¸ë¥´ë¤->í¸ë¥´ë¬)
+ * @param start
+ * @param end
+ * @return
+ * @throws MorphException
+ */
+ private static String[] restoreRUIrregular(String start, String end) throws MorphException {
+
+ if(start.length()<2) return null;
+
+ char ch1 = start.charAt(start.length()-1);
+ char ch2 = start.charAt(start.length()-2);
+
+ char[] jasos1 = MorphUtil.decompose(ch1);
+ char[] jasos2 = MorphUtil.decompose(ch2);
+ if(jasos1[0]!='ã¹'||jasos2[0]!='ã¹') return null;
+
+ ch2 = MorphUtil.makeChar(ch2, 0);
+ if(start.length()>2)
+ start = start.substring(0,start.length()-1);
+ else
+ start = Character.toString(ch2);
+
+ WordEntry entry = DictionaryUtil.getVerb(start);
+ if(entry!=null&&entry.getFeature(WordEntry.IDX_REGURA)==IRR_TYPE_RU)
+ return new String[]{start,end};
+
+ return null;
+ }
+
+ /**
+ * ã
íë½ ìíì ë³µìíë¤. (ê¹ë§£ë¤-->ê¹ë§,ê¹ë§¤ì)
+ * @param start
+ * @param end
+ * @return
+ * @throws MorphException
+ */
+ private static String[] restoreHIrregular(String start, String end) throws MorphException {
+ if(start==null||"".equals(start)||end==null||"".equals(end)) return null;
+ char ch1 = end.charAt(0);
+ char ch2 = start.charAt(start.length()-1);
+
+ char[] jasos1 = MorphUtil.decompose(ch1);
+ char[] jasos2 = MorphUtil.decompose(ch2);
+
+ if(jasos1.length==1) {
+ ch2 = MorphUtil.makeChar(ch2, 27);
+ }else {
+ if(jasos2.length!=2||jasos2[1]!='ã
') return null;
+ ch2 = MorphUtil.makeChar(ch2, 0, 27);
+ }
+
+ if(start.length()>1)
+ start = start.substring(0,start.length()-1)+ch2;
+ else
+ start = Character.toString(ch2);
+
+ WordEntry entry = DictionaryUtil.getVerb(start);
+ if(entry!=null&&entry.getFeature(WordEntry.IDX_REGURA)==IRR_TYPE_HIOOT)
+ return new String[]{start,end};
+
+ return null;
+ }
+
+ /**
+ * ì¼ íë½ ìíì ë³µìíë¤. (ë¨ë¤->ë, í¬ë¤-커)
+ * @param start
+ * @param end
+ * @return
+ * @throws MorphException
+ */
+ private static String[] restoreUIrregular(String start, String end) throws MorphException {
+ if(start==null||"".equals(start)) return null;
+ char ch = start.charAt(start.length()-1);
+ char[] jasos = MorphUtil.decompose(ch);
+ if(!(jasos.length==2&&jasos[1]=='ã
')) return null;
+
+ ch = MorphUtil.makeChar(ch, 18,0);
+
+ if(start.length()>1)
+ start = start.substring(0,start.length()-1)+ch;
+ else
+ start = Character.toString(ch);
+
+ WordEntry entry = DictionaryUtil.getVerb(start);
+ if(entry!=null) return new String[]{start,end};
+
+ return null;
+ }
+
+}
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/JarResources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/JarResources.java?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/JarResources.java (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/JarResources.java Sun May 5 03:39:51 2013
@@ -0,0 +1,152 @@
+package org.apache.lucene.analysis.kr.utils;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.*;
+import java.util.*;
+import java.util.zip.*;
+
+/**
+ * JarResources: JarResources maps all resources included in a
+ * Zip or Jar file. Additionaly, it provides a method to extract one
+ * as a blob.
+ */
+public final class JarResources {
+
+ // external debug flag
+ public boolean debugOn=false;
+
+ // jar resource mapping tables
+ private Hashtable htSizes=new Hashtable();
+
+ // a jar file
+ private String jarFileName;
+
+ /**
+ * creates a JarResources. It extracts all resources from a Jar
+ * into an internal hashtable, keyed by resource names.
+ * @param jarFileName a jar or zip file
+ */
+ public JarResources(String jarFileName) {
+ this.jarFileName=jarFileName;
+ }
+
+ /**
+ * Extracts a jar resource as a blob.
+ * @param name a resource name.
+ */
+ public byte[] getResource(String name) {
+ return read(name);
+ }
+
+ /**
+ * initializes internal hash tables with Jar file resources.
+ */
+ private byte[] read(String name) {
+ try {
+ // extracts just sizes only.
+ ZipFile zf=new ZipFile(jarFileName);
+ Enumeration e=zf.entries();
+ while (e.hasMoreElements()) {
+ ZipEntry ze=(ZipEntry)e.nextElement();
+ if (debugOn) {
+ System.out.println(dumpZipEntry(ze));
+ }
+ htSizes.put(ze.getName(),new Integer((int)ze.getSize()));
+ }
+ zf.close();
+
+ // extract resources and put them into the hashtable.
+ FileInputStream fis=new FileInputStream(jarFileName);
+ BufferedInputStream bis=new BufferedInputStream(fis);
+ ZipInputStream zis=new ZipInputStream(bis);
+ ZipEntry ze=null;
+ while ((ze=zis.getNextEntry())!=null) {
+ if (ze.isDirectory()) {
+ continue;
+ }
+ if (debugOn) {
+ System.out.println(
+ "ze.getName()="+ze.getName()+","+"getSize()="+ze.getSize()
+ );
+ }
+ int size=(int)ze.getSize();
+ // -1 means unknown size.
+ if (size==-1) {
+ size=((Integer)htSizes.get(ze.getName())).intValue();
+ }
+ byte[] b=new byte[(int)size];
+ int rb=0;
+ int chunk=0;
+ while (((int)size - rb) > 0) {
+ chunk=zis.read(b,rb,(int)size - rb);
+ if (chunk==-1) {
+ break;
+ }
+ rb+=chunk;
+ }
+
+ if (debugOn) {
+ System.out.println(
+ ze.getName()+" rb="+rb+
+ ",size="+size+
+ ",csize="+ze.getCompressedSize()
+ );
+ }
+
+ if(ze.getName().equals(name)) {
+ return b;
+ }
+ }
+ } catch (NullPointerException e) {
+ System.out.println("done.");
+ } catch (FileNotFoundException e) {
+ e.printStackTrace();
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ return null;
+ }
+
+ /**
+ * Dumps a zip entry into a string.
+ * @param ze a ZipEntry
+ */
+ private String dumpZipEntry(ZipEntry ze) {
+ StringBuffer sb=new StringBuffer();
+ if (ze.isDirectory()) {
+ sb.append("d ");
+ } else {
+ sb.append("f ");
+ }
+ if (ze.getMethod()==ZipEntry.STORED) {
+ sb.append("stored ");
+ } else {
+ sb.append("defalted ");
+ }
+ sb.append(ze.getName());
+ sb.append("\t");
+ sb.append(""+ze.getSize());
+ if (ze.getMethod()==ZipEntry.DEFLATED) {
+ sb.append("/"+ze.getCompressedSize());
+ }
+ return (sb.toString());
+ }
+
+}
\ No newline at end of file
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/KoreanEnv.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/KoreanEnv.java?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/KoreanEnv.java (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/KoreanEnv.java Sun May 5 03:39:51 2013
@@ -0,0 +1,140 @@
+package org.apache.lucene.analysis.kr.utils;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.util.Properties;
+
+import org.apache.lucene.analysis.kr.morph.MorphException;
+
+public class KoreanEnv {
+
+ public static final String FILE_SYLLABLE_FEATURE = "syllable.dic";
+
+ public static final String FILE_DICTIONARY = "dictionary.dic";
+
+ public static final String FILE_JOSA = "josa.dic";
+
+ public static final String FILE_EOMI = "eomi.dic";
+
+ public static final String FILE_EXTENSION = "extension.dic";
+
+ public static final String FILE_PREFIX = "prefix.dic";
+
+ public static final String FILE_SUFFIX = "suffix.dic";
+
+ public static final String FILE_COMPOUNDS = "compounds.dic";
+
+ public static final String FILE_UNCOMPOUNDS = "uncompounds.dic";
+
+ public static final String FILE_CJ = "cj.dic";
+
+ public static final String FILE_KOREAN_PROPERTY = "org/apache/lucene/analysis/kr/korean.properties";
+
+ private Properties defaults = null;
+
+ /**
+ * The props member gets its values from the configuration in the property file.
+ */
+ private Properties props = null;
+
+ private static KoreanEnv instance = null;
+
+ /**
+ * The constructor loads property values from the property file.
+ */
+ private KoreanEnv() throws MorphException {
+ try {
+ initDefaultProperties();
+ props = loadProperties(defaults);
+ } catch (MorphException e) {
+ throw new MorphException ("Failure while initializing property values:\n"+e.getMessage());
+ }
+ }
+
+ public static KoreanEnv getInstance() throws MorphException {
+ if(instance==null)
+ instance = new KoreanEnv();
+
+ return instance;
+ }
+
+ /**
+ * Initialize the default property values.
+ */
+ private void initDefaultProperties() {
+ defaults = new Properties();
+
+ defaults.setProperty(FILE_SYLLABLE_FEATURE,"org/apache/lucene/analysis/kr/dic/syllable.dic");
+ defaults.setProperty(FILE_DICTIONARY,"org/apache/lucene/analysis/kr/dic/dictionary.dic");
+ defaults.setProperty(FILE_DICTIONARY,"org/apache/lucene/analysis/kr/dic/extension.dic");
+ defaults.setProperty(FILE_JOSA,"org/apache/lucene/analysis/kr/dic/josa.dic");
+ defaults.setProperty(FILE_EOMI,"org/apache/lucene/analysis/kr/dic/eomi.dic");
+ defaults.setProperty(FILE_PREFIX,"org/apache/lucene/analysis/kr/dic/prefix.dic");
+ defaults.setProperty(FILE_SUFFIX,"org/apache/lucene/analysis/kr/dic/suffix.dic");
+ defaults.setProperty(FILE_COMPOUNDS,"org/apache/lucene/analysis/kr/dic/compounds.dic");
+ defaults.setProperty(FILE_UNCOMPOUNDS,"org/apache/lucene/analysis/kr/dic/uncompounds.dic");
+ defaults.setProperty(FILE_CJ,"org/apache/lucene/analysis/kr/dic/cj.dic");
+ }
+
+
+ /**
+ * Given a property file name, load the property file and return an object
+ * representing the property values.
+ *
+ * @param propertyFile The name of the property file to load.
+ * @param def Default property values, or <code>null</code> if there are no defaults.
+ * @return The loaded SortedProperties object.
+ */
+ private Properties loadProperties(Properties def) throws MorphException {
+ Properties properties = new Properties();
+
+ if (def != null) {
+ properties = new Properties(def);
+ }
+
+ File file = null;
+ try {
+ file = FileUtil.getClassLoaderFile(FILE_KOREAN_PROPERTY);
+
+ if (file != null) {
+ properties.load(new FileInputStream(file));
+ return properties;
+ }
+
+ byte[] in = FileUtil.readByteFromCurrentJar(FILE_KOREAN_PROPERTY);
+ properties.load(new ByteArrayInputStream(in));
+ } catch (Exception e) {
+ throw new MorphException("Failure while trying to load properties file " + file.getPath(), e);
+ }
+ return properties;
+ }
+
+
+ /**
+ * Returns the value of a property.
+ *
+ * @param name The name of the property whose value is to be retrieved.
+ * @return The value of the property.
+ */
+ public String getValue(String name) {
+ return props.getProperty(name);
+ }
+}
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/MorphUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/MorphUtil.java?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/MorphUtil.java (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/MorphUtil.java Sun May 5 03:39:51 2013
@@ -0,0 +1,246 @@
+package org.apache.lucene.analysis.kr.utils;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.lucene.analysis.kr.morph.AnalysisOutput;
+import org.apache.lucene.analysis.kr.morph.MorphException;
+import org.apache.lucene.analysis.kr.morph.PatternConstants;
+import org.apache.lucene.analysis.kr.morph.WordEntry;
+
+public class MorphUtil {
+
+ private static final char[] CHOSEONG = {
+ 'ã±','ã²','ã´','ã·','ã¸','ã¹','ã
','ã
','ã
','ã
',
+ 'ã
','ã
','ã
','ã
','ã
','ã
','ã
','ã
','ã
'
+ };
+
+ private static final char[] JUNGSEONG = {
+ 'ã
','ã
','ã
','ã
','ã
','ã
','ã
','ã
','ã
','ã
',
+ 'ã
','ã
','ã
','ã
','ã
','ã
','ã
','ã
','ã
¡','ã
¢',
+ 'ã
£'
+ };
+
+ private static final char[] JONGSEONG = {
+ '\0','ã±','ã²','ã³','ã´','ãµ','ã¶','ã·','ã¹','ãº',
+ 'ã»','ã¼','ã½','ã¾','ã¿','ã
','ã
','ã
','ã
','ã
',
+ 'ã
','ã
','ã
','ã
','ã
','ã
','ã
','ã
'
+ };
+
+ private static final int JUNG_JONG = JUNGSEONG.length * JONGSEONG.length;
+
+
+ /**
+ * íê¸ íê¸ì를 ì´ì±/ì¤ì±/ì¢
ì±ì ë°°ì´ë¡ ë§ë¤ì´ ë°ííë¤.
+ * @param c
+ * @return
+ */
+ public static char[] decompose(char c) {
+ char[] result = null;
+
+ if(c>0xD7A3||c<0xAC00) return new char[]{c};
+
+ c -= 0xAC00;
+
+ char choseong = CHOSEONG[c/JUNG_JONG];
+ c = (char)(c % JUNG_JONG);
+
+ char jungseong = JUNGSEONG[c/JONGSEONG.length];
+
+ char jongseong = JONGSEONG[c%JONGSEONG.length];
+
+ if(jongseong != 0) {
+ result = new char[] {choseong, jungseong, jongseong};
+ }else {
+ result = new char[] {choseong, jungseong};
+ }
+ return result;
+ }
+
+ public static char compound(int first, int middle, int last) {
+ return (char)(0xAC00 + first* JUNG_JONG + middle * JONGSEONG.length + last);
+ }
+
+
+ public static char makeChar(char ch, int mdl, int last) {
+ ch -= 0xAC00;
+ int first = ch/JUNG_JONG;
+ return compound(first,mdl,last);
+ }
+
+ public static char makeChar(char ch, int last) {
+ ch -= 0xAC00;
+ int first = ch/JUNG_JONG;
+ ch = (char)(ch % JUNG_JONG);
+ int middle = ch/JONGSEONG.length;
+
+ return compound(first,middle,last);
+ }
+
+ public static char replaceJongsung(char dest, char source) {
+ source -= 0xAC00;
+ int last = source % JONGSEONG.length;
+
+ return makeChar(dest,last);
+ }
+
+ /**
+ * ííì ì í ì¶ë ¥ì ìí 문ìì´ì ìì±íë¤.
+ * @param word
+ * @param type
+ * @return
+ */
+ public static String buildTypeString(String word, char type) {
+ StringBuffer sb = new StringBuffer();
+ sb.append(word);
+ sb.append("(");
+ sb.append(type);
+ sb.append(")");
+
+ return sb.toString();
+ }
+
+
+ public static void buildPtnVM(AnalysisOutput output, List candidates) throws MorphException {
+
+ String end = output.getEomi();
+ if(output.getPomi()!=null) end = output.getPomi();
+
+ output.setPatn(PatternConstants.PTN_VM);
+ output.setPos(PatternConstants.POS_VERB);
+
+ if(output.getScore()==AnalysisOutput.SCORE_CORRECT) {
+ candidates.add(output);
+ }else {
+ String[] irrs = IrregularUtil.restoreIrregularVerb(output.getStem(),end);
+ if(irrs!=null) {
+ output.setScore(AnalysisOutput.SCORE_CORRECT);
+ output.setStem(irrs[0]);
+ candidates.add(output);
+ }
+ }
+
+ }
+
+ /**
+ * ì©ì¸ + 'ì/기' + 'ì´' + ì´ë¯¸, ì²´ì¸ + 'ìì/ë¶í°/ììë¶í°' + 'ì´' + ì´ë¯¸
+ * @param output
+ * @param candidates
+ * @throws MorphException
+ */
+ public static void buildPtnCM(AnalysisOutput output, List candidates) throws MorphException {
+
+ char ch = output.getStem().charAt(output.getStem().length()-2);
+ char[] jasos = MorphUtil.decompose(ch);
+ if(jasos.length==3||ch=='기') {
+ buildPtnVMCM(output,candidates);
+ } else {
+
+ }
+ }
+
+ private static void buildPtnVMCM(AnalysisOutput output, List candidates) throws MorphException {
+ String stem = output.getStem();
+
+ output.setPatn(PatternConstants.PTN_VMCM);
+ output.setPos(PatternConstants.POS_VERB);
+
+ char ch = stem.charAt(stem.length()-2);
+ char[] jasos = MorphUtil.decompose(ch);
+
+ if(ch=='기') {
+ output.addElist("기");
+ output.addElist("ì´");
+ output.setStem(stem.substring(0,stem.length()-2));
+
+ if(DictionaryUtil.getVerb(output.getStem())!=null)
+ candidates.add(output);
+ }else if(jasos[2]=='ã
') {
+ if(stem.length()>1) stem = stem.substring(0,stem.length()-2);
+ stem += MorphUtil.makeChar(ch, 0);
+ output.addElist("ã
");
+ output.addElist("ì´");
+ output.setStem(stem);
+
+ if(DictionaryUtil.getVerb(stem)!=null)
+ candidates.add(output);
+ else {
+ String[] morphs = IrregularUtil.restoreIrregularVerb(stem,"ã
");
+ if(morphs!=null) {
+ output.setScore(AnalysisOutput.SCORE_CORRECT);
+ output.setStem(morphs[0]);
+ candidates.add(output);
+ }
+ }
+ }
+ }
+
+ public static boolean hasVerbOnly(String input) throws MorphException {
+
+ for(int i=input.length()-1;i>=0;i--) {
+ char[] feature = SyllableUtil.getFeature(input.charAt(i));
+ if(feature[SyllableUtil.IDX_WDSURF]=='1'&&input.length()>i) return true;
+ }
+ return false;
+ }
+
+ /**
+ * ìì ì ì´ë¯¸ë§ì ë§ë¤ì´ì ë°ííë¤.
+ * @param preword 'ì' ëë 'ì´'
+ * @param endword ì´ë¯¸[ì ì´ë¯¸ë§ì í¬í¨]
+ * @return 'ì' ëë 'ì'ì ë§ë¤ì´ì ë°ííë¤.
+ */
+ public static String makeTesnseEomi(String preword, String endword) {
+
+ if(preword==null||preword.length()==0) return endword;
+ if(endword==null||endword.length()==0) return preword;
+
+ if(endword.charAt(0)=='ã
') {
+ return preword.substring(0,preword.length()-1)+
+ makeChar(preword.charAt(preword.length()-1),20)+endword.substring(1,endword.length());
+ } else if(endword.charAt(0)=='ã´') {
+ return preword.substring(0,preword.length()-1)+
+ makeChar(preword.charAt(preword.length()-1),4)+endword.substring(1,endword.length());
+ } else if(endword.charAt(0)=='ã¹') {
+ return preword.substring(0,preword.length()-1)+
+ makeChar(preword.charAt(preword.length()-1),8)+endword.substring(1,endword.length());
+ } else if(endword.charAt(0)=='ã
') {
+ return preword.substring(0,preword.length()-1)+
+ makeChar(preword.charAt(preword.length()-1),16)+endword.substring(1,endword.length());
+ } else if(endword.charAt(0)=='ã
') {
+ return preword.substring(0,preword.length()-1)+
+ makeChar(preword.charAt(preword.length()-1),17)+endword.substring(1,endword.length());
+ }
+ return preword+endword;
+ }
+
+ /**
+ * ì©ì¸íì 미ì¬ê° ê²°í©ë ì ìëì§ ì¬ë¶ë¥¼ ì ê²íë¤.
+ * í¹í ì¬ì ì ë±ë¡ë ëë¤, íë¤í ì ì ìì´ ê°ë¥íì§ë¥¼ ì¡°ì¬íë¤.
+ * @param o
+ * @return
+ */
+ public static boolean isValidSuffix(WordEntry entry, AnalysisOutput o) {
+
+ return true;
+ }
+
+}
+
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/NounUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/NounUtil.java?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/NounUtil.java (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/NounUtil.java Sun May 5 03:39:51 2013
@@ -0,0 +1,407 @@
+package org.apache.lucene.analysis.kr.utils;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+
+import org.apache.lucene.analysis.kr.morph.AnalysisOutput;
+import org.apache.lucene.analysis.kr.morph.CompoundEntry;
+import org.apache.lucene.analysis.kr.morph.CompoundNounAnalyzer;
+import org.apache.lucene.analysis.kr.morph.MorphException;
+import org.apache.lucene.analysis.kr.morph.PatternConstants;
+import org.apache.lucene.analysis.kr.morph.WordEntry;
+
+public class NounUtil {
+
+ private static final List DNouns = new ArrayList();
+
+ static {
+ String[] strs = new String[]{"ë±", "ë¤","ì","ê°","ë¿","ë³"};
+ for(String str:strs) {
+ DNouns.add(str);
+ }
+ };
+
+ /**
+ *
+ * ì´ê°ë¶ê° ì/기 ë¡ ëëë ê²½ì°
+ *
+ * @param o
+ * @param candidates
+ * @return
+ * @throws MorphException
+ */
+ public static boolean analysisMJ(AnalysisOutput o, List candidates) throws MorphException {
+
+ int strlen = o.getStem().length();
+
+ if(strlen<2) return false;
+
+ char[] chrs = MorphUtil.decompose(o.getStem().charAt(strlen-1));
+ boolean success = false;
+
+ if(o.getStem().charAt(strlen-1)!='기'&&!(chrs.length==3&&chrs[2]=='ã
')) return false;
+
+ String start = o.getStem();
+ String end = "";
+ if(o.getStem().charAt(strlen-1)=='기') {
+ start = o.getStem().substring(0,strlen-1);
+ end = "기";
+ }else if(o.getStem().charAt(strlen-1)=='ì') {
+ start = o.getStem().substring(0,strlen-1);
+ end = "ì";
+ }
+
+ String[] eomis = EomiUtil.splitEomi(start, end);
+ if(eomis[0]==null) return false;
+ String[] pomis = EomiUtil.splitPomi(eomis[0]);
+ o.setStem(pomis[0]);
+ o.addElist(eomis[1]);
+ o.setPomi(pomis[1]);
+
+ try {
+ if(analysisVMJ(o.clone(),candidates)) return true;
+ if(analysisVMXMJ(o.clone(),candidates)) return true;
+ if(analysisNSMJ(o.clone(),candidates)) return true;
+ } catch (CloneNotSupportedException e) {
+ throw new MorphException(e.getMessage(),e);
+ }
+
+ if(DictionaryUtil.getVerb(o.getStem())!=null) {
+ o.setPos(PatternConstants.POS_VERB);
+ o.setPatn(PatternConstants.PTN_VMJ);
+ o.setScore(AnalysisOutput.SCORE_CORRECT);
+ candidates.add(o);
+ return true;
+ }
+
+ return false;
+
+ }
+
+ /**
+ * ì©ì¸ + 'ì/기' + ì¡°ì¬(PTN_VMXMJ)
+ * @param o
+ * @param candidates
+ * @return
+ * @throws MorphException
+ */
+ public static boolean analysisVMJ(AnalysisOutput o, List candidates) throws MorphException {
+
+ String[] irrs = IrregularUtil.restoreIrregularVerb(o.getStem(), o.getElist().get(0));
+ if(irrs!=null) {
+ o.setStem(irrs[0]);
+ o.setElist(irrs[1],0);
+ }
+
+ if(DictionaryUtil.getVerb(o.getStem())!=null) {
+ o.setPatn(PatternConstants.PTN_VMJ);
+ o.setPos(PatternConstants.POS_VERB);
+ o.setScore(AnalysisOutput.SCORE_CORRECT);
+ candidates.add(o);
+ return true;
+ }
+
+ return false;
+ }
+
+ /**
+ * ì©ì¸ + 'ì/ì´' + ë³´ì¡°ì©ì¸ + 'ì/기' + ì¡°ì¬(PTN_VMXMJ)
+ * @param o
+ * @param candidates
+ * @return
+ * @throws MorphException
+ */
+ public static boolean analysisVMXMJ(AnalysisOutput o, List candidates) throws MorphException {
+
+ int idxXVerb = VerbUtil.endsWithXVerb(o.getStem());
+
+ if(idxXVerb!=-1) { // 2. ì¬ëë°ìë³´ë¤
+ String eogan = o.getStem().substring(0,idxXVerb);
+ o.setXverb(o.getStem().substring(idxXVerb));
+
+ String[] stomis = null;
+ if(eogan.endsWith("ì")||eogan.endsWith("ì´"))
+ stomis = EomiUtil.splitEomi(eogan.substring(0,eogan.length()-1),eogan.substring(eogan.length()-1));
+ else
+ stomis = EomiUtil.splitEomi(eogan,"");
+ if(stomis[0]==null) return false;
+
+ String[] irrs = IrregularUtil.restoreIrregularVerb(stomis[0], stomis[1]);
+ if(irrs!=null) {
+ o.setStem(irrs[0]);
+ o.addElist(irrs[1]);
+ }else {
+ o.setStem(stomis[0]);
+ o.addElist(stomis[1]);
+ }
+
+ if(DictionaryUtil.getVerb(o.getStem())!=null) {
+ o.setPatn(PatternConstants.PTN_VMXMJ);
+ o.setPos(PatternConstants.POS_VERB);
+ o.setScore(AnalysisOutput.SCORE_CORRECT);
+ candidates.add(o);
+ return true;
+ }else if(analysisNSMXMJ(o, candidates)){
+ return true;
+ }
+
+ }
+
+ return false;
+ }
+
+ /**
+ * ì²´ì¸ + ì©ì¸íì ë¯¸ì¬ + 'ì/기' + ì¡°ì¬ (PTN_NSMJ)
+ * @param o
+ * @param candidates
+ * @return
+ * @throws MorphException
+ */
+ public static boolean analysisNSMJ(AnalysisOutput o, List candidates) throws MorphException {
+
+ int idxVbSfix = VerbUtil.endsWithVerbSuffix(o.getStem());
+ if(idxVbSfix==-1) return false;
+
+ o.setVsfx(o.getStem().substring(idxVbSfix));
+ o.setStem(o.getStem().substring(0,idxVbSfix));
+ o.setPatn(PatternConstants.PTN_NSMJ);
+ o.setPos(PatternConstants.POS_NOUN);
+
+ WordEntry entry = DictionaryUtil.getWordExceptVerb(o.getStem());
+
+ if(entry!=null) {
+ if(entry.getFeature(WordEntry.IDX_NOUN)=='0') return false;
+ else if(o.getVsfx().equals("í")&&entry.getFeature(WordEntry.IDX_DOV)!='1') return false;
+ else if(o.getVsfx().equals("ë")&&entry.getFeature(WordEntry.IDX_BEV)!='1') return false;
+ else if(o.getVsfx().equals("ë´")&&entry.getFeature(WordEntry.IDX_NE)!='1') return false;
+ o.setScore(AnalysisOutput.SCORE_CORRECT); // 'ì
ëë¤'ì¸ ê²½ì° ì¸ëª
ë± ë¯¸ë±ë¡ì´ê° ë§ì´ ë°ìëë¯ë¡ ë¶ìì±ê³µì¼ë¡ ê°ì íë¤.
+ }else {
+ o.setScore(AnalysisOutput.SCORE_ANALYSIS); // 'ì
ëë¤'ì¸ ê²½ì° ì¸ëª
ë± ë¯¸ë±ë¡ì´ê° ë§ì´ ë°ìëë¯ë¡ ë¶ìì±ê³µì¼ë¡ ê°ì íë¤.
+ }
+
+ candidates.add(o);
+
+ return true;
+ }
+
+ public static boolean analysisNSMXMJ(AnalysisOutput o, List candidates) throws MorphException {
+
+ int idxVbSfix = VerbUtil.endsWithVerbSuffix(o.getStem());
+ if(idxVbSfix==-1) return false;
+
+ o.setVsfx(o.getStem().substring(idxVbSfix));
+ o.setStem(o.getStem().substring(0,idxVbSfix));
+ o.setPatn(PatternConstants.PTN_NSMXMJ);
+ o.setPos(PatternConstants.POS_NOUN);
+
+ WordEntry entry = DictionaryUtil.getWordExceptVerb(o.getStem());
+
+ if(entry!=null) {
+ if(entry.getFeature(WordEntry.IDX_NOUN)=='0') return false;
+ else if(o.getVsfx().equals("í")&&entry.getFeature(WordEntry.IDX_DOV)!='1') return false;
+ else if(o.getVsfx().equals("ë")&&entry.getFeature(WordEntry.IDX_BEV)!='1') return false;
+ else if(o.getVsfx().equals("ë´")&&entry.getFeature(WordEntry.IDX_NE)!='1') return false;
+ o.setScore(AnalysisOutput.SCORE_CORRECT); // 'ì
ëë¤'ì¸ ê²½ì° ì¸ëª
ë± ë¯¸ë±ë¡ì´ê° ë§ì´ ë°ìëë¯ë¡ ë¶ìì±ê³µì¼ë¡ ê°ì íë¤.
+ }else {
+ o.setScore(AnalysisOutput.SCORE_ANALYSIS); // 'ì
ëë¤'ì¸ ê²½ì° ì¸ëª
ë± ë¯¸ë±ë¡ì´ê° ë§ì´ ë°ìëë¯ë¡ ë¶ìì±ê³µì¼ë¡ ê°ì íë¤.
+ }
+
+ candidates.add(o);
+
+ return true;
+ }
+
+
+ /**
+ * ë³µí©ëª
ì¬ì¸ì§ ì¡°ì¬íê³ , ë³µí©ëª
ì¬ì´ë©´ ë¨ìëª
ì¬ë¤ì ì°¾ëë¤.
+ * ë³µí©ëª
ì¬ì¸ì§ ì¬ë¶ë ë¨ìëª
ì¬ê° 모ë ì¬ì ì ìëì§ ì¬ë¶ë¡ íë¨íë¤.
+ * ë¨ìëª
ì¬ë 2ê¸ì ì´ì ë¨ì´ììë§ ì°¾ëë¤.
+ * @param o
+ * @return
+ * @throws MorphException
+ */
+// public static boolean confirmCNoun(AnalysisOutput o) throws MorphException {
+//
+// if(o.getStem().length()<3) return false;
+// if(o.getPatn()==PatternConstants.PTN_N
+// &&DictionaryUtil.existJosa(o.getStem().substring(o.getStem().length()-2))) return false;
+//
+// List<CompoundEntry> results = new ArrayList();
+// List<List> queue = new ArrayList();
+// String prefix = o.getStem().substring(0,1);
+//
+// int pos = 0;
+// boolean moreTwo = false;
+// while(pos<o.getStem().length()) {
+//
+// List<WordEntry> nList = findNouns(o.getStem().substring(pos),queue.size(),o);
+// if(nList==null) return false;
+//
+// if(pos==0&&DictionaryUtil.existPrefix(prefix)) nList.add(new WordEntry(prefix));
+//
+// if(nList.size()==0) {
+// if(queue.size()==0) return false;
+// List<WordEntry> tmpList = queue.get(queue.size()-1);
+//
+// tmpList.remove(tmpList.size()-1);
+// pos -= results.get(queue.size()-1).getWord().length();
+// if(tmpList.size()==0) {
+// while(tmpList.size()==0) {
+// results.remove(queue.size()-1);
+// queue.remove(tmpList);
+// if(queue.size()==0) return false;
+//
+// tmpList = queue.get(queue.size()-1);
+// tmpList.remove(tmpList.size()-1);
+// if(tmpList.size()==0) continue;
+//
+// pos -= results.get(queue.size()-1).getWord().length();
+// results.set(queue.size()-1, new CompoundEntry(tmpList.get(tmpList.size()-1).getWord(),pos));
+// pos += tmpList.get(tmpList.size()-1).getWord().length();
+//
+// }
+// }else {
+// results.set(queue.size()-1, new CompoundEntry(tmpList.get(tmpList.size()-1).getWord(),pos));
+// pos += tmpList.get(tmpList.size()-1).getWord().length();
+// }
+//
+// } else {
+// queue.add(nList);
+// WordEntry noun = nList.get(nList.size()-1);
+// results.add(new CompoundEntry(noun.getWord(),pos));
+// pos += noun.getWord().length();
+// if(noun.getCompounds().size()>0) o.addCNoun(noun.getCompounds());
+// if(noun.getWord().length()>1) moreTwo=true;
+// }
+// }
+//
+// if(results.size()>1&&DNouns.contains(results.get(results.size()-1).getWord())) {
+// CompoundEntry dnoun = results.remove(results.size()-1);
+// o.setStem(o.getStem().substring(0,o.getStem().length()-dnoun.getWord().length()));
+// o.setNsfx(dnoun.getWord());
+// }
+//
+// if(results.size()>1) o.addCNoun(results);
+//
+// o.setScore(AnalysisOutput.SCORE_CORRECT);
+// return true;
+// }
+
+ /**
+ * ë³µí©ëª
ì¬ìì ë¨ìëª
ì¬ë¥¼ ë¶ë¦¬í´ë¸ë¤.
+ * 리ì¤í¸ì ê°ì¥ ë§ì§ë§ì ìì¹í ë¨ì´ê° ìµì¥ë¨ì´ì´ë¤.
+ * @param str ë³µí©ëª
ì¬
+ * @param pos
+ * @param o ë¶ìê²°ê³¼
+ * @return ë¨ìëª
ì¬ ë¦¬ì¤í¸
+ * @throws MorphException
+ */
+ private static List findNouns(String str, int pos, AnalysisOutput o) throws MorphException {
+
+ List<WordEntry> nList = new ArrayList();
+
+ if(str.length()==2&&DictionaryUtil.existSuffix(str.substring(0,1))&&DNouns.contains(str.substring(1))) {
+ o.setStem(o.getStem().substring(0,o.getStem().length()-1));
+ o.setNsfx(str.substring(1));
+ nList.add(new WordEntry(str.substring(0,1)));
+ return nList;
+ }else if(str.length()==2&&DictionaryUtil.existSuffix(str.substring(0,1))&&DictionaryUtil.existJosa(str.substring(1))) {
+ return null;
+ }
+
+ if(pos>=2&&DictionaryUtil.existJosa(str)) return null;
+
+ if(str.length()==1&&(DictionaryUtil.existSuffix(str)||DNouns.contains(str))) {
+ nList.add(new WordEntry(str));
+ return nList;
+ }
+
+ for(int i=1;i<str.length();i++) {
+ String sub = str.substring(0,i+1);
+ if(!DictionaryUtil.findWithPrefix(sub).hasNext()) break;
+ WordEntry entry = DictionaryUtil.getCNoun(sub);
+ if(entry!=null) {
+ nList.add(entry);
+ }
+ }
+
+ return nList;
+ }
+
+ /*
+ * ë§ì§ë§ ìì ì´ ëª
ì¬í ì 미ì¬(ë±,ì..)ì¸ì§ ì¡°ì¬íë¤.
+ */
+ public static boolean confirmDNoun(AnalysisOutput output) throws MorphException {
+
+ int strlen = output.getStem().length();
+ String d = output.getStem().substring(strlen-1);
+ if(!DNouns.contains(d)) return false;
+
+ String s = output.getStem().substring(0, strlen-1);
+ output.setNsfx(d);
+ output.setStem(s);
+
+ WordEntry cnoun = DictionaryUtil.getCNoun(s);
+ if(cnoun != null) {
+ if(cnoun.getFeature(WordEntry.IDX_NOUN)=='2')
+ output.setCNoun(cnoun.getCompounds());
+ else
+ output.setCNoun(new ArrayList());
+ output.setScore(AnalysisOutput.SCORE_CORRECT);
+ }
+
+ return true;
+ }
+
+// public static int endsWithDNoun(String stem) {
+// for(int i = 0; i < DNouns.length; i++)
+// if(stem.endsWith(DNouns[i]))
+// return stem.lastIndexOf(DNouns[i]);
+//
+// return -1;
+// }
+
+ public static boolean endsWith2Josa(String input) throws MorphException {
+
+ boolean josaFlag = true;
+ for(int i=input.length()-2;i>0;i--) {
+
+ String josa = input.substring(i);
+
+ char[] feature = SyllableUtil.getFeature(josa.charAt(0));
+ if(josaFlag&&DictionaryUtil.existJosa(josa)) return true;
+
+
+ if(josaFlag&&feature[SyllableUtil.IDX_JOSA2]=='0') josaFlag = false;
+ if(!josaFlag) break;
+ }
+
+ return false;
+ }
+
+ public static double countFoundNouns(AnalysisOutput o) {
+ int count = 0;
+ for(int i=0;i<o.getCNounList().size();i++) {
+ if(o.getCNounList().get(i).isExist()) count++;
+ }
+ return (count*100)/o.getCNounList().size();
+ }
+}