You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cm...@apache.org on 2013/05/05 05:40:06 UTC
svn commit: r1479234 [1/15] - in /lucene/dev/branches/lucene4956:
dev-tools/idea/.idea/ dev-tools/idea/lucene/analysis/arirang/
lucene/analysis/ lucene/analysis/arirang/ lucene/analysis/arirang/src/
lucene/analysis/arirang/src/java/ lucene/analysis/ari...
Author: cm
Date: Sun May 5 03:39:51 2013
New Revision: 1479234
URL: http://svn.apache.org/r1479234
Log:
Initial check-in of donated code. Integrated into build and a basic test passes. Work-in-progress.
Added:
lucene/dev/branches/lucene4956/dev-tools/idea/lucene/analysis/arirang/
lucene/dev/branches/lucene4956/dev-tools/idea/lucene/analysis/arirang/arirang.iml
lucene/dev/branches/lucene4956/lucene/analysis/arirang/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/build.xml
lucene/dev/branches/lucene4956/lucene/analysis/arirang/ivy.xml
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/IndexWord.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanAnalyzer.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanFilter.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanFilterFactory.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizer.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerFactory.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerImpl.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerImpl.jflex
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/AnalysisOutput.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/AnalysisOutputComparator.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/CompoundEntry.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/CompoundNounAnalyzer.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/MorphAnalyzer.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/MorphAnalyzerManager.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/MorphException.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/NounProperty.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/PatternConstants.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/SpaceOutput.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/Status.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/WSAOutput.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/WSCandidateComparator.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/WSOuputComparator.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/WSOutput.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/WordEntry.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/morph/WordSpaceAnalyzer.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/tagging/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/tagging/Tagger.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/ConstraintUtil.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/DictionaryUtil.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/EomiUtil.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/FileUtil.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/HanjaUtils.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/IrregularUtil.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/JarResources.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/KoreanEnv.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/MorphUtil.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/NounUtil.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/StrBuilder.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/StringEscapeUtil.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/StringUtil.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/SyllableUtil.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/Trie.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/UnhandledException.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/UnmodifiableIterator.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/Utilities.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/utils/VerbUtil.java
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/dic/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/dic/cj.dic
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/dic/compounds.dic
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/dic/eomi.dic
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/dic/extension.dic
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/dic/josa.dic
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/dic/mapHanja.dic
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/dic/occurrence.dic
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/dic/prefix.dic
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/dic/suffix.dic
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/dic/syllable.dic
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/dic/total.dic
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/dic/uncompounds.dic
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/resources/org/apache/lucene/analysis/kr/korean.properties
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/test/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/test/org/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/test/org/apache/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/test/org/apache/lucene/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/test/org/apache/lucene/analysis/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/test/org/apache/lucene/analysis/kr/
lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/test/org/apache/lucene/analysis/kr/TestKoreanAnalyzer.java
Modified:
lucene/dev/branches/lucene4956/dev-tools/idea/.idea/modules.xml
lucene/dev/branches/lucene4956/lucene/analysis/build.xml
Modified: lucene/dev/branches/lucene4956/dev-tools/idea/.idea/modules.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/dev-tools/idea/.idea/modules.xml?rev=1479234&r1=1479233&r2=1479234&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/dev-tools/idea/.idea/modules.xml (original)
+++ lucene/dev/branches/lucene4956/dev-tools/idea/.idea/modules.xml Sun May 5 03:39:51 2013
@@ -10,6 +10,7 @@
<module filepath="$PROJECT_DIR$/lucene/analysis/common/analysis-common.iml" />
<module filepath="$PROJECT_DIR$/lucene/analysis/icu/icu.iml" />
+ <module filepath="$PROJECT_DIR$/lucene/analysis/arirang/arirang.iml" />
<module filepath="$PROJECT_DIR$/lucene/analysis/kuromoji/kuromoji.iml" />
<module filepath="$PROJECT_DIR$/lucene/analysis/morfologik/morfologik.iml" />
<module filepath="$PROJECT_DIR$/lucene/analysis/phonetic/phonetic.iml" />
Added: lucene/dev/branches/lucene4956/dev-tools/idea/lucene/analysis/arirang/arirang.iml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/dev-tools/idea/lucene/analysis/arirang/arirang.iml?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/dev-tools/idea/lucene/analysis/arirang/arirang.iml (added)
+++ lucene/dev/branches/lucene4956/dev-tools/idea/lucene/analysis/arirang/arirang.iml Sun May 5 03:39:51 2013
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="JAVA_MODULE" version="4">
+ <component name="NewModuleRootManager" inherit-compiler-output="false">
+ <output url="file://$MODULE_DIR$/../../../idea-build/lucene/analysis/arirang/classes/java" />
+ <output-test url="file://$MODULE_DIR$/../../../idea-build/lucene/analysis/arirang/classes/test" />
+ <exclude-output />
+ <content url="file://$MODULE_DIR$">
+ <sourceFolder url="file://$MODULE_DIR$/src/test" isTestSource="true" />
+ <sourceFolder url="file://$MODULE_DIR$/src/java" isTestSource="false" />
+ <sourceFolder url="file://$MODULE_DIR$/src/resources" isTestSource="false" />
+ </content>
+ <orderEntry type="inheritedJdk" />
+ <orderEntry type="sourceFolder" forTests="false" />
+ <orderEntry type="library" scope="TEST" name="JUnit" level="project" />
+ <orderEntry type="module" module-name="lucene-test-framework" scope="TEST" />
+ <orderEntry type="module" module-name="analysis-common" />
+ <orderEntry type="module" module-name="lucene-core" />
+ </component>
+</module>
+
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/build.xml?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/build.xml (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/build.xml Sun May 5 03:39:51 2013
@@ -0,0 +1,34 @@
+<?xml version="1.0"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+<project name="analyzers-arirang" default="default">
+
+ <description>
+ Korean Morphological Analyzer
+ </description>
+
+ <import file="../analysis-module-build.xml"/>
+
+ <path id="classpath">
+ <pathelement path="${analyzers-common.jar}"/>
+ <path refid="base.classpath"/>
+ </path>
+
+ <target name="compile-core" depends="jar-analyzers-common, common.compile-core"/>
+</project>
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/ivy.xml?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/ivy.xml (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/ivy.xml Sun May 5 03:39:51 2013
@@ -0,0 +1,21 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<ivy-module version="2.0">
+ <info organisation="org.apache.lucene" module="analyzers-arirang"/>
+</ivy-module>
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/IndexWord.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/IndexWord.java?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/IndexWord.java (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/IndexWord.java Sun May 5 03:39:51 2013
@@ -0,0 +1,55 @@
+package org.apache.lucene.analysis.kr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Index word extracted from a phrase.
+ */
+public class IndexWord {
+
+ private String word;
+
+ private int offset = 0;
+
+ public IndexWord() {
+
+ }
+
+ public IndexWord(String word, int pos) {
+ this.word = word;
+ this.offset = pos;
+ }
+
+ public String getWord() {
+ return word;
+ }
+
+ public void setWord(String word) {
+ this.word = word;
+ }
+
+ public int getOffset() {
+ return offset;
+ }
+
+ public void setOffset(int offset) {
+ this.offset = offset;
+ }
+
+
+}
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanAnalyzer.java?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanAnalyzer.java (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanAnalyzer.java Sun May 5 03:39:51 2013
@@ -0,0 +1,169 @@
+package org.apache.lucene.analysis.kr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.core.LowerCaseFilter;
+import org.apache.lucene.analysis.core.StopFilter;
+import org.apache.lucene.analysis.standard.StandardFilter;
+import org.apache.lucene.analysis.standard.StandardTokenizer;
+import org.apache.lucene.analysis.util.CharArraySet;
+import org.apache.lucene.analysis.util.StopwordAnalyzerBase;
+import org.apache.lucene.util.Version;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.Reader;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * Filters {@link StandardTokenizer} with {@link StandardFilter}, {@link
+ * LowerCaseFilter} and {@link StopFilter}, using a list of English stop words.
+ *
+ * @version $Id: KoreanAnalyzer.java,v 1.2 2013/04/07 13:09:33 smlee0818 Exp $
+ */
+public class KoreanAnalyzer extends StopwordAnalyzerBase {
+
+ /** Default maximum allowed token length */
+ public static final int DEFAULT_MAX_TOKEN_LENGTH = 255;
+
+ private int maxTokenLength = DEFAULT_MAX_TOKEN_LENGTH;
+
+ /**
+ * Specifies whether deprecated acronyms should be replaced with HOST type.
+ * See {@linkplain "https://issues.apache.org/jira/browse/LUCENE-1068"}
+ */
+ private final boolean replaceInvalidAcronym;
+
+ private Set<String> stopSet;
+
+ private boolean bigrammable = true;
+
+ private boolean hasOrigin = true;
+
+ private boolean exactMatch = false;
+
+ private boolean originCNoun = true;
+
+ public static final String DIC_ENCODING = "UTF-8";
+
+ /** An unmodifiable set containing some common English words that are usually not
+ useful for searching. */
+ public static final CharArraySet STOP_WORDS_SET;
+
+
+ static
+ {
+ List<String> stopWords = Arrays.asList(new String[] { "a", "an", "and", "are", "as", "at", "be", "but", "by",
+ "for", "if", "in", "into", "is", "it", "no", "not", "of", "on", "or", "such", "that", "the",
+ "their", "then", "there", "these", "they", "this", "to", "was", "will", "with",
+ "ì´","ê·¸","ì ","ê²","ì","ë±","ë¤","ë°","ìì","ê·¸ë¦¬ê³ ","ê·¸ëì","ë","ëë"}
+ );
+
+ CharArraySet stopSet = new CharArraySet(Version.LUCENE_42, stopWords.size(), false);
+
+ stopSet.addAll(stopWords);
+ STOP_WORDS_SET = CharArraySet.unmodifiableSet(stopSet);
+ }
+
+ public KoreanAnalyzer() {
+ this(Version.LUCENE_42, STOP_WORDS_SET);
+ }
+
+ /**
+ * ê²ìì ìí ííìë¶ì
+ */
+ public KoreanAnalyzer(boolean exactMatch) {
+ this(Version.LUCENE_42, STOP_WORDS_SET);
+ this.exactMatch = exactMatch;
+ }
+
+ public KoreanAnalyzer(Version matchVersion, String[] stopWords) throws IOException {
+ this(matchVersion, StopFilter.makeStopSet(matchVersion, stopWords));
+ }
+
+ public KoreanAnalyzer(Version matchVersion) throws IOException {
+ this(matchVersion, STOP_WORDS_SET);
+ }
+
+ public KoreanAnalyzer(Version matchVersion, File stopwords) throws IOException {
+ this(matchVersion, loadStopwordSet(stopwords, matchVersion));
+ }
+
+ public KoreanAnalyzer(Version matchVersion, File stopwords, String encoding) throws IOException {
+ this(matchVersion, loadStopwordSet(stopwords, matchVersion));
+ }
+
+ public KoreanAnalyzer(Version matchVersion, Reader stopwords) throws IOException {
+ this(matchVersion, loadStopwordSet(stopwords, matchVersion));
+ }
+
+ public KoreanAnalyzer(Version matchVersion, CharArraySet stopWords) {
+ super(matchVersion, stopWords);
+ replaceInvalidAcronym = matchVersion.onOrAfter(Version.LUCENE_42);
+ }
+
+
+ @Override
+ protected TokenStreamComponents createComponents(final String fieldName, final Reader reader) {
+ final KoreanTokenizer src = new KoreanTokenizer(matchVersion, reader);
+ src.setMaxTokenLength(maxTokenLength);
+ TokenStream tok = new KoreanFilter(src, bigrammable, hasOrigin, exactMatch, originCNoun);
+ tok = new LowerCaseFilter(matchVersion, tok);
+ tok = new StopFilter(matchVersion, tok, stopwords);
+ return new TokenStreamComponents(src, tok) {
+ @Override
+ protected void setReader(final Reader reader) throws IOException {
+ src.setMaxTokenLength(KoreanAnalyzer.this.maxTokenLength);
+ super.setReader(reader);
+ }
+ };
+ }
+
+ /**
+ * determine whether the bigram index term is returned or not if a input word is failed to analysis
+ * If true is set, the bigram index term is returned. If false is set, the bigram index term is not returned.
+ */
+ public void setBigrammable(boolean is) {
+ bigrammable = is;
+ }
+
+ /**
+ * determin whether the original term is returned or not if a input word is analyzed morphically.
+ */
+ public void setHasOrigin(boolean has) {
+ hasOrigin = has;
+ }
+
+ /**
+ * determin whether the original compound noun is returned or not if a input word is analyzed morphically.
+ */
+ public void setOriginCNoun(boolean cnoun) {
+ originCNoun = cnoun;
+ }
+
+ /**
+ * determin whether the original compound noun is returned or not if a input word is analyzed morphically.
+ */
+ public void setExactMatch(boolean exact) {
+ exactMatch = exact;
+ }
+
+}
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanFilter.java?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanFilter.java (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanFilter.java Sun May 5 03:39:51 2013
@@ -0,0 +1,447 @@
+package org.apache.lucene.analysis.kr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Stack;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.kr.morph.AnalysisOutput;
+import org.apache.lucene.analysis.kr.morph.CompoundEntry;
+import org.apache.lucene.analysis.kr.morph.CompoundNounAnalyzer;
+import org.apache.lucene.analysis.kr.morph.MorphAnalyzer;
+import org.apache.lucene.analysis.kr.morph.MorphException;
+import org.apache.lucene.analysis.kr.morph.PatternConstants;
+import org.apache.lucene.analysis.kr.morph.WordEntry;
+import org.apache.lucene.analysis.kr.morph.WordSpaceAnalyzer;
+import org.apache.lucene.analysis.kr.utils.DictionaryUtil;
+import org.apache.lucene.analysis.kr.utils.HanjaUtils;
+import org.apache.lucene.analysis.standard.ClassicTokenizer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionLengthAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.util.Version;
+
+public class KoreanFilter extends TokenFilter {
+
+ private LinkedList<IndexWord> morphQueue;
+
+ private MorphAnalyzer morph;
+
+ private WordSpaceAnalyzer wsAnal;
+
+ private boolean bigrammable = true;
+
+ private boolean hasOrigin = true;
+
+ private boolean originCNoun = true;
+
+ private boolean exactMatch = false;
+
+ private char[] curTermBuffer;
+
+ private int curTermLength;
+
+ private String curType;
+
+ private String curSource;
+
+ private int tokStart;
+
+ private int hanStart = 0; // íê¸ì ìì ìì¹, ë³µí©ëª
ì¬ì¼ê²½ì°
+
+ private int chStart = 0;
+
+ private CompoundNounAnalyzer cnAnalyzer = new CompoundNounAnalyzer();
+
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+ private final PositionLengthAttribute posLenAtt = addAttribute(PositionLengthAttribute.class);
+ private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+
+ private static final String APOSTROPHE_TYPE = ClassicTokenizer.TOKEN_TYPES[ClassicTokenizer.APOSTROPHE];
+ private static final String ACRONYM_TYPE = ClassicTokenizer.TOKEN_TYPES[ClassicTokenizer.ACRONYM];
+
+ public KoreanFilter(TokenStream input) {
+ super(input);
+ morphQueue = new LinkedList<IndexWord>();
+ morph = new MorphAnalyzer();
+ wsAnal = new WordSpaceAnalyzer();
+ cnAnalyzer.setExactMach(false);
+ }
+
+ /**
+ *
+ * @param input input token stream
+ * @param bigram Whether the bigram index term return or not.
+ */
+ public KoreanFilter(TokenStream input, boolean bigram) {
+ this(input);
+ bigrammable = bigram;
+ }
+
+ public KoreanFilter(TokenStream input, boolean bigram, boolean has) {
+ this(input, bigram);
+ hasOrigin = has;
+ }
+
+ public KoreanFilter(TokenStream input, boolean bigram, boolean has, boolean match) {
+ this(input, bigram,has);
+ this.exactMatch = match;
+ }
+
+ public KoreanFilter(TokenStream input, boolean bigram, boolean has, boolean match, boolean cnoun) {
+ this(input, bigram,has, match);
+ this.originCNoun = cnoun;
+ }
+
+ public final boolean incrementToken() throws IOException {
+
+ if(curTermBuffer!=null&&morphQueue.size()>0) {
+ setTermBufferByQueue(false);
+ return true;
+ }
+
+ if(!input.incrementToken()) return false;
+
+ curTermBuffer = termAtt.buffer().clone();
+ curTermLength = termAtt.length();
+ tokStart = offsetAtt.startOffset();
+ curType = typeAtt.type();
+
+ try {
+ if(KoreanTokenizer.TOKEN_TYPES[KoreanTokenizer.KOREAN].equals(curType)) {
+ analysisKorean(new String(curTermBuffer,0,termAtt.length()));
+ } else if(KoreanTokenizer.TOKEN_TYPES[KoreanTokenizer.CHINESE].equals(curType)) {
+ analysisChinese(new String(curTermBuffer,0,termAtt.length()));
+ } else {
+ analysisETC(new String(curTermBuffer,0,termAtt.length()));
+ }
+ }catch(MorphException e) {
+ throw new IOException("Korean Filter MorphException\n"+e.getMessage());
+ }
+
+ if(morphQueue!=null&&morphQueue.size()>0) {
+ setTermBufferByQueue(true);
+ } else {
+ return incrementToken();
+ }
+
+ return true;
+
+ }
+
+ /**
+ * queueì ì ì¥ë ê°ì¼ë¡ bufferì ê°ì ë³µì¬íë¤.
+ */
+ private void setTermBufferByQueue(boolean isFirst) {
+
+ clearAttributes();
+
+ IndexWord iw = morphQueue.removeFirst();
+ int pos = iw.getOffset();
+
+ termAtt.copyBuffer(iw.getWord().toCharArray(), 0, iw.getWord().length());
+ offsetAtt.setOffset(tokStart+pos, tokStart + pos + iw.getWord().length());
+
+ if(!isFirst && iw.getOffset()==0) {
+ posIncrAtt.setPositionIncrement(0);
+// posLenAtt.setPositionLength(iw.getWord().length());
+ }
+
+ }
+
+ /**
+ * íê¸ì ë¶ìíë¤.
+ * @throws MorphException
+ */
+ private void analysisKorean(String input) throws MorphException {
+
+ List<AnalysisOutput> outputs = morph.analyze(input);
+ if(outputs.size()==0) return;
+
+ Map<String,IndexWord> map = new LinkedHashMap<String,IndexWord>();
+ if(hasOrigin) map.put(input, new IndexWord(input,0));
+
+ if(outputs.get(0).getScore()>=AnalysisOutput.SCORE_COMPOUNDS)
+ {
+ extractKeyword(outputs, map);
+ }
+ else
+ {
+ try
+ {
+ List<AnalysisOutput> list = wsAnal.analyze(input);
+
+ List<AnalysisOutput> results = new ArrayList<AnalysisOutput>();
+ if(list.size()>1) {
+ for(AnalysisOutput o : list) {
+ if(hasOrigin) map.put(o.getSource(), new IndexWord(o.getSource(),0));
+ results.addAll(morph.analyze(o.getSource()));
+ }
+ } else {
+ results.addAll(list);
+ }
+
+ extractKeyword(results, map);
+
+ }
+ catch(Exception e)
+ {
+ extractKeyword(outputs, map);
+ }
+
+ }
+
+ Iterator<String> iter = map.keySet().iterator();
+
+ while(iter.hasNext()) {
+ String text = iter.next();
+ if(text.length()<=1) continue;
+ morphQueue.add(map.get(text));
+ }
+
+ }
+
+ private void extractKeyword(List<AnalysisOutput> outputs, Map<String,IndexWord> map) throws MorphException {
+
+ for(AnalysisOutput output : outputs) {
+
+ if(output.getPos()!=PatternConstants.POS_VERB) {
+ if(originCNoun || (!originCNoun&&output.getCNounList().size()==0)) {
+ map.put(output.getStem(), new IndexWord(output.getStem(),0));
+ }
+// }else {
+// map.put(output.getStem()+"ë¤", new Integer(1));
+ }
+
+ if(exactMatch) continue;
+
+ if(output.getScore()>=AnalysisOutput.SCORE_COMPOUNDS) {
+
+ List<CompoundEntry> cnouns = output.getCNounList();
+ int start = 0;
+
+ for(int jj=0;jj<cnouns.size();jj++) {
+
+ CompoundEntry cnoun = cnouns.get(jj);
+
+ if(cnoun.getWord().length()>1) map.put(cnoun.getWord(), new IndexWord(cnoun.getWord(), start));
+
+ if(jj==0 && cnoun.getWord().length()==1) {
+ map.put(cnoun.getWord()+cnouns.get(jj+1).getWord(), new IndexWord(cnoun.getWord(), start));
+ } else if(jj>1 && cnoun.getWord().length()==1) {
+ String iw = cnouns.get(jj-1).getWord()+cnoun.getWord();
+ map.put(iw, new IndexWord(iw, start-cnouns.get(jj-1).getWord().length()));
+ }
+
+ start += cnoun.getWord().length();
+ }
+
+ } else if(bigrammable) {
+ addBiagramToMap(output.getStem(),map);
+ }
+
+ }
+
+ }
+
+ private void addBiagramToMap(String input, Map<String,IndexWord> map) {
+
+ int offset = 0;
+ int strlen = input.length();
+
+ while(offset<strlen-1) {
+
+ if(isAlphaNumChar(input.charAt(offset))) {
+ String text = findAlphaNumeric(input.substring(offset));
+ map.put(text, new IndexWord(text, offset));
+ offset += text.length();
+ } else {
+ String text = input.substring(offset,
+ offset+2>strlen?strlen:offset+2);
+ map.put(text, new IndexWord(text, offset));
+ offset++;
+ }
+
+ }
+ }
+
+ private String findAlphaNumeric(String text) {
+ int pos = 0;
+ for(int i=0;i<text.length();i++) {
+ if(!isAlphaNumChar(text.charAt(i))) break;
+ pos++;
+ }
+ return text.substring(0,pos);
+ }
+
+ /**
+ * íìë 2ê°ì´ìì íê¸ ìì¼ë¡ ì½íì§ ì ìë¤.
+ * ëìë²ì¹ì´ ìë.
+ * @param term
+ * @throws MorphException
+ */
+ private void analysisChinese(String term) throws MorphException {
+
+ morphQueue.add(new IndexWord(term,0));
+ if(term.length()<2) return; // 1ê¸ì íìë ìì¸ì´ë¡ íê¸ì ì¶ì¶íì§ ìëë¤.
+
+ List<StringBuffer> candiList = new ArrayList<StringBuffer>();
+ candiList.add(new StringBuffer());
+
+
+
+ for(int i=0;i<term.length();i++) {
+
+ char[] chs = HanjaUtils.convertToHangul(term.charAt(i));
+ if(chs==null) continue;
+
+ List<StringBuffer> removeList = new ArrayList<StringBuffer>(); // ì ê±°ë í보를 ì ì¥
+
+ int caniSize = candiList.size();
+
+ for(int j=0;j<caniSize;j++) {
+ String origin = candiList.get(j).toString();
+
+ for(int k=0;k<chs.length;k++) { // ì¶ê°ë¡ ìì±ë ìì ëí´ì ìë¡ì´ í
ì¤í¸ë¥¼ ìì±íë¤.
+
+ if(k==4) break; // 4ê° ì´ìì ìì ê°ì§ê³ ìë ê²½ì° ì²«ë²ì§¸ ìì¼ë¡ë§ ì²ë¦¬ë¥¼ íë¤.
+
+ StringBuffer sb = candiList.get(j);
+ if(k>0) sb = new StringBuffer(origin);
+
+ sb.append(chs[k]);
+ if(k>0) candiList.add(sb);
+
+ Iterator iter = DictionaryUtil.findWithPrefix(sb.toString());
+ if(!iter.hasNext()) // ì¬ì ì ìì¼ë©´ ìì íë³´
+ removeList.add(sb);
+
+ }
+
+ }
+
+ if(removeList.size()==candiList.size()) { // ì¬ì ìì ì°¾ì ë¨ì´ê° íëë ìë¤ë©´..
+ candiList = candiList.subList(0, 1); // 첫ë²ì§¸ë§ ìì±íê³ ë머ì§ë ë²ë¦¼
+ }
+
+ for(StringBuffer rsb : removeList) {
+ if(candiList.size()>1) candiList.remove(rsb);
+ }
+
+ }
+
+ int maxCandidate = 5;
+ if(candiList.size()<maxCandidate) maxCandidate=candiList.size();
+
+ for(int i=0;i<maxCandidate;i++) {
+ morphQueue.add(new IndexWord(candiList.get(i).toString(),0));
+ }
+
+ Map<String, String> cnounMap = new HashMap<String, String>();
+
+ // ì¶ì¶ë ëª
ì¬ê° ë³µí©ëª
ì¬ì¸ ê²½ì° ë¶ë¦¬íë¤.
+ for(int i=0;i<maxCandidate;i++) {
+ List<CompoundEntry> results = confirmCNoun(candiList.get(i).toString());
+
+ int pos = 0;
+ int offset = 0;
+ for(CompoundEntry entry : results) {
+ pos += entry.getWord().length();
+ if(cnounMap.get(entry.getWord())!=null) continue;
+
+ // íê¸ê³¼ 매ì¹ëë íì를 짤ë¼ì íì ì ì¥íë¤.
+ morphQueue.add(new IndexWord(term.substring(offset,pos),offset));
+
+ cnounMap.put(entry.getWord(), entry.getWord());
+
+ if(entry.getWord().length()<2) continue; // íê¸ì 2ê¸ì ì´ìë§ ì ì¥íë¤.
+
+ // ë¶ë¦¬ë íê¸ì íì ì ì¥íë¤.
+ morphQueue.add(new IndexWord(entry.getWord(),offset));
+
+ offset = pos;
+ }
+ }
+ }
+
+ private List<CompoundEntry> confirmCNoun(String input) throws MorphException {
+
+ WordEntry cnoun = DictionaryUtil.getCNoun(input);
+ if(cnoun!=null && cnoun.getFeature(WordEntry.IDX_NOUN)=='2') {
+ return cnoun.getCompounds();
+ }
+
+ return cnAnalyzer.analyze(input);
+
+ }
+
+ private void analysisETC(String term) throws MorphException {
+
+ final char[] buffer = termAtt.buffer();
+ final int bufferLength = termAtt.length();
+ final String type = typeAtt.type();
+
+ if (type == APOSTROPHE_TYPE && // remove 's
+ bufferLength >= 2 &&
+ buffer[bufferLength-2] == '\'' &&
+ (buffer[bufferLength-1] == 's' || buffer[bufferLength-1] == 'S')) {
+ // Strip last 2 characters off
+ morphQueue.add(new IndexWord(term.substring(0,bufferLength - 2),0));
+ } else if (type == ACRONYM_TYPE) { // remove dots
+ int upto = 0;
+ for(int i=0;i<bufferLength;i++) {
+ char c = buffer[i];
+ if (c != '.')
+ buffer[upto++] = c;
+ }
+ morphQueue.add(new IndexWord(term.substring(0,upto),0));
+ } else {
+ morphQueue.add(new IndexWord(term,0));
+ }
+
+ }
+
+ private boolean isAlphaNumChar(int c) {
+ if((c>=48&&c<=57)||(c>=65&&c<=122)) return true;
+ return false;
+ }
+
+ public void setHasOrigin(boolean has) {
+ hasOrigin = has;
+ }
+
+ public void setExactMatch(boolean match) {
+ this.exactMatch = match;
+ }
+}
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanFilterFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanFilterFactory.java?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanFilterFactory.java (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanFilterFactory.java Sun May 5 03:39:51 2013
@@ -0,0 +1,77 @@
+package org.apache.lucene.analysis.kr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.util.Map;
+
+import org.apache.lucene.analysis.TokenStream;
+
+
+import org.apache.lucene.analysis.kr.KoreanFilter;
+import org.apache.lucene.analysis.util.TokenFilterFactory;
+
+public class KoreanFilterFactory extends TokenFilterFactory {
+
+ private boolean bigrammable = true;
+
+ private boolean hasOrigin = true;
+
+ private boolean hasCNoun = true;
+
+ private boolean exactMatch = false;
+
+ /**
+ * Initialize this factory via a set of key-value pairs.
+ */
+ protected KoreanFilterFactory(Map<String, String> args) {
+ super(args);
+ init(args);
+ }
+
+
+ public void init(Map<String, String> args) {
+// bigrammable = getBoolean("bigrammable", true);
+// hasOrigin = getBoolean("hasOrigin", true);
+// exactMatch = getBoolean("exactMatch", false);
+// hasCNoun = getBoolean("hasCNoun", true);
+ }
+
+ public TokenStream create(TokenStream tokenstream) {
+ return new KoreanFilter(tokenstream, bigrammable, hasOrigin, exactMatch, hasCNoun);
+ }
+
+ public void setBigrammable(boolean bool) {
+ this.bigrammable = bool;
+ }
+
+ public void setHasOrigin(boolean bool) {
+ this.hasOrigin = bool;
+ }
+
+ public void setHasCNoun(boolean bool) {
+ this.hasCNoun = bool;
+ }
+
+ /**
+ * determin whether the original compound noun is returned or not if a input word is analyzed morphically.
+// * @param has
+ */
+ public void setExactMatch(boolean bool) {
+ exactMatch = bool;
+ }
+}
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizer.java?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizer.java (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizer.java Sun May 5 03:39:51 2013
@@ -0,0 +1,178 @@
+package org.apache.lucene.analysis.kr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.IOException;
+
+import java.io.Reader;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.kr.morph.AnalysisOutput;
+import org.apache.lucene.analysis.kr.morph.CompoundEntry;
+import org.apache.lucene.analysis.kr.morph.MorphAnalyzer;
+import org.apache.lucene.analysis.kr.morph.MorphException;
+import org.apache.lucene.analysis.kr.morph.PatternConstants;
+import org.apache.lucene.analysis.kr.morph.WordSpaceAnalyzer;
+import org.apache.lucene.analysis.kr.utils.DictionaryUtil;
+import org.apache.lucene.analysis.standard.StandardAnalyzer;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
+import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
+import org.apache.lucene.analysis.tokenattributes.TypeAttribute;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.Version;
+
+public class KoreanTokenizer extends Tokenizer {
+
+ /** A private instance of the JFlex-constructed scanner */
+ private final KoreanTokenizerImpl scanner;
+
+ public static final int ALPHANUM = 0;
+ public static final int APOSTROPHE = 1;
+ public static final int ACRONYM = 2;
+ public static final int COMPANY = 3;
+ public static final int EMAIL = 4;
+ public static final int HOST = 5;
+ public static final int NUM = 6;
+ public static final int CJ = 7;
+ public static final int ACRONYM_DEP = 8;
+ public static final int KOREAN = 9;
+ public static final int CHINESE = 10;
+
+
+ /** String token types that correspond to token type int constants */
+ public static final String [] TOKEN_TYPES = new String [] {
+ "<ALPHANUM>",
+ "<APOSTROPHE>",
+ "<ACRONYM>",
+ "<COMPANY>",
+ "<EMAIL>",
+ "<HOST>",
+ "<NUM>",
+ "<CJ>",
+ "<ACRONYM_DEP>",
+ "<KOREAN>",
+ "<CHINESE>"
+ };
+
+ private boolean replaceInvalidAcronym;
+
+ private int maxTokenLength = StandardAnalyzer.DEFAULT_MAX_TOKEN_LENGTH;
+
+ /** Set the max allowed token length. Any token longer
+ * than this is skipped. */
+ public void setMaxTokenLength(int length) {
+ this.maxTokenLength = length;
+ }
+
+ /** @see #setMaxTokenLength */
+ public int getMaxTokenLength() {
+ return maxTokenLength;
+ }
+
+ /**
+ * Creates a new instance of the {@link org.apache.lucene.analysis.standard.StandardTokenizer}. Attaches
+ * the <code>input</code> to the newly created JFlex scanner.
+ *
+ * @param input The input reader
+ *
+ * See http://issues.apache.org/jira/browse/LUCENE-1068
+ */
+ public KoreanTokenizer(Version matchVersion, Reader input) {
+ super(input);
+ this.scanner = new KoreanTokenizerImpl(input);
+ init(input, matchVersion);
+ }
+
+ /**
+ * Creates a new StandardTokenizer with a given {@link org.apache.lucene.util.AttributeSource.AttributeFactory}
+ */
+ public KoreanTokenizer(Version matchVersion, AttributeFactory factory, Reader input) {
+ super(factory, input);
+ this.scanner = new KoreanTokenizerImpl(input);
+ init(input, matchVersion);
+ }
+
+ private final void init(Reader input, Version matchVersion) {
+ if (matchVersion.onOrAfter(Version.LUCENE_42)) {
+ replaceInvalidAcronym = true;
+ } else {
+ replaceInvalidAcronym = false;
+ }
+ this.input = input;
+ }
+
+ // this tokenizer generates three attributes:
+ // term offset, positionIncrement and type
+ private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final OffsetAttribute offsetAtt = addAttribute(OffsetAttribute.class);
+ private final PositionIncrementAttribute posIncrAtt = addAttribute(PositionIncrementAttribute.class);
+ private final TypeAttribute typeAtt = addAttribute(TypeAttribute.class);
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see org.apache.lucene.analysis.TokenStream#next()
+ */
+ @Override
+ public final boolean incrementToken() throws IOException {
+ clearAttributes();
+ int posIncr = 1;
+
+ while(true) {
+ int tokenType = scanner.getNextToken();
+
+ if (tokenType == KoreanTokenizerImpl.YYEOF) {
+ return false;
+ }
+
+ if (scanner.yylength() <= maxTokenLength) {
+ posIncrAtt.setPositionIncrement(posIncr);
+ scanner.getText(termAtt);
+ final int start = scanner.yychar();
+ offsetAtt.setOffset(correctOffset(start), correctOffset(start+termAtt.length()));
+ typeAtt.setType(KoreanTokenizer.TOKEN_TYPES[tokenType]);
+
+ return true;
+ } else
+ // When we skip a too-long term, we still increment the
+ // position increment
+ posIncr++;
+ }
+ }
+
+ @Override
+ public final void end() {
+ // set final offset
+ int finalOffset = correctOffset(scanner.yychar() + scanner.yylength());
+ offsetAtt.setOffset(finalOffset, finalOffset);
+ }
+
+ @Override
+ public void reset() throws IOException {
+ scanner.yyreset(input);
+ }
+
+
+}
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerFactory.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerFactory.java?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerFactory.java (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerFactory.java Sun May 5 03:39:51 2013
@@ -0,0 +1,59 @@
+package org.apache.lucene.analysis.kr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.Reader;
+import java.util.Map;
+
+
+import org.apache.lucene.analysis.Tokenizer;
+import org.apache.lucene.analysis.kr.KoreanTokenizer;
+import org.apache.lucene.analysis.util.TokenizerFactory;
+import org.apache.lucene.util.AttributeSource;
+import org.apache.lucene.util.Version;
+
+public class KoreanTokenizerFactory extends TokenizerFactory {
+
+ private Version version;
+
+ /**
+ * Initialize this factory via a set of key-value pairs.
+ */
+ protected KoreanTokenizerFactory(Map<String, String> args) {
+ super(args);
+ }
+
+ @Override
+ public Tokenizer create(AttributeSource.AttributeFactory factory, Reader input) {
+ return null; //To change body of implemented methods use File | Settings | File Templates.
+ }
+
+// public KoreanTokenizerFactory() {
+// version = Version.LUCENE_42;
+// }
+//
+//
+// public KoreanTokenizerFactory(Version v) {
+// version = v;
+// }
+//
+// public Tokenizer create(Reader input) {
+// return new KoreanTokenizer(version, input);
+// }
+
+}
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerImpl.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerImpl.java?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerImpl.java (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerImpl.java Sun May 5 03:39:51 2013
@@ -0,0 +1,836 @@
+/* The following code was generated by JFlex 1.4.1 on 12. 1. 3 ì¤ì 3:51 */
+
+package org.apache.lucene.analysis.kr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+
+/**
+ * This class is a scanner generated by
+ * <a href="http://www.jflex.de/">JFlex</a> 1.4.1
+ * on 12. 1. 3 ì¤ì 3:51 from the specification file
+ * <tt>D:/eclipse-workspace/search/kr.analyzer.3x/src/org/apache/lucene/analysis/kr/KoreanTokenizerImpl.jflex</tt>
+ */
+class KoreanTokenizerImpl {
+
+ /** This character denotes the end of file */
+ public static final int YYEOF = -1;
+
+ /** initial size of the lookahead buffer */
+ private static final int ZZ_BUFFERSIZE = 16384;
+
+ /** lexical states */
+ public static final int YYINITIAL = 0;
+
+ /**
+ * Translates characters to character classes
+ */
+ private static final String ZZ_CMAP_PACKED =
+ "\11\0\1\0\1\20\1\0\1\0\1\17\22\0\1\0\5\0\1\3"+
+ "\1\1\4\0\1\7\1\5\1\2\1\7\12\11\6\0\1\4\32\10"+
+ "\4\0\1\6\1\0\32\10\105\0\27\10\1\0\37\10\1\0\u0568\10"+
+ "\12\12\206\10\12\12\u026c\10\12\12\166\10\12\12\166\10\12\12\166\10"+
+ "\12\12\166\10\12\12\167\10\11\12\166\10\12\12\166\10\12\12\166\10"+
+ "\12\12\340\10\12\12\166\10\12\12\u0166\10\12\12\266\10\u0100\14\u0e00\10"+
+ "\u1040\0\u0150\16\140\0\20\16\u0100\0\200\16\200\0\u19c0\15\100\0\u5200\15"+
+ "\u0c00\0\u2bb0\13\u2150\0\u0200\15\u0465\0\73\16\75\10\43\0";
+
+ /**
+ * Translates characters to character classes
+ */
+ private static final char [] ZZ_CMAP = zzUnpackCMap(ZZ_CMAP_PACKED);
+
+ /**
+ * Translates DFA states to action switch labels.
+ */
+ private static final int [] ZZ_ACTION = zzUnpackAction();
+
+ private static final String ZZ_ACTION_PACKED_0 =
+ "\1\0\2\1\3\2\2\3\1\4\1\5\1\1\1\6"+
+ "\6\0\3\2\1\3\1\6\3\0\1\2\1\3\1\6"+
+ "\1\2\4\3\1\4\3\0\1\3\1\7\3\10\2\11"+
+ "\2\0\2\6\2\0\2\10\1\3\4\6\1\0\2\10"+
+ "\1\3\4\0\2\3\1\12\1\0\1\10\1\13\2\0"+
+ "\2\10\1\3\1\12\1\0\2\6\1\3\2\6\1\3"+
+ "\2\13\1\10\2\3\1\6\2\3\1\6\2\3\2\12"+
+ "\1\0\2\10\1\0\1\13\1\10\2\6\1\13\2\0"+
+ "\1\13\1\3\1\10\1\14\1\0\1\3\1\0\1\3"+
+ "\1\13\1\0\1\3\1\0\1\3\1\0\1\10\1\3"+
+ "\1\6\1\3\1\6\1\3\1\10\2\6\1\13\2\0"+
+ "\2\3\1\0\2\3\1\0\2\3\1\13\1\10\1\3"+
+ "\2\0\2\3\1\13\1\0\2\10\2\0\1\13\1\0"+
+ "\2\10\1\3\2\0\1\3\1\10\1\0\1\13\1\0"+
+ "\1\13\1\3\1\0\3\3";
+
+ private static int [] zzUnpackAction() {
+ int [] result = new int[174];
+ int offset = 0;
+ offset = zzUnpackAction(ZZ_ACTION_PACKED_0, offset, result);
+ return result;
+ }
+
+ private static int zzUnpackAction(String packed, int offset, int [] result) {
+ int i = 0; /* index in packed string */
+ int j = offset; /* index in unpacked array */
+ int l = packed.length();
+ while (i < l) {
+ int count = packed.charAt(i++);
+ int value = packed.charAt(i++);
+ do result[j++] = value; while (--count > 0);
+ }
+ return j;
+ }
+
+
+ /**
+ * Translates a state to a row index in the transition table
+ */
+ private static final int [] ZZ_ROWMAP = zzUnpackRowMap();
+
+ private static final String ZZ_ROWMAP_PACKED_0 =
+ "\0\0\0\21\0\42\0\63\0\104\0\125\0\146\0\167"+
+ "\0\210\0\231\0\252\0\273\0\314\0\335\0\356\0\377"+
+ "\0\u0110\0\u0121\0\u0132\0\u0143\0\u0154\0\u0165\0\u0176\0\u0187"+
+ "\0\u0198\0\u01a9\0\u01ba\0\u01cb\0\u01dc\0\u01ed\0\u01fe\0\u020f"+
+ "\0\u0220\0\u0231\0\u0242\0\u0253\0\u0264\0\u0275\0\u0286\0\u0297"+
+ "\0\u02a8\0\u02b9\0\u02ca\0\356\0\u02db\0\u02ec\0\u02fd\0\u030e"+
+ "\0\u031f\0\u0330\0\u0176\0\u0341\0\u0352\0\u0363\0\u0374\0\u0385"+
+ "\0\u0396\0\u03a7\0\u03b8\0\u03c9\0\u03da\0\u03eb\0\273\0\u03fc"+
+ "\0\u040d\0\u041e\0\u042f\0\u0440\0\u0451\0\u0462\0\u0473\0\u0484"+
+ "\0\u0495\0\u04a6\0\u04b7\0\u04c8\0\u04d9\0\u04ea\0\u04fb\0\u050c"+
+ "\0\u051d\0\u052e\0\u053f\0\u0550\0\u0561\0\u0330\0\u0176\0\u0572"+
+ "\0\u0583\0\u0594\0\u05a5\0\u05b6\0\u05c7\0\u05d8\0\u05e9\0\u05fa"+
+ "\0\335\0\u01dc\0\u060b\0\u061c\0\u062d\0\u063e\0\u064f\0\u0660"+
+ "\0\u0671\0\u0682\0\u0693\0\u06a4\0\u06b5\0\u06c6\0\u06d7\0\u06e8"+
+ "\0\u06f9\0\u070a\0\u071b\0\u072c\0\u073d\0\u03b8\0\u074e\0\u075f"+
+ "\0\u0770\0\u0781\0\u0792\0\u07a3\0\u07b4\0\u07c5\0\u07d6\0\u07e7"+
+ "\0\u07f8\0\u0809\0\u081a\0\u082b\0\u083c\0\u084d\0\u085e\0\u086f"+
+ "\0\u0880\0\u0891\0\u08a2\0\u08b3\0\u08c4\0\u08d5\0\u08e6\0\u08f7"+
+ "\0\u0908\0\u0919\0\u092a\0\u093b\0\u094c\0\u095d\0\u096e\0\u097f"+
+ "\0\u0990\0\u09a1\0\u09b2\0\u09c3\0\u09d4\0\u09e5\0\u09f6\0\u0a07"+
+ "\0\u0a18\0\u0a29\0\u0a3a\0\u0a4b\0\u0a5c\0\u0a6d\0\u0a7e\0\u0a8f"+
+ "\0\u0aa0\0\u0ab1\0\u0ac2\0\u0ad3\0\u0ae4\0\u0af5";
+
+ private static int [] zzUnpackRowMap() {
+ int [] result = new int[174];
+ int offset = 0;
+ offset = zzUnpackRowMap(ZZ_ROWMAP_PACKED_0, offset, result);
+ return result;
+ }
+
+ private static int zzUnpackRowMap(String packed, int offset, int [] result) {
+ int i = 0; /* index in packed string */
+ int j = offset; /* index in unpacked array */
+ int l = packed.length();
+ while (i < l) {
+ int high = packed.charAt(i++) << 16;
+ result[j++] = high | packed.charAt(i++);
+ }
+ return j;
+ }
+
+ /**
+ * The transition table of the DFA
+ */
+ private static final int [] ZZ_TRANS = zzUnpackTrans();
+
+ private static final String ZZ_TRANS_PACKED_0 =
+ "\2\2\1\3\5\2\1\4\1\5\1\6\1\7\1\10"+
+ "\1\11\1\12\1\13\1\2\32\0\2\14\7\0\1\15"+
+ "\1\16\1\17\1\20\2\21\1\22\1\23\1\24\1\25"+
+ "\1\7\1\26\6\0\1\27\1\0\1\30\2\31\1\32"+
+ "\1\33\2\24\1\7\1\34\5\0\1\15\1\35\1\17"+
+ "\1\20\2\31\1\32\1\36\1\24\1\25\1\7\1\37"+
+ "\14\0\3\40\2\7\5\0\1\15\1\16\1\17\1\20"+
+ "\2\21\1\22\1\26\1\41\1\42\1\7\1\26\14\0"+
+ "\3\43\1\0\1\43\1\11\21\0\1\12\22\0\1\2"+
+ "\2\0\1\44\5\0\1\45\2\46\1\7\1\47\14\0"+
+ "\1\50\1\0\1\50\1\0\1\50\14\0\1\51\1\52"+
+ "\1\53\1\0\1\51\14\0\1\54\1\0\1\54\1\0"+
+ "\1\54\14\0\1\55\1\56\1\55\1\0\1\55\14\0"+
+ "\1\57\2\60\1\0\1\57\14\0\1\22\2\61\1\0"+
+ "\1\22\5\0\1\15\1\62\1\17\1\20\2\21\1\22"+
+ "\1\23\1\24\1\25\1\7\1\26\6\0\1\63\1\0"+
+ "\1\30\2\31\1\32\1\33\2\24\1\7\1\34\5\0"+
+ "\1\15\1\63\1\17\1\20\2\31\1\32\1\36\1\24"+
+ "\1\25\1\7\1\37\5\0\1\15\1\62\1\17\1\20"+
+ "\2\21\1\22\1\26\1\41\1\42\1\7\1\26\6\0"+
+ "\1\44\5\0\1\64\2\65\1\7\1\66\14\0\3\56"+
+ "\1\0\1\56\14\0\1\67\2\70\1\0\1\67\14\0"+
+ "\1\71\2\72\1\0\1\71\6\0\1\73\1\0\1\30"+
+ "\2\31\1\32\1\33\2\24\1\7\1\34\6\0\1\73"+
+ "\1\0\1\30\2\31\1\32\1\34\2\41\1\7\1\34"+
+ "\6\0\1\44\5\0\1\74\1\65\1\75\1\7\1\76"+
+ "\5\0\1\15\1\73\1\17\1\20\2\31\1\32\1\36"+
+ "\1\24\1\25\1\7\1\37\5\0\1\15\1\73\1\17"+
+ "\1\20\2\31\1\32\1\37\1\41\1\42\1\7\1\37"+
+ "\14\0\3\40\1\0\1\40\6\0\1\63\1\0\1\30"+
+ "\2\31\1\32\1\34\2\41\1\7\1\34\5\0\1\15"+
+ "\1\63\1\17\1\20\2\31\1\32\1\37\1\41\1\42"+
+ "\1\7\1\37\14\0\3\43\1\0\1\43\15\0\2\77"+
+ "\10\0\1\100\2\0\3\100\1\45\2\46\1\7\1\47"+
+ "\6\0\1\77\2\0\3\101\1\102\2\46\1\7\1\103"+
+ "\6\0\1\100\2\0\3\100\1\47\2\104\1\7\1\47"+
+ "\5\0\1\15\6\0\1\50\1\0\1\50\1\0\1\50"+
+ "\6\0\1\105\1\0\1\30\2\106\1\0\1\107\2\52"+
+ "\1\0\1\107\6\0\1\110\1\0\1\30\2\111\1\112"+
+ "\1\113\2\114\1\7\1\115\6\0\1\116\1\0\1\30"+
+ "\2\111\1\112\1\113\2\114\1\7\1\115\6\0\1\117"+
+ "\2\0\1\117\2\0\1\55\1\56\1\55\1\0\1\55"+
+ "\6\0\1\117\2\0\1\117\2\0\3\56\1\0\1\56"+
+ "\6\0\1\106\1\0\1\30\2\106\1\0\1\57\2\60"+
+ "\1\0\1\57\6\0\1\111\1\0\1\30\2\111\1\112"+
+ "\1\120\2\121\1\7\1\122\6\0\1\112\2\0\3\112"+
+ "\1\123\2\124\1\7\1\125\14\0\1\107\2\52\1\0"+
+ "\1\107\6\0\1\126\1\0\1\30\2\21\1\22\1\64"+
+ "\2\65\1\7\1\66\6\0\1\127\1\0\1\30\2\31"+
+ "\1\32\1\130\2\65\1\7\1\131\6\0\1\126\1\0"+
+ "\1\30\2\21\1\22\1\66\2\132\1\7\1\66\6\0"+
+ "\1\21\1\0\1\30\2\21\1\22\1\67\2\133\1\7"+
+ "\1\134\6\0\1\31\1\0\1\30\2\31\1\32\1\70"+
+ "\2\133\1\7\1\135\6\0\1\22\2\0\3\22\1\71"+
+ "\2\136\1\7\1\137\6\0\1\32\2\0\3\32\1\72"+
+ "\2\136\1\7\1\140\14\0\1\64\2\130\1\0\1\64"+
+ "\6\0\1\141\1\0\1\30\2\21\1\22\1\64\2\65"+
+ "\1\7\1\66\6\0\1\142\1\0\1\30\2\31\1\32"+
+ "\1\130\2\65\1\7\1\131\6\0\1\141\1\0\1\30"+
+ "\2\21\1\22\1\66\2\132\1\7\1\66\14\0\1\100"+
+ "\2\143\1\0\1\100\14\0\1\45\2\102\1\0\1\45"+
+ "\6\0\1\101\2\0\3\101\1\102\2\46\1\7\1\103"+
+ "\6\0\1\101\2\0\3\101\1\103\2\104\1\7\1\103"+
+ "\6\0\1\77\2\0\3\101\1\103\2\104\1\7\1\103"+
+ "\14\0\1\144\1\145\1\144\1\0\1\144\14\0\3\146"+
+ "\1\0\1\146\6\0\1\147\1\0\1\30\2\106\1\0"+
+ "\1\107\2\52\1\0\1\107\14\0\3\150\1\0\1\150"+
+ "\14\0\3\151\1\0\1\151\14\0\3\152\1\0\1\152"+
+ "\6\0\1\153\1\0\1\30\2\154\1\155\1\113\2\114"+
+ "\1\7\1\115\6\0\1\156\1\0\1\30\2\154\1\155"+
+ "\1\113\2\114\1\7\1\115\6\0\1\153\1\0\1\30"+
+ "\2\154\1\155\1\115\2\157\1\7\1\115\14\0\1\160"+
+ "\1\150\1\160\1\0\1\160\14\0\3\161\1\0\1\161"+
+ "\6\0\1\154\1\0\1\30\2\154\1\155\1\120\2\121"+
+ "\1\7\1\122\6\0\1\162\1\0\1\30\2\154\1\155"+
+ "\1\120\2\121\1\7\1\122\6\0\1\154\1\0\1\30"+
+ "\2\154\1\155\1\122\2\163\1\7\1\122\6\0\1\155"+
+ "\2\0\3\155\1\123\2\124\1\7\1\125\6\0\1\164"+
+ "\2\0\3\155\1\123\2\124\1\7\1\125\6\0\1\155"+
+ "\2\0\3\155\1\125\2\165\1\7\1\125\6\0\1\166"+
+ "\1\0\1\30\2\31\1\32\1\130\2\65\1\7\1\131"+
+ "\6\0\1\166\1\0\1\30\2\31\1\32\1\131\2\132"+
+ "\1\7\1\131\6\0\1\127\1\0\1\30\2\31\1\32"+
+ "\1\131\2\132\1\7\1\131\6\0\1\167\1\0\1\30"+
+ "\2\31\1\32\1\70\2\133\1\7\1\135\6\0\1\21"+
+ "\1\0\1\30\2\21\1\22\1\134\2\170\1\7\1\134"+
+ "\6\0\1\31\1\0\1\30\2\31\1\32\1\135\2\170"+
+ "\1\7\1\135\6\0\1\171\2\0\3\32\1\72\2\136"+
+ "\1\7\1\140\6\0\1\22\2\0\3\22\1\137\2\172"+
+ "\1\7\1\137\6\0\1\32\2\0\3\32\1\140\2\172"+
+ "\1\7\1\140\6\0\1\173\2\0\3\173\1\102\2\46"+
+ "\1\7\1\103\6\0\1\105\1\0\1\30\2\106\1\0"+
+ "\3\145\1\0\1\145\6\0\1\147\1\0\1\30\2\106"+
+ "\1\0\3\145\1\0\1\145\6\0\1\106\1\0\1\30"+
+ "\2\106\1\0\3\146\1\0\1\146\14\0\3\145\1\0"+
+ "\1\145\6\0\1\126\1\0\1\30\2\21\1\22\1\150"+
+ "\2\174\1\7\1\175\6\0\1\21\1\0\1\30\2\21"+
+ "\1\22\1\151\2\176\1\7\1\177\6\0\1\22\2\0"+
+ "\3\22\1\152\2\200\1\7\1\201\14\0\1\150\2\202"+
+ "\1\0\1\150\14\0\1\151\2\203\1\0\1\151\14\0"+
+ "\1\152\2\204\1\0\1\152\6\0\1\44\5\0\1\150"+
+ "\2\174\1\7\1\175\6\0\1\156\1\0\1\30\2\154"+
+ "\1\155\1\115\2\157\1\7\1\115\6\0\1\141\1\0"+
+ "\1\30\2\21\1\22\1\150\2\174\1\7\1\175\6\0"+
+ "\1\117\2\0\1\117\2\0\3\161\1\0\1\161\6\0"+
+ "\1\44\5\0\1\151\2\176\1\7\1\177\6\0\1\162"+
+ "\1\0\1\30\2\154\1\155\1\122\2\163\1\7\1\122"+
+ "\6\0\1\44\5\0\1\152\2\200\1\7\1\201\6\0"+
+ "\1\164\2\0\3\155\1\125\2\165\1\7\1\125\6\0"+
+ "\1\44\5\0\1\67\2\133\1\7\1\134\6\0\1\167"+
+ "\1\0\1\30\2\31\1\32\1\135\2\170\1\7\1\135"+
+ "\6\0\1\44\5\0\1\71\2\136\1\7\1\137\6\0"+
+ "\1\171\2\0\3\32\1\140\2\172\1\7\1\140\14\0"+
+ "\3\45\1\0\1\45\6\0\1\205\1\0\1\30\2\206"+
+ "\1\207\1\202\2\174\1\7\1\210\6\0\1\126\1\0"+
+ "\1\30\2\21\1\22\1\175\2\211\1\7\1\175\6\0"+
+ "\1\212\1\0\1\30\2\206\1\207\1\203\2\176\1\7"+
+ "\1\213\6\0\1\21\1\0\1\30\2\21\1\22\1\177"+
+ "\2\214\1\7\1\177\6\0\1\215\2\0\3\207\1\204"+
+ "\2\200\1\7\1\216\6\0\1\22\2\0\3\22\1\201"+
+ "\2\217\1\7\1\201\6\0\1\220\1\0\1\30\2\206"+
+ "\1\207\1\202\2\174\1\7\1\210\6\0\1\206\1\0"+
+ "\1\30\2\206\1\207\1\203\2\176\1\7\1\213\6\0"+
+ "\1\207\2\0\3\207\1\204\2\200\1\7\1\216\6\0"+
+ "\1\44\5\0\1\221\2\114\1\7\1\222\14\0\1\223"+
+ "\2\120\1\0\1\223\14\0\1\224\2\123\1\0\1\224"+
+ "\6\0\1\220\1\0\1\30\2\206\1\207\1\210\2\211"+
+ "\1\7\1\210\6\0\1\205\1\0\1\30\2\206\1\207"+
+ "\1\210\2\211\1\7\1\210\6\0\1\44\5\0\1\223"+
+ "\2\121\1\7\1\225\6\0\1\206\1\0\1\30\2\206"+
+ "\1\207\1\213\2\214\1\7\1\213\6\0\1\212\1\0"+
+ "\1\30\2\206\1\207\1\213\2\214\1\7\1\213\6\0"+
+ "\1\44\5\0\1\224\2\124\1\7\1\226\6\0\1\207"+
+ "\2\0\3\207\1\216\2\217\1\7\1\216\6\0\1\215"+
+ "\2\0\3\207\1\216\2\217\1\7\1\216\14\0\1\221"+
+ "\2\113\1\0\1\221\6\0\1\227\1\0\1\30\2\230"+
+ "\1\100\1\221\2\114\1\7\1\222\6\0\1\227\1\0"+
+ "\1\30\2\230\1\100\1\222\2\157\1\7\1\222\6\0"+
+ "\1\230\1\0\1\30\2\230\1\100\1\223\2\121\1\7"+
+ "\1\225\6\0\1\100\2\0\3\100\1\224\2\124\1\7"+
+ "\1\226\6\0\1\230\1\0\1\30\2\230\1\100\1\225"+
+ "\2\163\1\7\1\225\6\0\1\100\2\0\3\100\1\226"+
+ "\2\165\1\7\1\226\14\0\1\231\2\232\1\0\1\231"+
+ "\14\0\1\233\2\234\1\0\1\233\6\0\1\147\1\0"+
+ "\1\30\2\106\1\0\1\231\2\232\1\0\1\231\6\0"+
+ "\1\235\1\0\1\30\2\236\1\173\1\237\2\240\1\7"+
+ "\1\241\6\0\1\106\1\0\1\30\2\106\1\0\1\233"+
+ "\2\234\1\0\1\233\6\0\1\236\1\0\1\30\2\236"+
+ "\1\173\1\242\2\243\1\7\1\244\14\0\3\245\1\0"+
+ "\1\245\14\0\3\246\1\0\1\246\6\0\1\247\1\0"+
+ "\1\30\2\250\1\101\1\237\2\240\1\7\1\241\6\0"+
+ "\1\251\1\0\1\30\2\250\1\101\1\237\2\240\1\7"+
+ "\1\241\6\0\1\247\1\0\1\30\2\250\1\101\1\241"+
+ "\2\252\1\7\1\241\6\0\1\250\1\0\1\30\2\250"+
+ "\1\101\1\242\2\243\1\7\1\244\6\0\1\253\1\0"+
+ "\1\30\2\250\1\101\1\242\2\243\1\7\1\244\6\0"+
+ "\1\250\1\0\1\30\2\250\1\101\1\244\2\254\1\7"+
+ "\1\244\6\0\1\227\1\0\1\30\2\230\1\100\1\245"+
+ "\2\240\1\7\1\255\6\0\1\230\1\0\1\30\2\230"+
+ "\1\100\1\246\2\243\1\7\1\256\14\0\1\245\2\237"+
+ "\1\0\1\245\14\0\1\246\2\242\1\0\1\246\6\0"+
+ "\1\44\5\0\1\245\2\240\1\7\1\255\6\0\1\251"+
+ "\1\0\1\30\2\250\1\101\1\241\2\252\1\7\1\241"+
+ "\6\0\1\44\5\0\1\246\2\243\1\7\1\256\6\0"+
+ "\1\253\1\0\1\30\2\250\1\101\1\244\2\254\1\7"+
+ "\1\244\6\0\1\227\1\0\1\30\2\230\1\100\1\255"+
+ "\2\252\1\7\1\255\6\0\1\230\1\0\1\30\2\230"+
+ "\1\100\1\256\2\254\1\7\1\256\4\0";
+
+ private static int [] zzUnpackTrans() {
+ int [] result = new int[2822];
+ int offset = 0;
+ offset = zzUnpackTrans(ZZ_TRANS_PACKED_0, offset, result);
+ return result;
+ }
+
+ private static int zzUnpackTrans(String packed, int offset, int [] result) {
+ int i = 0; /* index in packed string */
+ int j = offset; /* index in unpacked array */
+ int l = packed.length();
+ while (i < l) {
+ int count = packed.charAt(i++);
+ int value = packed.charAt(i++);
+ value--;
+ do result[j++] = value; while (--count > 0);
+ }
+ return j;
+ }
+
+
+ /* error codes */
+ private static final int ZZ_UNKNOWN_ERROR = 0;
+ private static final int ZZ_NO_MATCH = 1;
+ private static final int ZZ_PUSHBACK_2BIG = 2;
+
+ /* error messages for the codes above */
+ private static final String ZZ_ERROR_MSG[] = {
+ "Unkown internal scanner error",
+ "Error: could not match input",
+ "Error: pushback value was too large"
+ };
+
+ /**
+ * ZZ_ATTRIBUTE[aState] contains the attributes of state <code>aState</code>
+ */
+ private static final int [] ZZ_ATTRIBUTE = zzUnpackAttribute();
+
+ private static final String ZZ_ATTRIBUTE_PACKED_0 =
+ "\1\0\1\11\12\1\6\0\5\1\3\0\11\1\3\0"+
+ "\7\1\2\0\2\1\2\0\7\1\1\0\3\1\4\0"+
+ "\3\1\1\0\2\1\2\0\4\1\1\0\23\1\1\0"+
+ "\2\1\1\0\5\1\2\0\4\1\1\0\1\1\1\0"+
+ "\2\1\1\0\1\1\1\0\1\1\1\0\12\1\2\0"+
+ "\2\1\1\0\2\1\1\0\5\1\2\0\3\1\1\0"+
+ "\2\1\2\0\1\1\1\0\3\1\2\0\2\1\1\0"+
+ "\1\1\1\0\2\1\1\0\3\1";
+
+ private static int [] zzUnpackAttribute() {
+ int [] result = new int[174];
+ int offset = 0;
+ offset = zzUnpackAttribute(ZZ_ATTRIBUTE_PACKED_0, offset, result);
+ return result;
+ }
+
+ private static int zzUnpackAttribute(String packed, int offset, int [] result) {
+ int i = 0; /* index in packed string */
+ int j = offset; /* index in unpacked array */
+ int l = packed.length();
+ while (i < l) {
+ int count = packed.charAt(i++);
+ int value = packed.charAt(i++);
+ do result[j++] = value; while (--count > 0);
+ }
+ return j;
+ }
+
+ /** the input device */
+ private java.io.Reader zzReader;
+
+ /** the current state of the DFA */
+ private int zzState;
+
+ /** the current lexical state */
+ private int zzLexicalState = YYINITIAL;
+
+ /** this buffer contains the current text to be matched and is
+ the source of the yytext() string */
+ private char zzBuffer[] = new char[ZZ_BUFFERSIZE];
+
+ /** the textposition at the last accepting state */
+ private int zzMarkedPos;
+
+ /** the textposition at the last state to be included in yytext */
+ private int zzPushbackPos;
+
+ /** the current text position in the buffer */
+ private int zzCurrentPos;
+
+ /** startRead marks the beginning of the yytext() string in the buffer */
+ private int zzStartRead;
+
+ /** endRead marks the last character in the buffer, that has been read
+ from input */
+ private int zzEndRead;
+
+ /** number of newlines encountered up to the start of the matched text */
+ private int yyline;
+
+ /** the number of characters up to the start of the matched text */
+ private int yychar;
+
+ /**
+ * the number of characters from the last newline up to the start of the
+ * matched text
+ */
+ private int yycolumn;
+
+ /**
+ * zzAtBOL == true <=> the scanner is currently at the beginning of a line
+ */
+ private boolean zzAtBOL = true;
+
+ /** zzAtEOF == true <=> the scanner is at the EOF */
+ private boolean zzAtEOF;
+
+ /* user code: */
+
+public static final int ALPHANUM = 0;
+public static final int APOSTROPHE = 1;
+public static final int ACRONYM = 2;
+public static final int COMPANY = 3;
+public static final int EMAIL = 4;
+public static final int HOST = 5;
+public static final int NUM = 6;
+public static final int CJ = 7;
+/**
+ * @deprecated this solves a bug where HOSTs that end with '.' are identified
+ * as ACRONYMs. It is deprecated and will be removed in the next
+ * release.
+ */
+public static final int ACRONYM_DEP = 8;
+public static final int KOREAN = 9;
+public static final int CHINESE = 10;
+
+public static final String [] TOKEN_TYPES = new String [] {
+ "<ALPHANUM>",
+ "<APOSTROPHE>",
+ "<ACRONYM>",
+ "<COMPANY>",
+ "<EMAIL>",
+ "<HOST>",
+ "<NUM>",
+ "<CJ>",
+ "<ACRONYM_DEP>",
+ "<KOREAN>" ,
+ "<CHINESE>"
+};
+
+public final int yychar()
+{
+ return yychar;
+}
+
+/**
+ * Fills Lucene token with the current token text.
+ */
+final void getText(CharTermAttribute t) {
+ t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
+}
+
+
+ /**
+ * Creates a new scanner
+ * There is also a java.io.InputStream version of this constructor.
+ *
+ * @param in the java.io.Reader to read input from.
+ */
+ KoreanTokenizerImpl(java.io.Reader in) {
+ this.zzReader = in;
+ }
+
+ /**
+ * Creates a new scanner.
+ * There is also java.io.Reader version of this constructor.
+ *
+ * @param in the java.io.Inputstream to read input from.
+ */
+ KoreanTokenizerImpl(java.io.InputStream in) {
+ this(new java.io.InputStreamReader(in));
+ }
+
+ /**
+ * Unpacks the compressed character translation table.
+ *
+ * @param packed the packed character translation table
+ * @return the unpacked character translation table
+ */
+ private static char [] zzUnpackCMap(String packed) {
+ char [] map = new char[0x10000];
+ int i = 0; /* index in packed string */
+ int j = 0; /* index in unpacked array */
+ while (i < 156) {
+ int count = packed.charAt(i++);
+ char value = packed.charAt(i++);
+ do map[j++] = value; while (--count > 0);
+ }
+ return map;
+ }
+
+
+ /**
+ * Refills the input buffer.
+ *
+ * @return <code>false</code>, iff there was new input.
+ *
+ * @exception java.io.IOException if any I/O-Error occurs
+ */
+ private boolean zzRefill() throws java.io.IOException {
+
+ /* first: make room (if you can) */
+ if (zzStartRead > 0) {
+ System.arraycopy(zzBuffer, zzStartRead,
+ zzBuffer, 0,
+ zzEndRead-zzStartRead);
+
+ /* translate stored positions */
+ zzEndRead-= zzStartRead;
+ zzCurrentPos-= zzStartRead;
+ zzMarkedPos-= zzStartRead;
+ zzPushbackPos-= zzStartRead;
+ zzStartRead = 0;
+ }
+
+ /* is the buffer big enough? */
+ if (zzCurrentPos >= zzBuffer.length) {
+ /* if not: blow it up */
+ char newBuffer[] = new char[zzCurrentPos*2];
+ System.arraycopy(zzBuffer, 0, newBuffer, 0, zzBuffer.length);
+ zzBuffer = newBuffer;
+ }
+
+ /* finally: fill the buffer with new input */
+ int numRead = zzReader.read(zzBuffer, zzEndRead,
+ zzBuffer.length-zzEndRead);
+
+ if (numRead < 0) {
+ return true;
+ }
+ else {
+ zzEndRead+= numRead;
+ return false;
+ }
+ }
+
+
+ /**
+ * Closes the input stream.
+ */
+ public final void yyclose() throws java.io.IOException {
+ zzAtEOF = true; /* indicate end of file */
+ zzEndRead = zzStartRead; /* invalidate buffer */
+
+ if (zzReader != null)
+ zzReader.close();
+ }
+
+
+ /**
+ * Resets the scanner to read from a new input stream.
+ * Does not close the old reader.
+ *
+ * All internal variables are reset, the old input stream
+ * <b>cannot</b> be reused (internal buffer is discarded and lost).
+ * Lexical state is set to <tt>ZZ_INITIAL</tt>.
+ *
+ * @param reader the new input stream
+ */
+ public final void yyreset(java.io.Reader reader) {
+ zzReader = reader;
+ zzAtBOL = true;
+ zzAtEOF = false;
+ zzEndRead = zzStartRead = 0;
+ zzCurrentPos = zzMarkedPos = zzPushbackPos = 0;
+ yyline = yychar = yycolumn = 0;
+ zzLexicalState = YYINITIAL;
+ }
+
+
+ /**
+ * Returns the current lexical state.
+ */
+ public final int yystate() {
+ return zzLexicalState;
+ }
+
+
+ /**
+ * Enters a new lexical state
+ *
+ * @param newState the new lexical state
+ */
+ public final void yybegin(int newState) {
+ zzLexicalState = newState;
+ }
+
+
+ /**
+ * Returns the text matched by the current regular expression.
+ */
+ public final String yytext() {
+ return new String( zzBuffer, zzStartRead, zzMarkedPos-zzStartRead );
+ }
+
+
+ /**
+ * Returns the character at position <tt>pos</tt> from the
+ * matched text.
+ *
+ * It is equivalent to yytext().charAt(pos), but faster
+ *
+ * @param pos the position of the character to fetch.
+ * A value from 0 to yylength()-1.
+ *
+ * @return the character at position pos
+ */
+ public final char yycharat(int pos) {
+ return zzBuffer[zzStartRead+pos];
+ }
+
+
+ /**
+ * Returns the length of the matched text region.
+ */
+ public final int yylength() {
+ return zzMarkedPos-zzStartRead;
+ }
+
+
+ /**
+ * Reports an error that occured while scanning.
+ *
+ * In a wellformed scanner (no or only correct usage of
+ * yypushback(int) and a match-all fallback rule) this method
+ * will only be called with things that "Can't Possibly Happen".
+ * If this method is called, something is seriously wrong
+ * (e.g. a JFlex bug producing a faulty scanner etc.).
+ *
+ * Usual syntax/scanner level error handling should be done
+ * in error fallback rules.
+ *
+ * @param errorCode the code of the errormessage to display
+ */
+ private void zzScanError(int errorCode) {
+ String message;
+ try {
+ message = ZZ_ERROR_MSG[errorCode];
+ }
+ catch (ArrayIndexOutOfBoundsException e) {
+ message = ZZ_ERROR_MSG[ZZ_UNKNOWN_ERROR];
+ }
+
+ throw new Error(message);
+ }
+
+
+ /**
+ * Pushes the specified amount of characters back into the input stream.
+ *
+ * They will be read again by then next call of the scanning method
+ *
+ * @param number the number of characters to be read again.
+ * This number must not be greater than yylength()!
+ */
+ public void yypushback(int number) {
+ if ( number > yylength() )
+ zzScanError(ZZ_PUSHBACK_2BIG);
+
+ zzMarkedPos -= number;
+ }
+
+
+ /**
+ * Resumes scanning until the next regular expression is matched,
+ * the end of input is encountered or an I/O-Error occurs.
+ *
+ * @return the next token
+ * @exception java.io.IOException if any I/O-Error occurs
+ */
+ public int getNextToken() throws java.io.IOException {
+ int zzInput;
+ int zzAction;
+
+ // cached fields:
+ int zzCurrentPosL;
+ int zzMarkedPosL;
+ int zzEndReadL = zzEndRead;
+ char [] zzBufferL = zzBuffer;
+ char [] zzCMapL = ZZ_CMAP;
+
+ int [] zzTransL = ZZ_TRANS;
+ int [] zzRowMapL = ZZ_ROWMAP;
+ int [] zzAttrL = ZZ_ATTRIBUTE;
+
+ while (true) {
+ zzMarkedPosL = zzMarkedPos;
+
+ yychar+= zzMarkedPosL-zzStartRead;
+
+ zzAction = -1;
+
+ zzCurrentPosL = zzCurrentPos = zzStartRead = zzMarkedPosL;
+
+ zzState = zzLexicalState;
+
+
+ zzForAction: {
+ while (true) {
+
+ if (zzCurrentPosL < zzEndReadL)
+ zzInput = zzBufferL[zzCurrentPosL++];
+ else if (zzAtEOF) {
+ zzInput = YYEOF;
+ break zzForAction;
+ }
+ else {
+ // store back cached positions
+ zzCurrentPos = zzCurrentPosL;
+ zzMarkedPos = zzMarkedPosL;
+ boolean eof = zzRefill();
+ // get translated positions and possibly new buffer
+ zzCurrentPosL = zzCurrentPos;
+ zzMarkedPosL = zzMarkedPos;
+ zzBufferL = zzBuffer;
+ zzEndReadL = zzEndRead;
+ if (eof) {
+ zzInput = YYEOF;
+ break zzForAction;
+ }
+ else {
+ zzInput = zzBufferL[zzCurrentPosL++];
+ }
+ }
+ int zzNext = zzTransL[ zzRowMapL[zzState] + zzCMapL[zzInput] ];
+ if (zzNext == -1) break zzForAction;
+ zzState = zzNext;
+
+ int zzAttributes = zzAttrL[zzState];
+ if ( (zzAttributes & 1) == 1 ) {
+ zzAction = zzState;
+ zzMarkedPosL = zzCurrentPosL;
+ if ( (zzAttributes & 8) == 8 ) break zzForAction;
+ }
+
+ }
+ }
+
+ // store back cached position
+ zzMarkedPos = zzMarkedPosL;
+
+ switch (zzAction < 0 ? zzAction : ZZ_ACTION[zzAction]) {
+ case 8:
+ { return HOST;
+ }
+ case 13: break;
+ case 11:
+ { return ACRONYM_DEP;
+ }
+ case 14: break;
+ case 10:
+ { return ACRONYM;
+ }
+ case 15: break;
+ case 1:
+ { /* ignore */
+ }
+ case 16: break;
+ case 6:
+ { return NUM;
+ }
+ case 17: break;
+ case 5:
+ { return CJ;
+ }
+ case 18: break;
+ case 2:
+ { return ALPHANUM;
+ }
+ case 19: break;
+ case 9:
+ { return COMPANY;
+ }
+ case 20: break;
+ case 7:
+ { return APOSTROPHE;
+ }
+ case 21: break;
+ case 4:
+ { return CHINESE;
+ }
+ case 22: break;
+ case 3:
+ { return KOREAN;
+ }
+ case 23: break;
+ case 12:
+ { return EMAIL;
+ }
+ case 24: break;
+ default:
+ if (zzInput == YYEOF && zzStartRead == zzCurrentPos) {
+ zzAtEOF = true;
+ return YYEOF;
+ }
+ else {
+ zzScanError(ZZ_NO_MATCH);
+ }
+ }
+ }
+ }
+
+
+}
Added: lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerImpl.jflex
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerImpl.jflex?rev=1479234&view=auto
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerImpl.jflex (added)
+++ lucene/dev/branches/lucene4956/lucene/analysis/arirang/src/java/org/apache/lucene/analysis/kr/KoreanTokenizerImpl.jflex Sun May 5 03:39:51 2013
@@ -0,0 +1,157 @@
+package org.apache.lucene.analysis.kr;
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+
+%%
+
+%class KoreanTokenizerImpl
+%unicode
+%integer
+%function getNextToken
+%pack
+%char
+
+%{
+
+public static final int ALPHANUM = 0;
+public static final int APOSTROPHE = 1;
+public static final int ACRONYM = 2;
+public static final int COMPANY = 3;
+public static final int EMAIL = 4;
+public static final int HOST = 5;
+public static final int NUM = 6;
+public static final int CJ = 7;
+/**
+ * @deprecated this solves a bug where HOSTs that end with '.' are identified
+ * as ACRONYMs. It is deprecated and will be removed in the next
+ * release.
+ */
+public static final int ACRONYM_DEP = 8;
+public static final int KOREAN = 9;
+public static final int CHINESE = 10;
+
+public static final String [] TOKEN_TYPES = new String [] {
+ "<ALPHANUM>",
+ "<APOSTROPHE>",
+ "<ACRONYM>",
+ "<COMPANY>",
+ "<EMAIL>",
+ "<HOST>",
+ "<NUM>",
+ "<CJ>",
+ "<ACRONYM_DEP>",
+ "<KOREAN>" ,
+ "<CHINESE>"
+};
+
+public final int yychar()
+{
+ return yychar;
+}
+
+/**
+ * Fills Lucene token with the current token text.
+ */
+final void getText(CharTermAttribute t) {
+ t.copyBuffer(zzBuffer, zzStartRead, zzMarkedPos-zzStartRead);
+}
+%}
+
+// korean word: a sequence of digits & letters &
+KOREAN = ({LETTER}|{NUM}|{DIGIT})* {HANLETTER}+ ({LETTER}|{DIGIT})*
+
+// basic word: a sequence of digits & letters
+ALPHANUM = ({LETTER}|{DIGIT})+
+
+// chinese word: a sequence of digits & letters &
+CHINESE = {CHINESELETTER}+ ({LETTER}|{DIGIT})*
+
+// internal apostrophes: O'Reilly, you're, O'Reilly's
+// use a post-filter to remove possesives
+APOSTROPHE = {ALPHA} ("'" {ALPHA})+
+
+// acronyms: U.S.A., I.B.M., etc.
+// use a post-filter to remove dots
+ACRONYM = {LETTER} "." ({LETTER} ".")+
+
+ACRONYM_DEP = {ALPHANUM} "." ({ALPHANUM} ".")+
+
+// company names like AT&T and Excite@Home.
+COMPANY = {ALPHA} ("&"|"@") {ALPHA}
+
+// email addresses
+EMAIL = {ALPHANUM} (("."|"-"|"_") {ALPHANUM})* "@" {ALPHANUM} (("."|"-") {ALPHANUM})+
+
+// hostname
+HOST = {ALPHANUM} ((".") {ALPHANUM})+
+
+// floating point, serial, model numbers, ip addresses, etc.
+// every other segment must have at least one digit
+NUM = ({ALPHANUM} {P} {HAS_DIGIT}
+ | "." {DIGIT}
+ | {DIGIT} "."
+ | {HAS_DIGIT} {P} {ALPHANUM}
+ | {ALPHANUM} ({P} {HAS_DIGIT} {P} {ALPHANUM})+
+ | {HAS_DIGIT} ({P} {ALPHANUM} {P} {HAS_DIGIT})+
+ | {ALPHANUM} {P} {HAS_DIGIT} ({P} {ALPHANUM} {P} {HAS_DIGIT})+
+ | {HAS_DIGIT} {P} {ALPHANUM} ({P} {HAS_DIGIT} {P} {ALPHANUM})+)
+
+// punctuation
+P = ("_"|"-"|"/"|"."|",")
+
+// at least one digit
+HAS_DIGIT =
+ ({LETTER}|{DIGIT})*
+ {DIGIT}
+ ({LETTER}|{DIGIT})*
+
+ALPHA = ({LETTER})+
+
+
+LETTER = [\u0041-\u005a\u0061-\u007a\u00c0-\u00d6\u00d8-\u00f6\u00f8-\u00ff\u0100-\u1fff\uffa0-\uffdc]
+
+DIGIT = [\u0030-\u0039\u0660-\u0669\u06f0-\u06f9\u0966-\u096f\u09e6-\u09ef\u0a66-\u0a6f\u0ae6-\u0aef\u0b66-\u0b6f\u0be7-\u0bef\u0c66-\u0c6f\u0ce6-\u0cef\u0d66-\u0d6f\u0e50-\u0e59\u0ed0-\u0ed9\u1040-\u1049]
+
+HANLETTER = [\uac00-\ud7af\u1100-\u11ff]
+
+CHINESELETTER = [\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff]
+
+// Chinese, Japanese
+CJ = ([\u3040-\u318f\u3100-\u312f\u3040-\u309F\u30A0-\u30FF\u31F0-\u31FF\u3300-\u337f\uff65-\uff9f])+
+
+WHITESPACE = \r\n | [ \r\n\t\f]
+
+%%
+
+{KOREAN} { return KOREAN; }
+{CHINESE} { return CHINESE; }
+{ALPHANUM} { return ALPHANUM; }
+{APOSTROPHE} { return APOSTROPHE; }
+{ACRONYM} { return ACRONYM; }
+{COMPANY} { return COMPANY; }
+{EMAIL} { return EMAIL; }
+{HOST} { return HOST; }
+{NUM} { return NUM; }
+{CJ} { return CJ; }
+{ACRONYM_DEP} { return ACRONYM_DEP; }
+
+
+/** Ignore the rest */
+. | {WHITESPACE} { /* ignore */ }