You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by us...@apache.org on 2012/01/06 13:18:25 UTC
svn commit: r1228141 - in
/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src:
java/org/apache/lucene/analysis/kuromoji/dict/
tools/java/org/apache/lucene/analysis/kuromoji/util/
Author: uschindler
Date: Fri Jan 6 12:18:25 2012
New Revision: 1228141
URL: http://svn.apache.org/viewvc?rev=1228141&view=rev
Log:
LUCENE-3305: Separate readers and writers for ConnectionCosts and CharacterDefinition
Added:
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/CharacterDefinitionWriter.java (with props)
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsWriter.java (with props)
Modified:
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/CharacterDefinition.java
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/ConnectionCosts.java
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsBuilder.java
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/DictionaryBuilder.java
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/UnknownDictionaryWriter.java
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/CharacterDefinition.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/CharacterDefinition.java?rev=1228141&r1=1228140&r2=1228141&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/CharacterDefinition.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/CharacterDefinition.java Fri Jan 6 12:18:25 2012
@@ -18,19 +18,12 @@ package org.apache.lucene.analysis.kurom
*/
import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.FileNotFoundException;
import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.Arrays;
import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.InputStreamDataInput;
-import org.apache.lucene.store.OutputStreamDataOutput;
import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.IOUtils;
@@ -40,7 +33,7 @@ public final class CharacterDefinition {
public static final String HEADER = "kuromoji_cd";
public static final int VERSION = 1;
- private static final int CLASS_COUNT = CharacterClass.values().length;
+ public static final int CLASS_COUNT = CharacterClass.values().length;
// only used internally for lookup:
private static enum CharacterClass {
@@ -66,17 +59,7 @@ public final class CharacterDefinition {
public static final byte KANJI = (byte) CharacterClass.KANJI.ordinal();
public static final byte KANJINUMERIC = (byte) CharacterClass.KANJINUMERIC.ordinal();
- /**
- * Constructor for building. TODO: remove write access
- */
- public CharacterDefinition() {
- Arrays.fill(characterCategoryMap, DEFAULT);
- }
-
- // used only for singleton
- private CharacterDefinition(boolean dummy) throws IOException {
- assert dummy;
-
+ private CharacterDefinition() throws IOException {
IOException priorE = null;
InputStream is = null;
try {
@@ -116,60 +99,13 @@ public final class CharacterDefinition {
return characterClass == KANJI || characterClass == KANJINUMERIC;
}
- /**
- * Put mapping from unicode code point to character class.
- *
- * @param codePoint
- * code point
- * @param characterClassName character class name
- */
- public void putCharacterCategory(int codePoint, String characterClassName) {
- characterClassName = characterClassName.split(" ")[0]; // use first
- // category
- // class
-
- // Override Nakaguro
- if (codePoint == 0x30FB) {
- characterClassName = "SYMBOL";
- }
- characterCategoryMap[codePoint] = lookupCharacterClass(characterClassName);
- }
-
- public void putInvokeDefinition(String characterClassName, int invoke, int group, int length) {
- final byte characterClass = lookupCharacterClass(characterClassName);
- invokeMap[characterClass] = invoke == 1;
- groupMap[characterClass] = group == 1;
- // TODO: length def ignored
- }
-
public static byte lookupCharacterClass(String characterClassName) {
return (byte) CharacterClass.valueOf(characterClassName).ordinal();
}
- public void write(String baseDir) throws IOException {
- String filename = baseDir + File.separator + getClass().getName().replace('.', File.separatorChar) + FILENAME_SUFFIX;
- new File(filename).getParentFile().mkdirs();
- OutputStream os = new FileOutputStream(filename);
- try {
- os = new BufferedOutputStream(os);
- final DataOutput out = new OutputStreamDataOutput(os);
- CodecUtil.writeHeader(out, HEADER, VERSION);
- out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
- for (int i = 0; i < CLASS_COUNT; i++) {
- final byte b = (byte) (
- (invokeMap[i] ? 0x01 : 0x00) |
- (groupMap[i] ? 0x02 : 0x00)
- );
- out.writeByte(b);
- }
- } finally {
- os.close();
- }
- }
-
public synchronized static CharacterDefinition getInstance() {
if (singleton == null) try {
- singleton = new CharacterDefinition(true);
+ singleton = new CharacterDefinition();
} catch (IOException ioe) {
throw new RuntimeException("Cannot load CharacterDefinition.", ioe);
}
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/ConnectionCosts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/ConnectionCosts.java?rev=1228141&r1=1228140&r2=1228141&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/ConnectionCosts.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/ConnectionCosts.java Fri Jan 6 12:18:25 2012
@@ -18,18 +18,12 @@ package org.apache.lucene.analysis.kurom
*/
import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.FileNotFoundException;
import java.io.InputStream;
-import java.io.OutputStream;
import org.apache.lucene.store.DataInput;
-import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.InputStreamDataInput;
-import org.apache.lucene.store.OutputStreamDataOutput;
import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.IOUtils;
@@ -41,9 +35,7 @@ public final class ConnectionCosts {
private final short[][] costs; // array is backward IDs first since get is called using the same backward ID consecutively. maybe doesn't matter.
- private ConnectionCosts(boolean dummy) throws IOException {
- assert dummy;
-
+ private ConnectionCosts() throws IOException {
IOException priorE = null;
InputStream is = null;
short[][] costs = null;
@@ -72,17 +64,6 @@ public final class ConnectionCosts {
this.costs = costs;
}
- /**
- * Constructor for building. TODO: remove write access
- */
- public ConnectionCosts(int forwardSize, int backwardSize) {
- this.costs = new short[backwardSize][forwardSize];
- }
-
- public void add(int forwardId, int backwardId, int cost) {
- this.costs[backwardId][forwardId] = (short)cost;
- }
-
public int get(int forwardId, int backwardId) {
// FIXME: There seems to be something wrong with the double array trie in some rare
// cases causing and IndexOutOfBoundsException. Use a guard as a temporary work-around
@@ -94,29 +75,9 @@ public final class ConnectionCosts {
}
}
- public void write(String baseDir) throws IOException {
- String filename = baseDir + File.separator + getClass().getName().replace('.', File.separatorChar) + FILENAME_SUFFIX;
- new File(filename).getParentFile().mkdirs();
- OutputStream os = new FileOutputStream(filename);
- try {
- os = new BufferedOutputStream(os);
- final DataOutput out = new OutputStreamDataOutput(os);
- CodecUtil.writeHeader(out, HEADER, VERSION);
- out.writeVInt(costs.length);
- for (short[] a : costs) {
- out.writeVInt(a.length);
- for (int i = 0; i < a.length; i++) {
- out.writeShort(a[i]);
- }
- }
- } finally {
- os.close();
- }
- }
-
public synchronized static ConnectionCosts getInstance() {
if (singleton == null) try {
- singleton = new ConnectionCosts(true);
+ singleton = new ConnectionCosts();
} catch (IOException ioe) {
throw new RuntimeException("Cannot load ConnectionCosts.", ioe);
}
Added: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/CharacterDefinitionWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/CharacterDefinitionWriter.java?rev=1228141&view=auto
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/CharacterDefinitionWriter.java (added)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/CharacterDefinitionWriter.java Fri Jan 6 12:18:25 2012
@@ -0,0 +1,95 @@
+package org.apache.lucene.analysis.kuromoji.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+
+import org.apache.lucene.analysis.kuromoji.dict.CharacterDefinition;
+
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.OutputStreamDataOutput;
+import org.apache.lucene.util.CodecUtil;
+
+public final class CharacterDefinitionWriter {
+
+ private final byte[] characterCategoryMap = new byte[0x10000];
+
+ private final boolean[] invokeMap = new boolean[CharacterDefinition.CLASS_COUNT];
+ private final boolean[] groupMap = new boolean[CharacterDefinition.CLASS_COUNT];
+
+ /**
+ * Constructor for building. TODO: remove write access
+ */
+ public CharacterDefinitionWriter() {
+ Arrays.fill(characterCategoryMap, CharacterDefinition.DEFAULT);
+ }
+
+ /**
+ * Put mapping from unicode code point to character class.
+ *
+ * @param codePoint
+ * code point
+ * @param characterClassName character class name
+ */
+ public void putCharacterCategory(int codePoint, String characterClassName) {
+ characterClassName = characterClassName.split(" ")[0]; // use first
+ // category
+ // class
+
+ // Override Nakaguro
+ if (codePoint == 0x30FB) {
+ characterClassName = "SYMBOL";
+ }
+ characterCategoryMap[codePoint] = CharacterDefinition.lookupCharacterClass(characterClassName);
+ }
+
+ public void putInvokeDefinition(String characterClassName, int invoke, int group, int length) {
+ final byte characterClass = CharacterDefinition.lookupCharacterClass(characterClassName);
+ invokeMap[characterClass] = invoke == 1;
+ groupMap[characterClass] = group == 1;
+ // TODO: length def ignored
+ }
+
+ public void write(String baseDir) throws IOException {
+ String filename = baseDir + File.separator +
+ CharacterDefinition.class.getName().replace('.', File.separatorChar) + CharacterDefinition.FILENAME_SUFFIX;
+ new File(filename).getParentFile().mkdirs();
+ OutputStream os = new FileOutputStream(filename);
+ try {
+ os = new BufferedOutputStream(os);
+ final DataOutput out = new OutputStreamDataOutput(os);
+ CodecUtil.writeHeader(out, CharacterDefinition.HEADER, CharacterDefinition.VERSION);
+ out.writeBytes(characterCategoryMap, 0, characterCategoryMap.length);
+ for (int i = 0; i < CharacterDefinition.CLASS_COUNT; i++) {
+ final byte b = (byte) (
+ (invokeMap[i] ? 0x01 : 0x00) |
+ (groupMap[i] ? 0x02 : 0x00)
+ );
+ out.writeByte(b);
+ }
+ } finally {
+ os.close();
+ }
+ }
+
+}
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsBuilder.java?rev=1228141&r1=1228140&r2=1228141&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsBuilder.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsBuilder.java Fri Jan 6 12:18:25 2012
@@ -25,15 +25,12 @@ import java.nio.charset.Charset;
import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
-import org.apache.lucene.analysis.kuromoji.dict.ConnectionCosts;
-
public class ConnectionCostsBuilder {
- public ConnectionCostsBuilder() {
-
+ private ConnectionCostsBuilder() {
}
- public static ConnectionCosts build(String filename) throws IOException {
+ public static ConnectionCostsWriter build(String filename) throws IOException {
FileInputStream inputStream = new FileInputStream(filename);
Charset cs = Charset.forName("US-ASCII");
CharsetDecoder decoder = cs.newDecoder()
@@ -52,7 +49,7 @@ public class ConnectionCostsBuilder {
assert forwardSize > 0 && backwardSize > 0;
- ConnectionCosts costs = new ConnectionCosts(forwardSize, backwardSize);
+ ConnectionCostsWriter costs = new ConnectionCostsWriter(forwardSize, backwardSize);
while ((line = lineReader.readLine()) != null) {
String[] fields = line.split("\\s+");
Added: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsWriter.java?rev=1228141&view=auto
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsWriter.java (added)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsWriter.java Fri Jan 6 12:18:25 2012
@@ -0,0 +1,68 @@
+package org.apache.lucene.analysis.kuromoji.util;
+
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.lucene.analysis.kuromoji.dict.ConnectionCosts;
+
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.OutputStreamDataOutput;
+import org.apache.lucene.util.CodecUtil;
+
+public final class ConnectionCostsWriter {
+
+ private final short[][] costs; // array is backward IDs first since get is called using the same backward ID consecutively. maybe doesn't matter.
+
+ /**
+ * Constructor for building. TODO: remove write access
+ */
+ public ConnectionCostsWriter(int forwardSize, int backwardSize) {
+ this.costs = new short[backwardSize][forwardSize];
+ }
+
+ public void add(int forwardId, int backwardId, int cost) {
+ this.costs[backwardId][forwardId] = (short)cost;
+ }
+
+ public void write(String baseDir) throws IOException {
+ String filename = baseDir + File.separator +
+ ConnectionCosts.class.getName().replace('.', File.separatorChar) + ConnectionCosts.FILENAME_SUFFIX;
+ new File(filename).getParentFile().mkdirs();
+ OutputStream os = new FileOutputStream(filename);
+ try {
+ os = new BufferedOutputStream(os);
+ final DataOutput out = new OutputStreamDataOutput(os);
+ CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);
+ out.writeVInt(costs.length);
+ for (short[] a : costs) {
+ out.writeVInt(a.length);
+ for (int i = 0; i < a.length; i++) {
+ out.writeShort(a[i]);
+ }
+ }
+ } finally {
+ os.close();
+ }
+ }
+
+}
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/DictionaryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/DictionaryBuilder.java?rev=1228141&r1=1228140&r2=1228141&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/DictionaryBuilder.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/DictionaryBuilder.java Fri Jan 6 12:18:25 2012
@@ -21,9 +21,6 @@ import java.io.File;
import java.io.IOException;
import java.util.Map.Entry;
-import org.apache.lucene.analysis.kuromoji.dict.ConnectionCosts;
-import org.apache.lucene.analysis.kuromoji.dict.TokenInfoDictionary;
-import org.apache.lucene.analysis.kuromoji.dict.UnknownDictionary;
import org.apache.lucene.analysis.kuromoji.trie.DoubleArrayTrie;
public class DictionaryBuilder {
@@ -70,7 +67,7 @@ public class DictionaryBuilder {
System.out.println("done");
System.out.print("building connection costs...");
- ConnectionCosts connectionCosts
+ ConnectionCostsWriter connectionCosts
= ConnectionCostsBuilder.build(inputDirname + File.separator + "matrix.def");
connectionCosts.write(outputDirname);
System.out.println("done");
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/UnknownDictionaryWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/UnknownDictionaryWriter.java?rev=1228141&r1=1228140&r2=1228141&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/UnknownDictionaryWriter.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/UnknownDictionaryWriter.java Fri Jan 6 12:18:25 2012
@@ -8,7 +8,7 @@ import org.apache.lucene.analysis.kuromo
import org.apache.lucene.analysis.kuromoji.dict.UnknownDictionary;
public class UnknownDictionaryWriter extends TokenInfoDictionaryWriter {
- private final CharacterDefinition characterDefinition = new CharacterDefinition();
+ private final CharacterDefinitionWriter characterDefinition = new CharacterDefinitionWriter();
public UnknownDictionaryWriter(int size) {
super(size);