You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/01/06 08:26:30 UTC
svn commit: r1228000 - in
/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src:
java/org/apache/lucene/analysis/kuromoji/
java/org/apache/lucene/analysis/kuromoji/dict/
resources/org/apache/lucene/analysis/kuromoji/dict/ test/org/apache/lucene...
Author: rmuir
Date: Fri Jan 6 07:26:28 2012
New Revision: 1228000
URL: http://svn.apache.org/viewvc?rev=1228000&view=rev
Log:
LUCENE-3305: more efficient tokeninfodictionary (4x faster tokenization if you are pulling things like POS)
Modified:
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Token.java
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/BinaryDictionary.java
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/Dictionary.java
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/UserDictionary.java
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/TokenInfoDictionary$buffer.dat
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/TokenInfoDictionary$posDict.dat
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/TokenInfoDictionary$targetMap.dat
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/UnknownDictionary$buffer.dat
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/UnknownDictionary$posDict.dat
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/UnknownDictionary$targetMap.dat
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TokenizerTest.java
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/TokenInfoDictionaryBuilder.java
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/TokenInfoDictionaryWriter.java
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/UnknownDictionaryBuilder.java
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/test/org/apache/lucene/analysis/kuromoji/dict/UnknownDictionaryTest.java
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Token.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Token.java?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Token.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Token.java Fri Jan 6 07:26:28 2012
@@ -56,6 +56,13 @@ public class Token {
}
/**
+ * @return pronunciation. null if token doesn't have pronunciation.
+ */
+ public String getPronunciation() {
+ return dictionary.getPronunciation(wordId);
+ }
+
+ /**
* @return part of speech.
*/
public String getPartOfSpeech() {
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/BinaryDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/BinaryDictionary.java?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/BinaryDictionary.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/BinaryDictionary.java Fri Jan 6 07:26:28 2012
@@ -31,8 +31,6 @@ import org.apache.lucene.store.InputStre
import org.apache.lucene.util.CodecUtil;
import org.apache.lucene.util.IOUtils;
-import org.apache.lucene.analysis.kuromoji.util.CSVUtil;
-
public abstract class BinaryDictionary implements Dictionary {
public static final String DICT_FILENAME_SUFFIX = "$buffer.dat";
@@ -130,67 +128,65 @@ public abstract class BinaryDictionary i
public int getWordCost(int wordId) {
return buffer.getShort(wordId + 4); // Skip left id and right id
}
-
- // TODO: this method will likely never be efficient, do we need it?
- @Override
- public String[] getAllFeaturesArray(int wordId) {
- char posIndex = buffer.getChar(wordId + 6); // read index into posDict
- String pos = posDict[posIndex];
- int posLen = pos.length();
- int size = buffer.getShort(wordId + 8) / 2; // Read length of feature String. Skip 8 bytes, see data structure.
- char[] targetArr = new char[posLen + 1 + size]; // pos + separator + the rest
- int offset = wordId + 8 + 2; // offset is position where features string starts
- for (int i = 0; i < pos.length(); i++) {
- targetArr[i] = pos.charAt(i);
- }
- int upto = posLen;
- targetArr[upto++] = INTERNAL_SEPARATOR.charAt(0);
- for(int i = 0; i < size; i++){
- targetArr[upto++] = buffer.getChar(offset + i * 2);
- }
- String allFeatures = new String(targetArr);
- return allFeatures.split(INTERNAL_SEPARATOR);
- }
-
- @Override
- public String getFeature(int wordId, int... fields) {
- String[] allFeatures = getAllFeaturesArray(wordId);
- StringBuilder sb = new StringBuilder();
-
- if(fields.length == 0){ // All features
- for(String feature : allFeatures) {
- sb.append(CSVUtil.quoteEscape(feature)).append(",");
+
+ private String readString(int offset, int length, boolean kana) {
+ char text[] = new char[length];
+ if (kana) {
+ for (int i = 0; i < length; i++) {
+ text[i] = (char) (0x30A0 + (buffer.get(offset + i) & 0xff));
}
- } else if(fields.length == 1) { // One feature doesn't need to escape value
- sb.append(allFeatures[fields[0]]).append(",");
} else {
- for(int field : fields){
- sb.append(CSVUtil.quoteEscape(allFeatures[field])).append(",");
+ for (int i = 0; i < length; i++) {
+ text[i] = buffer.getChar(offset + (i << 1));
}
}
-
- return sb.deleteCharAt(sb.length() - 1).toString();
+ return new String(text);
}
@Override
public String getReading(int wordId) {
- return getFeature(wordId, 7);
+ int offset = wordId + 7;
+ int baseFormLength = buffer.get(offset++) & 0xff;
+ offset += baseFormLength << 1;
+ int readingData = buffer.get(offset++) & 0xff;
+ return readString(offset, readingData >>> 1, (readingData & 1) == 1);
}
@Override
- public String getAllFeatures(int wordId) {
- return getFeature(wordId);
+ public String getPronunciation(int wordId) {
+ int offset = wordId + 7;
+ int baseFormLength = buffer.get(offset++) & 0xff;
+ offset += baseFormLength << 1;
+ int readingData = buffer.get(offset++) & 0xff;
+ int readingLength = readingData >>> 1;
+ int readingOffset = offset;
+ if ((readingData & 1) == 0) {
+ offset += readingLength << 1;
+ } else {
+ offset += readingLength;
+ }
+ int pronunciationData = buffer.get(offset++) & 0xff;
+ if (pronunciationData == 0) {
+ return readString(readingOffset, readingLength, (readingData & 1) == 1);
+ } else {
+ return readString(offset, pronunciationData >>> 1, (pronunciationData & 1) == 1);
+ }
}
@Override
public String getPartOfSpeech(int wordId) {
- return getFeature(wordId, 0, 1, 2, 3);
+ int posIndex = buffer.get(wordId + 6) & 0xff; // read index into posDict
+ return posDict[posIndex];
}
@Override
public String getBaseForm(int wordId) {
- String form = getFeature(wordId, 6);
- return "*".equals(form) ? null : form;
+ int offset = wordId + 7;
+ int length = buffer.get(offset++) & 0xff;
+ if (length == 0) {
+ return null; // same as surface form
+ } else {
+ return readString(offset, length, false);
+ }
}
-
}
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/Dictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/Dictionary.java?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/Dictionary.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/Dictionary.java Fri Jan 6 07:26:28 2012
@@ -43,24 +43,11 @@ public interface Dictionary {
public int getWordCost(int wordId);
/**
- * Get all features of tokens
- * @param wordId word ID of token
- * @return All features of the token
- */
- public String getAllFeatures(int wordId);
-
- /**
- * Get all features as array
- * @param wordId word ID of token
- * @return Array containing all features of the token
- */
- public String[] getAllFeaturesArray(int wordId);
-
- /**
* Get Part-Of-Speech of tokens
* @param wordId word ID of token
* @return Part-Of-Speech of the token
*/
+ // TODO: split into the type-safe components
public String getPartOfSpeech(int wordId);
/**
@@ -78,10 +65,12 @@ public interface Dictionary {
public String getBaseForm(int wordId);
/**
- * Get feature(s) of tokens
- * @param wordId word ID token
- * @param fields array of index. If this is empty, return all features.
- * @return Features of the token
+ * Get pronunciation of tokens
+ * @param wordId word ID of token
+ * @return Pronunciation of the token
*/
- public String getFeature(int wordId, int... fields);
+ public String getPronunciation(int wordId);
+
+ // TODO: maybe we should have a optimal method, a non-typesafe
+ // 'getAdditionalData' if other dictionaries like unidic have additional data
}
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/UserDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/UserDictionary.java?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/UserDictionary.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/UserDictionary.java Fri Jan 6 07:26:28 2012
@@ -117,17 +117,16 @@ public class UserDictionary implements D
}
@Override
- public String getAllFeatures(int wordId) {
- return getFeature(wordId);
+ public String getBaseForm(int wordId) {
+ return null; // TODO: add support?
}
@Override
- public String getBaseForm(int wordId) {
+ public String getPronunciation(int wordId) {
return null; // TODO: add support?
}
- @Override
- public String[] getAllFeaturesArray(int wordId) {
+ private String[] getAllFeaturesArray(int wordId) {
String allFeatures = featureEntries.get(wordId);
if(allFeatures == null) {
return null;
@@ -137,8 +136,7 @@ public class UserDictionary implements D
}
- @Override
- public String getFeature(int wordId, int... fields) {
+ private String getFeature(int wordId, int... fields) {
String[] allFeatures = getAllFeaturesArray(wordId);
if (allFeatures == null) {
return null;
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/TokenInfoDictionary$buffer.dat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/TokenInfoDictionary%24buffer.dat?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/TokenInfoDictionary$posDict.dat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/TokenInfoDictionary%24posDict.dat?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/TokenInfoDictionary$targetMap.dat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/TokenInfoDictionary%24targetMap.dat?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/UnknownDictionary$buffer.dat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/UnknownDictionary%24buffer.dat?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/UnknownDictionary$posDict.dat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/UnknownDictionary%24posDict.dat?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/UnknownDictionary$targetMap.dat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/UnknownDictionary%24targetMap.dat?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TokenizerTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TokenizerTest.java?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TokenizerTest.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TokenizerTest.java Fri Jan 6 07:26:28 2012
@@ -63,7 +63,6 @@ public class TokenizerTest extends Lucen
}
}
-
@Test
public void testReadings() {
List<Token> tokens = tokenizer.tokenize("寿å¸ãé£ã¹ããã§ãã");
@@ -77,6 +76,49 @@ public class TokenizerTest extends Lucen
}
@Test
+ public void testReadings2() {
+ List<Token> tokens = tokenizer.tokenize("å¤ãã®å¦çã試é¨ã«è½ã¡ãã");
+ assertEquals(9, tokens.size());
+ assertEquals("ãªãªã¯", tokens.get(0).getReading());
+ assertEquals("ã", tokens.get(1).getReading());
+ assertEquals("ã¬ã¯ã»ã¤", tokens.get(2).getReading());
+ assertEquals("ã¬", tokens.get(3).getReading());
+ assertEquals("ã·ã±ã³", tokens.get(4).getReading());
+ assertEquals("ã", tokens.get(5).getReading());
+ assertEquals("ãªã", tokens.get(6).getReading());
+ assertEquals("ã¿", tokens.get(7).getReading());
+ assertEquals("ã", tokens.get(8).getReading());
+ }
+
+ @Test
+ public void testPronunciations() {
+ List<Token> tokens = tokenizer.tokenize("寿å¸ãé£ã¹ããã§ãã");
+ assertTrue(tokens.size() == 6);
+ assertEquals("ã¹ã·", tokens.get(0).getPronunciation());
+ assertEquals("ã¬", tokens.get(1).getPronunciation());
+ assertEquals("ã¿ã", tokens.get(2).getPronunciation());
+ assertEquals("ã¿ã¤", tokens.get(3).getPronunciation());
+ assertEquals("ãã¹", tokens.get(4).getPronunciation());
+ assertEquals("ã", tokens.get(5).getPronunciation());
+ }
+
+ @Test
+ public void testPronunciations2() {
+ List<Token> tokens = tokenizer.tokenize("å¤ãã®å¦çã試é¨ã«è½ã¡ãã");
+ assertEquals(9, tokens.size());
+ // pronunciation differs from reading here
+ assertEquals("ãªã¼ã¯", tokens.get(0).getPronunciation());
+ assertEquals("ã", tokens.get(1).getPronunciation());
+ assertEquals("ã¬ã¯ã»ã¤", tokens.get(2).getPronunciation());
+ assertEquals("ã¬", tokens.get(3).getPronunciation());
+ assertEquals("ã·ã±ã³", tokens.get(4).getPronunciation());
+ assertEquals("ã", tokens.get(5).getPronunciation());
+ assertEquals("ãªã", tokens.get(6).getPronunciation());
+ assertEquals("ã¿", tokens.get(7).getPronunciation());
+ assertEquals("ã", tokens.get(8).getPronunciation());
+ }
+
+ @Test
public void testBasicForms() {
List<Token> tokens = tokenizer.tokenize("ããã¯ã¾ã å®é¨æ®µéã«ããã¾ãã");
assertEquals(9, tokens.size());
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/TokenInfoDictionaryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/TokenInfoDictionaryBuilder.java?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/TokenInfoDictionaryBuilder.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/TokenInfoDictionaryBuilder.java Fri Jan 6 07:26:28 2012
@@ -154,44 +154,9 @@ public class TokenInfoDictionaryBuilder
* 13 - surface reading
*/
- // TODO: encoding in the current way (basically as CSV) is transparent, but we can
- // save a lot of space, RAM, and some cpu if we use a binary format (via ByteBuffer)
- // instead: (example entry from IPADIC):
- //
- // é¶ã話,1285,1285,5622,åè©,ä¸è¬,*,*,*,*,é¶ã話,ã³ãã¬ããã·,ã³ãã¬ããã·
- //
- // 1. (åè©,ä¸è¬,*,*,*,*) POS parts are huge but don't have many unique entries.
- // they can be deduplicated into separate 'indexes' (arrays) and encoded as a byte or vint refs.
- // 2. surface (é¶ã話) == basicForm (é¶ã話) very often, i think all words except adjectives/verbs.
- // in these common cases we can use a bit (or even '*' as basic form) to mark this.
- // 3. reading (ã³ãã¬ããã·) == pronunciation (ã³ãã¬ããã·) often, we can do the same as 2 here.
- // 4. readings and pronunciations are often completely katakana, cuts their size in half
- // to mark this as a bit and encode singlebyte instead of utf-16: writeByte(ch - 0x30A0)
- //
- // we can associate lazy decoding of these additional features in Token and speed things up a lot:
- // in lucene-gosen we have an enum 'loadState' (3-stages of laziness):
- // NONE, BASIC (pos and base forms), FULL (BASIC + reading/pronunciation).
- // the idea is that reading/pronunciation is rarely used by consumers, so its worth it to lazily
- // only load whats needed, and in the rarer case where someone asks for reading/pronunciation its
- // not too slow to skip over the BASIC stuff we already read in previously.
- // in some cases people just tokenize and don't even use POS or anything and this really helps then.
- public String[] compactEntry(String[] features) {
- // 2. when basicForm == surfaceForm, just indicate this with a * instead of duplicating it.
- if (features[0].equals(features[10])) {
- features[10] = "*";
- }
- // 3. when pronunciation == reading, just indicate this with a * instead of duplicating it.
- // TODO: is there also additional redundancy in the unidic case here that we can improve?
- if (features[12].equals(features[11])) {
- features[12] = "*";
- }
-
- return features;
- }
-
public String[] formatEntry(String[] features) {
if (this.format == DictionaryFormat.IPADIC) {
- return compactEntry(features);
+ return features;
} else {
String[] features2 = new String[13];
features2[0] = features[0];
@@ -215,7 +180,7 @@ public class TokenInfoDictionaryBuilder
features2[11] = features[13];
features2[12] = features[13];
}
- return compactEntry(features2);
+ return features2;
}
}
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/TokenInfoDictionaryWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/TokenInfoDictionaryWriter.java?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/TokenInfoDictionaryWriter.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/TokenInfoDictionaryWriter.java Fri Jan 6 07:26:28 2012
@@ -59,10 +59,10 @@ public class TokenInfoDictionaryWriter {
StringBuilder sb = new StringBuilder();
// build up the POS string
- for (int i = 4; i < 10; i++) {
- sb.append(entry[i]);
- if (i < 9) {
- sb.append(Dictionary.INTERNAL_SEPARATOR);
+ for (int i = 4; i < 8; i++) {
+ sb.append(CSVUtil.quoteEscape(entry[i]));
+ if (i < 7) {
+ sb.append(',');
}
}
String pos = sb.toString();
@@ -72,16 +72,18 @@ public class TokenInfoDictionaryWriter {
posDict.add(pos);
}
- sb.setLength(0);
- for (int i = 10; i < entry.length; i++){
- sb.append(entry[i]).append(Dictionary.INTERNAL_SEPARATOR);
- }
- String features = sb.deleteCharAt(sb.length() - 1).toString();
- int featuresSize = features.length()* 2;
+ // TODO: what are the parts 9 and 10 that kuromoji does not expose via Token?
+ // we need to break all these out (we can structure them inside posdict)
+
+ String baseForm = entry[10];
+ String reading = entry[11];
+ String pronunciation = entry[12];
// extend buffer if necessary
int left = buffer.remaining();
- if (10 + featuresSize > left) { // five short and features
+ // worst case: three short, 4 bytes and features (all as utf-16)
+ int worstCase = 6 + 4 + 2*(baseForm.length() + reading.length() + pronunciation.length());
+ if (worstCase > left) {
ByteBuffer newBuffer = ByteBuffer.allocate(ArrayUtil.oversize(buffer.limit() + 1, 1));
buffer.flip();
newBuffer.put(buffer);
@@ -91,16 +93,61 @@ public class TokenInfoDictionaryWriter {
buffer.putShort(leftId);
buffer.putShort(rightId);
buffer.putShort(wordCost);
- assert posIndex < Character.MAX_VALUE;
- buffer.putChar((char)posIndex);
- buffer.putShort((short)featuresSize);
- for (char c : features.toCharArray()){
- buffer.putChar(c);
+ assert posIndex < 256;
+ buffer.put((byte)posIndex);
+
+ if (baseForm.equals(entry[0])) {
+ buffer.put((byte)0); // base form is the same as surface form
+ } else {
+ buffer.put((byte)baseForm.length());
+ for (int i = 0; i < baseForm.length(); i++) {
+ buffer.putChar(baseForm.charAt(i));
+ }
+ }
+
+ if (isKatakana(reading)) {
+ buffer.put((byte) (reading.length() << 1 | 1));
+ writeKatakana(reading);
+ } else {
+ buffer.put((byte) (reading.length() << 1));
+ for (int i = 0; i < reading.length(); i++) {
+ buffer.putChar(reading.charAt(i));
+ }
+ }
+
+ if (pronunciation.equals(reading)) {
+ buffer.put((byte)0); // pronunciation is the same as reading
+ } else {
+ if (isKatakana(pronunciation)) {
+ buffer.put((byte) (pronunciation.length() << 1 | 1));
+ writeKatakana(pronunciation);
+ } else {
+ buffer.put((byte) (pronunciation.length() << 1));
+ for (int i = 0; i < pronunciation.length(); i++) {
+ buffer.putChar(pronunciation.charAt(i));
+ }
+ }
}
return buffer.position();
}
+ private boolean isKatakana(String s) {
+ for (int i = 0; i < s.length(); i++) {
+ char ch = s.charAt(i);
+ if (ch < 0x30A0 || ch > 0x30FF) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private void writeKatakana(String s) {
+ for (int i = 0; i < s.length(); i++) {
+ buffer.put((byte) (s.charAt(i) - 0x30A0));
+ }
+ }
+
public void addMapping(int sourceId, int wordId) {
if(targetMap.length <= sourceId) {
int[][] newArray = new int[sourceId + 1][];
@@ -186,6 +233,7 @@ public class TokenInfoDictionaryWriter {
} finally {
os.close();
}
+ System.out.println("Info: wrote " + posDict.size() + " unique POS entries");
}
protected void writeDictionary(String filename) throws IOException {
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/UnknownDictionaryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/UnknownDictionaryBuilder.java?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/UnknownDictionaryBuilder.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/UnknownDictionaryBuilder.java Fri Jan 6 07:26:28 2012
@@ -27,7 +27,7 @@ import java.nio.charset.CharsetDecoder;
import java.nio.charset.CodingErrorAction;
public class UnknownDictionaryBuilder {
- private static final String NGRAM_DICTIONARY_ENTRY = "NGRAM,5,5,-32768,-,*,*,*,*,*,*";
+ private static final String NGRAM_DICTIONARY_ENTRY = "NGRAM,5,5,-32768,-,*,*,*,*,*,*,*,*";
private String encoding = "euc-jp";
@@ -66,7 +66,9 @@ public class UnknownDictionaryBuilder {
String line = null;
while ((line = lineReader.readLine()) != null) {
- dictionary.put(CSVUtil.parse(line)); // Probably we don't need to validate entry
+ // note: unk.def only has 10 fields, it simplifies the writer to just append empty reading and pronunciation,
+ // even though the unknown dictionary returns hardcoded null here.
+ dictionary.put(CSVUtil.parse(line + ",*,*")); // Probably we don't need to validate entry
}
return dictionary;
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/test/org/apache/lucene/analysis/kuromoji/dict/UnknownDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/test/org/apache/lucene/analysis/kuromoji/dict/UnknownDictionaryTest.java?rev=1228000&r1=1227999&r2=1228000&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/test/org/apache/lucene/analysis/kuromoji/dict/UnknownDictionaryTest.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/test/org/apache/lucene/analysis/kuromoji/dict/UnknownDictionaryTest.java Fri Jan 6 07:26:28 2012
@@ -54,15 +54,15 @@ public class UnknownDictionaryTest exten
public void testPut() {
UnknownDictionaryWriter unkDic = new UnknownDictionaryWriter(10 * 1024 * 1024);
try{
- unkDic.put(CSVUtil.parse("KANJI,1285,11426,åè©,ä¸è¬,*,*,*,*,*"));
+ unkDic.put(CSVUtil.parse("KANJI,1285,11426,åè©,ä¸è¬,*,*,*,*,*,*,*"));
fail();
} catch(Exception e){
}
- String entry1 = "KANJI,1285,1285,11426,åè©,ä¸è¬,*,*,*,*,*";
- String entry2 = "ALPHA,1285,1285,13398,åè©,ä¸è¬,*,*,*,*,*";
- String entry3 = "HIRAGANA,1285,1285,13069,åè©,ä¸è¬,*,*,*,*,*";
+ String entry1 = "KANJI,1285,1285,11426,åè©,ä¸è¬,*,*,*,*,*,*,*";
+ String entry2 = "ALPHA,1285,1285,13398,åè©,ä¸è¬,*,*,*,*,*,*,*";
+ String entry3 = "HIRAGANA,1285,1285,13069,åè©,ä¸è¬,*,*,*,*,*,*,*";
unkDic.putCharacterCategory(0, "KANJI");
unkDic.putCharacterCategory(1, "ALPHA");