You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/02/14 12:37:46 UTC
svn commit: r1243882 - in /lucene/dev/branches/lucene3767:
lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/
modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/
modules/analysis/kuromoji/src/java/org/apache/lucene/ana...
Author: mikemccand
Date: Tue Feb 14 11:37:45 2012
New Revision: 1243882
URL: http://svn.apache.org/viewvc?rev=1243882&view=rev
Log:
LUCENE-3767: fix bug in 2nd best path; add Mode.SEARCH_WITH_COMPOUNDS; fix tests to test it; remove nocommits; add comments
Added:
lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/search-segmentation-with-compounds-tests.txt
Modified:
lucene/dev/branches/lucene3767/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java
lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiTokenizer2.java
lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Segmenter.java
lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Token.java
lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/viterbi/Viterbi.java
lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/SegmenterTest.java
lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java
lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiTokenizer.java
lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestQuality.java
lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestSearchMode.java
lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/search-segmentation-tests.txt
Modified: lucene/dev/branches/lucene3767/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3767/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java?rev=1243882&r1=1243881&r2=1243882&view=diff
==============================================================================
--- lucene/dev/branches/lucene3767/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java (original)
+++ lucene/dev/branches/lucene3767/lucene/core/src/java/org/apache/lucene/analysis/tokenattributes/PositionIncrementAttribute.java Tue Feb 14 11:37:45 2012
@@ -57,6 +57,7 @@ public interface PositionIncrementAttrib
*/
public int getPositionIncrement();
+ // nocommit break into new attr...
// nocommit better names...? getLength? getSpan?
/** @param positionLength how many positions this token
* spans. NOTE: this is optional, and most analyzers
Modified: lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiTokenizer2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiTokenizer2.java?rev=1243882&r1=1243881&r2=1243882&view=diff
==============================================================================
--- lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiTokenizer2.java (original)
+++ lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/KuromojiTokenizer2.java Tue Feb 14 11:37:45 2012
@@ -20,14 +20,10 @@ package org.apache.lucene.analysis.kurom
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
-import java.util.Collection;
import java.util.Collections;
import java.util.EnumMap;
-import java.util.HashSet;
import java.util.List;
-import java.util.Set;
-import org.apache.lucene.analysis.CharStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.kuromoji.Segmenter.Mode;
import org.apache.lucene.analysis.kuromoji.dict.CharacterDefinition;
@@ -43,7 +39,6 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.util.ArrayUtil;
-import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.fst.FST;
@@ -53,20 +48,12 @@ import org.apache.lucene.util.fst.FST;
// nocommit add toDot and look at 1st pass intersection
-// nocommit need better testing of compound output... need
-// assertAnalyzesTo to accept posLength too
-
-// nocommit add comment explaining this isn't quite a real viterbi
+// nocommit needs assertAnalyzersTo to accept
+// posLength... then fix compound tests to verify it
// nocomit explain how the 2nd best tokenization is
// "contextual"...
-// nocommit -- need a test that doesn't pre-split by
-// sentence... ie, we don't BOS/EOS on each sentence
-// break any more... so this can change the results
-// depending on whether ipadic was "trained" with sentence
-// breaks?
-
// nocommit -- should we use the sentence breakiterator
// too..? we can simply use it to slip an EOS/BOS token
// in...
@@ -87,9 +74,6 @@ public final class KuromojiTokenizer2 ex
private static final int SEARCH_MODE_OTHER_PENALTY = 1700;
- // nocommit
- public static boolean DO_OUTPUT_COMPOUND = false;
-
private final EnumMap<Type, Dictionary> dictionaryMap = new EnumMap<Type, Dictionary>(Type.class);
private final TokenInfoFST fst;
@@ -106,29 +90,36 @@ public final class KuromojiTokenizer2 ex
private final FST.BytesReader userFSTReader;
private final TokenInfoFST userFST;
- private Reader reader;
+ private final WrappedCharArray buffer = new WrappedCharArray();
- // Next absolute position to process:
- private int pos;
+ private final WrappedPositionArray positions = new WrappedPositionArray();
- private WrappedCharArray buffer = new WrappedCharArray();
+ private final boolean discardPunctuation;
+ private final boolean searchMode;
+ private final boolean extendedMode;
+ private final boolean outputCompounds;
// index of the last character of unknown word:
- // nocommit put back:
- // int unknownWordEndIndex = -1;
-
- private WrappedPositionArray positions = new WrappedPositionArray();
+ private int unknownWordEndIndex = -1;
+ // True once we've hit the EOF from the input reader:
private boolean end;
- private final boolean discardPunctuation;
- private final boolean searchMode;
- private final boolean extendedMode;
+
+ // Used to determine if all extensions from a given node
+ // originated from the same backtrace point:
private int sameLeastIndex;
+ // Last absolute position we backtraced from:
private int lastBackTracePos;
+
+ // Position of last token we returned; we use this to
+ // figure out whether to set posIncr to 0 or 1:
private int lastTokenPos;
- // Already parsed but not yet passed to caller:
+ // Next absolute position to process:
+ private int pos;
+
+ // Already parsed, but not yet passed to caller, tokens:
private final List<Token> pending = new ArrayList<Token>();
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
@@ -160,14 +151,22 @@ public final class KuromojiTokenizer2 ex
case SEARCH:
searchMode = true;
extendedMode = false;
+ outputCompounds = false;
+ break;
+ case SEARCH_WITH_COMPOUNDS:
+ searchMode = true;
+ extendedMode = false;
+ outputCompounds = true;
break;
case EXTENDED:
searchMode = true;
extendedMode = true;
+ outputCompounds = false;
break;
default:
searchMode = false;
extendedMode = false;
+ outputCompounds = false;
break;
}
buffer.reset(input);
@@ -187,8 +186,7 @@ public final class KuromojiTokenizer2 ex
private void resetState() {
positions.reset();
- // nocommit put back
- //unknownWordEndIndex = -1;
+ unknownWordEndIndex = -1;
pos = 0;
end = false;
lastBackTracePos = 0;
@@ -200,11 +198,24 @@ public final class KuromojiTokenizer2 ex
}
@Override
- public final void end() {
+ public void end() {
// set final offset
offsetAtt.setOffset(correctOffset(pos), correctOffset(pos));
}
+ // Returns the added cost that a 2nd best segmentation is
+ // allowed to have. Ie, if we see path with cost X,
+ // ending in a compound word, and this method returns
+ // threshold > 0, then we will also find the 2nd best
+ // segmentation and if its path score is within this
+ // threshold of X, we'll include it in the output:
+ private int computeSecondBestThreshold(int pos, int length) throws IOException {
+ // TODO: maybe we do something else here, instead of just
+ // using the penalty...? EG we can be more aggressive on
+ // when to also test for 2nd best path
+ return computePenalty(pos, length);
+ }
+
private int computePenalty(int pos, int length) throws IOException {
if (length > SEARCH_MODE_KANJI_LENGTH) {
boolean allKanji = true;
@@ -216,7 +227,6 @@ public final class KuromojiTokenizer2 ex
break;
}
}
- final int penalty;
if (allKanji) { // Process only Kanji keywords
return (length - SEARCH_MODE_KANJI_LENGTH) * SEARCH_MODE_KANJI_PENALTY;
} else if (length > SEARCH_MODE_OTHER_LENGTH) {
@@ -279,9 +289,14 @@ public final class KuromojiTokenizer2 ex
}
public void add(int cost, int nodeID, int backPos, int backIndex, int backID, Type backType) {
- // nocommit in theory, we should check if nodeID is
- // already present here, and update it if
- // so... instead of just always adding:
+ // NOTE: this isn't quite a true Viterbit search,
+ // becase we should check if nodeID is
+ // already present here, and only update if the new
+ // cost is less than the current cost, instead of
+ // simply appending. However, that will likely hurt
+ // performance (usually we add a nodeID only once),
+ // and it means we actually create the full graph
+ // intersection instead of a "normal" Viterbi lattice:
if (count == costs.length) {
grow();
}
@@ -310,37 +325,36 @@ public final class KuromojiTokenizer2 ex
// forwardCount naturally resets after it runs:
assert forwardCount == 0: "pos=" + pos + " forwardCount=" + forwardCount;
}
-
- // nocommit maybe?
- // public void update(int cost, int nodeID, int backPos, int backIndex, int backID)
}
- // nocommit absorb into add...?
- private void add(Dictionary dict, Position posData, int endPos, int wordID, Type type) throws IOException {
+ private void add(Dictionary dict, Position fromPosData, int endPos, int wordID, Type type, boolean addPenalty) throws IOException {
final int wordCost = dict.getWordCost(wordID);
final int leftID = dict.getLeftId(wordID);
int leastCost = Integer.MAX_VALUE;
int leastIDX = -1;
- assert posData.count > 0;
- for(int idx=0;idx<posData.count;idx++) {
- // Cost is path cost so far, plus word cost, plus
- // bigram cost:
- final int cost = posData.costs[idx] + wordCost + costs.get(posData.nodeID[idx], leftID);
+ assert fromPosData.count > 0;
+ for(int idx=0;idx<fromPosData.count;idx++) {
+ // Cost is path cost so far, plus word cost (added at
+ // end of loop), plus bigram cost:
+ final int cost = fromPosData.costs[idx] + costs.get(fromPosData.nodeID[idx], leftID);
+ if (VERBOSE) {
+ System.out.println(" fromIDX=" + idx + ": cost=" + cost + " (prevCost=" + fromPosData.costs[idx] + " wordCost=" + wordCost + " bgCost=" + costs.get(fromPosData.nodeID[idx], leftID) + " leftID=" + leftID);
+ }
if (cost < leastCost) {
leastCost = cost;
leastIDX = idx;
+ //System.out.println(" **");
}
}
+ leastCost += wordCost;
+
if (VERBOSE) {
- System.out.println(" + cost=" + leastCost + " wordID=" + wordID + " leftID=" + leftID + " tok=" + new String(buffer.get(posData.pos, endPos-posData.pos)) + " leastIDX=" + leastIDX + " toPos.idx=" + positions.get(endPos).count);
+ System.out.println(" + cost=" + leastCost + " wordID=" + wordID + " leftID=" + leftID + " leastIDX=" + leastIDX + " toPos.idx=" + positions.get(endPos).count);
}
- // nocommit ideally we don't have to do this ... just
- // putting it here to confirm same results as current
- // segmenter:
- if (!DO_OUTPUT_COMPOUND && searchMode && type != Type.USER) {
- final int penalty = computePenalty(posData.pos, endPos - posData.pos);
+ if ((addPenalty || (!outputCompounds && searchMode)) && type != Type.USER) {
+ final int penalty = computePenalty(fromPosData.pos, endPos - fromPosData.pos);
if (VERBOSE) {
if (penalty > 0) {
System.out.println(" + penalty=" + penalty + " cost=" + (leastCost+penalty));
@@ -349,14 +363,15 @@ public final class KuromojiTokenizer2 ex
leastCost += penalty;
}
- positions.get(endPos).add(leastCost, dict.getRightId(wordID), posData.pos, leastIDX, wordID, type);
- /*
+ //positions.get(endPos).add(leastCost, dict.getRightId(wordID), fromPosData.pos, leastIDX, wordID, type);
+ assert leftID == dict.getRightId(wordID);
+ positions.get(endPos).add(leastCost, leftID, fromPosData.pos, leastIDX, wordID, type);
+
if (sameLeastIndex == -1) {
sameLeastIndex = leastIDX;
} else if (sameLeastIndex != leastIDX) {
- sameLeastIndex = Integer.MAX_VALUE;
+ sameLeastIndex = -2;
}
- */
}
@Override
@@ -651,7 +666,8 @@ public final class KuromojiTokenizer2 ex
// nocommit if I change 40 below it changes number of
// tokens in TestQuality!
- if ((pos - lastBackTracePos >= 100) && posData.count == 1 && isFrontier) {
+ //if ((pos - lastBackTracePos >= 100) && posData.count == 1 && isFrontier) {
+ if (pos > lastBackTracePos && posData.count == 1 && isFrontier) {
// We are at a "frontier", and only one node is
// alive, so whatever the eventual best path is must
// come through this node. So we can safely commit
@@ -664,7 +680,6 @@ public final class KuromojiTokenizer2 ex
if (pending.size() != 0) {
return;
} else {
- // nocommit: make punctuation-only testcase
// This means the backtrace only produced
// punctuation tokens, so we must keep parsing.
}
@@ -689,11 +704,7 @@ public final class KuromojiTokenizer2 ex
System.out.println(" " + posData.count + " arcs in");
}
- // nocommit must also 1) detect a
- // less-obvious-yet-still-committable backtrace op,
- // when, even though N > 1 states are/were alive, they
- // all back through a single state, and also 2) maybe
- // need to "force" a "when all else fails" backtrace
+ // nocommit may need "force" a "when all else fails" backtrace?
boolean anyMatches = false;
@@ -711,22 +722,21 @@ public final class KuromojiTokenizer2 ex
}
output += arc.output.intValue();
if (arc.isFinal()) {
- add(userDictionary, posData, posAhead+1, output + arc.nextFinalOutput.intValue(), Type.USER);
+ add(userDictionary, posData, posAhead+1, output + arc.nextFinalOutput.intValue(), Type.USER, false);
anyMatches = true;
}
}
}
- // nocommit we can be more aggressive about user
+ // TODO: we can be more aggressive about user
// matches? if we are "under" a user match then don't
- // extend these paths?
+ // extend KNOWN/UNKNOWN paths?
if (!anyMatches) {
// Next, try known dictionary matches
fst.getFirstArc(arc);
int output = 0;
- boolean allKanji = true;
for(int posAhead=posData.pos;;posAhead++) {
final int ch = buffer.get(posAhead);
if (ch == -1) {
@@ -740,10 +750,11 @@ public final class KuromojiTokenizer2 ex
output += arc.output.intValue();
- // NOTE: for known words that are too-long
+ // Optimization: for known words that are too-long
// (compound), we should pre-compute the 2nd
// best segmentation and store it in the
- // dictionary. This is just a time/space opto...
+ // dictionary instead of recomputing it each time a
+ // match is found.
if (arc.isFinal()) {
dictionary.lookupWordIds(output + arc.nextFinalOutput.intValue(), wordIdRef);
@@ -751,7 +762,7 @@ public final class KuromojiTokenizer2 ex
System.out.println(" KNOWN word " + new String(buffer.get(pos, posAhead - pos + 1)) + " toPos=" + (posAhead + 1) + " " + wordIdRef.length + " wordIDs");
}
for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
- add(dictionary, posData, posAhead+1, wordIdRef.ints[wordIdRef.offset + ofs], Type.KNOWN);
+ add(dictionary, posData, posAhead+1, wordIdRef.ints[wordIdRef.offset + ofs], Type.KNOWN, false);
anyMatches = true;
}
}
@@ -760,12 +771,10 @@ public final class KuromojiTokenizer2 ex
// In the case of normal mode, it doesn't process unknown word greedily.
- // nocommit: fix
- /*
if (!searchMode && unknownWordEndIndex > posData.pos) {
continue;
}
- */
+
final char firstCharacter = (char) buffer.get(pos);
// nocommit -- can't we NOT pursue unk if a known
// token "covers" us...?
@@ -799,20 +808,18 @@ public final class KuromojiTokenizer2 ex
System.out.println(" UNKNOWN word len=" + unknownWordLength + " " + wordIdRef.length + " wordIDs");
}
for (int ofs = 0; ofs < wordIdRef.length; ofs++) {
- add(unkDictionary, posData, posData.pos + unknownWordLength, wordIdRef.ints[wordIdRef.offset + ofs], Type.UNKNOWN);
+ add(unkDictionary, posData, posData.pos + unknownWordLength, wordIdRef.ints[wordIdRef.offset + ofs], Type.UNKNOWN, false);
}
- // nocommit fixme
- //unknownWordEndIndex = posData.pos + unknownWordLength;
+ unknownWordEndIndex = posData.pos + unknownWordLength;
}
// nocommit explainme:
- if (false && (pos - lastBackTracePos >= 100) && sameLeastIndex != -1 && sameLeastIndex != Integer.MAX_VALUE &&
- pos != lastBackTracePos && isFrontier) {
+ if ((pos - lastBackTracePos >= 100) && sameLeastIndex >= 0 && isFrontier) {
final int sav = sameLeastIndex;
- //System.out.println("**SAME: " + sameLeastIndex);
+ //System.out.println("**SAME: " + sameLeastIndex + " vs " + posData.count);
backtrace(posData, sameLeastIndex);
// Re-base cost so we don't risk int overflow:
@@ -822,7 +829,6 @@ public final class KuromojiTokenizer2 ex
if (pending.size() != 0) {
return;
} else {
- // nocommit: make punctuation-only testcase
// This means the backtrace only produced
// punctuation tokens, so we must keep parsing.
}
@@ -871,7 +877,6 @@ public final class KuromojiTokenizer2 ex
int minBackPos = Integer.MAX_VALUE;
- final int compoundLength = endPos - startPos;
// First pass: walk backwards, building up the forward
// arcs and pruning inadmissible arcs:
for(int pos=endPos; pos >= startPos; pos--) {
@@ -881,14 +886,7 @@ public final class KuromojiTokenizer2 ex
}
for(int arcIDX=0;arcIDX<posData.count;arcIDX++) {
final int backPos = posData.backPos[arcIDX];
- // nocommit: O(N^2) cost here!! adversary could
- // exploit this i think...? how to protect? hmm
- // maybe i need to track lastKanjiCount at each
- // position..
- // nocommit hacky:
- // if (backPos >= startPos && computePenalty(backPos, pos-backPos) == 0) {
- if (backPos >= startPos && (pos-backPos) <= 2*compoundLength/3) {
- //if (backPos != -1 && (pos-backPos) <= 2*compoundLength/3) {
+ if (backPos >= startPos) {
// Keep this arc:
//System.out.println(" keep backPos=" + backPos);
minBackPos = Math.min(backPos, minBackPos);
@@ -907,8 +905,14 @@ public final class KuromojiTokenizer2 ex
}
}
+ // nocommit shouldn't we just "run w/ penalties" in this
+ // pass...?
+
// nocommit what if minBackPos not set...?
+ // nocommit it must be startPos...?
+ assert minBackPos == startPos;
+
// Second pass: walk forward, re-scoring:
//for(int pos=startPos; pos < endPos; pos++) {
for(int pos=minBackPos; pos < endPos; pos++) {
@@ -924,23 +928,29 @@ public final class KuromojiTokenizer2 ex
posData.forwardCount = 0;
continue;
}
+
if (pos == startPos) {
// On the initial position, only consider the best
// path so we "force congruence": the
// sub-segmentation is "in context" of what the best
// path (compound token) had matched:
+
+ // nocommit could be -1?: don't i need to check
+ // startPos==0 too...?
final Dictionary dict = getDict(posData.backType[bestStartIDX]);
+
final int rightID = startPos == 0 ? 0 : dict.getRightId(posData.backID[bestStartIDX]);
- final int pathCost = posData.backID[bestStartIDX];
+ final int pathCost = posData.costs[bestStartIDX];
for(int forwardArcIDX=0;forwardArcIDX<posData.forwardCount;forwardArcIDX++) {
final Type forwardType = posData.forwardType[forwardArcIDX];
final Dictionary dict2 = getDict(forwardType);
final int wordID = posData.forwardID[forwardArcIDX];
- final int newCost = pathCost + dict2.getWordCost(wordID) +
- costs.get(rightID, dict2.getLeftId(wordID));
final int toPos = posData.forwardPos[forwardArcIDX];
+ final int newCost = pathCost + dict2.getWordCost(wordID) +
+ costs.get(rightID, dict2.getLeftId(wordID)) +
+ computePenalty(pos, toPos-pos);
if (VERBOSE) {
- System.out.println(" + " + forwardType + " word " + new String(buffer.get(pos, toPos-pos)) + " toPos=" + toPos);
+ System.out.println(" + " + forwardType + " word " + new String(buffer.get(pos, toPos-pos)) + " toPos=" + toPos + " cost=" + newCost + " penalty=" + computePenalty(pos, toPos-pos) + " toPos.idx=" + positions.get(toPos).count);
}
positions.get(toPos).add(newCost,
dict2.getRightId(wordID),
@@ -962,7 +972,8 @@ public final class KuromojiTokenizer2 ex
posData,
toPos,
posData.forwardID[forwardArcIDX],
- forwardType);
+ forwardType,
+ true);
}
}
posData.forwardCount = 0;
@@ -1003,8 +1014,9 @@ public final class KuromojiTokenizer2 ex
int backPos = posData.backPos[bestIDX];
int length = pos - backPos;
Type backType = posData.backType[bestIDX];
+ int backID = posData.backID[bestIDX];
- if (DO_OUTPUT_COMPOUND && searchMode && altToken == null && backType != Type.USER) {
+ if (outputCompounds && searchMode && altToken == null && backType != Type.USER) {
// In searchMode, if best path had picked a too-long
// token, we use the "penalty" to compute the allowed
@@ -1013,10 +1025,7 @@ public final class KuromojiTokenizer2 ex
// threshold, we pursue it instead (but also output
// the long token).
- // nocommit -- maybe re-tune this "penalty", now
- // that it means something very different ("output
- // smaller segmentation"):
- final int penalty = computePenalty(backPos, pos-backPos);
+ final int penalty = computeSecondBestThreshold(backPos, pos-backPos);
if (penalty > 0) {
if (VERBOSE) {
@@ -1025,38 +1034,45 @@ public final class KuromojiTokenizer2 ex
// Use the penalty to set maxCost on the 2nd best
// segmentation:
- final int maxCost = posData.costs[bestIDX] + penalty;
-
- final int backID = posData.backID[bestIDX];
+ int maxCost = posData.costs[bestIDX] + penalty;
+ if (lastLeftWordID != -1) {
+ maxCost += costs.get(getDict(backType).getRightId(backID), lastLeftWordID);
+ }
// Now, prune all too-long tokens from the graph:
pruneAndRescore(backPos, pos,
posData.backIndex[bestIDX]);
- if (VERBOSE) {
- System.out.println(" afterPrune: " + posData.count + " arcs arriving");
- }
-
// Finally, find 2nd best back-trace and resume
// backtrace there:
int leastCost = Integer.MAX_VALUE;
int leastIDX = -1;
for(int idx=0;idx<posData.count;idx++) {
+ // nocommit needs wordCost added here?
int cost = posData.costs[idx];
+ //System.out.println(" idx=" + idx + " prevCost=" + cost);
+
if (lastLeftWordID != -1) {
cost += costs.get(getDict(posData.backType[idx]).getRightId(posData.backID[idx]),
lastLeftWordID);
+ //System.out.println(" += bgCost=" + costs.get(getDict(posData.backType[idx]).getRightId(posData.backID[idx]),
+ //lastLeftWordID) + " -> " + cost);
}
+ //System.out.println("penalty " + posData.backPos[idx] + " to " + pos);
+ //cost += computePenalty(posData.backPos[idx], pos - posData.backPos[idx]);
if (cost < leastCost) {
+ //System.out.println(" ** ");
leastCost = cost;
leastIDX = idx;
}
}
//System.out.println(" leastIDX=" + leastIDX);
- // nocommit -- must also check whether 2nd best score falls w/in threshold?
+ if (VERBOSE) {
+ System.out.println(" afterPrune: " + posData.count + " arcs arriving; leastCost=" + leastCost + " vs threshold=" + maxCost + " lastLeftWordID=" + lastLeftWordID);
+ }
- if (leastIDX != -1 && posData.backPos[leastIDX] <= maxCost) {
+ if (leastIDX != -1 && leastCost <= maxCost && posData.backPos[leastIDX] != backPos) {
// We should have pruned the altToken from the graph:
assert posData.backPos[leastIDX] != backPos;
@@ -1076,6 +1092,7 @@ public final class KuromojiTokenizer2 ex
backPos = posData.backPos[bestIDX];
length = pos - backPos;
backType = posData.backType[bestIDX];
+ backID = posData.backID[bestIDX];
backCount = 0;
} else {
@@ -1104,10 +1121,14 @@ public final class KuromojiTokenizer2 ex
if (VERBOSE) {
System.out.println(" add altToken=" + altToken);
}
- assert backCount >= 1;
- backCount++;
- altToken.setPositionLength(backCount);
- pending.add(altToken);
+ if (backCount > 0) {
+ backCount++;
+ altToken.setPositionLength(backCount);
+ pending.add(altToken);
+ } else {
+ // This means alt token was all punct tokens:
+ assert discardPunctuation;
+ }
altToken = null;
}
@@ -1117,7 +1138,7 @@ public final class KuromojiTokenizer2 ex
// Expand the phraseID we recorded into the actual
// segmentation:
- final int[] wordIDAndLength = userDictionary.lookupSegmentation(posData.backID[bestIDX]);
+ final int[] wordIDAndLength = userDictionary.lookupSegmentation(backID);
int wordID = wordIDAndLength[0];
int current = backPos;
for(int j=1; j < wordIDAndLength.length; j++) {
@@ -1142,7 +1163,7 @@ public final class KuromojiTokenizer2 ex
backCount += wordIDAndLength.length-1;
} else {
- if (extendedMode && posData.backType[bestIDX] == Type.UNKNOWN) {
+ if (extendedMode && backType == Type.UNKNOWN) {
// nocommit what if the altToken is unknonwn?
// In EXTENDED mode we convert unknown word into
// unigrams:
@@ -1153,21 +1174,24 @@ public final class KuromojiTokenizer2 ex
i--;
charLen = 2;
}
- //System.out.println(" extended tok offset=" + (offset + i));
- pending.add(new Token(CharacterDefinition.NGRAM,
- fragment,
- offset + i,
- charLen,
- Type.UNKNOWN,
- backPos + i,
- unkDictionary));
- unigramTokenCount++;
+ //System.out.println(" extended tok offset="
+ //+ (offset + i));
+ if (!discardPunctuation || !isPunctuation(fragment[offset+i])) {
+ pending.add(new Token(CharacterDefinition.NGRAM,
+ fragment,
+ offset + i,
+ charLen,
+ Type.UNKNOWN,
+ backPos + i,
+ unkDictionary));
+ unigramTokenCount++;
+ }
}
backCount += unigramTokenCount;
} else if (!discardPunctuation || length == 0 || !isPunctuation(fragment[offset])) {
//System.out.println("backPos=" + backPos);
- pending.add(new Token(posData.backID[bestIDX],
+ pending.add(new Token(backID,
fragment,
offset,
length,
@@ -1185,7 +1209,7 @@ public final class KuromojiTokenizer2 ex
}
}
- lastLeftWordID = dict.getLeftId(posData.backID[bestIDX]);
+ lastLeftWordID = dict.getLeftId(backID);
pos = backPos;
bestIDX = posData.backIndex[bestIDX];
}
@@ -1209,7 +1233,7 @@ public final class KuromojiTokenizer2 ex
return dictionaryMap.get(type);
}
- private static final boolean isPunctuation(char ch) {
+ private static boolean isPunctuation(char ch) {
// TODO: somehow this is slowish.. takes ~5% off
// chars/msec from Perf.java; maybe we
// can spend RAM...
Modified: lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Segmenter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Segmenter.java?rev=1243882&r1=1243881&r2=1243882&view=diff
==============================================================================
--- lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Segmenter.java (original)
+++ lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Segmenter.java Tue Feb 14 11:37:45 2012
@@ -38,7 +38,7 @@ import org.apache.lucene.analysis.kuromo
*/
public class Segmenter {
public static enum Mode {
- NORMAL, SEARCH, EXTENDED
+ NORMAL, SEARCH, SEARCH_WITH_COMPOUNDS, EXTENDED
}
public static final Mode DEFAULT_MODE = Mode.SEARCH;
@@ -128,8 +128,7 @@ public class Segmenter {
position = Math.min(indexOfMaru, indexOfTen);
}
- // nocommit
- if(false && position >= 0) {
+ if(position >= 0) {
splitPositions.add(position);
currentPosition = position + 1;
} else {
Modified: lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Token.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Token.java?rev=1243882&r1=1243881&r2=1243882&view=diff
==============================================================================
--- lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Token.java (original)
+++ lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/Token.java Tue Feb 14 11:37:45 2012
@@ -47,7 +47,7 @@ public class Token {
@Override
public String toString() {
- return "Token(\"" + new String(surfaceForm, offset, length) + "\" pos=" + position + " type=" + type + " wordId=" + wordId + ")";
+ return "Token(\"" + new String(surfaceForm, offset, length) + "\" pos=" + position + " type=" + type + " wordId=" + wordId + " leftID=" + dictionary.getLeftId(wordId) + ")";
}
/**
Modified: lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/viterbi/Viterbi.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/viterbi/Viterbi.java?rev=1243882&r1=1243881&r2=1243882&view=diff
==============================================================================
--- lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/viterbi/Viterbi.java (original)
+++ lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/viterbi/Viterbi.java Tue Feb 14 11:37:45 2012
@@ -126,7 +126,7 @@ public class Viterbi {
break;
}
- //System.out.println(" leaving node.wordID=" + node.getWordId() + " leftID=" + node.getLeftId());
+ //System.out.println(" leaving node.wordID=" + node.getWordId() + " leftID=" + node.getLeftId() + " toPos=" + (node.getOffset()+node.getLength()));
int backwardConnectionId = node.getLeftId();
int wordCost = node.getWordCost();
@@ -160,13 +160,14 @@ public class Viterbi {
if (allKanji) { // Process only Kanji keywords
pathCost += (length - SEARCH_MODE_KANJI_LENGTH) * SEARCH_MODE_KANJI_PENALTY;
//System.out.println(" + kanji penalty=" + (length - SEARCH_MODE_KANJI_LENGTH) * SEARCH_MODE_KANJI_PENALTY + " cost=" + pathCost);
+ //System.out.println(" after penalty pathCost=" + pathCost);
} else if (length > SEARCH_MODE_OTHER_LENGTH) {
pathCost += (length - SEARCH_MODE_OTHER_LENGTH) * SEARCH_MODE_OTHER_PENALTY;
//System.out.println(" + non-kanji penalty=" + (length - SEARCH_MODE_OTHER_LENGTH) * SEARCH_MODE_OTHER_PENALTY + " cost=" + pathCost);
+ //System.out.println(" after penalty pathCost=" + pathCost);
}
}
}
- //System.out.println(" after penalty pathCost=" + pathCost);
if (pathCost < leastPathCost){ // If total cost is lower than before, set current previous node as best left node (previous means left).
//System.out.println(" **");
@@ -318,7 +319,7 @@ public class Viterbi {
private void processUserDictionary(char text[], int offset, int len, ViterbiNode[][] startIndexArr, ViterbiNode[][] endIndexArr, int[] startSizeArr, int[] endSizeArr) throws IOException {
int[][] result = userDictionary.lookup(text, offset, len);
for(int[] segmentation : result) {
- System.out.println("SEG=" + Arrays.toString(segmentation));
+ //System.out.println("SEG=" + Arrays.toString(segmentation));
int wordId = segmentation[0];
int index = segmentation[1];
int length = segmentation[2];
Modified: lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/SegmenterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/SegmenterTest.java?rev=1243882&r1=1243881&r2=1243882&view=diff
==============================================================================
--- lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/SegmenterTest.java (original)
+++ lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/SegmenterTest.java Tue Feb 14 11:37:45 2012
@@ -26,8 +26,6 @@ import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
-// nocommit cut these over to KT2
-
public class SegmenterTest extends LuceneTestCase {
private static Segmenter segmenter;
Modified: lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java?rev=1243882&r1=1243881&r2=1243882&view=diff
==============================================================================
--- lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java (original)
+++ lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiAnalyzer.java Tue Feb 14 11:37:45 2012
@@ -23,7 +23,7 @@ import java.io.StringReader;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.TokenStream;
-import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.kuromoji.Segmenter.Mode;
public class TestKuromojiAnalyzer extends BaseTokenStreamTestCase {
/** This test fails with NPE when the
@@ -51,101 +51,92 @@ public class TestKuromojiAnalyzer extend
*/
public void testDecomposition() throws IOException {
- // nocommit
+ // Senior software engineer:
+ assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢",
+ new String[] { "ã·ãã¢",
+ "ã½ããã¦ã§ã¢",
+ "ã¨ã³ã¸ãã¢" },
+ new int[] { 1, 1, 1}
+ );
+ // Kansai International Airport:
+ assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "é¢è¥¿å½é空港",
+ new String[] { "é¢è¥¿",
+ "å½é",
+ "空港" },
+ new int[] {1, 1, 1}
+ );
+
+ // Konika Minolta Holdings; not quite the right
+ // segmentation (see LUCENE-3726):
+ assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "ã³ãã«ããã«ã¿ãã¼ã«ãã£ã³ã°ã¹",
+ new String[] { "ã³ãã«",
+ "ããã«ã¿",
+ "ãã¼ã«ãã£ã³ã°ã¹"},
+ new int[] {1, 1, 1}
+ );
+
+ // Narita Airport
+ assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "æç°ç©ºæ¸¯",
+ new String[] { "æç°",
+ "空港" },
+ new int[] {1, 1});
+
+ // Kyoto University Baseball Club
+ assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "京é½å¤§å¦ç¡¬å¼éçé¨",
+ new String[] { "京é½",
+ "大å¦",
+ "硬å¼",
+ "éç",
+ "é¨" },
+ new int[] {1, 1, 1, 1, 1});
+ }
+
+ public void testDecompositionWithCompounds() throws IOException {
+
+ final Analyzer a = new KuromojiAnalyzer(TEST_VERSION_CURRENT, null, Mode.SEARCH_WITH_COMPOUNDS,
+ KuromojiAnalyzer.getDefaultStopSet(),
+ KuromojiAnalyzer.getDefaultStopTags());
+
/*
- TokenStream ts = new KuromojiAnalyzer(TEST_VERSION_CURRENT)
- .tokenStream("foo",
- //new StringReader("楽å¨ã»æ©æã®è³ªã¯å®å
¨ã«ããä»æ§ï¼ã§ããå¤æ®µã¯ä½å¦ãããå®ãï¼"));
- //new StringReader("ã§ããå¤æ®µã¯ä½å¦ãããå®ãï¼"));
- new StringReader("æ¨å·¥è¸ã§ã¯ãæ¨ãã硬æ¨ãè»æ¨ã¨ãåæ¨ã«åé¡ãã¾ãã"));
+ TokenStream ts = a.tokenStream("foo", new StringReader("京é½å¤§å¦ç¡¬å¼éçé¨"));
while(ts.incrementToken());
*/
// Senior software engineer:
- if (KuromojiTokenizer2.DO_OUTPUT_COMPOUND) {
- assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢",
- new String[] { "ã·ãã¢",
- "ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢",
- "ã½ããã¦ã§ã¢",
- "ã¨ã³ã¸ãã¢" },
- new int[] { 1, 0, 1, 1}
- );
-
- // Kansai International Airport:
- assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "é¢è¥¿å½é空港",
- new String[] { "é¢è¥¿",
- "é¢è¥¿å½é空港", // zero pos inc
- "å½é",
- "空港" },
- new int[] {1, 0, 1, 1}
- );
-
- // Konika Minolta Holdings; not quite the right
- // segmentation (see LUCENE-3726):
- assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "ã³ãã«ããã«ã¿ãã¼ã«ãã£ã³ã°ã¹",
- new String[] { "ã³ãã«",
- "ã³ãã«ããã«ã¿ãã¼ã«ãã£ã³ã°ã¹", // zero pos inc
- "ããã«ã¿",
- "ãã¼ã«ãã£ã³ã°ã¹"},
- new int[] {1, 0, 1, 1}
- );
-
- // Narita Airport
- assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "æç°ç©ºæ¸¯",
- new String[] { "æç°",
- "æç°ç©ºæ¸¯",
- "空港" },
- new int[] {1, 0, 1});
-
- // Kyoto University Baseball Club
- // nocommit --segments differently but perhaps OK
- assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "京é½å¤§å¦ç¡¬å¼éçé¨",
- new String[] { "京é½",
- "京é½å¤§å¦ç¡¬å¼éçé¨",
- "大å¦",
- "硬å¼",
- "éç",
- "é¨" },
- new int[] {1, 0, 1, 1, 1, 1});
- } else {
- assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢",
- new String[] { "ã·ãã¢",
- "ã½ããã¦ã§ã¢",
- "ã¨ã³ã¸ãã¢" },
- new int[] { 1, 1, 1}
- );
- // Kansai International Airport:
- assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "é¢è¥¿å½é空港",
- new String[] { "é¢è¥¿",
- "å½é",
- "空港" },
- new int[] {1, 1, 1}
- );
-
- // Konika Minolta Holdings; not quite the right
- // segmentation (see LUCENE-3726):
- assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "ã³ãã«ããã«ã¿ãã¼ã«ãã£ã³ã°ã¹",
- new String[] { "ã³ãã«",
- "ããã«ã¿",
- "ãã¼ã«ãã£ã³ã°ã¹"},
- new int[] {1, 1, 1}
- );
-
- // Narita Airport
- assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "æç°ç©ºæ¸¯",
- new String[] { "æç°",
- "空港" },
- new int[] {1, 1});
-
- // Kyoto University Baseball Club
- assertAnalyzesTo(new KuromojiAnalyzer(TEST_VERSION_CURRENT), "京é½å¤§å¦ç¡¬å¼éçé¨",
- new String[] { "京é½",
- "大å¦",
- "硬å¼",
- "éç",
- "é¨" },
- new int[] {1, 1, 1, 1, 1});
- }
+ assertAnalyzesTo(a, "ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢",
+ new String[] { "ã·ãã¢",
+ "ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢",
+ "ã½ããã¦ã§ã¢",
+ "ã¨ã³ã¸ãã¢" },
+ new int[] { 1, 0, 1, 1}
+ );
+
+ // Kansai International Airport:
+ assertAnalyzesTo(a, "é¢è¥¿å½é空港",
+ new String[] { "é¢è¥¿",
+ "é¢è¥¿å½é空港", // zero pos inc
+ "å½é",
+ "空港" },
+ new int[] {1, 0, 1, 1}
+ );
+
+ // Konika Minolta Holdings; not quite the right
+ // segmentation (see LUCENE-3726):
+ assertAnalyzesTo(a, "ã³ãã«ããã«ã¿ãã¼ã«ãã£ã³ã°ã¹",
+ new String[] { "ã³ãã«",
+ "ã³ãã«ããã«ã¿ãã¼ã«ãã£ã³ã°ã¹", // zero pos inc
+ "ããã«ã¿",
+ "ãã¼ã«ãã£ã³ã°ã¹"},
+ new int[] {1, 0, 1, 1}
+ );
+
+ // Narita Airport
+ assertAnalyzesTo(a, "æç°ç©ºæ¸¯",
+ new String[] { "æç°",
+ "æç°ç©ºæ¸¯",
+ "空港" },
+ new int[] {1, 0, 1});
+
}
@@ -153,6 +144,16 @@ public class TestKuromojiAnalyzer extend
* blast random strings against the analyzer
*/
public void testRandom() throws IOException {
- checkRandomData(random, new KuromojiAnalyzer(TEST_VERSION_CURRENT), atLeast(10000));
+ final Analyzer a = new KuromojiAnalyzer(TEST_VERSION_CURRENT, null, Mode.SEARCH,
+ KuromojiAnalyzer.getDefaultStopSet(),
+ KuromojiAnalyzer.getDefaultStopTags());
+ checkRandomData(random, a, atLeast(10000));
+ }
+
+ public void testRandomWithCompounds() throws IOException {
+ final Analyzer a = new KuromojiAnalyzer(TEST_VERSION_CURRENT, null, Mode.SEARCH_WITH_COMPOUNDS,
+ KuromojiAnalyzer.getDefaultStopSet(),
+ KuromojiAnalyzer.getDefaultStopTags());
+ checkRandomData(random, a, atLeast(10000));
}
}
Modified: lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiTokenizer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiTokenizer.java?rev=1243882&r1=1243881&r2=1243882&view=diff
==============================================================================
--- lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiTokenizer.java (original)
+++ lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestKuromojiTokenizer.java Tue Feb 14 11:37:45 2012
@@ -17,10 +17,10 @@ package org.apache.lucene.analysis.kurom
* limitations under the License.
*/
-import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.io.LineNumberReader;
import java.io.Reader;
import java.io.StringReader;
@@ -30,6 +30,7 @@ import org.apache.lucene.analysis.TokenS
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.kuromoji.Segmenter.Mode;
import org.apache.lucene.analysis.kuromoji.dict.UserDictionary;
+import org.apache.lucene.analysis.kuromoji.tokenattributes.*;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.UnicodeUtil;
@@ -57,16 +58,37 @@ public class TestKuromojiTokenizer exten
private Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- //Tokenizer tokenizer = new KuromojiTokenizer(new Segmenter(readDict()), reader);
- //Tokenizer tokenizer = new KuromojiTokenizer(reader);
+ Tokenizer tokenizer = new KuromojiTokenizer2(reader, readDict(), false, Mode.SEARCH);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+
+ private Analyzer analyzerNoPunct = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
Tokenizer tokenizer = new KuromojiTokenizer2(reader, readDict(), true, Mode.SEARCH);
- //Tokenizer tokenizer = new KuromojiTokenizer2(reader, null, true, true);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+
+ private Analyzer analyzerWithCompounds = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new KuromojiTokenizer2(reader, readDict(), false, Mode.SEARCH_WITH_COMPOUNDS);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
+
+ private Analyzer extendedModeAnalyzerNoPunct = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new KuromojiTokenizer2(reader, readDict(), true, Mode.EXTENDED);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
public void testDecomposition1() throws Exception {
- assertAnalyzesTo(analyzer, "æ¬æ¥ã¯ã貧å°å±¤ã®å¥³æ§ãåä¾ã«å»çä¿è·ãæä¾ããããã«åµè¨ãããå¶åº¦ã§ããã" +
+ assertAnalyzesTo(analyzerNoPunct, "æ¬æ¥ã¯ã貧å°å±¤ã®å¥³æ§ãåä¾ã«å»çä¿è·ãæä¾ããããã«åµè¨ãããå¶åº¦ã§ããã" +
"ã¢ã¡ãªã«ä½æå¾è
å»çæ´å©å¶åº¦ããä»æ¥ã§ã¯ããã®äºç®ã®ç´ï¼åã®ï¼ãè人ã«è²»ããã¦ããã",
new String[] { "æ¬æ¥", "ã¯", "貧å°", "層", "ã®", "女æ§", "ã", "åä¾", "ã«", "å»ç", "ä¿è·", "ã",
"æä¾", "ãã", "ãã", "ã«", "åµè¨", "ã", "ã", "ã", "å¶åº¦", "ã§", "ãã", "ã¢ã¡ãªã«",
@@ -82,7 +104,7 @@ public class TestKuromojiTokenizer exten
}
public void testDecomposition2() throws Exception {
- assertAnalyzesTo(analyzer, "麻è¬ã®å¯å£²ã¯æ ¹ããã絶ãããªããã°ãªããªã",
+ assertAnalyzesTo(analyzerNoPunct, "麻è¬ã®å¯å£²ã¯æ ¹ããã絶ãããªããã°ãªããªã",
new String[] { "麻è¬", "ã®", "å¯å£²", "ã¯", "æ ¹ããã", "絶ãã", "ãªãã", "ã°", "ãªã", "ãªã" },
new int[] { 0, 2, 3, 5, 6, 10, 13, 16, 17, 19 },
new int[] { 2, 3, 5, 6, 10, 13, 16, 17, 19, 21 }
@@ -90,7 +112,7 @@ public class TestKuromojiTokenizer exten
}
public void testDecomposition3() throws Exception {
- assertAnalyzesTo(analyzer, "é女ç©å¤§å°ãã·ã¥ã¼ã»ãããã³ã¹ã",
+ assertAnalyzesTo(analyzerNoPunct, "é女ç©å¤§å°ãã·ã¥ã¼ã»ãããã³ã¹ã",
new String[] { "é女", "ç©", "大å°", "ãã·ã¥ã¼", "ãããã³ã¹" },
new int[] { 0, 2, 3, 5, 10 },
new int[] { 2, 3, 5, 9, 15 }
@@ -132,7 +154,7 @@ public class TestKuromojiTokenizer exten
/** Tests that sentence offset is incorporated into the resulting offsets */
public void testTwoSentences() throws Exception {
- assertAnalyzesTo(analyzer, "é女ç©å¤§å°ãã·ã¥ã¼ã»ãããã³ã¹ã é女ç©å¤§å°ãã·ã¥ã¼ã»ãããã³ã¹ã",
+ assertAnalyzesTo(analyzerNoPunct, "é女ç©å¤§å°ãã·ã¥ã¼ã»ãããã³ã¹ã é女ç©å¤§å°ãã·ã¥ã¼ã»ãããã³ã¹ã",
new String[] { "é女", "ç©", "大å°", "ãã·ã¥ã¼", "ãããã³ã¹", "é女", "ç©", "大å°", "ãã·ã¥ã¼", "ãããã³ã¹" },
new int[] { 0, 2, 3, 5, 10, 17, 19, 20, 22, 27 },
new int[] { 2, 3, 5, 9, 15, 19, 20, 22, 26, 32 }
@@ -176,18 +198,32 @@ public class TestKuromojiTokenizer exten
}
}
}
+
+ public void testOnlyPunctuation() throws IOException {
+ TokenStream ts = analyzerNoPunct.tokenStream("foo", new StringReader("ãããã"));
+ ts.reset();
+ assertFalse(ts.incrementToken());
+ ts.end();
+ }
+
+ public void testOnlyPunctuationExtended() throws IOException {
+ TokenStream ts = extendedModeAnalyzerNoPunct.tokenStream("foo", new StringReader("......"));
+ ts.reset();
+ assertFalse(ts.incrementToken());
+ ts.end();
+ }
// note: test is kinda silly since kuromoji emits punctuation tokens.
// but, when/if we filter these out it will be useful.
public void testEnd() throws Exception {
- assertTokenStreamContents(analyzer.tokenStream("foo", new StringReader("ããã¯æ¬ã§ã¯ãªã")),
+ assertTokenStreamContents(analyzerNoPunct.tokenStream("foo", new StringReader("ããã¯æ¬ã§ã¯ãªã")),
new String[] { "ãã", "ã¯", "æ¬", "ã§", "ã¯", "ãªã" },
new int[] { 0, 2, 3, 4, 5, 6 },
new int[] { 2, 3, 4, 5, 6, 8 },
new Integer(8)
);
- assertTokenStreamContents(analyzer.tokenStream("foo", new StringReader("ããã¯æ¬ã§ã¯ãªã ")),
+ assertTokenStreamContents(analyzerNoPunct.tokenStream("foo", new StringReader("ããã¯æ¬ã§ã¯ãªã ")),
new String[] { "ãã", "ã¯", "æ¬", "ã§", "ã¯", "ãªã" },
new int[] { 0, 2, 3, 4, 5, 6, 8 },
new int[] { 2, 3, 4, 5, 6, 8, 9 },
@@ -215,6 +251,273 @@ public class TestKuromojiTokenizer exten
new Integer(3)
);
}
+
+ public void testSegmentation() throws Exception {
+ // Skip tests for Michelle Kwan -- UniDic segments Kwan as 㯠ã¯ã³
+ // String input = "ãã·ã§ã«ã»ã¯ã¯ã³ãåªåãã¾ãããã¹ãã¼ã¹ã¹ãã¼ã·ã§ã³ã«è¡ãã¾ããããããããã";
+ // String[] surfaceForms = {
+ // "ãã·ã§ã«", "ã»", "ã¯ã¯ã³", "ã", "åªå", "ã", "ã¾ã", "ã", "ã",
+ // "ã¹ãã¼ã¹", "ã¹ãã¼ã·ã§ã³", "ã«", "è¡ã", "ã¾ã", "ã",
+ // "ãããããã", "ã"
+ // };
+ String input = "ã¹ãã¼ã¹ã¹ãã¼ã·ã§ã³ã«è¡ãã¾ããããããããã";
+ String[] surfaceForms = {
+ "ã¹ãã¼ã¹", "ã¹ãã¼ã·ã§ã³", "ã«", "è¡ã", "ã¾ã", "ã",
+ "ãããããã", "ã"
+ };
+ assertAnalyzesTo(analyzer,
+ input,
+ surfaceForms);
+ assertAnalyzesTo(analyzerWithCompounds,
+ input,
+ surfaceForms);
+ }
+
+ private void assertReadings(String input, String... readings) throws IOException {
+ TokenStream ts = analyzer.tokenStream("ignored", new StringReader(input));
+ ReadingAttribute readingAtt = ts.addAttribute(ReadingAttribute.class);
+ ts.reset();
+ for(String reading : readings) {
+ assertTrue(ts.incrementToken());
+ assertEquals(reading, readingAtt.getReading());
+ }
+ assertFalse(ts.incrementToken());
+ ts.end();
+ }
+
+ private void assertPronunciations(String input, String... pronunciations) throws IOException {
+ TokenStream ts = analyzer.tokenStream("ignored", new StringReader(input));
+ ReadingAttribute readingAtt = ts.addAttribute(ReadingAttribute.class);
+ ts.reset();
+ for(String pronunciation : pronunciations) {
+ assertTrue(ts.incrementToken());
+ assertEquals(pronunciation, readingAtt.getPronunciation());
+ }
+ assertFalse(ts.incrementToken());
+ ts.end();
+ }
+
+ private void assertBaseForms(String input, String... baseForms) throws IOException {
+ TokenStream ts = analyzer.tokenStream("ignored", new StringReader(input));
+ BaseFormAttribute baseFormAtt = ts.addAttribute(BaseFormAttribute.class);
+ ts.reset();
+ for(String baseForm : baseForms) {
+ assertTrue(ts.incrementToken());
+ assertEquals(baseForm, baseFormAtt.getBaseForm());
+ }
+ assertFalse(ts.incrementToken());
+ ts.end();
+ }
+
+ private void assertInflectionTypes(String input, String... inflectionTypes) throws IOException {
+ TokenStream ts = analyzer.tokenStream("ignored", new StringReader(input));
+ InflectionAttribute inflectionAtt = ts.addAttribute(InflectionAttribute.class);
+ ts.reset();
+ for(String inflectionType : inflectionTypes) {
+ assertTrue(ts.incrementToken());
+ assertEquals(inflectionType, inflectionAtt.getInflectionType());
+ }
+ assertFalse(ts.incrementToken());
+ ts.end();
+ }
+
+ private void assertInflectionForms(String input, String... inflectionForms) throws IOException {
+ TokenStream ts = analyzer.tokenStream("ignored", new StringReader(input));
+ InflectionAttribute inflectionAtt = ts.addAttribute(InflectionAttribute.class);
+ ts.reset();
+ for(String inflectionForm : inflectionForms) {
+ assertTrue(ts.incrementToken());
+ assertEquals(inflectionForm, inflectionAtt.getInflectionForm());
+ }
+ assertFalse(ts.incrementToken());
+ ts.end();
+ }
+
+ private void assertPartsOfSpeech(String input, String... partsOfSpeech) throws IOException {
+ TokenStream ts = analyzer.tokenStream("ignored", new StringReader(input));
+ PartOfSpeechAttribute partOfSpeechAtt = ts.addAttribute(PartOfSpeechAttribute.class);
+ ts.reset();
+ for(String partOfSpeech : partsOfSpeech) {
+ assertTrue(ts.incrementToken());
+ assertEquals(partOfSpeech, partOfSpeechAtt.getPartOfSpeech());
+ }
+ assertFalse(ts.incrementToken());
+ ts.end();
+ }
+
+ public void testReadings() throws Exception {
+ assertReadings("寿å¸ãé£ã¹ããã§ãã",
+ "ã¹ã·",
+ "ã¬",
+ "ã¿ã",
+ "ã¿ã¤",
+ "ãã¹",
+ "ã");
+ }
+
+ public void testReadings2() throws Exception {
+ assertReadings("å¤ãã®å¦çã試é¨ã«è½ã¡ãã",
+ "ãªãªã¯",
+ "ã",
+ "ã¬ã¯ã»ã¤",
+ "ã¬",
+ "ã·ã±ã³",
+ "ã",
+ "ãªã",
+ "ã¿",
+ "ã");
+ }
+
+ public void testPronunciations() throws Exception {
+ assertPronunciations("寿å¸ãé£ã¹ããã§ãã",
+ "ã¹ã·",
+ "ã¬",
+ "ã¿ã",
+ "ã¿ã¤",
+ "ãã¹",
+ "ã");
+ }
+
+ public void testPronunciations2() throws Exception {
+ // pronunciation differs from reading here
+ assertPronunciations("å¤ãã®å¦çã試é¨ã«è½ã¡ãã",
+ "ãªã¼ã¯",
+ "ã",
+ "ã¬ã¯ã»ã¤",
+ "ã¬",
+ "ã·ã±ã³",
+ "ã",
+ "ãªã",
+ "ã¿",
+ "ã");
+ }
+
+ public void testBasicForms() throws Exception {
+ assertBaseForms("ããã¯ã¾ã å®é¨æ®µéã«ããã¾ãã",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ "ãã",
+ null,
+ null);
+ }
+
+ public void testInflectionTypes() throws Exception {
+ assertInflectionTypes("ããã¯ã¾ã å®é¨æ®µéã«ããã¾ãã",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ "äºæ®µã»ã©è¡",
+ "ç¹æ®ã»ãã¹",
+ null);
+ }
+
+ public void testInflectionForms() throws Exception {
+ assertInflectionForms("ããã¯ã¾ã å®é¨æ®µéã«ããã¾ãã",
+ null,
+ null,
+ null,
+ null,
+ null,
+ null,
+ "é£ç¨å½¢",
+ "åºæ¬å½¢",
+ null);
+ }
+
+ public void testPartOfSpeech() throws Exception {
+ assertPartsOfSpeech("ããã¯ã¾ã å®é¨æ®µéã«ããã¾ãã",
+ "åè©-代åè©-ä¸è¬",
+ "å©è©-ä¿å©è©",
+ "å¯è©-å©è©é¡æ¥ç¶",
+ "åè©-ãµå¤æ¥ç¶",
+ "åè©-ä¸è¬",
+ "å©è©-æ ¼å©è©-ä¸è¬",
+ "åè©-èªç«",
+ "å©åè©",
+ "è¨å·-å¥ç¹");
+ }
+
+ // TODO: the next 2 tests are no longer using the first/last word ids, maybe lookup the words and fix?
+ // do we have a possibility to actually lookup the first and last word from dictionary?
+ public void testYabottai() throws Exception {
+ assertAnalyzesTo(analyzer, "ãã¼ã£ãã",
+ new String[] {"ãã¼ã£ãã"});
+ assertAnalyzesTo(analyzerWithCompounds, "ãã¼ã£ãã",
+ new String[] {"ãã¼ã£ãã"});
+ }
+
+ public void testTsukitosha() throws Exception {
+ assertAnalyzesTo(analyzer, "çªãéãã",
+ new String[] {"çªãéãã"});
+ assertAnalyzesTo(analyzerWithCompounds, "çªãéãã",
+ new String[] {"çªãéãã"});
+ }
+
+ public void testBocchan() throws Exception {
+ doTestBocchan(1, true);
+ doTestBocchan(1, false);
+ }
+
+ @Nightly
+ public void testBocchanBig() throws Exception {
+ doTestBocchan(100, true);
+ doTestBocchan(100, false);
+ }
+
+ private void doTestBocchan(int numIterations, boolean includeCompounds) throws Exception {
+ LineNumberReader reader = new LineNumberReader(new InputStreamReader(
+ this.getClass().getResourceAsStream("bocchan.utf-8")));
+
+ String line = reader.readLine();
+ reader.close();
+
+ if (VERBOSE) {
+ System.out.println("Test for Bocchan without pre-splitting sentences");
+ }
+
+ /*
+ if (numIterations > 1) {
+ // warmup
+ for (int i = 0; i < numIterations; i++) {
+ final TokenStream ts = analyzer.tokenStream("ignored", new StringReader(line));
+ ts.reset();
+ while(ts.incrementToken());
+ }
+ }
+ */
+
+ long totalStart = System.currentTimeMillis();
+ for (int i = 0; i < numIterations; i++) {
+ final Analyzer a = includeCompounds ? analyzerWithCompounds : analyzer;
+ final TokenStream ts = a.tokenStream("ignored", new StringReader(line));
+ ts.reset();
+ while(ts.incrementToken());
+ }
+ String[] sentences = line.split("ã|ã");
+ if (VERBOSE) {
+ System.out.println("Total time : " + (System.currentTimeMillis() - totalStart));
+ System.out.println("Test for Bocchan with pre-splitting sentences (" + sentences.length + " sentences)");
+ }
+ totalStart = System.currentTimeMillis();
+ for (int i = 0; i < numIterations; i++) {
+ for (String sentence: sentences) {
+ final TokenStream ts = analyzer.tokenStream("ignored", new StringReader(sentence));
+ ts.reset();
+ while(ts.incrementToken());
+ }
+ }
+ if (VERBOSE) {
+ System.out.println("Total time : " + (System.currentTimeMillis() - totalStart));
+ }
+ }
+
// nocommit need a still better test, where an input is
// tokenized to multiple tokens that would be different
Modified: lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestQuality.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestQuality.java?rev=1243882&r1=1243881&r2=1243882&view=diff
==============================================================================
--- lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestQuality.java (original)
+++ lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestQuality.java Tue Feb 14 11:37:45 2012
@@ -59,7 +59,6 @@ public class TestQuality extends LuceneT
sentence agreement?: 0.998161495317142
word agreement?: 0.999587584716181
*/
- //final Tokenizer tokenizer = new KuromojiTokenizer(null, null, true, Mode.SEARCH);
StringBuilder sb = new StringBuilder();
@@ -73,15 +72,17 @@ public class TestQuality extends LuceneT
// end of each sentence...
maxLen = Math.max(line1.length(), maxLen);
sb.append(line1);
+ /*
if (ONE_TIME && count++ == 100) {
// nocommit;
break;
}
+ */
}
- System.out.println("maxLen=" + maxLen);
+ //System.out.println("maxLen=" + maxLen);
- //final Tokenizer tokenizer = new KuromojiTokenizer2(new StringReader(""), null, true, Mode.SEARCH);
- final Tokenizer tokenizer = new KuromojiTokenizer(new StringReader(""));
+ final Tokenizer tokenizer = new KuromojiTokenizer2(new StringReader(""), null, true, Mode.SEARCH_WITH_COMPOUNDS);
+ //final Tokenizer tokenizer = new KuromojiTokenizer(new StringReader(""));
final String all = sb.toString();
final int ITERS = 20;
CharTermAttribute termAtt = tokenizer.addAttribute(CharTermAttribute.class);
@@ -123,7 +124,7 @@ public class TestQuality extends LuceneT
//final Tokenizer tokenizer = new KuromojiTokenizer(new StringReader(""));
//final Tokenizer tokenizer = new KuromojiTokenizer(new Segmenter(Mode.NORMAL), new StringReader(""));
- final Tokenizer tokenizer = new KuromojiTokenizer2(new StringReader(""), null, true, Mode.SEARCH);
+ final Tokenizer tokenizer = new KuromojiTokenizer2(new StringReader(""), null, true, Mode.SEARCH_WITH_COMPOUNDS);
String line1 = null;
String line2 = null;
@@ -256,6 +257,7 @@ public class TestQuality extends LuceneT
}
}
assert minPath != null;
+ /*
if (minEdits > 0) {
if (!VERBOSE) {
System.out.println("\nTEST " + lineCount + ": input " + unseg + "; " + minEdits + " edits");
@@ -263,6 +265,7 @@ public class TestQuality extends LuceneT
System.out.println(" expected: " + expectedTokens);
System.out.println(" actual: " + minPath);
}
+ */
stats.numChars += seg.length();
stats.numEdits += minEdits;
stats.numWords += expectedTokens.size();
Modified: lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestSearchMode.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestSearchMode.java?rev=1243882&r1=1243881&r2=1243882&view=diff
==============================================================================
--- lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestSearchMode.java (original)
+++ lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/TestSearchMode.java Tue Feb 14 11:37:45 2012
@@ -33,15 +33,22 @@ import org.apache.lucene.util.IOUtils;
public class TestSearchMode extends BaseTokenStreamTestCase {
private final static String SEGMENTATION_FILENAME = "search-segmentation-tests.txt";
- private final Segmenter segmenter = new Segmenter(Mode.SEARCH);
+ private final static String SEGMENTATION_WITH_COMPOUNDS_FILENAME = "search-segmentation-with-compounds-tests.txt";
private final Analyzer analyzer = new Analyzer() {
@Override
protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
- //Tokenizer tokenizer = new KuromojiTokenizer(segmenter, reader);
Tokenizer tokenizer = new KuromojiTokenizer2(reader, null, true, Mode.SEARCH);
return new TokenStreamComponents(tokenizer, tokenizer);
}
};
+
+ private final Analyzer analyzerWithCompounds = new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ Tokenizer tokenizer = new KuromojiTokenizer2(reader, null, true, Mode.SEARCH_WITH_COMPOUNDS);
+ return new TokenStreamComponents(tokenizer, tokenizer);
+ }
+ };
/** Test search mode segmentation */
public void testSearchSegmentation() throws IOException {
@@ -65,26 +72,46 @@ public class TestSearchMode extends Base
String[] fields = line.split("\t", 2);
String sourceText = fields[0];
String[] expectedTokens = fields[1].split("\\s+");
- int[] expectedPosIncrs;
- if (KuromojiTokenizer2.DO_OUTPUT_COMPOUND && expectedTokens.length > 1) {
- String[] newTokens = new String[expectedTokens.length+1];
- newTokens[0] = expectedTokens[0];
- newTokens[1] = fields[0];
- System.arraycopy(expectedTokens, 1, newTokens, 2, expectedTokens.length-1);
- expectedTokens = newTokens;
- expectedPosIncrs = new int[expectedTokens.length];
- Arrays.fill(expectedPosIncrs, 1);
- expectedPosIncrs[1] = 0;
- System.out.println("fixup: " + Arrays.toString(expectedTokens));
- } else {
- expectedPosIncrs = new int[expectedTokens.length];
- Arrays.fill(expectedPosIncrs, 1);
- }
- try {
+ int[] expectedPosIncrs = new int[expectedTokens.length];
+ Arrays.fill(expectedPosIncrs, 1);
assertAnalyzesTo(analyzer, sourceText, expectedTokens, expectedPosIncrs);
- } catch (Throwable t) {
- System.out.println(" diffs: " + t);
+ }
+ } finally {
+ is.close();
+ }
+ }
+
+ /** Test search_with_compounds mode segmentation */
+ public void testSearchSegmentationWithCompounds() throws IOException {
+ InputStream is = TestSearchMode.class.getResourceAsStream(SEGMENTATION_WITH_COMPOUNDS_FILENAME);
+ if (is == null) {
+ throw new FileNotFoundException("Cannot find " + SEGMENTATION_FILENAME + " in test classpath");
+ }
+ try {
+ LineNumberReader reader = new LineNumberReader(new InputStreamReader(is, IOUtils.CHARSET_UTF_8));
+ String line = null;
+ while ((line = reader.readLine()) != null) {
+ // Remove comments
+ line = line.replaceAll("#.*$", "");
+ // Skip empty lines or comment lines
+ if (line.trim().isEmpty()) {
+ continue;
+ }
+ if (VERBOSE) {
+ System.out.println("Line no. " + reader.getLineNumber() + ": " + line);
+ }
+ String[] fields = line.split("\t", 2);
+ String sourceText = fields[0];
+ String[] expectedTokens = fields[1].split("\\s+");
+ int[] expectedPosIncrs = new int[expectedTokens.length];
+ for(int tokIDX=0;tokIDX<expectedTokens.length;tokIDX++) {
+ if (expectedTokens[tokIDX].endsWith("/0")) {
+ expectedTokens[tokIDX] = expectedTokens[tokIDX].replace("/0", "");
+ } else {
+ expectedPosIncrs[tokIDX] = 1;
+ }
}
+ assertAnalyzesTo(analyzerWithCompounds, sourceText, expectedTokens, expectedPosIncrs);
}
} finally {
is.close();
Modified: lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/search-segmentation-tests.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/search-segmentation-tests.txt?rev=1243882&r1=1243881&r2=1243882&view=diff
==============================================================================
--- lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/search-segmentation-tests.txt (original)
+++ lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/search-segmentation-tests.txt Tue Feb 14 11:37:45 2012
@@ -100,7 +100,7 @@
ã¹ãã£ã¼ãã¸ã§ã㺠ã¹ãã£ã¼ã ã¸ã§ããº
# Harry Potter NOTE: Becomes one token (short word)
ããªã¼ããã¿ã¼ ããªã¼ããã¿ã¼
-# Bill Gates NOTE: Becomes one token (short work)
+# Bill Gates NOTE: Becomes one token (short word)
ãã«ã²ã¤ã ãã«ã²ã¤ã
# Sean Connery NOTE: Becomes one token (okay)
ã·ã§ã¼ã³ã³ããªã¼ ã·ã§ã¼ã³ã³ããªã¼
Added: lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/search-segmentation-with-compounds-tests.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/search-segmentation-with-compounds-tests.txt?rev=1243882&view=auto
==============================================================================
--- lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/search-segmentation-with-compounds-tests.txt (added)
+++ lucene/dev/branches/lucene3767/modules/analysis/kuromoji/src/test/org/apache/lucene/analysis/kuromoji/search-segmentation-with-compounds-tests.txt Tue Feb 14 11:37:45 2012
@@ -0,0 +1,142 @@
+###
+### Tests for Kuromoji's search mode heuristic
+###
+### In search-mode, Kuromoji uses a heuristic to do extra splitting of words
+### to get a decompounding effect useful for search. This file includes tests
+### for this heuristic and demonstrates its usefulness, but also weaknesses.
+###
+### This file's format is as follows:
+### <text><tab><token1> <token2> ... <token>
+###
+### This file should use UTF-8 encoding and there is one test per line. The
+### text to be segmented and its expected surface form token sequence is
+### separated by a tab ('\t'). Tokens are separated by a half-width space.
+### Whitespace lines and lines starting with a '#' are ignored. Comments
+### are not allowed on entry line.
+###
+### NOTE: These tests depends on IPADIC
+###
+### Revision history:
+### - 2012-01-29: Initial version
+###
+
+##
+## Organizations
+##
+
+# Kansai Internationl Airport
+é¢è¥¿å½é空港 é¢è¥¿ é¢è¥¿å½é空港/0 å½é 空港
+# Narita Airport
+æç°ç©ºæ¸¯ æç° æç°ç©ºæ¸¯/0 空港
+# Haneda Airport
+ç¾½ç°ç©ºæ¸¯ ç¾½ç° ç¾½ç°ç©ºæ¸¯/0 空港
+# Nara Institute of Science and Technology
+å¥è¯å
端ç§å¦æè¡å¤§å¦é¢å¤§å¦ å¥è¯ å¥è¯å
端ç§å¦æè¡å¤§å¦é¢å¤§å¦/0 å
端 ç§å¦ æè¡ å¤§å¦é¢ 大å¦
+# Tokyo University
+æ±äº¬å¤§å¦ æ±äº¬ æ±äº¬å¤§å¦/0 大å¦
+# Kyoto University
+京é½å¤§å¦ äº¬é½ äº¬é½å¤§å¦/0 大å¦
+
+# NOTE: differs from non-compound mode:
+# Kyoto University Baseball Club
+京é½å¤§å¦ç¡¬å¼éçé¨ äº¬é½å¤§ å¦ ç¡¬å¼ éç é¨
+
+##
+## Katakana titles
+##
+
+# Senior Software Engineer
+ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ã㢠ã·ã㢠ã·ãã¢ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢/0 ã½ããã¦ã§ã¢ ã¨ã³ã¸ãã¢
+# Software Engineer
+ã½ããã¦ã§ã¢ã¨ã³ã¸ã㢠ã½ããã¦ã§ã¢ ã¨ã³ã¸ãã¢
+# Senior Project Manager
+ã·ãã¢ããã¸ã§ã¯ãããã¸ã£ã¼ ã·ã㢠ã·ãã¢ããã¸ã§ã¯ãããã¸ã£ã¼/0 ããã¸ã§ã¯ã ããã¸ã£ã¼
+# Project Manager
+ããã¸ã§ã¯ãããã¸ã£ã¼ ããã¸ã§ã¯ã ããã¸ã£ã¼
+# Senior Sales Engineer
+ã·ãã¢ã»ã¼ã«ã¹ã¨ã³ã¸ã㢠ã·ã㢠ã·ãã¢ã»ã¼ã«ã¹ã¨ã³ã¸ãã¢/0 ã»ã¼ã«ã¹ ã¨ã³ã¸ãã¢
+# System Architect
+ã·ã¹ãã ã¢ã¼ããã¯ã ã·ã¹ãã ã·ã¹ãã ã¢ã¼ããã¯ã/0 ã¢ã¼ããã¯ã
+# Senior System Architect
+ã·ãã¢ã·ã¹ãã ã¢ã¼ããã¯ã ã·ã㢠ã·ãã¢ã·ã¹ãã ã¢ã¼ããã¯ã/0 ã·ã¹ãã ã¢ã¼ããã¯ã
+# System Administrator
+ã·ã¹ãã ã¢ãããã¹ãã¬ã¼ã¿ ã·ã¹ãã ã¢ãããã¹ãã¬ã¼ã¿
+ã·ã¹ãã ã¢ãããã¹ãã¬ã¼ã¿ã¼ ã·ã¹ãã ã·ã¹ãã ã¢ãããã¹ãã¬ã¼ã¿ã¼/0 ã¢ãããã¹ãã¬ã¼ã¿ã¼
+# Senior System Administrator
+ã·ãã¢ã·ã¹ãã ã¢ãããã¹ãã¬ã¼ã¿ã¼ ã·ã㢠ã·ãã¢ã·ã¹ãã ã¢ãããã¹ãã¬ã¼ã¿ã¼/0 ã·ã¹ãã ã¢ãããã¹ãã¬ã¼ã¿ã¼
+
+##
+## Company names (several are fictitious)
+##
+
+# SoftBank Mobile
+ã½ãããã³ã¯ã¢ãã¤ã« ã½ãããã³ã¯ ã¢ãã¤ã«
+# Alpine Materials
+ã¢ã«ãã¤ã³ãããªã¢ã«ãº ã¢ã«ãã¤ã³ ã¢ã«ãã¤ã³ãããªã¢ã«ãº/0 ãããªã¢ã«ãº
+# Sapporo Holdings
+ãµããããã¼ã«ãã£ã³ã°ã¹ ãµããã ãã¼ã«ãã£ã³ã°ã¹
+# Yamada Corporation
+ã¤ããã³ã¼ãã¬ã¼ã·ã§ã³ ã¤ãã ã¤ããã³ã¼ãã¬ã¼ã·ã§ã³/0 ã³ã¼ãã¬ã¼ã·ã§ã³
+# Canon Semiconductor equipement NOTE: Semiconductor becomes semi + conductor
+ãã¤ãã³ã»ãã³ã³ãã¯ã¿ã¼ã¨ã¯ã£ããã¡ã³ã ãã¤ãã³ ãã¤ãã³ã»ãã³ã³ãã¯ã¿ã¼ã¨ã¯ã£ããã¡ã³ã/0 ã»ã ã³ã³ãã¯ã¿ã¼ ã¨ã¯ã£ããã¡ã³ã
+# Orental Chain
+ãªãªã¨ã³ã¿ã«ãã¨ã³ ãªãªã¨ã³ã¿ã« ãªãªã¨ã³ã¿ã«ãã¨ã³/0 ãã¨ã³
+# Ally Projects Japan NOTE: Becomes one token as ããã¸ã§ã¯ã is not in IPADIC
+ã¢ã¼ãªã¼ããã¸ã§ã¯ãã¸ã£ãã³ ã¢ã¼ãªã¼ããã¸ã§ã¯ãã¸ã£ãã³
+# Peter Pan Corporation
+ãã¼ã¿ã¼ãã³ã³ã¼ãã¬ã¼ã·ã§ã³ ãã¼ã¿ã¼ ãã¼ã¿ã¼ãã³ã³ã¼ãã¬ã¼ã·ã§ã³/0 ãã³ ã³ã¼ãã¬ã¼ã·ã§ã³
+# AIM Create
+ã¨ã¤ã ã¯ãªã¨ã¤ã ã¨ã¤ã ã¯ãªã¨ã¤ã
+# Mars Engineering
+ãã¼ã¹ã¨ã³ã¸ãã¢ãªã³ã° ãã¼ã¹ ãã¼ã¹ã¨ã³ã¸ãã¢ãªã³ã°/0 ã¨ã³ã¸ãã¢ãªã³ã°
+# Fuji Protein Technology
+ãã¸ãããã¤ã³ãã¯ããã¸ã¼ ã㸠ãã¸ãããã¤ã³ãã¯ããã¸ã¼/0 ãããã¤ã³ ãã¯ããã¸ã¼
+
+##
+## Person names
+##
+
+# Michael Jackson
+ãã¤ã±ã«ã¸ã£ã¯ã½ã³ ãã¤ã±ã« ã¸ã£ã¯ã½ã³
+# Steve Jobs
+ã¹ãã£ã¼ãã¸ã§ã㺠ã¹ãã£ã¼ã ã¸ã§ããº
+# Harry Potter NOTE: Becomes one token (short word)
+ããªã¼ããã¿ã¼ ããªã¼ããã¿ã¼
+# Bill Gates NOTE: Becomes one token (short word)
+ãã«ã²ã¤ã ãã«ã²ã¤ã
+# Sean Connery NOTE: Becomes one token (okay)
+ã·ã§ã¼ã³ã³ããªã¼ ã·ã§ã¼ã³ã³ããªã¼
+
+##
+## Other nouns
+##
+
+# Holdings
+ãã¼ã«ãã£ã³ã°ã¹ ãã¼ã«ãã£ã³ã°ã¹
+# Engineering
+ã¨ã³ã¸ãã¢ãªã³ã° ã¨ã³ã¸ãã¢ãªã³ã°
+# Software Engineering
+ã½ããã¦ã§ã¢ã¨ã³ã¸ãã¢ãªã³ã° ã½ããã¦ã§ã¢ ã¨ã³ã¸ãã¢ãªã³ã°
+# Shopping center
+ã·ã§ããã³ã°ã»ã³ã¿ã¼ ã·ã§ããã³ã° ã»ã³ã¿ã¼
+# Game center (arcade) NOTE: One token because of short word
+ã²ã¼ã ã»ã³ã¿ã¼ ã²ã¼ã ã»ã³ã¿ã¼
+# Christmas shopping
+ã¯ãªã¹ãã¹ã·ã§ããã³ã° ã¯ãªã¹ãã¹ ã·ã§ããã³ã°
+# Download file
+ãã¦ã³ãã¼ããã¡ã¤ã« ãã¦ã³ãã¼ã ãã¡ã¤ã«
+# Technology
+ãã¯ããã¸ã¼ ãã¯ããã¸ã¼
+# Lillehammer Olympics
+ãªã¬ãã³ã¡ã«ãªãªã³ãã㯠ãªã¬ãã³ã¡ã« ãªãªã³ããã¯
+
+##
+## Problematic terms
+##
+
+# JT Engineering NOTE: Becomes J Tien ginia ring (substrings are in IPADIC)
+ã¸ã§ã¤ãã£ã¨ã³ã¸ãã¢ãªã³ã° ã¸ã§ã¤ ã¸ã§ã¤ãã£ã¨ã³ã¸ãã¢ãªã³ã°/0 ãã£ã¨ã³ ã¸ã㢠ãªã³ã°
+# Anchovy pasta NOTE: Become Anch yvipasta
+ã¢ã³ãã§ããã¹ã¿ ã¢ã³ã ã¢ã³ãã§ããã¹ã¿/0 ã§ããã¹ã¿
+# Surprise gift NOTE: Becomes one token (surprise not in IPADIC)
+ãµãã©ã¤ãºã®ãã ãµãã©ã¤ãºã®ãã