You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by rm...@apache.org on 2012/01/09 02:41:37 UTC
svn commit: r1229001 - in
/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src:
java/org/apache/lucene/analysis/kuromoji/dict/
resources/org/apache/lucene/analysis/kuromoji/dict/
tools/java/org/apache/lucene/analysis/kuromoji/util/
Author: rmuir
Date: Mon Jan 9 01:41:37 2012
New Revision: 1229001
URL: http://svn.apache.org/viewvc?rev=1229001&view=rev
Log:
LUCENE-3305: compress connectioncosts
Modified:
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/ConnectionCosts.java
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/ConnectionCosts.dat
lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsWriter.java
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/ConnectionCosts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/ConnectionCosts.java?rev=1229001&r1=1229000&r2=1229001&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/ConnectionCosts.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/java/org/apache/lucene/analysis/kuromoji/dict/ConnectionCosts.java Mon Jan 9 01:41:37 2012
@@ -49,10 +49,13 @@ public final class ConnectionCosts {
int forwardSize = in.readVInt();
int backwardSize = in.readVInt();
costs = new short[backwardSize][forwardSize];
+ int accum = 0;
for (int j = 0; j < costs.length; j++) {
final short[] a = costs[j];
for (int i = 0; i < a.length; i++) {
- a[i] = in.readShort();
+ int raw = in.readVInt();
+ accum += (raw >>> 1) ^ -(raw & 1);
+ a[i] = (short)accum;
}
}
} catch (IOException ioe) {
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/ConnectionCosts.dat
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/resources/org/apache/lucene/analysis/kuromoji/dict/ConnectionCosts.dat?rev=1229001&r1=1229000&r2=1229001&view=diff
==============================================================================
Binary files - no diff available.
Modified: lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsWriter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsWriter.java?rev=1229001&r1=1229000&r2=1229001&view=diff
==============================================================================
--- lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsWriter.java (original)
+++ lucene/dev/branches/lucene3305/modules/analysis/kuromoji/src/tools/java/org/apache/lucene/analysis/kuromoji/util/ConnectionCostsWriter.java Mon Jan 9 01:41:37 2012
@@ -58,11 +58,15 @@ public final class ConnectionCostsWriter
CodecUtil.writeHeader(out, ConnectionCosts.HEADER, ConnectionCosts.VERSION);
out.writeVInt(forwardSize);
out.writeVInt(backwardSize);
+ int last = 0;
assert costs.length == backwardSize;
for (short[] a : costs) {
assert a.length == forwardSize;
for (int i = 0; i < a.length; i++) {
- out.writeShort(a[i]);
+ // TODO: when delta is 0, maybe we should RLE
+ int delta = (int)a[i] - last;
+ out.writeVInt((delta >> 31) ^ (delta << 1));
+ last = a[i];
}
}
} finally {