You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/05/20 21:04:30 UTC
svn commit: r1596368 - in /lucene/dev/branches/branch_4x/lucene: ./
codecs/src/java/org/apache/lucene/codecs/memory/
core/src/java/org/apache/lucene/util/fst/
misc/src/java/org/apache/lucene/util/fst/
Author: mikemccand
Date: Tue May 20 19:04:29 2014
New Revision: 1596368
URL: http://svn.apache.org/r1596368
Log:
LUCENE-5670: add skip/FinalOutput to FST Outputs
Modified:
lucene/dev/branches/branch_4x/lucene/CHANGES.txt
lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ForwardBytesReader.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ReverseBytesReader.java
lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1596368&r1=1596367&r2=1596368&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Tue May 20 19:04:29 2014
@@ -89,6 +89,9 @@ Optimizations
to 8 (for int/float) and 16 (for long/double), for faster indexing
time and smaller indices. (Robert Muir, Uwe Schindler, Mike McCandless)
+* LUCENE-5670: Add skip/FinalOutput to FST Outputs. (Christian
+ Ziech via Mike McCandless).
+
Bug fixes
* LUCENE-5673: MMapDirectory: Work around a "bug" in the JDK that throws
Modified: lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java?rev=1596368&r1=1596367&r2=1596368&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/codecs/src/java/org/apache/lucene/codecs/memory/FSTTermOutputs.java Tue May 20 19:04:29 2014
@@ -25,7 +25,6 @@ import org.apache.lucene.index.FieldInfo
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.fst.Outputs;
-import org.apache.lucene.util.LongsRef;
/**
* An FST {@link Outputs} implementation for
@@ -294,6 +293,33 @@ class FSTTermOutputs extends Outputs<FST
}
return new TermData(longs, bytes, docFreq, totalTermFreq);
}
+
+
+ @Override
+ public void skipOutput(DataInput in) throws IOException {
+ int bits = in.readByte() & 0xff;
+ int bit0 = bits & 1;
+ int bit1 = bits & 2;
+ int bit2 = bits & 4;
+ int bytesSize = (bits >>> 3);
+ if (bit1 > 0 && bytesSize == 0) { // determine extra length
+ bytesSize = in.readVInt();
+ }
+ if (bit0 > 0) { // not all-zero case
+ for (int pos = 0; pos < longsSize; pos++) {
+ in.readVLong();
+ }
+ }
+ if (bit1 > 0) { // bytes exists
+ in.skipBytes(bytesSize);
+ }
+ if (bit2 > 0) { // stats exist
+ int code = in.readVInt();
+ if (hasPos && (code & 1) == 0) {
+ in.readVLong();
+ }
+ }
+ }
@Override
public TermData getNoOutput() {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java?rev=1596368&r1=1596367&r2=1596368&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java Tue May 20 19:04:29 2014
@@ -129,6 +129,14 @@ public final class ByteSequenceOutputs e
}
@Override
+ public void skipOutput(DataInput in) throws IOException {
+ final int len = in.readVInt();
+ if (len != 0) {
+ in.skipBytes(len);
+ }
+ }
+
+ @Override
public BytesRef getNoOutput() {
return NO_OUTPUT;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java?rev=1596368&r1=1596367&r2=1596368&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/BytesStore.java Tue May 20 19:04:29 2014
@@ -362,7 +362,7 @@ class BytesStore extends DataOutput {
}
@Override
- public void skipBytes(int count) {
+ public void skipBytes(long count) {
setPosition(getPosition() + count);
}
@@ -430,7 +430,7 @@ class BytesStore extends DataOutput {
}
@Override
- public void skipBytes(int count) {
+ public void skipBytes(long count) {
setPosition(getPosition() - count);
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java?rev=1596368&r1=1596367&r2=1596368&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java Tue May 20 19:04:29 2014
@@ -132,6 +132,14 @@ public final class CharSequenceOutputs e
return output;
}
}
+
+ @Override
+ public void skipOutput(DataInput in) throws IOException {
+ final int len = in.readVInt();
+ for(int idx=0;idx<len;idx++) {
+ in.readVInt();
+ }
+ }
@Override
public CharsRef getNoOutput() {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java?rev=1596368&r1=1596367&r2=1596368&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java Tue May 20 19:04:29 2014
@@ -893,10 +893,10 @@ public final class FST<T> {
// skip this arc:
readLabel(in);
if (arc.flag(BIT_ARC_HAS_OUTPUT)) {
- outputs.read(in);
+ outputs.skipOutput(in);
}
if (arc.flag(BIT_ARC_HAS_FINAL_OUTPUT)) {
- outputs.readFinalOutput(in);
+ outputs.skipFinalOutput(in);
}
if (arc.flag(BIT_STOP_NODE)) {
} else if (arc.flag(BIT_TARGET_NEXT)) {
@@ -1252,11 +1252,11 @@ public final class FST<T> {
readLabel(in);
if (flag(flags, BIT_ARC_HAS_OUTPUT)) {
- outputs.read(in);
+ outputs.skipOutput(in);
}
if (flag(flags, BIT_ARC_HAS_FINAL_OUTPUT)) {
- outputs.readFinalOutput(in);
+ outputs.skipFinalOutput(in);
}
if (!flag(flags, BIT_STOP_NODE) && !flag(flags, BIT_TARGET_NEXT)) {
@@ -1330,9 +1330,6 @@ public final class FST<T> {
/** Returns true if this reader uses reversed bytes
* under-the-hood. */
public abstract boolean reversed();
-
- /** Skips bytes. */
- public abstract void skipBytes(int count);
}
private static class ArcAndState<T> {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ForwardBytesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ForwardBytesReader.java?rev=1596368&r1=1596367&r2=1596368&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ForwardBytesReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ForwardBytesReader.java Tue May 20 19:04:29 2014
@@ -41,7 +41,7 @@ final class ForwardBytesReader extends F
}
@Override
- public void skipBytes(int count) {
+ public void skipBytes(long count) {
pos += count;
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java?rev=1596368&r1=1596367&r2=1596368&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java Tue May 20 19:04:29 2014
@@ -131,6 +131,17 @@ public final class IntSequenceOutputs ex
return output;
}
}
+
+ @Override
+ public void skipOutput(DataInput in) throws IOException {
+ final int len = in.readVInt();
+ if (len == 0) {
+ return;
+ }
+ for(int idx=0;idx<len;idx++) {
+ in.readVInt();
+ }
+ }
@Override
public IntsRef getNoOutput() {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java?rev=1596368&r1=1596367&r2=1596368&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java Tue May 20 19:04:29 2014
@@ -63,12 +63,25 @@ public abstract class Outputs<T> {
* #write(Object, DataOutput)}. */
public abstract T read(DataInput in) throws IOException;
+ /** Skip the output; defaults to just calling {@link #read}
+ * and discarding the result. */
+ public void skipOutput(DataInput in) throws IOException {
+ read(in);
+ }
+
/** Decode an output value previously written with {@link
* #writeFinalOutput(Object, DataOutput)}. By default this
* just calls {@link #read(DataInput)}. */
public T readFinalOutput(DataInput in) throws IOException {
return read(in);
}
+
+ /** Skip the output previously written with {@link #writeFinalOutput};
+ * defaults to just calling {@link #readFinalOutput} and discarding
+ * the result. */
+ public void skipFinalOutput(DataInput in) throws IOException {
+ skipOutput(in);
+ }
/** NOTE: this output is compared with == so you must
* ensure that all methods return the single object if
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java?rev=1596368&r1=1596367&r2=1596368&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java Tue May 20 19:04:29 2014
@@ -148,6 +148,12 @@ public class PairOutputs<A,B> extends Ou
B output2 = outputs2.read(in);
return newPair(output1, output2);
}
+
+ @Override
+ public void skipOutput(DataInput in) throws IOException {
+ outputs1.skipOutput(in);
+ outputs2.skipOutput(in);
+ }
@Override
public Pair<A,B> getNoOutput() {
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ReverseBytesReader.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ReverseBytesReader.java?rev=1596368&r1=1596367&r2=1596368&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ReverseBytesReader.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/ReverseBytesReader.java Tue May 20 19:04:29 2014
@@ -39,7 +39,7 @@ final class ReverseBytesReader extends F
}
@Override
- public void skipBytes(int count) {
+ public void skipBytes(long count) {
pos -= count;
}
Modified: lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java?rev=1596368&r1=1596367&r2=1596368&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java Tue May 20 19:04:29 2014
@@ -122,6 +122,11 @@ public final class ListOfOutputs<T> exte
public Object read(DataInput in) throws IOException {
return outputs.read(in);
}
+
+ @Override
+ public void skipOutput(DataInput in) throws IOException {
+ outputs.skipOutput(in);
+ }
@Override
public Object readFinalOutput(DataInput in) throws IOException {
@@ -136,6 +141,14 @@ public final class ListOfOutputs<T> exte
return outputList;
}
}
+
+ @Override
+ public void skipFinalOutput(DataInput in) throws IOException {
+ int count = in.readVInt();
+ for(int i=0;i<count;i++) {
+ outputs.skipOutput(in);
+ }
+ }
@Override
public Object getNoOutput() {