You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2012/06/09 12:44:44 UTC
svn commit: r1348355 - in /lucene/dev/branches/branch_4x: ./ lucene/
lucene/core/ lucene/core/src/java/org/apache/lucene/util/fst/
lucene/core/src/test/org/apache/lucene/util/fst/ lucene/suggest/
lucene/suggest/src/java/org/apache/lucene/search/suggest...
Author: mikemccand
Date: Sat Jun 9 10:44:43 2012
New Revision: 1348355
URL: http://svn.apache.org/viewvc?rev=1348355&view=rev
Log:
LUCENE-4117: add BytesReader to all FST arc reading APIs
Modified:
lucene/dev/branches/branch_4x/ (props changed)
lucene/dev/branches/branch_4x/lucene/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/ (props changed)
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java
lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/Util.java
lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
lucene/dev/branches/branch_4x/lucene/suggest/ (props changed)
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java?rev=1348355&r1=1348354&r2=1348355&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FST.java Sat Jun 9 10:44:43 2012
@@ -246,7 +246,7 @@ public final class FST<T> {
}
};
- private final static boolean flag(int flags, int bit) {
+ private static boolean flag(int flags, int bit) {
return (flags & bit) != 0;
}
@@ -755,7 +755,7 @@ public final class FST<T> {
*
* @return Returns the second argument
* (<code>arc</code>). */
- public Arc<T> readLastTargetArc(Arc<T> follow, Arc<T> arc) throws IOException {
+ public Arc<T> readLastTargetArc(Arc<T> follow, Arc<T> arc, FST.BytesReader in) throws IOException {
//System.out.println("readLast");
if (!targetHasArcs(follow)) {
//System.out.println(" end node");
@@ -766,7 +766,7 @@ public final class FST<T> {
arc.flags = BIT_LAST_ARC;
return arc;
} else {
- final BytesReader in = getBytesReader(getNodeAddress(follow.target));
+ in.pos = getNodeAddress(follow.target);
arc.node = follow.target;
final byte b = in.readByte();
if (b == ARCS_AS_FIXED_ARRAY) {
@@ -822,7 +822,7 @@ public final class FST<T> {
*
* @return Returns the second argument (<code>arc</code>).
*/
- public Arc<T> readFirstTargetArc(Arc<T> follow, Arc<T> arc) throws IOException {
+ public Arc<T> readFirstTargetArc(Arc<T> follow, Arc<T> arc, BytesReader in) throws IOException {
//int pos = address;
//System.out.println(" readFirstTarget follow.target=" + follow.target + " isFinal=" + follow.isFinal());
if (follow.isFinal()) {
@@ -841,7 +841,7 @@ public final class FST<T> {
//System.out.println(" insert isFinal; nextArc=" + follow.target + " isLast=" + arc.isLast() + " output=" + outputs.outputToString(arc.output));
return arc;
} else {
- return readFirstRealTargetArc(follow.target, arc, getBytesReader(0));
+ return readFirstRealTargetArc(follow.target, arc, in);
}
}
@@ -881,37 +881,36 @@ public final class FST<T> {
* @return Returns <code>true</code> if <code>arc</code> points to a state in an
* expanded array format.
*/
- boolean isExpandedTarget(Arc<T> follow) throws IOException {
+ boolean isExpandedTarget(Arc<T> follow, FST.BytesReader in) throws IOException {
if (!targetHasArcs(follow)) {
return false;
} else {
- final BytesReader in = getBytesReader(getNodeAddress(follow.target));
+ in.pos = getNodeAddress(follow.target);
return in.readByte() == ARCS_AS_FIXED_ARRAY;
}
}
/** In-place read; returns the arc. */
- public Arc<T> readNextArc(Arc<T> arc) throws IOException {
+ public Arc<T> readNextArc(Arc<T> arc, BytesReader in) throws IOException {
if (arc.label == END_LABEL) {
// This was a fake inserted "final" arc
if (arc.nextArc <= 0) {
throw new IllegalArgumentException("cannot readNextArc when arc.isLast()=true");
}
- return readFirstRealTargetArc(arc.nextArc, arc, getBytesReader(0));
+ return readFirstRealTargetArc(arc.nextArc, arc, in);
} else {
- return readNextRealArc(arc, getBytesReader(0));
+ return readNextRealArc(arc, in);
}
}
/** Peeks at next arc's label; does not alter arc. Do
* not call this if arc.isLast()! */
- public int readNextArcLabel(Arc<T> arc) throws IOException {
+ public int readNextArcLabel(Arc<T> arc, BytesReader in) throws IOException {
assert !arc.isLast();
- final BytesReader in;
if (arc.label == END_LABEL) {
//System.out.println(" nextArc fake " + arc.nextArc);
- in = getBytesReader(getNodeAddress(arc.nextArc));
+ in.pos = getNodeAddress(arc.nextArc);
final byte b = bytes[in.pos];
if (b == ARCS_AS_FIXED_ARRAY) {
//System.out.println(" nextArc fake array");
@@ -927,12 +926,12 @@ public final class FST<T> {
if (arc.bytesPerArc != 0) {
//System.out.println(" nextArc real array");
// arcs are at fixed entries
- in = getBytesReader(arc.posArcsStart);
+ in.pos = arc.posArcsStart;
in.skip((1+arc.arcIdx)*arc.bytesPerArc);
} else {
// arcs are packed
//System.out.println(" nextArc real packed");
- in = getBytesReader(arc.nextArc);
+ in.pos = arc.nextArc;
}
}
// skip flags
@@ -1223,7 +1222,7 @@ public final class FST<T> {
}
}
- public final BytesReader getBytesReader(int pos) {
+ public BytesReader getBytesReader(int pos) {
// TODO: maybe re-use via ThreadLocal?
if (packed) {
return new ForwardBytesReader(bytes, pos);
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java?rev=1348355&r1=1348354&r2=1348355&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/FSTEnum.java Sat Jun 9 10:44:43 2012
@@ -35,6 +35,7 @@ abstract class FSTEnum<T> {
@SuppressWarnings({"rawtypes","unchecked"}) protected T[] output = (T[]) new Object[10];
protected final T NO_OUTPUT;
+ protected final FST.BytesReader fstReader;
protected final FST.Arc<T> scratchArc = new FST.Arc<T>();
protected int upto;
@@ -45,6 +46,7 @@ abstract class FSTEnum<T> {
* term before target. */
protected FSTEnum(FST<T> fst) {
this.fst = fst;
+ fstReader = fst.getBytesReader(0);
NO_OUTPUT = fst.outputs.getNoOutput();
fst.getFirstArc(getArc(0));
output[0] = NO_OUTPUT;
@@ -62,7 +64,7 @@ abstract class FSTEnum<T> {
if (upto == 0) {
//System.out.println(" init");
upto = 1;
- fst.readFirstTargetArc(getArc(0), getArc(1));
+ fst.readFirstTargetArc(getArc(0), getArc(1), fstReader);
return;
}
//System.out.println(" rewind upto=" + upto + " vs targetLength=" + targetLength);
@@ -78,7 +80,7 @@ abstract class FSTEnum<T> {
} else if (cmp > 0) {
// seek backwards -- reset this arc to the first arc
final FST.Arc<T> arc = getArc(upto);
- fst.readFirstTargetArc(getArc(upto-1), arc);
+ fst.readFirstTargetArc(getArc(upto-1), arc, fstReader);
//System.out.println(" seek first arc");
break;
}
@@ -92,7 +94,7 @@ abstract class FSTEnum<T> {
if (upto == 0) {
//System.out.println(" init");
upto = 1;
- fst.readFirstTargetArc(getArc(0), getArc(1));
+ fst.readFirstTargetArc(getArc(0), getArc(1), fstReader);
} else {
// pop
//System.out.println(" check pop curArc target=" + arcs[upto].target + " label=" + arcs[upto].label + " isLast?=" + arcs[upto].isLast());
@@ -103,7 +105,7 @@ abstract class FSTEnum<T> {
return;
}
}
- fst.readNextArc(arcs[upto]);
+ fst.readNextArc(arcs[upto], fstReader);
}
pushFirst();
@@ -180,7 +182,7 @@ abstract class FSTEnum<T> {
}
setCurrentLabel(arc.label);
incr();
- arc = fst.readFirstTargetArc(arc, getArc(upto));
+ arc = fst.readFirstTargetArc(arc, getArc(upto), fstReader);
targetLabel = getTargetLabel();
continue;
} else if (low == arc.numArcs) {
@@ -198,7 +200,7 @@ abstract class FSTEnum<T> {
final FST.Arc<T> prevArc = getArc(upto);
//System.out.println(" rollback upto=" + upto + " arc.label=" + prevArc.label + " isLast?=" + prevArc.isLast());
if (!prevArc.isLast()) {
- fst.readNextArc(prevArc);
+ fst.readNextArc(prevArc, fstReader);
pushFirst();
return;
}
@@ -221,7 +223,7 @@ abstract class FSTEnum<T> {
}
setCurrentLabel(arc.label);
incr();
- arc = fst.readFirstTargetArc(arc, getArc(upto));
+ arc = fst.readFirstTargetArc(arc, getArc(upto), fstReader);
targetLabel = getTargetLabel();
} else if (arc.label > targetLabel) {
pushFirst();
@@ -237,7 +239,7 @@ abstract class FSTEnum<T> {
final FST.Arc<T> prevArc = getArc(upto);
//System.out.println(" rollback upto=" + upto + " arc.label=" + prevArc.label + " isLast?=" + prevArc.isLast());
if (!prevArc.isLast()) {
- fst.readNextArc(prevArc);
+ fst.readNextArc(prevArc, fstReader);
pushFirst();
return;
}
@@ -246,7 +248,7 @@ abstract class FSTEnum<T> {
} else {
// keep scanning
//System.out.println(" next scan");
- fst.readNextArc(arc);
+ fst.readNextArc(arc, fstReader);
}
}
}
@@ -320,7 +322,7 @@ abstract class FSTEnum<T> {
}
setCurrentLabel(arc.label);
incr();
- arc = fst.readFirstTargetArc(arc, getArc(upto));
+ arc = fst.readFirstTargetArc(arc, getArc(upto), fstReader);
targetLabel = getTargetLabel();
continue;
} else if (high == -1) {
@@ -333,12 +335,12 @@ abstract class FSTEnum<T> {
while(true) {
// First, walk backwards until we find a first arc
// that's before our target label:
- fst.readFirstTargetArc(getArc(upto-1), arc);
+ fst.readFirstTargetArc(getArc(upto-1), arc, fstReader);
if (arc.label < targetLabel) {
// Then, scan forwards to the arc just before
// the targetLabel:
- while(!arc.isLast() && fst.readNextArcLabel(arc) < targetLabel) {
- fst.readNextArc(arc);
+ while(!arc.isLast() && fst.readNextArcLabel(arc, in) < targetLabel) {
+ fst.readNextArc(arc, fstReader);
}
pushLast();
return;
@@ -355,7 +357,7 @@ abstract class FSTEnum<T> {
arc.arcIdx = (low > high ? high : low)-1;
//System.out.println(" hasFloor arcIdx=" + (arc.arcIdx+1));
fst.readNextRealArc(arc, in);
- assert arc.isLast() || fst.readNextArcLabel(arc) > targetLabel;
+ assert arc.isLast() || fst.readNextArcLabel(arc, in) > targetLabel;
assert arc.label < targetLabel: "arc.label=" + arc.label + " vs targetLabel=" + targetLabel;
pushLast();
return;
@@ -370,7 +372,7 @@ abstract class FSTEnum<T> {
}
setCurrentLabel(arc.label);
incr();
- arc = fst.readFirstTargetArc(arc, getArc(upto));
+ arc = fst.readFirstTargetArc(arc, getArc(upto), fstReader);
targetLabel = getTargetLabel();
} else if (arc.label > targetLabel) {
// TODO: if each arc could somehow read the arc just
@@ -380,12 +382,12 @@ abstract class FSTEnum<T> {
while(true) {
// First, walk backwards until we find a first arc
// that's before our target label:
- fst.readFirstTargetArc(getArc(upto-1), arc);
+ fst.readFirstTargetArc(getArc(upto-1), arc, fstReader);
if (arc.label < targetLabel) {
// Then, scan forwards to the arc just before
// the targetLabel:
- while(!arc.isLast() && fst.readNextArcLabel(arc) < targetLabel) {
- fst.readNextArc(arc);
+ while(!arc.isLast() && fst.readNextArcLabel(arc, fstReader) < targetLabel) {
+ fst.readNextArc(arc, fstReader);
}
pushLast();
return;
@@ -399,12 +401,12 @@ abstract class FSTEnum<T> {
}
} else if (!arc.isLast()) {
//System.out.println(" check next label=" + fst.readNextArcLabel(arc) + " (" + (char) fst.readNextArcLabel(arc) + ")");
- if (fst.readNextArcLabel(arc) > targetLabel) {
+ if (fst.readNextArcLabel(arc, fstReader) > targetLabel) {
pushLast();
return;
} else {
// keep scanning
- fst.readNextArc(arc);
+ fst.readNextArc(arc, fstReader);
}
} else {
pushLast();
@@ -441,7 +443,7 @@ abstract class FSTEnum<T> {
// short circuit
//upto--;
//upto = 0;
- fst.readFirstTargetArc(arc, getArc(upto));
+ fst.readFirstTargetArc(arc, getArc(upto), fstReader);
//System.out.println(" no match upto=" + upto);
return false;
}
@@ -493,7 +495,7 @@ abstract class FSTEnum<T> {
incr();
final FST.Arc<T> nextArc = getArc(upto);
- fst.readFirstTargetArc(arc, nextArc);
+ fst.readFirstTargetArc(arc, nextArc, fstReader);
arc = nextArc;
}
}
@@ -514,7 +516,7 @@ abstract class FSTEnum<T> {
}
incr();
- arc = fst.readLastTargetArc(arc, getArc(upto));
+ arc = fst.readLastTargetArc(arc, getArc(upto), fstReader);
}
}
Modified: lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/Util.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/Util.java?rev=1348355&r1=1348354&r2=1348355&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/Util.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/java/org/apache/lucene/util/fst/Util.java Sat Jun 9 10:44:43 2012
@@ -335,6 +335,7 @@ public final class Util {
final List<MinResult<T>> results = new ArrayList<MinResult<T>>();
+ final FST.BytesReader fstReader = fst.getBytesReader(0);
final T NO_OUTPUT = fst.outputs.getNoOutput();
// TODO: we could enable FST to sorting arcs by weight
@@ -366,7 +367,7 @@ public final class Util {
FST.Arc<T> minArc = null;
path = new FSTPath<T>(NO_OUTPUT, fromNode, comparator);
- fst.readFirstTargetArc(fromNode, path.arc);
+ fst.readFirstTargetArc(fromNode, path.arc, fstReader);
// Bootstrap: find the min starting arc
while (true) {
@@ -383,7 +384,7 @@ public final class Util {
if (path.arc.isLast()) {
break;
}
- fst.readNextArc(path.arc);
+ fst.readNextArc(path.arc, fstReader);
}
assert minArc != null;
@@ -439,7 +440,7 @@ public final class Util {
while (true) {
//System.out.println("\n cycle path: " + path);
- fst.readFirstTargetArc(path.arc, path.arc);
+ fst.readFirstTargetArc(path.arc, path.arc, fstReader);
// For each arc leaving this node:
boolean foundZero = false;
@@ -463,7 +464,7 @@ public final class Util {
if (path.arc.isLast()) {
break;
}
- fst.readNextArc(path.arc);
+ fst.readNextArc(path.arc, fstReader);
}
assert foundZero;
@@ -598,12 +599,13 @@ public final class Util {
emitDotState(out, "initial", "point", "white", "");
final T NO_OUTPUT = fst.outputs.getNoOutput();
+ final FST.BytesReader r = fst.getBytesReader(0);
// final FST.Arc<T> scratchArc = new FST.Arc<T>();
{
final String stateColor;
- if (fst.isExpandedTarget(startArc)) {
+ if (fst.isExpandedTarget(startArc, r)) {
stateColor = expandedNodeColor;
} else {
stateColor = null;
@@ -626,8 +628,6 @@ public final class Util {
int level = 0;
- final FST.BytesReader r = fst.getBytesReader(0);
-
while (!nextLevelQueue.isEmpty()) {
// we could double buffer here, but it doesn't matter probably.
//System.out.println("next level=" + level);
@@ -666,7 +666,7 @@ public final class Util {
}
*/
final String stateColor;
- if (fst.isExpandedTarget(arc)) {
+ if (fst.isExpandedTarget(arc, r)) {
stateColor = expandedNodeColor;
} else {
stateColor = null;
Modified: lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java?rev=1348355&r1=1348354&r2=1348355&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java (original)
+++ lucene/dev/branches/branch_4x/lucene/core/src/test/org/apache/lucene/util/fst/TestFSTs.java Sat Jun 9 10:44:43 2012
@@ -436,13 +436,14 @@ public class TestFSTs extends LuceneTest
in.offset = 0;
final T NO_OUTPUT = fst.outputs.getNoOutput();
T output = NO_OUTPUT;
+ final FST.BytesReader fstReader = fst.getBytesReader(0);
while(true) {
// read all arcs:
- fst.readFirstTargetArc(arc, arc);
+ fst.readFirstTargetArc(arc, arc, fstReader);
arcs.add(new FST.Arc<T>().copyFrom(arc));
while(!arc.isLast()) {
- fst.readNextArc(arc);
+ fst.readNextArc(arc, fstReader);
arcs.add(new FST.Arc<T>().copyFrom(arc));
}
@@ -1847,10 +1848,11 @@ public class TestFSTs extends LuceneTest
throws IOException {
if (FST.targetHasArcs(arc)) {
int childCount = 0;
- for (arc = fst.readFirstTargetArc(arc, arc);;
- arc = fst.readNextArc(arc), childCount++)
+ FST.BytesReader fstReader = fst.getBytesReader(0);
+ for (arc = fst.readFirstTargetArc(arc, arc, fstReader);;
+ arc = fst.readNextArc(arc, fstReader), childCount++)
{
- boolean expanded = fst.isExpandedTarget(arc);
+ boolean expanded = fst.isExpandedTarget(arc, fstReader);
int children = verifyStateAndBelow(fst, new FST.Arc<Object>().copyFrom(arc), depth + 1);
assertEquals(
@@ -1982,12 +1984,13 @@ public class TestFSTs extends LuceneTest
assertEquals(nothing, startArc.output);
assertEquals(nothing, startArc.nextFinalOutput);
- FST.Arc<Long> arc = fst.readFirstTargetArc(startArc, new FST.Arc<Long>());
+ FST.Arc<Long> arc = fst.readFirstTargetArc(startArc, new FST.Arc<Long>(),
+ fst.getBytesReader(0));
assertEquals('a', arc.label);
assertEquals(17, arc.nextFinalOutput.longValue());
assertTrue(arc.isFinal());
- arc = fst.readNextArc(arc);
+ arc = fst.readNextArc(arc, fst.getBytesReader(0));
assertEquals('b', arc.label);
assertFalse(arc.isFinal());
assertEquals(42, arc.output.longValue());
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java?rev=1348355&r1=1348354&r2=1348355&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletion.java Sat Jun 9 10:44:43 2012
@@ -135,11 +135,12 @@ public class FSTCompletion {
try {
List<Arc<Object>> rootArcs = new ArrayList<Arc<Object>>();
Arc<Object> arc = automaton.getFirstArc(new Arc<Object>());
- automaton.readFirstTargetArc(arc, arc);
+ FST.BytesReader fstReader = automaton.getBytesReader(0);
+ automaton.readFirstTargetArc(arc, arc, fstReader);
while (true) {
rootArcs.add(new Arc<Object>().copyFrom(arc));
if (arc.isLast()) break;
- automaton.readNextArc(arc);
+ automaton.readNextArc(arc, fstReader);
}
Collections.reverse(rootArcs); // we want highest weights first.
@@ -168,13 +169,14 @@ public class FSTCompletion {
// Get the UTF-8 bytes representation of the input key.
try {
final FST.Arc<Object> scratch = new FST.Arc<Object>();
+ FST.BytesReader fstReader = automaton.getBytesReader(0);
for (; rootArcIndex < rootArcs.length; rootArcIndex++) {
final FST.Arc<Object> rootArc = rootArcs[rootArcIndex];
final FST.Arc<Object> arc = scratch.copyFrom(rootArc);
// Descend into the automaton using the key as prefix.
if (descendWithPrefix(arc, utf8)) {
- automaton.readFirstTargetArc(arc, arc);
+ automaton.readFirstTargetArc(arc, arc, fstReader);
if (arc.label == FST.END_LABEL) {
// Normalize prefix-encoded weight.
return rootArc.label;
@@ -356,8 +358,8 @@ public class FSTCompletion {
}
assert output.offset == 0;
output.bytes[output.length++] = (byte) arc.label;
-
- automaton.readFirstTargetArc(arc, arc);
+ FST.BytesReader fstReader = automaton.getBytesReader(0);
+ automaton.readFirstTargetArc(arc, arc, fstReader);
while (true) {
if (arc.label == FST.END_LABEL) {
res.add(new Completion(output, bucket));
@@ -373,7 +375,7 @@ public class FSTCompletion {
if (arc.isLast()) {
break;
}
- automaton.readNextArc(arc);
+ automaton.readNextArc(arc, fstReader);
}
return false;
}