You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2014/08/14 15:39:30 UTC
svn commit: r1617940 - in /lucene/dev/trunk/lucene: ./
core/src/java/org/apache/lucene/util/fst/
misc/src/java/org/apache/lucene/util/fst/
Author: mikemccand
Date: Thu Aug 14 13:39:29 2014
New Revision: 1617940
URL: http://svn.apache.org/r1617940
Log:
LUCENE-5884: optimize FST.ramBytesUsed
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/NoOutputs.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java
lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java
lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1617940&r1=1617939&r2=1617940&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Thu Aug 14 13:39:29 2014
@@ -214,6 +214,9 @@ Optimizations
* LUCENE-5856: Optimize Fixed/Open/LongBitSet to remove unnecessary AND.
(Robert Muir)
+* LUCENE-5884: Optimize FST.ramBytesUsed. (Adrien Grand, Robert Muir,
+ Mike McCandless)
+
Bug Fixes
* LUCENE-5796: Fixes the Scorer.getChildren() method for two combinations
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java?rev=1617940&r1=1617939&r2=1617940&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/ByteSequenceOutputs.java Thu Aug 14 13:39:29 2014
@@ -151,8 +151,10 @@ public final class ByteSequenceOutputs e
return output.toString();
}
+ private static final long BASE_NUM_BYTES = RamUsageEstimator.shallowSizeOf(NO_OUTPUT);
+
@Override
public long ramBytesUsed(BytesRef output) {
- return super.ramBytesUsed(output) + RamUsageEstimator.sizeOf(output.bytes);
+ return BASE_NUM_BYTES + RamUsageEstimator.sizeOf(output.bytes);
}
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java?rev=1617940&r1=1617939&r2=1617940&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/CharSequenceOutputs.java Thu Aug 14 13:39:29 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.CharsRef;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* An FST {@link Outputs} implementation where each output
@@ -150,4 +151,11 @@ public final class CharSequenceOutputs e
public String outputToString(CharsRef output) {
return output.toString();
}
+
+ private static final long BASE_NUM_BYTES = RamUsageEstimator.shallowSizeOf(NO_OUTPUT);
+
+ @Override
+ public long ramBytesUsed(CharsRef output) {
+ return BASE_NUM_BYTES + RamUsageEstimator.sizeOf(output.chars);
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/FST.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/FST.java?rev=1617940&r1=1617939&r2=1617940&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/FST.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/FST.java Thu Aug 14 13:39:29 2014
@@ -428,6 +428,8 @@ public final class FST<T> implements Acc
return size;
}
+ private int cachedArcsBytesUsed;
+
@Override
public long ramBytesUsed() {
long size = BASE_RAM_BYTES_USED;
@@ -438,8 +440,7 @@ public final class FST<T> implements Acc
size += nodeAddress.ramBytesUsed();
size += inCounts.ramBytesUsed();
}
- size += ramBytesUsed(cachedRootArcs);
- size += ramBytesUsed(assertingCachedRootArcs);
+ size += cachedArcsBytesUsed;
size += RamUsageEstimator.sizeOf(bytesPerArc);
return size;
}
@@ -472,6 +473,7 @@ public final class FST<T> implements Acc
private void cacheRootArcs() throws IOException {
cachedRootArcs = (Arc<T>[]) new Arc[0x80];
readRootArcs(cachedRootArcs);
+ cachedArcsBytesUsed += ramBytesUsed(cachedRootArcs);
assert setAssertingRootArcs(cachedRootArcs);
assert assertRootArcs();
@@ -502,6 +504,7 @@ public final class FST<T> implements Acc
private boolean setAssertingRootArcs(Arc<T>[] arcs) throws IOException {
assertingCachedRootArcs = (Arc<T>[]) new Arc[arcs.length];
readRootArcs(assertingCachedRootArcs);
+ cachedArcsBytesUsed *= 2;
return true;
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java?rev=1617940&r1=1617939&r2=1617940&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/IntSequenceOutputs.java Thu Aug 14 13:39:29 2014
@@ -22,6 +22,7 @@ import java.io.IOException;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.IntsRef;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* An FST {@link Outputs} implementation where each output
@@ -152,4 +153,11 @@ public final class IntSequenceOutputs ex
public String outputToString(IntsRef output) {
return output.toString();
}
+
+ private static final long BASE_NUM_BYTES = RamUsageEstimator.shallowSizeOf(NO_OUTPUT);
+
+ @Override
+ public long ramBytesUsed(IntsRef output) {
+ return BASE_NUM_BYTES + RamUsageEstimator.sizeOf(output.ints);
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/NoOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/NoOutputs.java?rev=1617940&r1=1617939&r2=1617940&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/NoOutputs.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/NoOutputs.java Thu Aug 14 13:39:29 2014
@@ -101,4 +101,9 @@ public final class NoOutputs extends Out
public String outputToString(Object output) {
return "";
}
+
+ @Override
+ public long ramBytesUsed(Object output) {
+ return 0;
+ }
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java?rev=1617940&r1=1617939&r2=1617940&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/Outputs.java Thu Aug 14 13:39:29 2014
@@ -22,7 +22,6 @@ import java.io.IOException;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.Accountable;
-import org.apache.lucene.util.RamUsageEstimator;
/**
* Represents the outputs for an FST, providing the basic
@@ -100,7 +99,5 @@ public abstract class Outputs<T> {
/** Return memory usage for the provided output.
* @see Accountable */
- public long ramBytesUsed(T output) {
- return RamUsageEstimator.shallowSizeOf(output);
- }
+ public abstract long ramBytesUsed(T output);
}
Modified: lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java?rev=1617940&r1=1617939&r2=1617940&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java (original)
+++ lucene/dev/trunk/lucene/core/src/java/org/apache/lucene/util/fst/PairOutputs.java Thu Aug 14 13:39:29 2014
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* An FST {@link Outputs} implementation, holding two other outputs.
@@ -176,9 +177,11 @@ public class PairOutputs<A,B> extends Ou
return "PairOutputs<" + outputs1 + "," + outputs2 + ">";
}
+ private static final long BASE_NUM_BYTES = RamUsageEstimator.shallowSizeOf(new Pair<Object,Object>(null, null));
+
@Override
public long ramBytesUsed(Pair<A,B> output) {
- long ramBytesUsed = super.ramBytesUsed(output);
+ long ramBytesUsed = BASE_NUM_BYTES;
if (output.output1 != null) {
ramBytesUsed += outputs1.ramBytesUsed(output.output1);
}
Modified: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java?rev=1617940&r1=1617939&r2=1617940&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java (original)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/util/fst/ListOfOutputs.java Thu Aug 14 13:39:29 2014
@@ -24,6 +24,7 @@ import java.util.List;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.IntsRef; // javadocs
+import org.apache.lucene.util.RamUsageEstimator;
/**
* Wraps another Outputs implementation and encodes one or
@@ -208,4 +209,24 @@ public final class ListOfOutputs<T> exte
return (List<T>) output;
}
}
+
+ private static final long BASE_LIST_NUM_BYTES = RamUsageEstimator.shallowSizeOf(new ArrayList<Object>());
+
+ @Override
+ public long ramBytesUsed(Object output) {
+ long bytes = 0;
+ if (output instanceof List) {
+ bytes += BASE_LIST_NUM_BYTES;
+ List<T> outputList = (List<T>) output;
+ for(T _output : outputList) {
+ bytes += outputs.ramBytesUsed(_output);
+ }
+ // 2 * to allow for ArrayList's oversizing:
+ bytes += 2 * outputList.size() * RamUsageEstimator.NUM_BYTES_OBJECT_REF;
+ } else {
+ bytes += outputs.ramBytesUsed((T) output);
+ }
+
+ return bytes;
+ }
}
Modified: lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java?rev=1617940&r1=1617939&r2=1617940&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java (original)
+++ lucene/dev/trunk/lucene/misc/src/java/org/apache/lucene/util/fst/UpToTwoPositiveIntOutputs.java Thu Aug 14 13:39:29 2014
@@ -21,6 +21,7 @@ import java.io.IOException;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.util.RamUsageEstimator;
/**
* An FST {@link Outputs} implementation where each output
@@ -232,4 +233,16 @@ public final class UpToTwoPositiveIntOut
assert valid(second, false);
return new TwoLongs((Long) first, (Long) second);
}
+
+ private static final long TWO_LONGS_NUM_BYTES = RamUsageEstimator.shallowSizeOf(new TwoLongs(0, 0));
+
+ @Override
+ public long ramBytesUsed(Object o) {
+ if (o instanceof Long) {
+ return RamUsageEstimator.sizeOf((Long) o);
+ } else {
+ assert o instanceof TwoLongs;
+ return TWO_LONGS_NUM_BYTES;
+ }
+ }
}