You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2013/11/04 13:31:34 UTC
svn commit: r1538578 - in /lucene/dev/trunk/lucene: ./
suggest/src/java/org/apache/lucene/search/suggest/
suggest/src/java/org/apache/lucene/search/suggest/analyzing/
suggest/src/java/org/apache/lucene/search/suggest/fst/
suggest/src/java/org/apache/lu...
Author: mikemccand
Date: Mon Nov 4 12:31:33 2013
New Revision: 1538578
URL: http://svn.apache.org/r1538578
Log:
LUCENE-5323: add Lookup.sizeInBytes
Modified:
lucene/dev/trunk/lucene/CHANGES.txt
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
Modified: lucene/dev/trunk/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/CHANGES.txt?rev=1538578&r1=1538577&r2=1538578&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/CHANGES.txt (original)
+++ lucene/dev/trunk/lucene/CHANGES.txt Mon Nov 4 12:31:33 2013
@@ -139,6 +139,9 @@ New Features
a ValueSource, such as a NumericDocValuesField or an expression.
(Shai Erera)
+* LUCENE-5323: Add .sizeInBytes method to all suggesters (Lookup).
+ (Areek Zillur via Mike McCandless)
+
Bug Fixes
* LUCENE-4998: Fixed a few places to pass IOContext.READONCE instead
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java?rev=1538578&r1=1538577&r2=1538578&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java Mon Nov 4 12:31:33 2013
@@ -201,4 +201,9 @@ public abstract class Lookup {
*/
public abstract boolean load(InputStream input) throws IOException;
+ /**
+ * Get the size of the underlying lookup implementation in memory
+ * @return ram size of the lookup implementation in bytes
+ */
+ public abstract long sizeInBytes();
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java?rev=1538578&r1=1538577&r2=1538578&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java Mon Nov 4 12:31:33 2013
@@ -72,6 +72,7 @@ import org.apache.lucene.store.Directory
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.Version;
// TODO:
@@ -599,4 +600,9 @@ public class AnalyzingInfixSuggester ext
dir = null;
}
}
+
+ @Override
+ public long sizeInBytes() {
+ return RamUsageEstimator.sizeOf(this);
+ }
};
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java?rev=1538578&r1=1538577&r2=1538578&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java Mon Nov 4 12:31:33 2013
@@ -250,6 +250,7 @@ public class AnalyzingSuggester extends
}
/** Returns byte size of the underlying FST. */
+ @Override
public long sizeInBytes() {
return fst == null ? 0 : fst.sizeInBytes();
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java?rev=1538578&r1=1538577&r2=1538578&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java Mon Nov 4 12:31:33 2013
@@ -198,7 +198,8 @@ public class FreeTextSuggester extends L
this.separator = separator;
}
- /** Returns byte size of the underlying FST. */
+ /** Returns byte size of the underlying FST. */
+ @Override
public long sizeInBytes() {
if (fst == null) {
return 0;
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java?rev=1538578&r1=1538577&r2=1538578&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java Mon Nov 4 12:31:33 2013
@@ -288,4 +288,9 @@ public class FSTCompletionLookup extends
}
return true;
}
+
+ @Override
+ public long sizeInBytes() {
+ return RamUsageEstimator.sizeOf(this);
+ }
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java?rev=1538578&r1=1538577&r2=1538578&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java Mon Nov 4 12:31:33 2013
@@ -287,4 +287,10 @@ public class WFSTCompletionLookup extend
return left.compareTo(right);
}
};
+
+ /** Returns byte size of the underlying FST. */
+ @Override
+ public long sizeInBytes() {
+ return (fst == null) ? 0 : fst.sizeInBytes();
+ }
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java?rev=1538578&r1=1538577&r2=1538578&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java Mon Nov 4 12:31:33 2013
@@ -31,6 +31,7 @@ import org.apache.lucene.search.suggest.
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.UnicodeUtil;
/**
@@ -200,4 +201,10 @@ public class JaspellLookup extends Looku
}
return true;
}
+
+ /** Returns byte size of the underlying TST. */
+ @Override
+ public long sizeInBytes() {
+ return RamUsageEstimator.sizeOf(trie);
+ }
}
Modified: lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java?rev=1538578&r1=1538577&r2=1538578&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java Mon Nov 4 12:31:33 2013
@@ -31,6 +31,7 @@ import org.apache.lucene.search.suggest.
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
+import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.UnicodeUtil;
/**
@@ -219,5 +220,11 @@ public class TSTLookup extends Lookup {
}
return true;
}
+
+ /** Returns byte size of the underlying TST */
+ @Override
+ public long sizeInBytes() {
+ return RamUsageEstimator.sizeOf(autocomplete);
+ }
}
Modified: lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java?rev=1538578&r1=1538577&r2=1538578&view=diff
==============================================================================
--- lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java (original)
+++ lucene/dev/trunk/lucene/suggest/src/test/org/apache/lucene/search/suggest/LookupBenchmarkTest.java Mon Nov 4 12:31:33 2013
@@ -144,15 +144,7 @@ public class LookupBenchmarkTest extends
System.err.println("-- RAM consumption");
for (Class<? extends Lookup> cls : benchmarkClasses) {
Lookup lookup = buildLookup(cls, dictionaryInput);
- long sizeInBytes;
- if (lookup instanceof AnalyzingSuggester) {
- // Just get size of FST: else we are also measuring
- // size of MockAnalyzer which is non-trivial and
- // varies depending on test seed:
- sizeInBytes = ((AnalyzingSuggester) lookup).sizeInBytes();
- } else {
- sizeInBytes = RamUsageEstimator.sizeOf(lookup);
- }
+ long sizeInBytes = lookup.sizeInBytes();
System.err.println(
String.format(Locale.ROOT, "%-15s size[B]:%,13d",
lookup.getClass().getSimpleName(),