You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ar...@apache.org on 2014/02/07 21:58:26 UTC
svn commit: r1565811 - in /lucene/dev/branches/branch_4x/lucene: ./
suggest/src/java/org/apache/lucene/search/spell/
suggest/src/java/org/apache/lucene/search/suggest/
suggest/src/java/org/apache/lucene/search/suggest/analyzing/
suggest/src/java/org/ap...
Author: areek
Date: Fri Feb 7 20:58:25 2014
New Revision: 1565811
URL: http://svn.apache.org/r1565811
Log:
LUCENE-5404: Add .getCount method to all suggesters (Lookup), persist count metadata on .store(), Dictionary returns InputIterator, Dictionary.getWordIterator renamed to .getEntryIterator
Modified:
lucene/dev/branches/branch_4x/lucene/CHANGES.txt
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/Dictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/LuceneDictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentValueSourceDictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/FileDictionaryTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
Modified: lucene/dev/branches/branch_4x/lucene/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/CHANGES.txt?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/CHANGES.txt (original)
+++ lucene/dev/branches/branch_4x/lucene/CHANGES.txt Fri Feb 7 20:58:25 2014
@@ -18,6 +18,10 @@ New Features
(missing the term, weight or payload). (Areek Zillur via
Mike McCandless)
+* LUCENE-5404: Add .getCount method to all suggesters (Lookup); persist count
+ metadata on .store(); Dictionary returns InputIterator; Dictionary.getWordIterator
+ renamed to .getEntryIterator. (Areek Zillur)
+
* SOLR-1871: The RangeMapFloatFunction accepts an arbitrary ValueSource
as target and default values. (Chris Harris, shalin)
@@ -323,7 +327,7 @@ New Features
* LUCENE-5323: Add .sizeInBytes method to all suggesters (Lookup).
(Areek Zillur via Mike McCandless)
-
+
* LUCENE-5312: Add BlockJoinSorter, a new Sorter implementation that makes sure
to never split up blocks of documents indexed with IndexWriter.addDocuments.
(Adrien Grand)
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/Dictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/Dictionary.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/Dictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/Dictionary.java Fri Feb 7 20:58:25 2014
@@ -17,19 +17,20 @@ package org.apache.lucene.search.spell;
*/
import java.io.IOException;
-import org.apache.lucene.util.BytesRefIterator;
+
+import org.apache.lucene.search.suggest.InputIterator;
/**
* A simple interface representing a Dictionary. A Dictionary
- * here is just a list of words.
+ * here is a list of entries, where every entry consists of
+ * term, weight and payload.
*
- *
*/
public interface Dictionary {
/**
- * Return all words present in the dictionary
+ * Returns an iterator over all the entries
* @return Iterator
*/
- BytesRefIterator getWordsIterator() throws IOException;
+ InputIterator getEntryIterator() throws IOException;
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/HighFrequencyDictionary.java Fri Feb 7 20:58:25 2014
@@ -57,7 +57,7 @@ public class HighFrequencyDictionary imp
}
@Override
- public final BytesRefIterator getWordsIterator() throws IOException {
+ public final InputIterator getEntryIterator() throws IOException {
return new HighFrequencyIterator();
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/LuceneDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/LuceneDictionary.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/LuceneDictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/LuceneDictionary.java Fri Feb 7 20:58:25 2014
@@ -18,6 +18,7 @@ package org.apache.lucene.search.spell;
*/
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.util.BytesRefIterator;
import org.apache.lucene.index.Terms;
import org.apache.lucene.index.MultiFields;
@@ -42,12 +43,12 @@ public class LuceneDictionary implements
}
@Override
- public final BytesRefIterator getWordsIterator() throws IOException {
+ public final InputIterator getEntryIterator() throws IOException {
final Terms terms = MultiFields.getTerms(reader, field);
if (terms != null) {
- return terms.iterator(null);
+ return new InputIterator.InputIteratorWrapper(terms.iterator(null));
} else {
- return BytesRefIterator.EMPTY;
+ return InputIterator.EMPTY;
}
}
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/PlainTextDictionary.java Fri Feb 7 20:58:25 2014
@@ -21,6 +21,7 @@ package org.apache.lucene.search.spell;
import java.util.Comparator;
import java.io.*;
+import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.BytesRefIterator;
import org.apache.lucene.util.IOUtils;
@@ -64,8 +65,8 @@ public class PlainTextDictionary impleme
}
@Override
- public BytesRefIterator getWordsIterator() throws IOException {
- return new FileIterator();
+ public InputIterator getEntryIterator() throws IOException {
+ return new InputIterator.InputIteratorWrapper(new FileIterator());
}
final class FileIterator implements BytesRefIterator {
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/spell/SpellChecker.java Fri Feb 7 20:58:25 2014
@@ -512,7 +512,7 @@ public class SpellChecker implements jav
boolean isEmpty = termsEnums.isEmpty();
try {
- BytesRefIterator iter = dict.getWordsIterator();
+ BytesRefIterator iter = dict.getEntryIterator();
BytesRef currentTerm;
terms: while ((currentTerm = iter.next()) != null) {
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentDictionary.java Fri Feb 7 20:58:25 2014
@@ -87,7 +87,7 @@ public class DocumentDictionary implemen
}
@Override
- public BytesRefIterator getWordsIterator() throws IOException {
+ public InputIterator getEntryIterator() throws IOException {
return new DocumentInputIterator(payloadField!=null);
}
@@ -156,7 +156,7 @@ public class DocumentDictionary implemen
currentPayload = tempPayload;
currentWeight = getWeight(doc, currentDocId);
-
+
return tempTerm;
}
return null;
@@ -171,7 +171,7 @@ public class DocumentDictionary implemen
public boolean hasPayloads() {
return hasPayloads;
}
-
+
/**
* Returns the value of the <code>weightField</code> for the current document.
* Retrieves the value for the <code>weightField</code> if its stored (using <code>doc</code>)
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentValueSourceDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentValueSourceDictionary.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentValueSourceDictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/DocumentValueSourceDictionary.java Fri Feb 7 20:58:25 2014
@@ -92,7 +92,7 @@ public class DocumentValueSourceDictiona
}
@Override
- public BytesRefIterator getWordsIterator() throws IOException {
+ public InputIterator getEntryIterator() throws IOException {
return new DocumentValueSourceInputIterator(payloadField!=null);
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/FileDictionary.java Fri Feb 7 20:58:25 2014
@@ -107,7 +107,7 @@ public class FileDictionary implements D
}
@Override
- public InputIterator getWordsIterator() {
+ public InputIterator getEntryIterator() {
try {
return new FileIterator();
} catch (IOException e) {
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/InputIterator.java Fri Feb 7 20:58:25 2014
@@ -45,6 +45,9 @@ public interface InputIterator extends B
/** Returns true if the iterator has payloads */
public boolean hasPayloads();
+ /** Singleton InputIterator that iterates over 0 BytesRefs. */
+ public static final InputIterator EMPTY = new InputIteratorWrapper(BytesRefIterator.EMPTY);
+
/**
* Wraps a BytesRefIterator as a suggester InputIterator, with all weights
* set to <code>1</code> and carries no payload
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/Lookup.java Fri Feb 7 20:58:25 2014
@@ -24,8 +24,12 @@ import java.util.Comparator;
import java.util.List;
import org.apache.lucene.search.spell.Dictionary;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
+import org.apache.lucene.store.InputStreamDataInput;
+import org.apache.lucene.store.OutputStreamDataOutput;
import org.apache.lucene.util.BytesRef;
-import org.apache.lucene.util.BytesRefIterator;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.PriorityQueue;
/**
@@ -33,6 +37,7 @@ import org.apache.lucene.util.PriorityQu
* @lucene.experimental
*/
public abstract class Lookup {
+
/**
* Result of a lookup.
*/
@@ -157,21 +162,46 @@ public abstract class Lookup {
* {@link UnsortedInputIterator} in such case.
*/
public void build(Dictionary dict) throws IOException {
- BytesRefIterator it = dict.getWordsIterator();
- InputIterator tfit;
- if (it instanceof InputIterator) {
- tfit = (InputIterator)it;
- } else {
- tfit = new InputIterator.InputIteratorWrapper(it);
+ build(dict.getEntryIterator());
+ }
+
+ /**
+ * Calls {@link #load(DataInput)} after converting
+ * {@link InputStream} to {@link DataInput}
+ */
+ public boolean load(InputStream input) throws IOException {
+ DataInput dataIn = new InputStreamDataInput(input);
+ try {
+ return load(dataIn);
+ } finally {
+ IOUtils.close(input);
+ }
+ }
+
+ /**
+ * Calls {@link #store(DataOutput)} after converting
+ * {@link OutputStream} to {@link DataOutput}
+ */
+ public boolean store(OutputStream output) throws IOException {
+ DataOutput dataOut = new OutputStreamDataOutput(output);
+ try {
+ return store(dataOut);
+ } finally {
+ IOUtils.close(output);
}
- build(tfit);
}
/**
+ * Get the number of entries the lookup was built with
+ * @return total number of suggester entries
+ */
+ public abstract long getCount();
+
+ /**
* Builds up a new internal {@link Lookup} representation based on the given {@link InputIterator}.
* The implementation might re-sort the data internally.
*/
- public abstract void build(InputIterator tfit) throws IOException;
+ public abstract void build(InputIterator inputIterator) throws IOException;
/**
* Look up a key and return possible completion for this key.
@@ -183,23 +213,22 @@ public abstract class Lookup {
*/
public abstract List<LookupResult> lookup(CharSequence key, boolean onlyMorePopular, int num);
-
/**
* Persist the constructed lookup data to a directory. Optional operation.
- * @param output {@link OutputStream} to write the data to.
+ * @param output {@link DataOutput} to write the data to.
* @return true if successful, false if unsuccessful or not supported.
* @throws IOException when fatal IO error occurs.
*/
- public abstract boolean store(OutputStream output) throws IOException;
+ public abstract boolean store(DataOutput output) throws IOException;
/**
* Discard current lookup data and load it from a previously saved copy.
* Optional operation.
- * @param input the {@link InputStream} to load the lookup data.
+ * @param input the {@link DataInput} to load the lookup data.
* @return true if completed successfully, false if unsuccessful or not supported.
* @throws IOException when fatal IO error occurs.
*/
- public abstract boolean load(InputStream input) throws IOException;
+ public abstract boolean load(DataInput input) throws IOException;
/**
* Get the size of the underlying lookup implementation in memory
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggester.java Fri Feb 7 20:58:25 2014
@@ -20,8 +20,6 @@ package org.apache.lucene.search.suggest
import java.io.Closeable;
import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashSet;
@@ -68,6 +66,8 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.suggest.Lookup.LookupResult; // javadocs
import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.util.BytesRef;
@@ -107,6 +107,8 @@ public class AnalyzingInfixSuggester ext
private final File indexPath;
final int minPrefixChars;
private Directory dir;
+ /** Number of entries the lookup was built with */
+ private long count = 0;
/** {@link IndexSearcher} used for lookups. */
protected IndexSearcher searcher;
@@ -154,12 +156,14 @@ public class AnalyzingInfixSuggester ext
if (DirectoryReader.indexExists(dir)) {
// Already built; open it:
- searcher = new IndexSearcher(DirectoryReader.open(dir));
+ IndexReader reader = DirectoryReader.open(dir);
+ searcher = new IndexSearcher(reader);
// This will just be null if app didn't pass payloads to build():
// TODO: maybe just stored fields? they compress...
payloadsDV = MultiDocValues.getBinaryValues(searcher.getIndexReader(), "payloads");
weightsDV = MultiDocValues.getNumericValues(searcher.getIndexReader(), "weight");
textDV = MultiDocValues.getBinaryValues(searcher.getIndexReader(), TEXT_FIELD_NAME);
+ count = reader.numDocs();
assert textDV != null;
}
}
@@ -194,6 +198,7 @@ public class AnalyzingInfixSuggester ext
IndexWriter w2 = null;
AtomicReader r = null;
boolean success = false;
+ count = 0;
try {
Analyzer gramAnalyzer = new AnalyzerWrapper(Analyzer.PER_FIELD_REUSE_STRATEGY) {
@Override
@@ -239,7 +244,6 @@ public class AnalyzingInfixSuggester ext
} else {
payloadField = null;
}
-
//long t0 = System.nanoTime();
while ((text = iter.next()) != null) {
String textString = text.utf8ToString();
@@ -251,6 +255,7 @@ public class AnalyzingInfixSuggester ext
payloadField.setBytesValue(iter.payload());
}
w.addDocument(doc);
+ count++;
}
//System.out.println("initial indexing time: " + ((System.nanoTime()-t0)/1000000) + " msec");
@@ -619,12 +624,12 @@ public class AnalyzingInfixSuggester ext
}
@Override
- public boolean store(OutputStream out) {
+ public boolean store(DataOutput in) throws IOException {
return false;
}
@Override
- public boolean load(InputStream out) {
+ public boolean load(DataInput out) throws IOException {
return false;
}
@@ -644,4 +649,9 @@ public class AnalyzingInfixSuggester ext
public long sizeInBytes() {
return RamUsageEstimator.sizeOf(this);
}
+
+ @Override
+ public long getCount() {
+ return count;
+ }
};
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggester.java Fri Feb 7 20:58:25 2014
@@ -19,8 +19,6 @@ package org.apache.lucene.search.suggest
import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -38,8 +36,6 @@ import org.apache.lucene.store.ByteArray
import org.apache.lucene.store.ByteArrayDataOutput;
import org.apache.lucene.store.DataInput;
import org.apache.lucene.store.DataOutput;
-import org.apache.lucene.store.InputStreamDataInput;
-import org.apache.lucene.store.OutputStreamDataOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
@@ -187,6 +183,9 @@ public class AnalyzingSuggester extends
/** Whether position holes should appear in the automaton. */
private boolean preservePositionIncrements;
+ /** Number of entries the lookup was built with */
+ private long count = 0;
+
/**
* Calls {@link #AnalyzingSuggester(Analyzer,Analyzer,int,int,int,boolean)
* AnalyzingSuggester(analyzer, analyzer, EXACT_FIRST |
@@ -394,6 +393,7 @@ public class AnalyzingSuggester extends
TokenStreamToAutomaton ts2a = getTokenStreamToAutomaton();
boolean success = false;
+ count = 0;
byte buffer[] = new byte[8];
try {
ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);
@@ -458,6 +458,7 @@ public class AnalyzingSuggester extends
writer.write(buffer, 0, output.getPosition());
}
+ count++;
}
writer.close();
@@ -571,32 +572,24 @@ public class AnalyzingSuggester extends
}
@Override
- public boolean store(OutputStream output) throws IOException {
- DataOutput dataOut = new OutputStreamDataOutput(output);
- try {
- if (fst == null) {
- return false;
- }
-
- fst.save(dataOut);
- dataOut.writeVInt(maxAnalyzedPathsForOneInput);
- dataOut.writeByte((byte) (hasPayloads ? 1 : 0));
- } finally {
- IOUtils.close(output);
+ public boolean store(DataOutput output) throws IOException {
+ output.writeVLong(count);
+ if (fst == null) {
+ return false;
}
+
+ fst.save(output);
+ output.writeVInt(maxAnalyzedPathsForOneInput);
+ output.writeByte((byte) (hasPayloads ? 1 : 0));
return true;
}
@Override
- public boolean load(InputStream input) throws IOException {
- DataInput dataIn = new InputStreamDataInput(input);
- try {
- this.fst = new FST<Pair<Long,BytesRef>>(dataIn, new PairOutputs<Long,BytesRef>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
- maxAnalyzedPathsForOneInput = dataIn.readVInt();
- hasPayloads = dataIn.readByte() == 1;
- } finally {
- IOUtils.close(input);
- }
+ public boolean load(DataInput input) throws IOException {
+ count = input.readVLong();
+ this.fst = new FST<Pair<Long,BytesRef>>(input, new PairOutputs<Long,BytesRef>(PositiveIntOutputs.getSingleton(), ByteSequenceOutputs.getSingleton()));
+ maxAnalyzedPathsForOneInput = input.readVInt();
+ hasPayloads = input.readByte() == 1;
return true;
}
@@ -809,6 +802,11 @@ public class AnalyzingSuggester extends
throw new RuntimeException(bogus);
}
}
+
+ @Override
+ public long getCount() {
+ return count;
+ }
/** Returns all prefix paths to initialize the search. */
protected List<FSTUtil.Path<Pair<Long,BytesRef>>> getFullPrefixPaths(List<FSTUtil.Path<Pair<Long,BytesRef>>> prefixPaths,
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/analyzing/FreeTextSuggester.java Fri Feb 7 20:58:25 2014
@@ -22,8 +22,6 @@ package org.apache.lucene.search.suggest
// - add pruning of low-freq ngrams?
import java.io.File;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
//import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collections;
@@ -62,8 +60,6 @@ import org.apache.lucene.store.DataInput
import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
-import org.apache.lucene.store.InputStreamDataInput;
-import org.apache.lucene.store.OutputStreamDataOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
import org.apache.lucene.util.IOUtils;
@@ -153,6 +149,10 @@ public class FreeTextSuggester extends L
private final int grams;
private final byte separator;
+
+ /** Number of entries the lookup was built with */
+ private long count = 0;
+
/** The default character used to join multiple tokens
* into a single ngram token. The input tokens produced
* by the analyzer must not contain this character. */
@@ -320,6 +320,7 @@ public class FreeTextSuggester extends L
IndexReader reader = null;
boolean success = false;
+ count = 0;
try {
while (true) {
BytesRef surfaceForm = iterator.next();
@@ -328,6 +329,7 @@ public class FreeTextSuggester extends L
}
field.setStringValue(surfaceForm.utf8ToString());
writer.addDocument(doc);
+ count++;
}
reader = DirectoryReader.open(writer, false);
@@ -397,31 +399,31 @@ public class FreeTextSuggester extends L
}
@Override
- public boolean store(OutputStream output) throws IOException {
- DataOutput out = new OutputStreamDataOutput(output);
- CodecUtil.writeHeader(out, CODEC_NAME, VERSION_CURRENT);
- out.writeByte(separator);
- out.writeVInt(grams);
- out.writeVLong(totTokens);
- fst.save(out);
+ public boolean store(DataOutput output) throws IOException {
+ CodecUtil.writeHeader(output, CODEC_NAME, VERSION_CURRENT);
+ output.writeVLong(count);
+ output.writeByte(separator);
+ output.writeVInt(grams);
+ output.writeVLong(totTokens);
+ fst.save(output);
return true;
}
@Override
- public boolean load(InputStream input) throws IOException {
- DataInput in = new InputStreamDataInput(input);
- CodecUtil.checkHeader(in, CODEC_NAME, VERSION_START, VERSION_START);
- byte separatorOrig = in.readByte();
+ public boolean load(DataInput input) throws IOException {
+ CodecUtil.checkHeader(input, CODEC_NAME, VERSION_START, VERSION_START);
+ count = input.readVLong();
+ byte separatorOrig = input.readByte();
if (separatorOrig != separator) {
throw new IllegalStateException("separator=" + separator + " is incorrect: original model was built with separator=" + separatorOrig);
}
- int gramsOrig = in.readVInt();
+ int gramsOrig = input.readVInt();
if (gramsOrig != grams) {
throw new IllegalStateException("grams=" + grams + " is incorrect: original model was built with grams=" + gramsOrig);
}
- totTokens = in.readVLong();
+ totTokens = input.readVLong();
- fst = new FST<Long>(in, PositiveIntOutputs.getSingleton());
+ fst = new FST<Long>(input, PositiveIntOutputs.getSingleton());
return true;
}
@@ -436,6 +438,11 @@ public class FreeTextSuggester extends L
}
}
+ @Override
+ public long getCount() {
+ return count;
+ }
+
private int countGrams(BytesRef token) {
int count = 1;
for(int i=0;i<token.length;i++) {
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/FSTCompletionLookup.java Fri Feb 7 20:58:25 2014
@@ -32,6 +32,8 @@ import org.apache.lucene.search.suggest.
import org.apache.lucene.search.suggest.tst.TSTLookup;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.store.InputStreamDataInput;
import org.apache.lucene.store.OutputStreamDataOutput;
import org.apache.lucene.util.*;
@@ -93,6 +95,9 @@ public class FSTCompletionLookup extends
*/
private FSTCompletion normalCompletion;
+ /** Number of entries the lookup was built with */
+ private long count = 0;
+
/**
* This constructor prepares for creating a suggested FST using the
* {@link #build(InputIterator)} method. The number of weight
@@ -140,8 +145,8 @@ public class FSTCompletionLookup extends
}
@Override
- public void build(InputIterator tfit) throws IOException {
- if (tfit.hasPayloads()) {
+ public void build(InputIterator iterator) throws IOException {
+ if (iterator.hasPayloads()) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
File tempInput = File.createTempFile(
@@ -156,17 +161,18 @@ public class FSTCompletionLookup extends
// Push floats up front before sequences to sort them. For now, assume they are non-negative.
// If negative floats are allowed some trickery needs to be done to find their byte order.
boolean success = false;
+ count = 0;
try {
byte [] buffer = new byte [0];
ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);
BytesRef spare;
- while ((spare = tfit.next()) != null) {
+ while ((spare = iterator.next()) != null) {
if (spare.length + 4 >= buffer.length) {
buffer = ArrayUtil.grow(buffer, spare.length + 4);
}
output.reset(buffer);
- output.writeInt(encodeWeight(tfit.weight()));
+ output.writeInt(encodeWeight(iterator.weight()));
output.writeBytes(spare.bytes, spare.offset, spare.length);
writer.write(buffer, 0, output.getPosition());
}
@@ -207,6 +213,7 @@ public class FSTCompletionLookup extends
builder.add(tmp2, bucket);
line++;
+ count++;
}
// The two FSTCompletions share the same automaton.
@@ -264,28 +271,21 @@ public class FSTCompletionLookup extends
@Override
- public synchronized boolean store(OutputStream output) throws IOException {
-
- try {
- if (this.normalCompletion == null || normalCompletion.getFST() == null)
- return false;
- normalCompletion.getFST().save(new OutputStreamDataOutput(output));
- } finally {
- IOUtils.close(output);
- }
+ public synchronized boolean store(DataOutput output) throws IOException {
+ output.writeVLong(count);
+ if (this.normalCompletion == null || normalCompletion.getFST() == null)
+ return false;
+ normalCompletion.getFST().save(output);
return true;
}
@Override
- public synchronized boolean load(InputStream input) throws IOException {
- try {
- this.higherWeightsCompletion = new FSTCompletion(new FST<Object>(
- new InputStreamDataInput(input), NoOutputs.getSingleton()));
- this.normalCompletion = new FSTCompletion(
- higherWeightsCompletion.getFST(), false, exactMatchFirst);
- } finally {
- IOUtils.close(input);
- }
+ public synchronized boolean load(DataInput input) throws IOException {
+ count = input.readVLong();
+ this.higherWeightsCompletion = new FSTCompletion(new FST<Object>(
+ input, NoOutputs.getSingleton()));
+ this.normalCompletion = new FSTCompletion(
+ higherWeightsCompletion.getFST(), false, exactMatchFirst);
return true;
}
@@ -293,4 +293,9 @@ public class FSTCompletionLookup extends
public long sizeInBytes() {
return RamUsageEstimator.sizeOf(this);
}
+
+ @Override
+ public long getCount() {
+ return count;
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/fst/WFSTCompletionLookup.java Fri Feb 7 20:58:25 2014
@@ -18,8 +18,6 @@ package org.apache.lucene.search.suggest
*/
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Comparator;
@@ -31,12 +29,11 @@ import org.apache.lucene.search.suggest.
import org.apache.lucene.search.suggest.SortedInputIterator;
import org.apache.lucene.store.ByteArrayDataInput;
import org.apache.lucene.store.ByteArrayDataOutput;
-import org.apache.lucene.store.InputStreamDataInput;
-import org.apache.lucene.store.OutputStreamDataOutput;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.ArrayUtil;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.UnicodeUtil;
import org.apache.lucene.util.fst.Builder;
@@ -72,6 +69,9 @@ public class WFSTCompletionLookup extend
*/
private final boolean exactFirst;
+ /** Number of entries the lookup was built with */
+ private long count = 0;
+
/**
* Calls {@link #WFSTCompletionLookup(boolean) WFSTCompletionLookup(true)}
*/
@@ -96,6 +96,7 @@ public class WFSTCompletionLookup extend
if (iterator.hasPayloads()) {
throw new IllegalArgumentException("this suggester doesn't support payloads");
}
+ count = 0;
BytesRef scratch = new BytesRef();
InputIterator iter = new WFSTInputIterator(iterator);
IntsRef scratchInts = new IntsRef();
@@ -114,31 +115,26 @@ public class WFSTCompletionLookup extend
Util.toIntsRef(scratch, scratchInts);
builder.add(scratchInts, cost);
previous.copyBytes(scratch);
+ count++;
}
fst = builder.finish();
}
@Override
- public boolean store(OutputStream output) throws IOException {
- try {
- if (fst == null) {
- return false;
- }
- fst.save(new OutputStreamDataOutput(output));
- } finally {
- IOUtils.close(output);
+ public boolean store(DataOutput output) throws IOException {
+ output.writeVLong(count);
+ if (fst == null) {
+ return false;
}
+ fst.save(output);
return true;
}
@Override
- public boolean load(InputStream input) throws IOException {
- try {
- this.fst = new FST<Long>(new InputStreamDataInput(input), PositiveIntOutputs.getSingleton());
- } finally {
- IOUtils.close(input);
- }
+ public boolean load(DataInput input) throws IOException {
+ count = input.readVLong();
+ this.fst = new FST<Long>(input, PositiveIntOutputs.getSingleton());
return true;
}
@@ -293,4 +289,9 @@ public class WFSTCompletionLookup extend
public long sizeInBytes() {
return (fst == null) ? 0 : fst.sizeInBytes();
}
+
+ @Override
+ public long getCount() {
+ return count;
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/jaspell/JaspellLookup.java Fri Feb 7 20:58:25 2014
@@ -17,11 +17,7 @@ package org.apache.lucene.search.suggest
* limitations under the License.
*/
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
@@ -29,9 +25,10 @@ import org.apache.lucene.search.suggest.
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.UnsortedInputIterator;
import org.apache.lucene.search.suggest.jaspell.JaspellTernarySearchTrie.TSTNode;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.UnicodeUtil;
@@ -45,6 +42,9 @@ public class JaspellLookup extends Looku
JaspellTernarySearchTrie trie = new JaspellTernarySearchTrie();
private boolean usePrefix = true;
private int editDistance = 2;
+
+ /** Number of entries the lookup was built with */
+ private long count = 0;
/**
* Creates a new empty trie
@@ -137,8 +137,8 @@ public class JaspellLookup extends Looku
private static final byte HI_KID = 0x04;
private static final byte HAS_VALUE = 0x08;
- private void readRecursively(DataInputStream in, TSTNode node) throws IOException {
- node.splitchar = in.readChar();
+ private void readRecursively(DataInput in, TSTNode node) throws IOException {
+ node.splitchar = in.readString().charAt(0);
byte mask = in.readByte();
if ((mask & HAS_VALUE) != 0) {
node.data = Long.valueOf(in.readLong());
@@ -160,11 +160,11 @@ public class JaspellLookup extends Looku
}
}
- private void writeRecursively(DataOutputStream out, TSTNode node) throws IOException {
+ private void writeRecursively(DataOutput out, TSTNode node) throws IOException {
if (node == null) {
return;
}
- out.writeChar(node.splitchar);
+ out.writeString(new String(new char[] {node.splitchar}, 0, 1));
byte mask = 0;
if (node.relatives[TSTNode.LOKID] != null) mask |= LO_KID;
if (node.relatives[TSTNode.EQKID] != null) mask |= EQ_KID;
@@ -180,31 +180,22 @@ public class JaspellLookup extends Looku
}
@Override
- public boolean store(OutputStream output) throws IOException {
+ public boolean store(DataOutput output) throws IOException {
+ output.writeVLong(count);
TSTNode root = trie.getRoot();
if (root == null) { // empty tree
return false;
}
- DataOutputStream out = new DataOutputStream(output);
- try {
- writeRecursively(out, root);
- out.flush();
- } finally {
- IOUtils.close(out);
- }
+ writeRecursively(output, root);
return true;
}
@Override
- public boolean load(InputStream input) throws IOException {
- DataInputStream in = new DataInputStream(input);
+ public boolean load(DataInput input) throws IOException {
+ count = input.readVLong();
TSTNode root = trie.new TSTNode('\0', null);
- try {
- readRecursively(in, root);
- trie.setRoot(root);
- } finally {
- IOUtils.close(in);
- }
+ readRecursively(input, root);
+ trie.setRoot(root);
return true;
}
@@ -213,4 +204,9 @@ public class JaspellLookup extends Looku
public long sizeInBytes() {
return RamUsageEstimator.sizeOf(trie);
}
+
+ @Override
+ public long getCount() {
+ return count;
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/java/org/apache/lucene/search/suggest/tst/TSTLookup.java Fri Feb 7 20:58:25 2014
@@ -17,20 +17,17 @@ package org.apache.lucene.search.suggest
* limitations under the License.
*/
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.search.suggest.InputIterator;
import org.apache.lucene.search.suggest.Lookup;
import org.apache.lucene.search.suggest.SortedInputIterator;
+import org.apache.lucene.store.DataInput;
+import org.apache.lucene.store.DataOutput;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRef;
-import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.RamUsageEstimator;
import org.apache.lucene.util.UnicodeUtil;
@@ -43,6 +40,9 @@ import org.apache.lucene.util.UnicodeUti
public class TSTLookup extends Lookup {
TernaryTreeNode root = new TernaryTreeNode();
TSTAutocomplete autocomplete = new TSTAutocomplete();
+
+ /** Number of entries the lookup was built with */
+ private long count = 0;
/**
* Creates a new TSTLookup with an empty Ternary Search Tree.
@@ -150,11 +150,11 @@ public class TSTLookup extends Lookup {
private static final byte HAS_VALUE = 0x10;
// pre-order traversal
- private void readRecursively(DataInputStream in, TernaryTreeNode node) throws IOException {
- node.splitchar = in.readChar();
+ private void readRecursively(DataInput in, TernaryTreeNode node) throws IOException {
+ node.splitchar = in.readString().charAt(0);
byte mask = in.readByte();
if ((mask & HAS_TOKEN) != 0) {
- node.token = in.readUTF();
+ node.token = in.readString();
}
if ((mask & HAS_VALUE) != 0) {
node.val = Long.valueOf(in.readLong());
@@ -174,9 +174,9 @@ public class TSTLookup extends Lookup {
}
// pre-order traversal
- private void writeRecursively(DataOutputStream out, TernaryTreeNode node) throws IOException {
+ private void writeRecursively(DataOutput out, TernaryTreeNode node) throws IOException {
// write out the current node
- out.writeChar(node.splitchar);
+ out.writeString(new String(new char[] {node.splitchar}, 0, 1));
// prepare a mask of kids
byte mask = 0;
if (node.eqKid != null) mask |= EQ_KID;
@@ -185,7 +185,7 @@ public class TSTLookup extends Lookup {
if (node.token != null) mask |= HAS_TOKEN;
if (node.val != null) mask |= HAS_VALUE;
out.writeByte(mask);
- if (node.token != null) out.writeUTF(node.token);
+ if (node.token != null) out.writeString(node.token);
if (node.val != null) out.writeLong(((Number)node.val).longValue());
// recurse and write kids
if (node.loKid != null) {
@@ -200,26 +200,17 @@ public class TSTLookup extends Lookup {
}
@Override
- public synchronized boolean store(OutputStream output) throws IOException {
- DataOutputStream out = new DataOutputStream(output);
- try {
- writeRecursively(out, root);
- out.flush();
- } finally {
- IOUtils.close(output);
- }
+ public synchronized boolean store(DataOutput output) throws IOException {
+ output.writeVLong(count);
+ writeRecursively(output, root);
return true;
}
@Override
- public synchronized boolean load(InputStream input) throws IOException {
- DataInputStream in = new DataInputStream(input);
+ public synchronized boolean load(DataInput input) throws IOException {
+ count = input.readVLong();
root = new TernaryTreeNode();
- try {
- readRecursively(in, root);
- } finally {
- IOUtils.close(in);
- }
+ readRecursively(input, root);
return true;
}
@@ -229,4 +220,8 @@ public class TSTLookup extends Lookup {
return RamUsageEstimator.sizeOf(autocomplete);
}
+ @Override
+ public long getCount() {
+ return count;
+ }
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/spell/TestLuceneDictionary.java Fri Feb 7 20:58:25 2014
@@ -90,7 +90,7 @@ public class TestLuceneDictionary extend
indexReader = DirectoryReader.open(store);
ld = new LuceneDictionary(indexReader, "nonexistent_field");
- it = ld.getWordsIterator();
+ it = ld.getEntryIterator();
assertNull("More elements than expected", spare = it.next());
} finally {
@@ -103,7 +103,7 @@ public class TestLuceneDictionary extend
indexReader = DirectoryReader.open(store);
ld = new LuceneDictionary(indexReader, "aaa");
- it = ld.getWordsIterator();
+ it = ld.getEntryIterator();
assertNotNull("First element doesn't exist.", spare = it.next());
assertTrue("First element isn't correct", spare.utf8ToString().equals("foo"));
assertNull("More elements than expected", it.next());
@@ -117,7 +117,7 @@ public class TestLuceneDictionary extend
indexReader = DirectoryReader.open(store);
ld = new LuceneDictionary(indexReader, "contents");
- it = ld.getWordsIterator();
+ it = ld.getEntryIterator();
assertNotNull("First element doesn't exist.", spare = it.next());
assertTrue("First element isn't correct", spare.utf8ToString().equals("Jerry"));
@@ -126,7 +126,7 @@ public class TestLuceneDictionary extend
assertNull("More elements than expected", it.next());
ld = new LuceneDictionary(indexReader, "contents");
- it = ld.getWordsIterator();
+ it = ld.getEntryIterator();
int counter = 2;
while (it.next() != null) {
@@ -145,7 +145,7 @@ public class TestLuceneDictionary extend
indexReader = DirectoryReader.open(store);
ld = new LuceneDictionary(indexReader, "contents");
- it = ld.getWordsIterator();
+ it = ld.getEntryIterator();
// just iterate through words
assertEquals("First element isn't correct", "Jerry", it.next().utf8ToString());
@@ -162,7 +162,7 @@ public class TestLuceneDictionary extend
indexReader = DirectoryReader.open(store);
ld = new LuceneDictionary(indexReader, "zzz");
- it = ld.getWordsIterator();
+ it = ld.getEntryIterator();
assertNotNull("First element doesn't exist.", spare = it.next());
assertEquals("First element isn't correct", "bar", spare.utf8ToString());
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentDictionaryTest.java Fri Feb 7 20:58:25 2014
@@ -110,7 +110,7 @@ public class DocumentDictionaryTest exte
writer.close();
IndexReader ir = DirectoryReader.open(dir);
Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
- InputIterator inputIterator = (InputIterator) dictionary.getWordsIterator();
+ InputIterator inputIterator = dictionary.getEntryIterator();
assertNull(inputIterator.next());
assertEquals(inputIterator.weight(), 0);
@@ -136,7 +136,7 @@ public class DocumentDictionaryTest exte
writer.close();
IndexReader ir = DirectoryReader.open(dir);
Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME, PAYLOAD_FIELD_NAME);
- InputIterator inputIterator = (InputIterator) dictionary.getWordsIterator();
+ InputIterator inputIterator = dictionary.getEntryIterator();
BytesRef f;
while((f = inputIterator.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
@@ -171,7 +171,7 @@ public class DocumentDictionaryTest exte
writer.close();
IndexReader ir = DirectoryReader.open(dir);
Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME);
- InputIterator inputIterator = (InputIterator) dictionary.getWordsIterator();
+ InputIterator inputIterator = dictionary.getEntryIterator();
BytesRef f;
while((f = inputIterator.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
@@ -229,7 +229,7 @@ public class DocumentDictionaryTest exte
IndexReader ir = DirectoryReader.open(dir);
assertEquals(ir.numDocs(), docs.size());
Dictionary dictionary = new DocumentDictionary(ir, FIELD_NAME, WEIGHT_FIELD_NAME);
- InputIterator inputIterator = (InputIterator) dictionary.getWordsIterator();
+ InputIterator inputIterator = dictionary.getEntryIterator();
BytesRef f;
while((f = inputIterator.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/DocumentValueSourceDictionaryTest.java Fri Feb 7 20:58:25 2014
@@ -85,7 +85,7 @@ public class DocumentValueSourceDictiona
writer.close();
IndexReader ir = DirectoryReader.open(dir);
Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new DoubleConstValueSource(10), PAYLOAD_FIELD_NAME);
- InputIterator inputIterator = (InputIterator) dictionary.getWordsIterator();
+ InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
assertNull(inputIterator.next());
assertEquals(inputIterator.weight(), 0);
@@ -111,7 +111,7 @@ public class DocumentValueSourceDictiona
IndexReader ir = DirectoryReader.open(dir);
ValueSource[] toAdd = new ValueSource[] {new LongFieldSource(WEIGHT_FIELD_NAME_1), new LongFieldSource(WEIGHT_FIELD_NAME_2), new LongFieldSource(WEIGHT_FIELD_NAME_3)};
Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new SumFloatFunction(toAdd), PAYLOAD_FIELD_NAME);
- InputIterator inputIterator = (InputIterator) dictionary.getWordsIterator();
+ InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
BytesRef f;
while((f = inputIterator.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
@@ -143,7 +143,7 @@ public class DocumentValueSourceDictiona
IndexReader ir = DirectoryReader.open(dir);
ValueSource[] toAdd = new ValueSource[] {new LongFieldSource(WEIGHT_FIELD_NAME_1), new LongFieldSource(WEIGHT_FIELD_NAME_2), new LongFieldSource(WEIGHT_FIELD_NAME_3)};
Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new SumFloatFunction(toAdd));
- InputIterator inputIterator = (InputIterator) dictionary.getWordsIterator();
+ InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
BytesRef f;
while((f = inputIterator.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
@@ -197,7 +197,7 @@ public class DocumentValueSourceDictiona
ValueSource[] toAdd = new ValueSource[] {new LongFieldSource(WEIGHT_FIELD_NAME_1), new LongFieldSource(WEIGHT_FIELD_NAME_2)};
Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new SumFloatFunction(toAdd), PAYLOAD_FIELD_NAME);
- InputIterator inputIterator = (InputIterator) dictionary.getWordsIterator();
+ InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
BytesRef f;
while((f = inputIterator.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
@@ -228,7 +228,7 @@ public class DocumentValueSourceDictiona
IndexReader ir = DirectoryReader.open(dir);
Dictionary dictionary = new DocumentValueSourceDictionary(ir, FIELD_NAME, new DoubleConstValueSource(10), PAYLOAD_FIELD_NAME);
- InputIterator inputIterator = (InputIterator) dictionary.getWordsIterator();
+ InputIterator inputIterator = (InputIterator) dictionary.getEntryIterator();
BytesRef f;
while((f = inputIterator.next())!=null) {
Document doc = docs.remove(f.utf8ToString());
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/FileDictionaryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/FileDictionaryTest.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/FileDictionaryTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/FileDictionaryTest.java Fri Feb 7 20:58:25 2014
@@ -76,7 +76,7 @@ public class FileDictionaryTest extends
InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes("UTF-8"));
FileDictionary dictionary = new FileDictionary(inputReader);
List<List<String>> entries = fileInput.getKey();
- InputIterator inputIter = dictionary.getWordsIterator();
+ InputIterator inputIter = dictionary.getEntryIterator();
assertFalse(inputIter.hasPayloads());
BytesRef term;
int count = 0;
@@ -98,7 +98,7 @@ public class FileDictionaryTest extends
InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes("UTF-8"));
FileDictionary dictionary = new FileDictionary(inputReader);
List<List<String>> entries = fileInput.getKey();
- InputIterator inputIter = dictionary.getWordsIterator();
+ InputIterator inputIter = dictionary.getEntryIterator();
assertFalse(inputIter.hasPayloads());
BytesRef term;
int count = 0;
@@ -120,7 +120,7 @@ public class FileDictionaryTest extends
InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes("UTF-8"));
FileDictionary dictionary = new FileDictionary(inputReader);
List<List<String>> entries = fileInput.getKey();
- InputIterator inputIter = dictionary.getWordsIterator();
+ InputIterator inputIter = dictionary.getEntryIterator();
assertTrue(inputIter.hasPayloads());
BytesRef term;
int count = 0;
@@ -146,7 +146,7 @@ public class FileDictionaryTest extends
InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes("UTF-8"));
FileDictionary dictionary = new FileDictionary(inputReader);
List<List<String>> entries = fileInput.getKey();
- InputIterator inputIter = dictionary.getWordsIterator();
+ InputIterator inputIter = dictionary.getEntryIterator();
assertTrue(inputIter.hasPayloads());
BytesRef term;
int count = 0;
@@ -173,7 +173,7 @@ public class FileDictionaryTest extends
InputStream inputReader = new ByteArrayInputStream(fileInput.getValue().getBytes("UTF-8"));
FileDictionary dictionary = new FileDictionary(inputReader, " , ");
List<List<String>> entries = fileInput.getKey();
- InputIterator inputIter = dictionary.getWordsIterator();
+ InputIterator inputIter = dictionary.getEntryIterator();
assertTrue(inputIter.hasPayloads());
BytesRef term;
int count = 0;
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/TestHighFrequencyDictionary.java Fri Feb 7 20:58:25 2014
@@ -35,7 +35,7 @@ public class TestHighFrequencyDictionary
writer.close();
IndexReader ir = DirectoryReader.open(dir);
Dictionary dictionary = new HighFrequencyDictionary(ir, "bogus", 0.1f);
- BytesRefIterator tf = dictionary.getWordsIterator();
+ BytesRefIterator tf = dictionary.getEntryIterator();
assertNull(tf.getComparator());
assertNull(tf.next());
dir.close();
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingInfixSuggesterTest.java Fri Feb 7 20:58:25 2014
@@ -111,6 +111,7 @@ public class AnalyzingInfixSuggesterTest
}
};
suggester.build(new InputArrayIterator(keys));
+ assertEquals(2, suggester.getCount());
suggester.close();
suggester = new AnalyzingInfixSuggester(TEST_VERSION_CURRENT, tempDir, a, a, 3) {
@@ -124,6 +125,7 @@ public class AnalyzingInfixSuggesterTest
assertEquals("a penny saved is a penny <b>ear</b>ned", results.get(0).key);
assertEquals(10, results.get(0).value);
assertEquals(new BytesRef("foobaz"), results.get(0).payload);
+ assertEquals(2, suggester.getCount());
suggester.close();
}
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/AnalyzingSuggesterTest.java Fri Feb 7 20:58:25 2014
@@ -912,6 +912,7 @@ public class AnalyzingSuggesterTest exte
new Input("a c", 4),
}));
+ assertEquals(3, suggester.getCount());
List<LookupResult> results = suggester.lookup("a", false, 3);
assertEquals(3, results.size());
assertEquals("a", results.get(0).key);
@@ -935,6 +936,7 @@ public class AnalyzingSuggesterTest exte
suggester.load(is);
is.close();
+ assertEquals(3, suggester.getCount());
results = suggester.lookup("a", false, 3);
assertEquals(3, results.size());
assertEquals("a", results.get(0).key);
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/analyzing/TestFreeTextSuggester.java Fri Feb 7 20:58:25 2014
@@ -64,6 +64,7 @@ public class TestFreeTextSuggester exten
Analyzer a = new MockAnalyzer(random());
FreeTextSuggester sug = new FreeTextSuggester(a, a, 2, (byte) 0x20);
sug.build(new InputArrayIterator(keys));
+ assertEquals(2, sug.getCount());
for(int i=0;i<2;i++) {
@@ -97,6 +98,7 @@ public class TestFreeTextSuggester exten
sug = new FreeTextSuggester(a, a, 2, (byte) 0x20);
sug.load(is);
is.close();
+ assertEquals(2, sug.getCount());
}
}
@@ -178,6 +180,7 @@ public class TestFreeTextSuggester exten
public boolean hasPayloads() {
return false;
}
+
});
if (VERBOSE) {
System.out.println(sug.sizeInBytes() + " bytes");
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/FSTCompletionTest.java Fri Feb 7 20:58:25 2014
@@ -181,6 +181,7 @@ public class FSTCompletionTest extends L
FSTCompletionLookup lookup = new FSTCompletionLookup();
lookup.build(new InputArrayIterator(input));
+ assertEquals(input.size(), lookup.getCount());
for (Input tf : input) {
assertNotNull("Not found: " + tf.term.toString(), lookup.get(_TestUtil.bytesToCharSequence(tf.term, random())));
assertEquals(tf.term.utf8ToString(), lookup.lookup(_TestUtil.bytesToCharSequence(tf.term, random()), true, 1).get(0).key.toString());
Modified: lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java?rev=1565811&r1=1565810&r2=1565811&view=diff
==============================================================================
--- lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java (original)
+++ lucene/dev/branches/branch_4x/lucene/suggest/src/test/org/apache/lucene/search/suggest/fst/WFSTCompletionTest.java Fri Feb 7 20:58:25 2014
@@ -156,6 +156,7 @@ public class WFSTCompletionTest extends
WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
suggester.build(new InputArrayIterator(keys));
+ assertEquals(numWords, suggester.getCount());
Random random = new Random(random().nextLong());
for (String prefix : allPrefixes) {
final int topN = _TestUtil.nextInt(random, 1, 10);
@@ -215,6 +216,7 @@ public class WFSTCompletionTest extends
WFSTCompletionLookup suggester = new WFSTCompletionLookup(false);
suggester.build(new InputArrayIterator(new Input[0]));
+ assertEquals(0, suggester.getCount());
List<LookupResult> result = suggester.lookup("a", false, 20);
assertTrue(result.isEmpty());
}