You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by sy...@apache.org on 2014/09/16 00:24:49 UTC
[2/8] Porting Lucene.Net.Suggest (still not compiling)
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0ebac726/src/Lucene.Net.Suggest/Suggest/Jaspell/JaspellLookup.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Suggest/Suggest/Jaspell/JaspellLookup.cs b/src/Lucene.Net.Suggest/Suggest/Jaspell/JaspellLookup.cs
new file mode 100644
index 0000000..7b5a2d2
--- /dev/null
+++ b/src/Lucene.Net.Suggest/Suggest/Jaspell/JaspellLookup.cs
@@ -0,0 +1,258 @@
+using System;
+using System.Collections.Generic;
+using Lucene.Net.Util;
+
+namespace Lucene.Net.Search.Suggest.Jaspell
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ /// <summary>
+ /// Suggest implementation based on
+ /// <a href="http://jaspell.sourceforge.net/">JaSpell</a>.
+ /// </summary>
+ /// <seealso cref= JaspellTernarySearchTrie </seealso>
+ public class JaspellLookup : Lookup
+ {
+ internal JaspellTernarySearchTrie trie = new JaspellTernarySearchTrie();
+ private bool usePrefix = true;
+ private int editDistance = 2;
+
+ /// <summary>
+ /// Number of entries the lookup was built with </summary>
+ private long count = 0;
+
+ /// <summary>
+ /// Creates a new empty trie </summary>
+ /// <seealso cref= #build(InputIterator)
+ /// </seealso>
+ public JaspellLookup()
+ {
+ }
+
+ public override void Build(InputIterator tfit)
+ {
+ if (tfit.HasPayloads)
+ {
+ throw new ArgumentException("this suggester doesn't support payloads");
+ }
+ if (tfit.Comparator != null)
+ {
+ // make sure it's unsorted
+ // WTF - this could result in yet another sorted iteration....
+ tfit = new UnsortedInputIterator(tfit);
+ }
+ if (tfit.HasContexts)
+ {
+ throw new System.ArgumentException("this suggester doesn't support contexts");
+ }
+ count = 0;
+ trie = new JaspellTernarySearchTrie();
+ trie.MatchAlmostDiff = editDistance;
+ BytesRef spare;
+
+ CharsRef charsSpare = new CharsRef();
+
+ while ((spare = tfit.Next()) != null)
+ {
+
+ long weight = tfit.Weight;
+ if (spare.Length == 0)
+ {
+ continue;
+ }
+ charsSpare.Grow(spare.Length);
+ UnicodeUtil.UTF8toUTF16(spare.Bytes, spare.Offset, spare.Length, charsSpare);
+ trie.Put(charsSpare.ToString(), Convert.ToInt64(weight));
+ }
+ }
+
+ /// <summary>
+ /// Adds a new node if <code>key</code> already exists,
+ /// otherwise replaces its value.
+ /// <para>
+ /// This method always returns false.
+ /// </para>
+ /// </summary>
+ public virtual bool Add(string key, object value)
+ {
+ trie.Put(key, value);
+ // XXX
+ return false;
+ }
+
+ /// <summary>
+ /// Returns the value for the specified key, or null
+ /// if the key does not exist.
+ /// </summary>
+ public virtual object Get(string key)
+ {
+ return trie.Get(key);
+ }
+
+ public override IList<LookupResult> Lookup(string key, HashSet<BytesRef> contexts, bool onlyMorePopular, int num)
+ {
+ if (contexts != null)
+ {
+ throw new System.ArgumentException("this suggester doesn't support contexts");
+ }
+ IList<LookupResult> res = new List<LookupResult>();
+ IList<string> list;
+ int count = onlyMorePopular ? num * 2 : num;
+ if (usePrefix)
+ {
+ list = trie.MatchPrefix(key, count);
+ }
+ else
+ {
+ list = trie.MatchAlmost(key, count);
+ }
+ if (list == null || list.Count == 0)
+ {
+ return res;
+
+ }
+ int maxCnt = Math.Min(num, list.Count);
+ if (onlyMorePopular)
+ {
+ LookupPriorityQueue queue = new LookupPriorityQueue(num);
+ foreach (string s in list)
+ {
+ long freq = (long)((Number)trie.Get(s));
+ queue.InsertWithOverflow(new LookupResult(new CharsRef(s), freq));
+ }
+ foreach (LookupResult lr in queue.Results)
+ {
+ res.Add(lr);
+ }
+ }
+ else
+ {
+ for (int i = 0; i < maxCnt; i++)
+ {
+ string s = list[i];
+ long freq = (long)((Number)trie.Get(s));
+ res.Add(new LookupResult(new CharsRef(s), freq));
+ }
+ }
+ return res;
+ }
+
+ private const sbyte LO_KID = 0x01;
+ private const sbyte EQ_KID = 0x02;
+ private const sbyte HI_KID = 0x04;
+ private const sbyte HAS_VALUE = 0x08;
+
+ private void ReadRecursively(DataInput @in, JaspellTernarySearchTrie.TSTNode node)
+ {
+ node.splitchar = @in.ReadString().charAt(0);
+ sbyte mask = @in.ReadByte();
+ if ((mask & HAS_VALUE) != 0)
+ {
+ node.data = Convert.ToInt64(@in.readLong());
+ }
+ if ((mask & LO_KID) != 0)
+ {
+ JaspellTernarySearchTrie.TSTNode kid = new JaspellTernarySearchTrie.TSTNode(trie, '\0', node);
+ node.relatives[JaspellTernarySearchTrie.TSTNode.LOKID] = kid;
+ ReadRecursively(@in, kid);
+ }
+ if ((mask & EQ_KID) != 0)
+ {
+ JaspellTernarySearchTrie.TSTNode kid = new JaspellTernarySearchTrie.TSTNode(trie, '\0', node);
+ node.relatives[JaspellTernarySearchTrie.TSTNode.EQKID] = kid;
+ ReadRecursively(@in, kid);
+ }
+ if ((mask & HI_KID) != 0)
+ {
+ JaspellTernarySearchTrie.TSTNode kid = new JaspellTernarySearchTrie.TSTNode(trie, '\0', node);
+ node.relatives[JaspellTernarySearchTrie.TSTNode.HIKID] = kid;
+ ReadRecursively(@in, kid);
+ }
+ }
+
+ private void WriteRecursively(DataOutput @out, JaspellTernarySearchTrie.TSTNode node)
+ {
+ if (node == null)
+ {
+ return;
+ }
+ @out.writeString(new string(new char[] { node.splitchar }, 0, 1));
+ sbyte mask = 0;
+ if (node.relatives[JaspellTernarySearchTrie.TSTNode.LOKID] != null)
+ {
+ mask |= LO_KID;
+ }
+ if (node.relatives[JaspellTernarySearchTrie.TSTNode.EQKID] != null)
+ {
+ mask |= EQ_KID;
+ }
+ if (node.relatives[JaspellTernarySearchTrie.TSTNode.HIKID] != null)
+ {
+ mask |= HI_KID;
+ }
+ if (node.data != null)
+ {
+ mask |= HAS_VALUE;
+ }
+ @out.writeByte(mask);
+ if (node.data != null)
+ {
+ @out.writeLong((long)((Number)node.data));
+ }
+ WriteRecursively(@out, node.relatives[JaspellTernarySearchTrie.TSTNode.LOKID]);
+ WriteRecursively(@out, node.relatives[JaspellTernarySearchTrie.TSTNode.EQKID]);
+ WriteRecursively(@out, node.relatives[JaspellTernarySearchTrie.TSTNode.HIKID]);
+ }
+
+ public override bool Store(DataOutput output)
+ {
+ output.WriteVLong(count);
+ JaspellTernarySearchTrie.TSTNode root = trie.Root;
+ if (root == null) // empty tree
+ {
+ return false;
+ }
+ WriteRecursively(output, root);
+ return true;
+ }
+
+ public override bool load(DataInput input)
+ {
+ count = input.ReadVLong();
+ JaspellTernarySearchTrie.TSTNode root = new JaspellTernarySearchTrie.TSTNode(trie, '\0', null);
+ ReadRecursively(input, root);
+ trie.Root = root;
+ return true;
+ }
+
+ /// <summary>
+ /// Returns byte size of the underlying TST. </summary>
+ public override long SizeInBytes()
+ {
+ return trie.SizeInBytes();
+ }
+
+ public override long Count
+ {
+ get
+ {
+ return count;
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0ebac726/src/Lucene.Net.Suggest/Suggest/Jaspell/JaspellTernarySearchTrie.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Suggest/Suggest/Jaspell/JaspellTernarySearchTrie.cs b/src/Lucene.Net.Suggest/Suggest/Jaspell/JaspellTernarySearchTrie.cs
new file mode 100644
index 0000000..db9e2a7
--- /dev/null
+++ b/src/Lucene.Net.Suggest/Suggest/Jaspell/JaspellTernarySearchTrie.cs
@@ -0,0 +1,986 @@
+// Copyright (c) 2005 Bruno Martins
+// All rights reserved.
+//
+// Redistribution and use in source and binary forms, with or without
+// modification, are permitted provided that the following conditions
+// are met:
+// 1. Redistributions of source code must retain the above copyright
+// notice, this list of conditions and the following disclaimer.
+// 2. Redistributions in binary form must reproduce the above copyright
+// notice, this list of conditions and the following disclaimer in the
+// documentation and/or other materials provided with the distribution.
+// 3. Neither the name of the organization nor the names of its contributors
+// may be used to endorse or promote products derived from this software
+// without specific prior written permission.
+//
+// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+// AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+// ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+// LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+// CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+// SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+// INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+// CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+// ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
+// THE POSSIBILITY OF SUCH DAMAGE.
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+
+namespace Lucene.Net.Search.Suggest.Jaspell
+{
+ /// <summary>
+ /// Implementation of a Ternary Search Trie, a data structure for storing
+ /// <code>String</code> objects that combines the compact size of a binary search
+ /// tree with the speed of a digital search trie, and is therefore ideal for
+ /// practical use in sorting and searching data.</p>
+ /// <para>
+ ///
+ /// This data structure is faster than hashing for many typical search problems,
+ /// and supports a broader range of useful problems and operations. Ternary
+ /// searches are faster than hashing and more powerful, too.
+ /// </para>
+ /// <para>
+ ///
+ /// The theory of ternary search trees was described at a symposium in 1997 (see
+ /// "Fast Algorithms for Sorting and Searching Strings," by J.L. Bentley and R.
+ /// Sedgewick, Proceedings of the 8th Annual ACM-SIAM Symposium on Discrete
+ /// Algorithms, January 1997). Algorithms in C, Third Edition, by Robert
+ /// Sedgewick (Addison-Wesley, 1998) provides yet another view of ternary search
+ /// trees.
+ /// </para>
+ /// </summary>
+ public class JaspellTernarySearchTrie
+ {
+
+ /// <summary>
+ /// An inner class of Ternary Search Trie that represents a node in the trie.
+ /// </summary>
+ protected internal sealed class TSTNode
+ {
+ private readonly JaspellTernarySearchTrie outerInstance;
+
+
+ /// <summary>
+ /// Index values for accessing relatives array. </summary>
+ protected internal const int PARENT = 0, LOKID = 1, EQKID = 2, HIKID = 3;
+
+ /// <summary>
+ /// The key to the node. </summary>
+ protected internal object data;
+
+ /// <summary>
+ /// The relative nodes. </summary>
+ protected internal readonly TSTNode[] relatives = new TSTNode[4];
+
+ /// <summary>
+ /// The char used in the split. </summary>
+ protected internal char splitchar;
+
+ /// <summary>
+ /// Constructor method.
+ /// </summary>
+ /// <param name="splitchar">
+ /// The char used in the split. </param>
+ /// <param name="parent">
+ /// The parent node. </param>
+ protected internal TSTNode(JaspellTernarySearchTrie outerInstance, char splitchar, TSTNode parent)
+ {
+ this.outerInstance = outerInstance;
+ this.splitchar = splitchar;
+ relatives[PARENT] = parent;
+ }
+
+ /// <summary>
+ /// Return an approximate memory usage for this node and its sub-nodes. </summary>
+ public long SizeInBytes()
+ {
+ long mem = RamUsageEstimator.ShallowSizeOf(this) + RamUsageEstimator.ShallowSizeOf(relatives);
+ foreach (TSTNode node in relatives)
+ {
+ if (node != null)
+ {
+ mem += node.SizeInBytes();
+ }
+ }
+ return mem;
+ }
+
+ }
+
+ /// <summary>
+ /// Compares characters by alfabetical order.
+ /// </summary>
+ /// <param name="cCompare2">
+ /// The first char in the comparison. </param>
+ /// <param name="cRef">
+ /// The second char in the comparison. </param>
+ /// <returns> A negative number, 0 or a positive number if the second char is
+ /// less, equal or greater. </returns>
+ private static int compareCharsAlphabetically(char cCompare2, char cRef)
+ {
+ return char.ToLower(cCompare2) - char.ToLower(cRef);
+ }
+
+ /* what follows is the original Jaspell code.
+ private static int compareCharsAlphabetically(int cCompare2, int cRef) {
+ int cCompare = 0;
+ if (cCompare2 >= 65) {
+ if (cCompare2 < 89) {
+ cCompare = (2 * cCompare2) - 65;
+ } else if (cCompare2 < 97) {
+ cCompare = cCompare2 + 24;
+ } else if (cCompare2 < 121) {
+ cCompare = (2 * cCompare2) - 128;
+ } else cCompare = cCompare2;
+ } else cCompare = cCompare2;
+ if (cRef < 65) {
+ return cCompare - cRef;
+ }
+ if (cRef < 89) {
+ return cCompare - ((2 * cRef) - 65);
+ }
+ if (cRef < 97) {
+ return cCompare - (cRef + 24);
+ }
+ if (cRef < 121) {
+ return cCompare - ((2 * cRef) - 128);
+ }
+ return cCompare - cRef;
+ }
+ */
+
+ /// <summary>
+ /// The default number of values returned by the <code>matchAlmost</code>
+ /// method.
+ /// </summary>
+ private int defaultNumReturnValues = -1;
+
+ /// <summary>
+ /// the number of differences allowed in a call to the
+ /// <code>matchAlmostKey</code> method.
+ /// </summary>
+ private int matchAlmostDiff;
+
+ /// <summary>
+ /// The base node in the trie. </summary>
+ private TSTNode rootNode;
+
+ private readonly Locale locale;
+
+ /// <summary>
+ /// Constructs an empty Ternary Search Trie.
+ /// </summary>
+ public JaspellTernarySearchTrie()
+ : this(Locale.ROOT)
+ {
+ }
+
+ /// <summary>
+ /// Constructs an empty Ternary Search Trie,
+ /// specifying the Locale used for lowercasing.
+ /// </summary>
+ public JaspellTernarySearchTrie(Locale locale)
+ {
+ this.locale = locale;
+ }
+
+ // for loading
+ internal virtual TSTNode Root
+ {
+ set
+ {
+ rootNode = value;
+ }
+ get
+ {
+ return rootNode;
+ }
+ }
+
+
+ /// <summary>
+ /// Constructs a Ternary Search Trie and loads data from a <code>File</code>
+ /// into the Trie. The file is a normal text document, where each line is of
+ /// the form word TAB float.
+ /// </summary>
+ /// <param name="file">
+ /// The <code>File</code> with the data to load into the Trie. </param>
+ /// <exception cref="IOException">
+ /// A problem occured while reading the data. </exception>
+ public JaspellTernarySearchTrie(File file)
+ : this(file, false)
+ {
+ }
+
+ /// <summary>
+ /// Constructs a Ternary Search Trie and loads data from a <code>File</code>
+ /// into the Trie. The file is a normal text document, where each line is of
+ /// the form "word TAB float".
+ /// </summary>
+ /// <param name="file">
+ /// The <code>File</code> with the data to load into the Trie. </param>
+ /// <param name="compression">
+ /// If true, the file is compressed with the GZIP algorithm, and if
+ /// false, the file is a normal text document. </param>
+ /// <exception cref="IOException">
+ /// A problem occured while reading the data. </exception>
+ public JaspellTernarySearchTrie(File file, bool compression)
+ : this()
+ {
+ BufferedReader @in;
+ if (compression)
+ {
+ @in = new BufferedReader(IOUtils.getDecodingReader(new GZIPInputStream(new FileInputStream(file)), StandardCharsets.UTF_8));
+ }
+ else
+ {
+ @in = new BufferedReader(IOUtils.getDecodingReader((new FileInputStream(file)), StandardCharsets.UTF_8));
+ }
+ string word;
+ int pos;
+ float? occur, one = new float?(1);
+ while ((word = @in.readLine()) != null)
+ {
+ pos = word.IndexOf("\t", StringComparison.Ordinal);
+ occur = one;
+ if (pos != -1)
+ {
+ occur = Convert.ToSingle(word.Substring(pos + 1).Trim());
+ word = word.Substring(0, pos);
+ }
+ string key = word.ToLower(locale);
+ if (rootNode == null)
+ {
+ rootNode = new TSTNode(this, key[0], null);
+ }
+ TSTNode node = null;
+ if (key.Length > 0 && rootNode != null)
+ {
+ TSTNode currentNode = rootNode;
+ int charIndex = 0;
+ while (true)
+ {
+ if (currentNode == null)
+ {
+ break;
+ }
+ int charComp = compareCharsAlphabetically(key[charIndex], currentNode.splitchar);
+ if (charComp == 0)
+ {
+ charIndex++;
+ if (charIndex == key.Length)
+ {
+ node = currentNode;
+ break;
+ }
+ currentNode = currentNode.relatives[TSTNode.EQKID];
+ }
+ else if (charComp < 0)
+ {
+ currentNode = currentNode.relatives[TSTNode.LOKID];
+ }
+ else
+ {
+ currentNode = currentNode.relatives[TSTNode.HIKID];
+ }
+ }
+ float? occur2 = null;
+ if (node != null)
+ {
+ occur2 = ((float?)(node.data));
+ }
+ if (occur2 != null)
+ {
+ occur += (float)occur2;
+ }
+ currentNode = GetOrCreateNode(word.Trim().ToLower(locale));
+ currentNode.data = occur;
+ }
+ }
+ @in.close();
+ }
+
+ /// <summary>
+ /// Deletes the node passed in as an argument. If this node has non-null data,
+ /// then both the node and the data will be deleted. It also deletes any other
+ /// nodes in the trie that are no longer needed after the deletion of the node.
+ /// </summary>
+ /// <param name="nodeToDelete">
+ /// The node to delete. </param>
+ private void DeleteNode(TSTNode nodeToDelete)
+ {
+ if (nodeToDelete == null)
+ {
+ return;
+ }
+ nodeToDelete.data = null;
+ while (nodeToDelete != null)
+ {
+ nodeToDelete = DeleteNodeRecursion(nodeToDelete);
+ // deleteNodeRecursion(nodeToDelete);
+ }
+ }
+
+ /// <summary>
+ /// Recursively visits each node to be deleted.
+ ///
+ /// To delete a node, first set its data to null, then pass it into this
+ /// method, then pass the node returned by this method into this method (make
+ /// sure you don't delete the data of any of the nodes returned from this
+ /// method!) and continue in this fashion until the node returned by this
+ /// method is <code>null</code>.
+ ///
+ /// The TSTNode instance returned by this method will be next node to be
+ /// operated on by <code>deleteNodeRecursion</code> (This emulates recursive
+ /// method call while avoiding the JVM overhead normally associated with a
+ /// recursive method.)
+ /// </summary>
+ /// <param name="currentNode">
+ /// The node to delete. </param>
+ /// <returns> The next node to be called in deleteNodeRecursion. </returns>
+ private TSTNode DeleteNodeRecursion(TSTNode currentNode)
+ {
+ if (currentNode == null)
+ {
+ return null;
+ }
+ if (currentNode.relatives[TSTNode.EQKID] != null || currentNode.data != null)
+ {
+ return null;
+ }
+ // can't delete this node if it has a non-null eq kid or data
+ TSTNode currentParent = currentNode.relatives[TSTNode.PARENT];
+ bool lokidNull = currentNode.relatives[TSTNode.LOKID] == null;
+ bool hikidNull = currentNode.relatives[TSTNode.HIKID] == null;
+ int childType;
+ if (currentParent.relatives[TSTNode.LOKID] == currentNode)
+ {
+ childType = TSTNode.LOKID;
+ }
+ else if (currentParent.relatives[TSTNode.EQKID] == currentNode)
+ {
+ childType = TSTNode.EQKID;
+ }
+ else if (currentParent.relatives[TSTNode.HIKID] == currentNode)
+ {
+ childType = TSTNode.HIKID;
+ }
+ else
+ {
+ rootNode = null;
+ return null;
+ }
+ if (lokidNull && hikidNull)
+ {
+ currentParent.relatives[childType] = null;
+ return currentParent;
+ }
+ if (lokidNull)
+ {
+ currentParent.relatives[childType] = currentNode.relatives[TSTNode.HIKID];
+ currentNode.relatives[TSTNode.HIKID].relatives[TSTNode.PARENT] = currentParent;
+ return currentParent;
+ }
+ if (hikidNull)
+ {
+ currentParent.relatives[childType] = currentNode.relatives[TSTNode.LOKID];
+ currentNode.relatives[TSTNode.LOKID].relatives[TSTNode.PARENT] = currentParent;
+ return currentParent;
+ }
+ int deltaHi = currentNode.relatives[TSTNode.HIKID].splitchar - currentNode.splitchar;
+ int deltaLo = currentNode.splitchar - currentNode.relatives[TSTNode.LOKID].splitchar;
+ int movingKid;
+ TSTNode targetNode;
+ if (deltaHi == deltaLo)
+ {
+ if (new Random(1).NextDouble() < 0.5)
+ {
+ deltaHi++;
+ }
+ else
+ {
+ deltaLo++;
+ }
+ }
+ if (deltaHi > deltaLo)
+ {
+ movingKid = TSTNode.HIKID;
+ targetNode = currentNode.relatives[TSTNode.LOKID];
+ }
+ else
+ {
+ movingKid = TSTNode.LOKID;
+ targetNode = currentNode.relatives[TSTNode.HIKID];
+ }
+ while (targetNode.relatives[movingKid] != null)
+ {
+ targetNode = targetNode.relatives[movingKid];
+ }
+ targetNode.relatives[movingKid] = currentNode.relatives[movingKid];
+ currentParent.relatives[childType] = targetNode;
+ targetNode.relatives[TSTNode.PARENT] = currentParent;
+ if (!lokidNull)
+ {
+ currentNode.relatives[TSTNode.LOKID] = null;
+ }
+ if (!hikidNull)
+ {
+ currentNode.relatives[TSTNode.HIKID] = null;
+ }
+ return currentParent;
+ }
+
+ /// <summary>
+ /// Retrieve the object indexed by a key.
+ /// </summary>
+ /// <param name="key">
+ /// A <code>String</code> index. </param>
+ /// <returns> The object retrieved from the Ternary Search Trie. </returns>
+ public virtual object Get(string key)
+ {
+ TSTNode node = GetNode(key);
+ if (node == null)
+ {
+ return null;
+ }
+ return node.data;
+ }
+
+ /// <summary>
+ /// Retrieve the <code>Float</code> indexed by key, increment it by one unit
+ /// and store the new <code>Float</code>.
+ /// </summary>
+ /// <param name="key">
+ /// A <code>String</code> index. </param>
+ /// <returns> The <code>Float</code> retrieved from the Ternary Search Trie. </returns>
+ public virtual float? GetAndIncrement(string key)
+ {
+ string key2 = key.Trim().ToLower(locale);
+ TSTNode node = GetNode(key2);
+ if (node == null)
+ {
+ return null;
+ }
+ float? aux = (float?)(node.data);
+ if (aux == null)
+ {
+ aux = new float?(1);
+ }
+ else
+ {
+ aux = new float?((int)aux + 1);
+ }
+ Put(key2, aux);
+ return aux;
+ }
+
+ /// <summary>
+ /// Returns the key that indexes the node argument.
+ /// </summary>
+ /// <param name="node">
+ /// The node whose index is to be calculated. </param>
+ /// <returns> The <code>String</code> that indexes the node argument. </returns>
+ protected internal virtual string getKey(TSTNode node)
+ {
+ StringBuilder getKeyBuffer = new StringBuilder();
+ getKeyBuffer.Length = 0;
+ getKeyBuffer.Append("" + node.splitchar);
+ TSTNode currentNode;
+ TSTNode lastNode;
+ currentNode = node.relatives[TSTNode.PARENT];
+ lastNode = node;
+ while (currentNode != null)
+ {
+ if (currentNode.relatives[TSTNode.EQKID] == lastNode)
+ {
+ getKeyBuffer.Append("" + currentNode.splitchar);
+ }
+ lastNode = currentNode;
+ currentNode = currentNode.relatives[TSTNode.PARENT];
+ }
+
+ getKeyBuffer.Reverse();
+ return getKeyBuffer.ToString();
+ }
+
+ /// <summary>
+ /// Returns the node indexed by key, or <code>null</code> if that node doesn't
+ /// exist. Search begins at root node.
+ /// </summary>
+ /// <param name="key">
+ /// A <code>String</code> that indexes the node that is returned. </param>
+ /// <returns> The node object indexed by key. This object is an instance of an
+ /// inner class named <code>TernarySearchTrie.TSTNode</code>. </returns>
+ public virtual TSTNode GetNode(string key)
+ {
+ return GetNode(key, rootNode);
+ }
+
+ /// <summary>
+ /// Returns the node indexed by key, or <code>null</code> if that node doesn't
+ /// exist. The search begins at root node.
+ /// </summary>
+ /// <param name="key">
+ /// A <code>String</code> that indexes the node that is returned. </param>
+ /// <param name="startNode">
+ /// The top node defining the subtrie to be searched. </param>
+ /// <returns> The node object indexed by key. This object is an instance of an
+ /// inner class named <code>TernarySearchTrie.TSTNode</code>. </returns>
+ protected internal virtual TSTNode GetNode(string key, TSTNode startNode)
+ {
+ if (key == null || startNode == null || key.Length == 0)
+ {
+ return null;
+ }
+ TSTNode currentNode = startNode;
+ int charIndex = 0;
+ while (true)
+ {
+ if (currentNode == null)
+ {
+ return null;
+ }
+ int charComp = compareCharsAlphabetically(key.charAt(charIndex), currentNode.splitchar);
+ if (charComp == 0)
+ {
+ charIndex++;
+ if (charIndex == key.Length)
+ {
+ return currentNode;
+ }
+ currentNode = currentNode.relatives[TSTNode.EQKID];
+ }
+ else if (charComp < 0)
+ {
+ currentNode = currentNode.relatives[TSTNode.LOKID];
+ }
+ else
+ {
+ currentNode = currentNode.relatives[TSTNode.HIKID];
+ }
+ }
+ }
+
+ /// <summary>
+ /// Returns the node indexed by key, creating that node if it doesn't exist,
+ /// and creating any required intermediate nodes if they don't exist.
+ /// </summary>
+ /// <param name="key">
+ /// A <code>String</code> that indexes the node that is returned. </param>
+ /// <returns> The node object indexed by key. This object is an instance of an
+ /// inner class named <code>TernarySearchTrie.TSTNode</code>. </returns>
+ /// <exception cref="NullPointerException">
+ /// If the key is <code>null</code>. </exception>
+ /// <exception cref="IllegalArgumentException">
+ /// If the key is an empty <code>String</code>. </exception>
+ protected internal virtual TSTNode GetOrCreateNode(string key)
+ {
+ if (key == null)
+ {
+ throw new NullReferenceException("attempt to get or create node with null key");
+ }
+ if (key.Length == 0)
+ {
+ throw new System.ArgumentException("attempt to get or create node with key of zero length");
+ }
+ if (rootNode == null)
+ {
+ rootNode = new TSTNode(this, key.charAt(0), null);
+ }
+ TSTNode currentNode = rootNode;
+ int charIndex = 0;
+ while (true)
+ {
+ int charComp = compareCharsAlphabetically(key.charAt(charIndex), currentNode.splitchar);
+ if (charComp == 0)
+ {
+ charIndex++;
+ if (charIndex == key.Length)
+ {
+ return currentNode;
+ }
+ if (currentNode.relatives[TSTNode.EQKID] == null)
+ {
+ currentNode.relatives[TSTNode.EQKID] = new TSTNode(this, key.charAt(charIndex), currentNode);
+ }
+ currentNode = currentNode.relatives[TSTNode.EQKID];
+ }
+ else if (charComp < 0)
+ {
+ if (currentNode.relatives[TSTNode.LOKID] == null)
+ {
+ currentNode.relatives[TSTNode.LOKID] = new TSTNode(this, key.charAt(charIndex), currentNode);
+ }
+ currentNode = currentNode.relatives[TSTNode.LOKID];
+ }
+ else
+ {
+ if (currentNode.relatives[TSTNode.HIKID] == null)
+ {
+ currentNode.relatives[TSTNode.HIKID] = new TSTNode(this, key.charAt(charIndex), currentNode);
+ }
+ currentNode = currentNode.relatives[TSTNode.HIKID];
+ }
+ }
+ }
+
+ /// <summary>
+ /// Returns a <code>List</code> of keys that almost match the argument key.
+ /// Keys returned will have exactly diff characters that do not match the
+ /// target key, where diff is equal to the last value passed in as an argument
+ /// to the <code>setMatchAlmostDiff</code> method.
+ /// <para>
+ /// If the <code>matchAlmost</code> method is called before the
+ /// <code>setMatchAlmostDiff</code> method has been called for the first time,
+ /// then diff = 0.
+ ///
+ /// </para>
+ /// </summary>
+ /// <param name="key">
+ /// The target key. </param>
+ /// <returns> A <code>List</code> with the results. </returns>
+ public virtual IList<string> MatchAlmost(string key)
+ {
+ return MatchAlmost(key, defaultNumReturnValues);
+ }
+
+ /// <summary>
+ /// Returns a <code>List</code> of keys that almost match the argument key.
+ /// Keys returned will have exactly diff characters that do not match the
+ /// target key, where diff is equal to the last value passed in as an argument
+ /// to the <code>setMatchAlmostDiff</code> method.
+ /// <para>
+ /// If the <code>matchAlmost</code> method is called before the
+ /// <code>setMatchAlmostDiff</code> method has been called for the first time,
+ /// then diff = 0.
+ ///
+ /// </para>
+ /// </summary>
+ /// <param name="key">
+ /// The target key. </param>
+ /// <param name="numReturnValues">
+ /// The maximum number of values returned by this method. </param>
+ /// <returns> A <code>List</code> with the results </returns>
+ public virtual IList<string> MatchAlmost(string key, int numReturnValues)
+ {
+ return MatchAlmostRecursion(rootNode, 0, matchAlmostDiff, key, ((numReturnValues < 0) ? -1 : numReturnValues), new List<string>(), false);
+ }
+
+ /// <summary>
+ /// Recursivelly vists the nodes in order to find the ones that almost match a
+ /// given key.
+ /// </summary>
+ /// <param name="currentNode">
+ /// The current node. </param>
+ /// <param name="charIndex">
+ /// The current char. </param>
+ /// <param name="d">
+ /// The number of differences so far. </param>
+ /// <param name="matchAlmostNumReturnValues">
+ /// The maximum number of values in the result <code>List</code>. </param>
+ /// <param name="matchAlmostResult2">
+ /// The results so far. </param>
+ /// <param name="upTo">
+ /// If true all keys having up to and including matchAlmostDiff
+ /// mismatched letters will be included in the result (including a key
+ /// that is exactly the same as the target string) otherwise keys will
+ /// be included in the result only if they have exactly
+ /// matchAlmostDiff number of mismatched letters. </param>
+ /// <param name="matchAlmostKey">
+ /// The key being searched. </param>
+ /// <returns> A <code>List</code> with the results. </returns>
+ private IList<string> MatchAlmostRecursion(TSTNode currentNode, int charIndex, int d, string matchAlmostKey, int matchAlmostNumReturnValues, IList<string> matchAlmostResult2, bool upTo)
+ {
+ if ((currentNode == null) || (matchAlmostNumReturnValues != -1 && matchAlmostResult2.Count >= matchAlmostNumReturnValues) || (d < 0) || (charIndex >= matchAlmostKey.length()))
+ {
+ return matchAlmostResult2;
+ }
+ int charComp = compareCharsAlphabetically(matchAlmostKey.charAt(charIndex), currentNode.splitchar);
+ IList<string> matchAlmostResult = matchAlmostResult2;
+ if ((d > 0) || (charComp < 0))
+ {
+ matchAlmostResult = MatchAlmostRecursion(currentNode.relatives[TSTNode.LOKID], charIndex, d, matchAlmostKey, matchAlmostNumReturnValues, matchAlmostResult, upTo);
+ }
+ int nextD = (charComp == 0) ? d : d - 1;
+ bool cond = (upTo) ? (nextD >= 0) : (nextD == 0);
+ if ((matchAlmostKey.Length == charIndex + 1) && cond && (currentNode.data != null))
+ {
+ matchAlmostResult.Add(getKey(currentNode));
+ }
+ matchAlmostResult = MatchAlmostRecursion(currentNode.relatives[TSTNode.EQKID], charIndex + 1, nextD, matchAlmostKey, matchAlmostNumReturnValues, matchAlmostResult, upTo);
+ if ((d > 0) || (charComp > 0))
+ {
+ matchAlmostResult = MatchAlmostRecursion(currentNode.relatives[TSTNode.HIKID], charIndex, d, matchAlmostKey, matchAlmostNumReturnValues, matchAlmostResult, upTo);
+ }
+ return matchAlmostResult;
+ }
+
+ /// <summary>
+ /// Returns an alphabetical <code>List</code> of all keys in the trie that
+ /// begin with a given prefix. Only keys for nodes having non-null data are
+ /// included in the <code>List</code>.
+ /// </summary>
+ /// <param name="prefix">
+ /// Each key returned from this method will begin with the characters
+ /// in prefix. </param>
+ /// <returns> A <code>List</code> with the results. </returns>
+ public virtual IList<string> MatchPrefix(string prefix)
+ {
+ return MatchPrefix(prefix, defaultNumReturnValues);
+ }
+
+ /// <summary>
+ /// Returns an alphabetical <code>List</code> of all keys in the trie that
+ /// begin with a given prefix. Only keys for nodes having non-null data are
+ /// included in the <code>List</code>.
+ /// </summary>
+ /// <param name="prefix">
+ /// Each key returned from this method will begin with the characters
+ /// in prefix. </param>
+ /// <param name="numReturnValues">
+ /// The maximum number of values returned from this method. </param>
+ /// <returns> A <code>List</code> with the results </returns>
+ public virtual IList<string> MatchPrefix(string prefix, int numReturnValues)
+ {
+ List<string> sortKeysResult = new List<string>();
+ TSTNode startNode = GetNode(prefix);
+ if (startNode == null)
+ {
+ return sortKeysResult;
+ }
+ if (startNode.data != null)
+ {
+ sortKeysResult.Add(getKey(startNode));
+ }
+ return sortKeysRecursion(startNode.relatives[TSTNode.EQKID], ((numReturnValues < 0) ? -1 : numReturnValues), sortKeysResult);
+ }
+
+ /// <summary>
+ /// Returns the number of nodes in the trie that have non-null data.
+ /// </summary>
+ /// <returns> The number of nodes in the trie that have non-null data. </returns>
+ public virtual int NumDataNodes()
+ {
+ return NumDataNodes(rootNode);
+ }
+
+ /// <summary>
+ /// Returns the number of nodes in the subtrie below and including the starting
+ /// node. The method counts only nodes that have non-null data.
+ /// </summary>
+ /// <param name="startingNode">
+ /// The top node of the subtrie. the node that defines the subtrie. </param>
+ /// <returns> The total number of nodes in the subtrie. </returns>
+ protected internal virtual int NumDataNodes(TSTNode startingNode)
+ {
+ return RecursiveNodeCalculator(startingNode, true, 0);
+ }
+
+ /// <summary>
+ /// Returns the total number of nodes in the trie. The method counts nodes
+ /// whether or not they have data.
+ /// </summary>
+ /// <returns> The total number of nodes in the trie. </returns>
+ public virtual int NumNodes()
+ {
+ return NumNodes(rootNode);
+ }
+
+ /// <summary>
+ /// Returns the total number of nodes in the subtrie below and including the
+ /// starting Node. The method counts nodes whether or not they have data.
+ /// </summary>
+ /// <param name="startingNode">
+ /// The top node of the subtrie. The node that defines the subtrie. </param>
+ /// <returns> The total number of nodes in the subtrie. </returns>
+ protected internal virtual int NumNodes(TSTNode startingNode)
+ {
+ return RecursiveNodeCalculator(startingNode, false, 0);
+ }
+
+ /// <summary>
+ /// Stores a value in the trie. The value may be retrieved using the key.
+ /// </summary>
+ /// <param name="key">
+ /// A <code>String</code> that indexes the object to be stored. </param>
+ /// <param name="value">
+ /// The object to be stored in the Trie. </param>
+ public virtual void Put(string key, object value)
+ {
+ GetOrCreateNode(key).data = value;
+ }
+
+ /// <summary>
+ /// Recursivelly visists each node to calculate the number of nodes.
+ /// </summary>
+ /// <param name="currentNode">
+ /// The current node. </param>
+ /// <param name="checkData">
+ /// If true we check the data to be different of <code>null</code>. </param>
+ /// <param name="numNodes2">
+ /// The number of nodes so far. </param>
+ /// <returns> The number of nodes accounted. </returns>
+ private int RecursiveNodeCalculator(TSTNode currentNode, bool checkData, int numNodes2)
+ {
+ if (currentNode == null)
+ {
+ return numNodes2;
+ }
+ int numNodes = RecursiveNodeCalculator(currentNode.relatives[TSTNode.LOKID], checkData, numNodes2);
+ numNodes = RecursiveNodeCalculator(currentNode.relatives[TSTNode.EQKID], checkData, numNodes);
+ numNodes = RecursiveNodeCalculator(currentNode.relatives[TSTNode.HIKID], checkData, numNodes);
+ if (checkData)
+ {
+ if (currentNode.data != null)
+ {
+ numNodes++;
+ }
+ }
+ else
+ {
+ numNodes++;
+ }
+ return numNodes;
+ }
+
+ /// <summary>
+ /// Removes the value indexed by key. Also removes all nodes that are rendered
+ /// unnecessary by the removal of this data.
+ /// </summary>
+ /// <param name="key">
+ /// A <code>string</code> that indexes the object to be removed from
+ /// the Trie. </param>
+ public virtual void Remove(string key)
+ {
+ DeleteNode(GetNode(key.Trim().ToLower(locale)));
+ }
+
+ /// <summary>
+ /// Sets the number of characters by which words can differ from target word
+ /// when calling the <code>matchAlmost</code> method.
+ /// <para>
+ /// Arguments less than 0 will set the char difference to 0, and arguments
+ /// greater than 3 will set the char difference to 3.
+ ///
+ /// </para>
+ /// </summary>
+ /// <param name="diff">
+ /// The number of characters by which words can differ from target
+ /// word. </param>
+ public virtual int MatchAlmostDiff
+ {
+ set
+ {
+ if (value < 0)
+ {
+ matchAlmostDiff = 0;
+ }
+ else if (value > 3)
+ {
+ matchAlmostDiff = 3;
+ }
+ else
+ {
+ matchAlmostDiff = value;
+ }
+ }
+ }
+
+ /// <summary>
+ /// Sets the default maximum number of values returned from the
+ /// <code>matchPrefix</code> and <code>matchAlmost</code> methods.
+ /// <para>
+ /// The value should be set this to -1 to get an unlimited number of return
+ /// values. note that the methods mentioned above provide overloaded versions
+ /// that allow you to specify the maximum number of return values, in which
+ /// case this value is temporarily overridden.
+ ///
+ /// </para>
+ /// </summary>
+ /// **<param name="num">
+ /// The number of values that will be returned when calling the
+ /// methods above. </param>
+ public virtual int NumReturnValues
+ {
+ set
+ {
+ defaultNumReturnValues = (value < 0) ? -1 : value;
+ }
+ }
+
+ /// <summary>
+ /// Returns keys sorted in alphabetical order. This includes the start Node and
+ /// all nodes connected to the start Node.
+ /// <para>
+ /// The number of keys returned is limited to numReturnValues. To get a list
+ /// that isn't limited in size, set numReturnValues to -1.
+ ///
+ /// </para>
+ /// </summary>
+ /// <param name="startNode">
+ /// The top node defining the subtrie to be searched. </param>
+ /// <param name="numReturnValues">
+ /// The maximum number of values returned from this method. </param>
+ /// <returns> A <code>List</code> with the results. </returns>
+ protected internal virtual IList<string> sortKeys(TSTNode startNode, int numReturnValues)
+ {
+ return sortKeysRecursion(startNode, ((numReturnValues < 0) ? -1 : numReturnValues), new List<string>());
+ }
+
+ /// <summary>
+ /// Returns keys sorted in alphabetical order. This includes the current Node
+ /// and all nodes connected to the current Node.
+ /// <para>
+ /// Sorted keys will be appended to the end of the resulting <code>List</code>.
+ /// The result may be empty when this method is invoked, but may not be
+ /// <code>null</code>.
+ ///
+ /// </para>
+ /// </summary>
+ /// <param name="currentNode">
+ /// The current node. </param>
+ /// <param name="sortKeysNumReturnValues">
+ /// The maximum number of values in the result. </param>
+ /// <param name="sortKeysResult2">
+ /// The results so far. </param>
+ /// <returns> A <code>List</code> with the results. </returns>
+ private IList<string> sortKeysRecursion(TSTNode currentNode, int sortKeysNumReturnValues, IList<string> sortKeysResult2)
+ {
+ if (currentNode == null)
+ {
+ return sortKeysResult2;
+ }
+ IList<string> sortKeysResult = sortKeysRecursion(currentNode.relatives[TSTNode.LOKID], sortKeysNumReturnValues, sortKeysResult2);
+ if (sortKeysNumReturnValues != -1 && sortKeysResult.Count >= sortKeysNumReturnValues)
+ {
+ return sortKeysResult;
+ }
+ if (currentNode.data != null)
+ {
+ sortKeysResult.Add(getKey(currentNode));
+ }
+ sortKeysResult = sortKeysRecursion(currentNode.relatives[TSTNode.EQKID], sortKeysNumReturnValues, sortKeysResult);
+ return sortKeysRecursion(currentNode.relatives[TSTNode.HIKID], sortKeysNumReturnValues, sortKeysResult);
+ }
+
+ /// <summary>
+ /// Return an approximate memory usage for this trie. </summary>
+ public virtual long SizeInBytes()
+ {
+ long mem = RamUsageEstimator.ShallowSizeOf(this);
+ TSTNode root = Root;
+ if (root != null)
+ {
+ mem += root.SizeInBytes();
+ }
+ return mem;
+ }
+
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0ebac726/src/Lucene.Net.Suggest/Suggest/Lookup.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Suggest/Suggest/Lookup.cs b/src/Lucene.Net.Suggest/Suggest/Lookup.cs
new file mode 100644
index 0000000..67e0184
--- /dev/null
+++ b/src/Lucene.Net.Suggest/Suggest/Lookup.cs
@@ -0,0 +1,299 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Lucene.Net.Util;
+
+namespace Lucene.Net.Search.Suggest
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ /// <summary>
+ /// Simple Lookup interface for <seealso cref="string"/> suggestions.
+ /// @lucene.experimental
+ /// </summary>
+ public abstract class Lookup
+ {
+
+ /// <summary>
+ /// Result of a lookup.
+ /// @lucene.experimental
+ /// </summary>
+ public sealed class LookupResult : IComparable<LookupResult>
+ {
+ /// <summary>
+ /// the key's text </summary>
+ public readonly string key;
+
+ /// <summary>
+ /// Expert: custom Object to hold the result of a
+ /// highlighted suggestion.
+ /// </summary>
+ public readonly object highlightKey;
+
+ /// <summary>
+ /// the key's weight </summary>
+ public readonly long value;
+
+ /// <summary>
+ /// the key's payload (null if not present) </summary>
+ public readonly BytesRef payload;
+
+ /// <summary>
+ /// the key's contexts (null if not present) </summary>
+ public readonly HashSet<BytesRef> contexts;
+
+ /// <summary>
+ /// Create a new result from a key+weight pair.
+ /// </summary>
+ public LookupResult(string key, long value)
+ : this(key, null, value, null, null)
+ {
+ }
+
+ /// <summary>
+ /// Create a new result from a key+weight+payload triple.
+ /// </summary>
+ public LookupResult(string key, long value, BytesRef payload)
+ : this(key, null, value, payload, null)
+ {
+ }
+
+ /// <summary>
+ /// Create a new result from a key+highlightKey+weight+payload triple.
+ /// </summary>
+ public LookupResult(string key, object highlightKey, long value, BytesRef payload)
+ : this(key, highlightKey, value, payload, null)
+ {
+ }
+
+ /// <summary>
+ /// Create a new result from a key+weight+payload+contexts triple.
+ /// </summary>
+ public LookupResult(string key, long value, BytesRef payload, HashSet<BytesRef> contexts)
+ : this(key, null, value, payload, contexts)
+ {
+ }
+
+ /// <summary>
+ /// Create a new result from a key+weight+contexts triple.
+ /// </summary>
+ public LookupResult(string key, long value, HashSet<BytesRef> contexts)
+ : this(key, null, value, null, contexts)
+ {
+ }
+
+ /// <summary>
+ /// Create a new result from a key+highlightKey+weight+payload+contexts triple.
+ /// </summary>
+ public LookupResult(string key, object highlightKey, long value, BytesRef payload, HashSet<BytesRef> contexts)
+ {
+ this.key = key;
+ this.highlightKey = highlightKey;
+ this.value = value;
+ this.payload = payload;
+ this.contexts = contexts;
+ }
+
+ public override string ToString()
+ {
+ return key + "/" + value;
+ }
+
+ /// <summary>
+ /// Compare alphabetically. </summary>
+ public int CompareTo(LookupResult o)
+ {
+ return CHARSEQUENCE_COMPARATOR.Compare(key, o.key);
+ }
+ }
+
+ /// <summary>
+ /// A simple char-by-char comparator for <seealso cref="CharSequence"/>
+ /// </summary>
+ public static readonly IComparer<string> CHARSEQUENCE_COMPARATOR = new CharSequenceComparator();
+
+ private class CharSequenceComparator : IComparer<string>
+ {
+
+ public virtual int Compare(string o1, string o2)
+ {
+ int l1 = o1.Length;
+ int l2 = o2.Length;
+
+ int aStop = Math.Min(l1, l2);
+ for (int i = 0; i < aStop; i++)
+ {
+ int diff = o1[i] - o2[i];
+ if (diff != 0)
+ {
+ return diff;
+ }
+ }
+ // One is a prefix of the other, or, they are equal:
+ return l1 - l2;
+ }
+
+ }
+
+ /// <summary>
+ /// A <seealso cref="PriorityQueue"/> collecting a fixed size of high priority <seealso cref="LookupResult"/>
+ /// </summary>
+ public sealed class LookupPriorityQueue : PriorityQueue<LookupResult>
+ {
+ // TODO: should we move this out of the interface into a utility class?
+ /// <summary>
+ /// Creates a new priority queue of the specified size.
+ /// </summary>
+ public LookupPriorityQueue(int size)
+ : base(size)
+ {
+ }
+
+ public override bool LessThan(LookupResult a, LookupResult b)
+ {
+ return a.value < b.value;
+ }
+
+ /// <summary>
+ /// Returns the top N results in descending order. </summary>
+ /// <returns> the top N results in descending order. </returns>
+ public LookupResult[] Results
+ {
+ get
+ {
+ int size = Size();
+ var res = new LookupResult[size];
+ for (int i = size - 1; i >= 0; i--)
+ {
+ res[i] = Pop();
+ }
+ return res;
+ }
+ }
+ }
+
+ /// <summary>
+ /// Sole constructor. (For invocation by subclass
+ /// constructors, typically implicit.)
+ /// </summary>
+ public Lookup()
+ {
+ }
+
+ /// <summary>
+ /// Build lookup from a dictionary. Some implementations may require sorted
+ /// or unsorted keys from the dictionary's iterator - use
+ /// <seealso cref="SortedInputIterator"/> or
+ /// <seealso cref="UnsortedInputIterator"/> in such case.
+ /// </summary>
+ public virtual void Build(Dictionary dict)
+ {
+ Build(dict.EntryIterator);
+ }
+
+ /// <summary>
+ /// Calls <seealso cref="#load(DataInput)"/> after converting
+ /// <seealso cref="InputStream"/> to <seealso cref="DataInput"/>
+ /// </summary>
+ public virtual bool Load(InputStream input)
+ {
+ DataInput dataIn = new InputStreamDataInput(input);
+ try
+ {
+ return Load(dataIn);
+ }
+ finally
+ {
+ IOUtils.Close(input);
+ }
+ }
+
+ /// <summary>
+ /// Calls <seealso cref="#store(DataOutput)"/> after converting
+ /// <seealso cref="OutputStream"/> to <seealso cref="DataOutput"/>
+ /// </summary>
+ public virtual bool Store(OutputStream output)
+ {
+ DataOutput dataOut = new OutputStreamDataOutput(output);
+ try
+ {
+ return Store(dataOut);
+ }
+ finally
+ {
+ IOUtils.Close(output);
+ }
+ }
+
+ /// <summary>
+ /// Get the number of entries the lookup was built with </summary>
+ /// <returns> total number of suggester entries </returns>
+ public abstract long Count { get; }
+
+ /// <summary>
+ /// Builds up a new internal <seealso cref="Lookup"/> representation based on the given <seealso cref="InputIterator"/>.
+ /// The implementation might re-sort the data internally.
+ /// </summary>
+ public abstract void Build(InputIterator inputIterator);
+
+ /// <summary>
+ /// Look up a key and return possible completion for this key. </summary>
+ /// <param name="key"> lookup key. Depending on the implementation this may be
+ /// a prefix, misspelling, or even infix. </param>
+ /// <param name="onlyMorePopular"> return only more popular results </param>
+ /// <param name="num"> maximum number of results to return </param>
+ /// <returns> a list of possible completions, with their relative weight (e.g. popularity) </returns>
+ public virtual IList<LookupResult> Lookup(string key, bool onlyMorePopular, int num)
+ {
+ return Lookup(key, null, onlyMorePopular, num);
+ }
+
+ /// <summary>
+ /// Look up a key and return possible completion for this key. </summary>
+ /// <param name="key"> lookup key. Depending on the implementation this may be
+ /// a prefix, misspelling, or even infix. </param>
+ /// <param name="contexts"> contexts to filter the lookup by, or null if all contexts are allowed; if the suggestion contains any of the contexts, it's a match </param>
+ /// <param name="onlyMorePopular"> return only more popular results </param>
+ /// <param name="num"> maximum number of results to return </param>
+ /// <returns> a list of possible completions, with their relative weight (e.g. popularity) </returns>
+ public abstract IList<LookupResult> Lookup(string key, HashSet<BytesRef> contexts, bool onlyMorePopular, int num);
+
+ /// <summary>
+ /// Persist the constructed lookup data to a directory. Optional operation. </summary>
+ /// <param name="output"> <seealso cref="DataOutput"/> to write the data to. </param>
+ /// <returns> true if successful, false if unsuccessful or not supported. </returns>
+ /// <exception cref="IOException"> when fatal IO error occurs. </exception>
+ public abstract bool Store(DataOutput output);
+
+ /// <summary>
+ /// Discard current lookup data and load it from a previously saved copy.
+ /// Optional operation. </summary>
+ /// <param name="input"> the <seealso cref="DataInput"/> to load the lookup data. </param>
+ /// <returns> true if completed successfully, false if unsuccessful or not supported. </returns>
+ /// <exception cref="IOException"> when fatal IO error occurs. </exception>
+ public abstract bool Load(DataInput input);
+
+ /// <summary>
+ /// Get the size of the underlying lookup implementation in memory </summary>
+ /// <returns> ram size of the lookup implementation in bytes </returns>
+ public abstract long SizeInBytes();
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0ebac726/src/Lucene.Net.Suggest/Suggest/SortedInputIterator.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Suggest/Suggest/SortedInputIterator.cs b/src/Lucene.Net.Suggest/Suggest/SortedInputIterator.cs
new file mode 100644
index 0000000..f9a651e
--- /dev/null
+++ b/src/Lucene.Net.Suggest/Suggest/SortedInputIterator.cs
@@ -0,0 +1,353 @@
+using System.Collections.Generic;
+using Lucene.Net.Store;
+using Lucene.Net.Util;
+
+namespace Lucene.Net.Search.Suggest
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ /// <summary>
+ /// This wrapper buffers incoming elements and makes sure they are sorted based on given comparator.
+ /// @lucene.experimental
+ /// </summary>
+ public class SortedInputIterator : InputIterator
+ {
+
+ private readonly InputIterator source;
+ private File tempInput;
+ private File tempSorted;
+ private readonly OfflineSorter.ByteSequencesReader reader;
+ private readonly IComparer<BytesRef> comparator;
+ private readonly bool hasPayloads;
+ private readonly bool hasContexts;
+ private bool done = false;
+
+ private long weight;
+ private readonly BytesRef scratch = new BytesRef();
+ private BytesRef payload = new BytesRef();
+ private HashSet<BytesRef> contexts = null;
+
+ /// <summary>
+ /// Creates a new sorted wrapper, using {@link
+ /// BytesRef#getUTF8SortedAsUnicodeComparator} for
+ /// sorting.
+ /// </summary>
+ public SortedInputIterator(InputIterator source)
+ : this(source, BytesRef.UTF8SortedAsUnicodeComparator)
+ {
+ }
+
+ /// <summary>
+ /// Creates a new sorted wrapper, sorting by BytesRef
+ /// (ascending) then cost (ascending).
+ /// </summary>
+ public SortedInputIterator(InputIterator source, IComparer<BytesRef> comparator)
+ {
+ this.hasPayloads = source.HasPayloads;
+ this.hasContexts = source.HasContexts;
+ this.source = source;
+ this.comparator = comparator;
+ this.reader = Sort();
+ }
+
+ public BytesRef Next()
+ {
+ bool success = false;
+ if (done)
+ {
+ return null;
+ }
+ try
+ {
+ var input = new ByteArrayDataInput();
+ if (reader.Read(scratch))
+ {
+ weight = Decode(scratch, input);
+ if (hasPayloads)
+ {
+ payload = DecodePayload(scratch, input);
+ }
+ if (hasContexts)
+ {
+ contexts = DecodeContexts(scratch, input);
+ }
+ success = true;
+ return scratch;
+ }
+ Dispose();
+ success = done = true;
+ return null;
+ }
+ finally
+ {
+ if (!success)
+ {
+ done = true;
+ Dispose();
+ }
+ }
+ }
+
+ public virtual long Weight
+ {
+ get { return weight; }
+ }
+
+ public virtual BytesRef Payload
+ {
+ get
+ {
+ if (hasPayloads)
+ {
+ return payload;
+ }
+ return null;
+ }
+ }
+
+ public virtual bool HasPayloads
+ {
+ get { return hasPayloads; }
+ }
+
+ public virtual HashSet<BytesRef> Contexts
+ {
+ get { return contexts; }
+ }
+
+ public IComparer<BytesRef> Comparator
+ {
+ get
+ {
+ return tieBreakByCostComparator;
+ }
+ }
+
+ public virtual bool HasContexts
+ {
+ get { return hasContexts; }
+ }
+
+ /// <summary>
+ /// Sortes by BytesRef (ascending) then cost (ascending). </summary>
+ private readonly IComparer<BytesRef> tieBreakByCostComparator = new ComparatorAnonymousInnerClassHelper();
+
+ private class ComparatorAnonymousInnerClassHelper : IComparer<BytesRef>
+ {
+ public ComparatorAnonymousInnerClassHelper()
+ {
+ }
+
+
+ private readonly BytesRef leftScratch = new BytesRef();
+ private readonly BytesRef rightScratch = new BytesRef();
+ private readonly ByteArrayDataInput input = new ByteArrayDataInput();
+
+ public virtual int Compare(BytesRef left, BytesRef right)
+ {
+ // Make shallow copy in case decode changes the BytesRef:
+ leftScratch.Bytes = left.Bytes;
+ leftScratch.Offset = left.Offset;
+ leftScratch.Length = left.Length;
+ rightScratch.Bytes = right.Bytes;
+ rightScratch.Offset = right.Offset;
+ rightScratch.Length = right.Length;
+ long leftCost = outerInstance.decode(leftScratch, input);
+ long rightCost = outerInstance.decode(rightScratch, input);
+ if (outerInstance.hasPayloads_Renamed)
+ {
+ outerInstance.decodePayload(leftScratch, input);
+ outerInstance.decodePayload(rightScratch, input);
+ }
+ if (outerInstance.hasContexts_Renamed)
+ {
+ outerInstance.decodeContexts(leftScratch, input);
+ outerInstance.decodeContexts(rightScratch, input);
+ }
+ int cmp = outerInstance.comparator.Compare(leftScratch, rightScratch);
+ if (cmp != 0)
+ {
+ return cmp;
+ }
+ if (leftCost < rightCost)
+ {
+ return -1;
+ }
+ else if (leftCost > rightCost)
+ {
+ return 1;
+ }
+ else
+ {
+ return 0;
+ }
+
+ }
+ }
+
+ private OfflineSorter.ByteSequencesReader Sort()
+ {
+ string prefix = this.GetType().Name;
+ File directory = OfflineSorter.DefaultTempDir();
+ tempInput = File.createTempFile(prefix, ".input", directory);
+ tempSorted = File.createTempFile(prefix, ".sorted", directory);
+
+ var writer = new OfflineSorter.ByteSequencesWriter(tempInput);
+ bool success = false;
+ try
+ {
+ BytesRef spare;
+ sbyte[] buffer = new sbyte[0];
+ var output = new ByteArrayDataOutput(buffer);
+
+ while ((spare = source.Next()) != null)
+ {
+ Encode(writer, output, buffer, spare, source.Payload, source.Contexts, source.Weight);
+ }
+ writer.Dispose();
+ (new OfflineSorter(tieBreakByCostComparator)).Sort(tempInput, tempSorted);
+ var reader = new OfflineSorter.ByteSequencesReader(tempSorted);
+ success = true;
+ return reader;
+
+ }
+ finally
+ {
+ if (success)
+ {
+ IOUtils.Close(writer);
+ }
+ else
+ {
+ try
+ {
+ IOUtils.CloseWhileHandlingException(writer);
+ }
+ finally
+ {
+ Dispose();
+ }
+ }
+ }
+ }
+
+ private void Dispose()
+ {
+ IOUtils.Close(reader);
+ if (tempInput != null)
+ {
+ tempInput.Delete();
+ }
+ if (tempSorted != null)
+ {
+ tempSorted.Delete();
+ }
+ }
+
+ /// <summary>
+ /// encodes an entry (bytes+(contexts)+(payload)+weight) to the provided writer
+ /// </summary>
+ protected internal virtual void Encode(OfflineSorter.ByteSequencesWriter writer, ByteArrayDataOutput output, sbyte[] buffer, BytesRef spare, BytesRef payload, HashSet<BytesRef> contexts, long weight)
+ {
+ int requiredLength = spare.Length + 8 + ((hasPayloads) ? 2 + payload.Length : 0);
+ if (hasContexts)
+ {
+ foreach (BytesRef ctx in contexts)
+ {
+ requiredLength += 2 + ctx.Length;
+ }
+ requiredLength += 2; // for length of contexts
+ }
+ if (requiredLength >= buffer.Length)
+ {
+ buffer = ArrayUtil.Grow(buffer, requiredLength);
+ }
+ output.Reset(buffer);
+ output.WriteBytes(spare.Bytes, spare.Offset, spare.Length);
+ if (hasContexts)
+ {
+ foreach (BytesRef ctx in contexts)
+ {
+ output.WriteBytes(ctx.Bytes, ctx.Offset, ctx.Length);
+ output.WriteShort((short)ctx.Length);
+ }
+ output.WriteShort((short)contexts.Count);
+ }
+ if (hasPayloads)
+ {
+ output.WriteBytes(payload.Bytes, payload.Offset, payload.Length);
+ output.WriteShort((short)payload.Length);
+ }
+ output.WriteLong(weight);
+ writer.Write(buffer, 0, output.Position);
+ }
+
+ /// <summary>
+ /// decodes the weight at the current position </summary>
+ protected internal virtual long Decode(BytesRef scratch, ByteArrayDataInput tmpInput)
+ {
+ tmpInput.Reset(scratch.Bytes);
+ tmpInput.SkipBytes(scratch.Length - 8); // suggestion
+ scratch.Length -= 8; // long
+ return tmpInput.ReadLong();
+ }
+
+ /// <summary>
+ /// decodes the contexts at the current position </summary>
+ protected internal virtual HashSet<BytesRef> DecodeContexts(BytesRef scratch, ByteArrayDataInput tmpInput)
+ {
+ tmpInput.Reset(scratch.Bytes);
+ tmpInput.SkipBytes(scratch.Length - 2); //skip to context set size
+ short ctxSetSize = tmpInput.ReadShort();
+ scratch.Length -= 2;
+
+ var contextSet = new HashSet<BytesRef>();
+ for (short i = 0; i < ctxSetSize; i++)
+ {
+ tmpInput.Position = scratch.Length - 2;
+ short curContextLength = tmpInput.ReadShort();
+ scratch.Length -= 2;
+ tmpInput.Position = scratch.Length - curContextLength;
+ BytesRef contextSpare = new BytesRef(curContextLength);
+ tmpInput.ReadBytes(contextSpare.Bytes, 0, curContextLength);
+ contextSpare.Length = curContextLength;
+ contextSet.Add(contextSpare);
+ scratch.Length -= curContextLength;
+ }
+ return contextSet;
+ }
+
+ /// <summary>
+ /// decodes the payload at the current position
+ /// </summary>
+ protected internal virtual BytesRef DecodePayload(BytesRef scratch, ByteArrayDataInput tmpInput)
+ {
+ tmpInput.Reset(scratch.Bytes);
+ tmpInput.SkipBytes(scratch.Length - 2); // skip to payload size
+ short payloadLength = tmpInput.ReadShort(); // read payload size
+ tmpInput.Position = scratch.Length - 2 - payloadLength; // setPosition to start of payload
+ BytesRef payloadScratch = new BytesRef(payloadLength);
+ tmpInput.ReadBytes(payloadScratch.Bytes, 0, payloadLength); // read payload
+ payloadScratch.Length = payloadLength;
+ scratch.Length -= 2; // payload length info (short)
+ scratch.Length -= payloadLength; // payload
+ return payloadScratch;
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/0ebac726/src/Lucene.Net.Suggest/Suggest/SortedTermFreqIteratorWrapper.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Suggest/Suggest/SortedTermFreqIteratorWrapper.cs b/src/Lucene.Net.Suggest/Suggest/SortedTermFreqIteratorWrapper.cs
new file mode 100644
index 0000000..33c1091
--- /dev/null
+++ b/src/Lucene.Net.Suggest/Suggest/SortedTermFreqIteratorWrapper.cs
@@ -0,0 +1,230 @@
+using System.Collections.Generic;
+using Lucene.Net.Search.Spell;
+using Lucene.Net.Store;
+using Lucene.Net.Util;
+
+namespace Lucene.Net.Search.Suggest
+{
+
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ /// <summary>
+ /// This wrapper buffers incoming elements and makes sure they are sorted based on given comparator.
+ /// @lucene.experimental
+ /// </summary>
+ public class SortedTermFreqIteratorWrapper : TermFreqIterator
+ {
+
+ private readonly TermFreqIterator source;
+ private File tempInput;
+ private File tempSorted;
+ private readonly OfflineSorter.ByteSequencesReader reader;
+ private readonly IComparer<BytesRef> comparator;
+ private bool done = false;
+
+ private long weight;
+ private readonly BytesRef scratch = new BytesRef();
+
+ /// <summary>
+ /// Creates a new sorted wrapper, using {@link
+ /// BytesRef#getUTF8SortedAsUnicodeComparator} for
+ /// sorting.
+ /// </summary>
+ //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+ //ORIGINAL LINE: public SortedTermFreqIteratorWrapper(org.apache.lucene.search.spell.TermFreqIterator source) throws java.io.IOException
+ public SortedTermFreqIteratorWrapper(TermFreqIterator source)
+ : this(source, BytesRef.UTF8SortedAsUnicodeComparator)
+ {
+ }
+
+ /// <summary>
+ /// Creates a new sorted wrapper, sorting by BytesRef
+ /// (ascending) then cost (ascending).
+ /// </summary>
+ //JAVA TO C# CONVERTER WARNING: Method 'throws' clauses are not available in .NET:
+ //ORIGINAL LINE: public SortedTermFreqIteratorWrapper(org.apache.lucene.search.spell.TermFreqIterator source, java.util.Comparator<org.apache.lucene.util.BytesRef> comparator) throws java.io.IOException
+ public SortedTermFreqIteratorWrapper(TermFreqIterator source, IComparer<BytesRef> comparator)
+ {
+ this.source = source;
+ this.comparator = comparator;
+ this.reader = Sort();
+ }
+
+ public IComparer<BytesRef> Comparator
+ {
+ get
+ {
+ return comparator;
+ }
+ }
+
+ public BytesRef Next()
+ {
+ bool success = false;
+ if (done)
+ {
+ return null;
+ }
+ try
+ {
+ var input = new ByteArrayDataInput();
+ if (reader.Read(scratch))
+ {
+ weight = Decode(scratch, input);
+ success = true;
+ return scratch;
+ }
+ close();
+ success = done = true;
+ return null;
+ }
+ finally
+ {
+ if (!success)
+ {
+ done = true;
+ close();
+ }
+ }
+ }
+
+ public virtual long Weight
+ {
+ get { return weight; }
+ }
+
+ /// <summary>
+ /// Sortes by BytesRef (ascending) then cost (ascending).
+ /// </summary>
+ private readonly IComparer<BytesRef> tieBreakByCostComparator = new ComparatorAnonymousInnerClassHelper();
+
+ private class ComparatorAnonymousInnerClassHelper : IComparer<BytesRef>
+ {
+ public ComparatorAnonymousInnerClassHelper()
+ {
+ }
+
+
+ private readonly BytesRef leftScratch = new BytesRef();
+ private readonly BytesRef rightScratch = new BytesRef();
+ private readonly ByteArrayDataInput input = new ByteArrayDataInput();
+
+ public virtual int Compare(BytesRef left, BytesRef right)
+ {
+ // Make shallow copy in case decode changes the BytesRef:
+ leftScratch.bytes = left.bytes;
+ leftScratch.offset = left.offset;
+ leftScratch.length = left.length;
+ rightScratch.bytes = right.bytes;
+ rightScratch.offset = right.offset;
+ rightScratch.length = right.length;
+ long leftCost = outerInstance.decode(leftScratch, input);
+ long rightCost = outerInstance.decode(rightScratch, input);
+ int cmp = outerInstance.comparator.Compare(leftScratch, rightScratch);
+ if (cmp != 0)
+ {
+ return cmp;
+ }
+ return long.Compare(leftCost, rightCost);
+ }
+ }
+
+ private OfflineSorter.ByteSequencesReader Sort()
+ {
+ string prefix = this.GetType().Name;
+ File directory = OfflineSorter.DefaultTempDir();
+ tempInput = File.createTempFile(prefix, ".input", directory);
+ tempSorted = File.createTempFile(prefix, ".sorted", directory);
+
+ var writer = new OfflineSorter.ByteSequencesWriter(tempInput);
+ bool success = false;
+ try
+ {
+ BytesRef spare;
+ sbyte[] buffer = new sbyte[0];
+ ByteArrayDataOutput output = new ByteArrayDataOutput(buffer);
+
+ while ((spare = source.Next()) != null)
+ {
+ Encode(writer, output, buffer, spare, source.Weight);
+ }
+ writer.Dispose();
+ (new OfflineSorter(tieBreakByCostComparator)).Sort(tempInput, tempSorted);
+ OfflineSorter.ByteSequencesReader reader = new OfflineSorter.ByteSequencesReader(tempSorted);
+ success = true;
+ return reader;
+
+ }
+ finally
+ {
+ if (success)
+ {
+ IOUtils.Close(writer);
+ }
+ else
+ {
+ try
+ {
+ IOUtils.CloseWhileHandlingException(writer);
+ }
+ finally
+ {
+ Close();
+ }
+ }
+ }
+ }
+
+ private void Close()
+ {
+ IOUtils.Close(reader);
+ if (tempInput != null)
+ {
+ tempInput.delete();
+ }
+ if (tempSorted != null)
+ {
+ tempSorted.delete();
+ }
+ }
+
+ /// <summary>
+ /// encodes an entry (bytes+weight) to the provided writer
+ /// </summary>
+ protected internal virtual void Encode(OfflineSorter.ByteSequencesWriter writer, ByteArrayDataOutput output, sbyte[] buffer, BytesRef spare, long weight)
+ {
+ if (spare.Length + 8 >= buffer.Length)
+ {
+ buffer = ArrayUtil.Grow(buffer, spare.Length + 8);
+ }
+ output.Reset(buffer);
+ output.WriteBytes(spare.Bytes, spare.Offset, spare.Length);
+ output.WriteLong(weight);
+ writer.Write(buffer, 0, output.Position);
+ }
+
+ /// <summary>
+ /// decodes the weight at the current position </summary>
+ protected internal virtual long Decode(BytesRef scratch, ByteArrayDataInput tmpInput)
+ {
+ tmpInput.Reset(scratch.Bytes);
+ tmpInput.SkipBytes(scratch.Length - 8); // suggestion
+ scratch.Length -= 8; // long
+ return tmpInput.ReadLong();
+ }
+ }
+}
\ No newline at end of file