You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by pa...@apache.org on 2013/11/07 14:53:29 UTC
[14/53] [abbrv] Port Highlighter namespace
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/45ba8d83/src/contrib/Highlighter/Highlight/QueryTermScorer.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Highlighter/Highlight/QueryTermScorer.cs b/src/contrib/Highlighter/Highlight/QueryTermScorer.cs
new file mode 100644
index 0000000..5d4532d
--- /dev/null
+++ b/src/contrib/Highlighter/Highlight/QueryTermScorer.cs
@@ -0,0 +1,190 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Tokenattributes;
+using Lucene.Net.Index;
+using Lucene.Net.Support;
+
+namespace Lucene.Net.Search.Highlight
+{
+ /*
+ * {@link Scorer} implementation which scores text fragments by the number of
+ * unique query terms found. This class uses the {@link QueryTermExtractor}
+ * class to process determine the query terms and their boosts to be used.
+ */
+ // TODO: provide option to boost score of fragments near beginning of document
+ // based on fragment.getFragNum()
+ public class QueryTermScorer : IScorer
+ {
+ private TextFragment currentTextFragment = null;
+ private HashSet<String> uniqueTermsInFragment;
+
+ private float totalScore = 0;
+ private float maxTermWeight = 0;
+ private HashMap<String, WeightedTerm> termsToFind;
+
+ private ICharTermAttribute termAtt;
+
+ /*
+ *
+ * @param query a Lucene query (ideally rewritten using query.rewrite before
+ * being passed to this class and the searcher)
+ */
+
+ public QueryTermScorer(Query query)
+ : this(QueryTermExtractor.GetTerms(query))
+ {
+ }
+
+ /*
+ *
+ * @param query a Lucene query (ideally rewritten using query.rewrite before
+ * being passed to this class and the searcher)
+ * @param fieldName the Field name which is used to match Query terms
+ */
+
+ public QueryTermScorer(Query query, String fieldName)
+ : this(QueryTermExtractor.GetTerms(query, false, fieldName))
+ {
+ }
+
+ /*
+ *
+ * @param query a Lucene query (ideally rewritten using query.rewrite before
+ * being passed to this class and the searcher)
+ * @param reader used to compute IDF which can be used to a) score selected
+ * fragments better b) use graded highlights eg set font color
+ * intensity
+ * @param fieldName the field on which Inverse Document Frequency (IDF)
+ * calculations are based
+ */
+
+ public QueryTermScorer(Query query, IndexReader reader, String fieldName)
+ : this(QueryTermExtractor.GetIdfWeightedTerms(query, reader, fieldName))
+ {
+ }
+
+ public QueryTermScorer(WeightedTerm[] weightedTerms)
+ {
+ termsToFind = new HashMap<String, WeightedTerm>();
+ for (int i = 0; i < weightedTerms.Length; i++)
+ {
+ WeightedTerm existingTerm = termsToFind[weightedTerms[i].Term];
+ if ((existingTerm == null)
+ || (existingTerm.Weight < weightedTerms[i].Weight))
+ {
+ // if a term is defined more than once, always use the highest scoring
+ // Weight
+ termsToFind[weightedTerms[i].Term] = weightedTerms[i];
+ maxTermWeight = Math.Max(maxTermWeight, weightedTerms[i].Weight);
+ }
+ }
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.search.highlight.Scorer#init(org.apache.lucene.analysis.TokenStream)
+ */
+
+ public TokenStream Init(TokenStream tokenStream)
+ {
+ termAtt = tokenStream.AddAttribute<ICharTermAttribute>();
+ return null;
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.highlight.FragmentScorer#startFragment(org.apache
+ * .lucene.search.highlight.TextFragment)
+ */
+
+ public void StartFragment(TextFragment newFragment)
+ {
+ uniqueTermsInFragment = new HashSet<String>();
+ currentTextFragment = newFragment;
+ totalScore = 0;
+
+ }
+
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.search.highlight.Scorer#getTokenScore()
+ */
+
+ public float GetTokenScore()
+ {
+ String termText = termAtt.ToString();
+
+ WeightedTerm queryTerm = termsToFind[termText];
+ if (queryTerm == null)
+ {
+ // not a query term - return
+ return 0;
+ }
+ // found a query term - is it unique in this doc?
+ if (!uniqueTermsInFragment.Contains(termText))
+ {
+ totalScore += queryTerm.Weight;
+ uniqueTermsInFragment.Add(termText);
+ }
+ return queryTerm.Weight;
+ }
+
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.search.highlight.Scorer#getFragmentScore()
+ */
+
+ public float FragmentScore
+ {
+ get { return totalScore; }
+ }
+
+ /*
+ * (non-Javadoc)
+ *
+ * @see
+ * org.apache.lucene.search.highlight.FragmentScorer#allFragmentsProcessed()
+ */
+
+ public void AllFragmentsProcessed()
+ {
+ // this class has no special operations to perform at end of processing
+ }
+
+ /*
+ *
+ * @return The highest weighted term (useful for passing to GradientFormatter
+ * to set top end of coloring scale.
+ */
+
+ public float MaxTermWeight
+ {
+ get { return maxTermWeight; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/45ba8d83/src/contrib/Highlighter/Highlight/SimpleFragmenter.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Highlighter/Highlight/SimpleFragmenter.cs b/src/contrib/Highlighter/Highlight/SimpleFragmenter.cs
new file mode 100644
index 0000000..6318e93
--- /dev/null
+++ b/src/contrib/Highlighter/Highlight/SimpleFragmenter.cs
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Tokenattributes;
+
+namespace Lucene.Net.Search.Highlight
+{
+
+ /// <summary> <see cref="IFragmenter"/> implementation which breaks text up into same-size
+ /// fragments with no concerns over spotting sentence boundaries.
+ /// </summary>
+ /// <author> mark@searcharea.co.uk
+ /// </author>
+ public class SimpleFragmenter : IFragmenter
+ {
+ private static int DEFAULT_FRAGMENT_SIZE = 100;
+ private int currentNumFrags;
+ private int fragmentSize;
+ private IOffsetAttribute offsetAtt;
+
+ public SimpleFragmenter()
+ : this(DEFAULT_FRAGMENT_SIZE)
+ {
+ }
+
+ /*
+ *
+ * @param fragmentSize size in number of characters of each fragment
+ */
+
+ public SimpleFragmenter(int fragmentSize)
+ {
+ this.fragmentSize = fragmentSize;
+ }
+
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.search.highlight.Fragmenter#start(java.lang.String, org.apache.lucene.analysis.TokenStream)
+ */
+
+ public void Start(String originalText, TokenStream stream)
+ {
+ offsetAtt = stream.AddAttribute<IOffsetAttribute>();
+ currentNumFrags = 1;
+ }
+
+
+ /* (non-Javadoc)
+ * @see org.apache.lucene.search.highlight.Fragmenter#isNewFragment()
+ */
+
+ public bool IsNewFragment()
+ {
+ bool isNewFrag = offsetAtt.EndOffset >= (fragmentSize * currentNumFrags);
+ if (isNewFrag)
+ {
+ currentNumFrags++;
+ }
+ return isNewFrag;
+ }
+
+ /// <summary>
+ /// Gets or sets the size in number of characters of each fragment
+ /// </summary>
+ public int FragmentSize
+ {
+ get { return fragmentSize; }
+ set { fragmentSize = value; }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/45ba8d83/src/contrib/Highlighter/Highlight/SimpleHTMLEncoder.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Highlighter/Highlight/SimpleHTMLEncoder.cs b/src/contrib/Highlighter/Highlight/SimpleHTMLEncoder.cs
new file mode 100644
index 0000000..80833de
--- /dev/null
+++ b/src/contrib/Highlighter/Highlight/SimpleHTMLEncoder.cs
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Text;
+
+namespace Lucene.Net.Search.Highlight
+{
+ /// <summary> Simple <see cref="IEncoder"/> implementation to escape text for HTML output</summary>
+ public class SimpleHTMLEncoder : IEncoder
+ {
+ public SimpleHTMLEncoder()
+ {
+ }
+
+ public String EncodeText(String originalText)
+ {
+ return HtmlEncode(originalText);
+ }
+
+ /*
+ * Encode string into HTML
+ */
+ public static String HtmlEncode(String plainText)
+ {
+ if (string.IsNullOrEmpty(plainText))
+ {
+ return string.Empty;
+ }
+
+ var result = new StringBuilder(plainText.Length);
+
+ for (int index = 0; index < plainText.Length; index++)
+ {
+ char ch = plainText[index];
+
+ switch (ch)
+ {
+ case '"':
+ result.Append(""");
+ break;
+
+ case '&':
+ result.Append("&");
+ break;
+
+ case '<':
+ result.Append("<");
+ break;
+
+ case '>':
+ result.Append(">");
+ break;
+
+ default:
+ if (ch < 128)
+ {
+ result.Append(ch);
+ }
+ else
+ {
+ result.Append("&#").Append((int)ch).Append(";");
+ }
+ break;
+ }
+ }
+
+ return result.ToString();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/45ba8d83/src/contrib/Highlighter/Highlight/SimpleHTMLFormatter.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Highlighter/Highlight/SimpleHTMLFormatter.cs b/src/contrib/Highlighter/Highlight/SimpleHTMLFormatter.cs
new file mode 100644
index 0000000..a2c3f95
--- /dev/null
+++ b/src/contrib/Highlighter/Highlight/SimpleHTMLFormatter.cs
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+using System;
+using System.Text;
+
+namespace Lucene.Net.Search.Highlight
+{
+ /// <summary> Simple <see cref="IFormatter"/> implementation to highlight terms with a pre and post tag</summary>
+ /// <author> MAHarwood
+ ///
+ /// </author>
+ public class SimpleHTMLFormatter : IFormatter
+ {
+ private const string DEFAULT_PRE_TAG = "<B>";
+ private const string DEFAULT_POST_TAG = "</B>";
+
+ internal string preTag;
+ internal string postTag;
+
+ public SimpleHTMLFormatter(string preTag, string postTag)
+ {
+ this.preTag = preTag;
+ this.postTag = postTag;
+ }
+
+ /// <summary> Default constructor uses HTML: <B> tags to markup terms
+ ///
+ ///
+ /// </summary>
+ public SimpleHTMLFormatter()
+ : this(DEFAULT_PRE_TAG, DEFAULT_POST_TAG)
+ {
+ }
+
+ /* (non-Javadoc)
+ * <see cref="Lucene.Net.Highlight.Formatter.highlightTerm(java.lang.String, Lucene.Net.Highlight.TokenGroup)"/>
+ */
+ public virtual string HighlightTerm(string originalText, TokenGroup tokenGroup)
+ {
+ if (tokenGroup.TotalScore <= 0)
+ return originalText;
+
+ // Allocate StringBuilder with the right number of characters from the
+ // beginning, to avoid char[] allocations in the middle of appends.
+ StringBuilder returnBuffer = new StringBuilder(preTag.Length + originalText.Length + postTag.Length);
+ returnBuffer.Append(preTag);
+ returnBuffer.Append(originalText);
+ returnBuffer.Append(postTag);
+ return returnBuffer.ToString();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/45ba8d83/src/contrib/Highlighter/Highlight/SimpleSpanFragmenter.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Highlighter/Highlight/SimpleSpanFragmenter.cs b/src/contrib/Highlighter/Highlight/SimpleSpanFragmenter.cs
new file mode 100644
index 0000000..f07a850
--- /dev/null
+++ b/src/contrib/Highlighter/Highlight/SimpleSpanFragmenter.cs
@@ -0,0 +1,112 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Tokenattributes;
+
+namespace Lucene.Net.Search.Highlight
+{
+ public class SimpleSpanFragmenter : IFragmenter
+ {
+ private static int DEFAULT_FRAGMENT_SIZE = 100;
+ private int fragmentSize;
+ private int currentNumFrags;
+ private int position = -1;
+ private QueryScorer queryScorer;
+ private int waitForPos = -1;
+ private int textSize;
+ private ICharTermAttribute termAtt;
+ private IPositionIncrementAttribute posIncAtt;
+ private IOffsetAttribute offsetAtt;
+
+ /// <param name="queryScorer">QueryScorer that was used to score hits</param>
+ public SimpleSpanFragmenter(QueryScorer queryScorer)
+ : this(queryScorer, DEFAULT_FRAGMENT_SIZE)
+ {
+
+ }
+
+ /// <param name="queryScorer">QueryScorer that was used to score hits</param>
+ /// <param name="fragmentSize">size in bytes of each fragment</param>
+ public SimpleSpanFragmenter(QueryScorer queryScorer, int fragmentSize)
+ {
+ this.fragmentSize = fragmentSize;
+ this.queryScorer = queryScorer;
+ }
+
+ /// <seealso cref="IFragmenter.IsNewFragment"/>
+ public bool IsNewFragment()
+ {
+ position += posIncAtt.PositionIncrement;
+
+ if (waitForPos == position)
+ {
+ waitForPos = -1;
+ }
+ else if (waitForPos != -1)
+ {
+ return false;
+ }
+
+ WeightedSpanTerm wSpanTerm = queryScorer.GetWeightedSpanTerm(termAtt.ToString());
+
+ if (wSpanTerm != null)
+ {
+ IList<PositionSpan> positionSpans = wSpanTerm.GetPositionSpans();
+
+ for (int i = 0; i < positionSpans.Count; i++)
+ {
+ if (positionSpans[i].Start == position)
+ {
+ waitForPos = positionSpans[i].End + 1;
+ break;
+ }
+ }
+ }
+
+ bool isNewFrag = offsetAtt.EndOffset >= (fragmentSize * currentNumFrags)
+ && (textSize - offsetAtt.EndOffset) >= ((uint)fragmentSize >> 1);
+
+
+ if (isNewFrag)
+ {
+ currentNumFrags++;
+ }
+
+ return isNewFrag;
+ }
+
+ /// <seealso cref="IFragmenter.Start(string, TokenStream)"/>
+ public void Start(string originalText, TokenStream tokenStream)
+ {
+ position = -1;
+ currentNumFrags = 1;
+ textSize = originalText.Length;
+ termAtt = tokenStream.AddAttribute<ICharTermAttribute>();
+ posIncAtt = tokenStream.AddAttribute<IPositionIncrementAttribute>();
+ offsetAtt = tokenStream.AddAttribute<IOffsetAttribute>();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/45ba8d83/src/contrib/Highlighter/Highlight/SpanGradientFormatter.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Highlighter/Highlight/SpanGradientFormatter.cs b/src/contrib/Highlighter/Highlight/SpanGradientFormatter.cs
new file mode 100644
index 0000000..d0a3c3d
--- /dev/null
+++ b/src/contrib/Highlighter/Highlight/SpanGradientFormatter.cs
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Text;
+
+namespace Lucene.Net.Search.Highlight
+{
+ /// <summary>
+ /// Formats text with different color intensity depending on the score of the
+ /// term using the span tag. GradientFormatter uses a bgcolor argument to the font tag which
+ /// doesn't work in Mozilla, thus this class.
+ /// </summary>
+ /// <seealso cref="GradientFormatter"/>
+ public class SpanGradientFormatter : GradientFormatter
+ {
+ // guess how much extra text we'll add to the text we're highlighting to try to avoid a StringBuilder resize
+ private const string TEMPLATE = "<span style=\"background: #EEEEEE; color: #000000;\">...</span>";
+ private const int EXTRA = TEMPLATE.Length;
+
+ public SpanGradientFormatter(float maxScore, string minForegroundColor,
+ string maxForegroundColor, string minBackgroundColor,
+ string maxBackgroundColor)
+ : base(maxScore, minForegroundColor, maxForegroundColor, minBackgroundColor, maxBackgroundColor)
+ {
+ }
+
+ public override string HighlightTerm(string originalText, TokenGroup tokenGroup)
+ {
+ if (tokenGroup.TotalScore == 0)
+ return originalText;
+ float score = tokenGroup.TotalScore;
+ if (score == 0)
+ {
+ return originalText;
+ }
+
+ // try to size sb correctly
+ var sb = new StringBuilder(originalText.Length + EXTRA);
+
+ sb.Append("<span style=\"");
+ if (highlightForeground)
+ {
+ sb.Append("color: ");
+ sb.Append(GetForegroundColorString(score));
+ sb.Append("; ");
+ }
+ if (highlightBackground)
+ {
+ sb.Append("background: ");
+ sb.Append(GetBackgroundColorString(score));
+ sb.Append("; ");
+ }
+ sb.Append("\">");
+ sb.Append(originalText);
+ sb.Append("</span>");
+ return sb.ToString();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/45ba8d83/src/contrib/Highlighter/Highlight/TextFragment.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Highlighter/Highlight/TextFragment.cs b/src/contrib/Highlighter/Highlight/TextFragment.cs
new file mode 100644
index 0000000..7915497
--- /dev/null
+++ b/src/contrib/Highlighter/Highlight/TextFragment.cs
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Text;
+
+namespace Lucene.Net.Search.Highlight
+{
+ /// <summary> Low-level class used to record information about a section of a document
+ /// with a score.
+ /// </summary>
+ public class TextFragment
+ {
+ private StringBuilder markedUpText;
+ private int fragNum;
+ private int textStartPos;
+ private int textEndPos;
+ private float score;
+
+ public TextFragment(StringBuilder markedUpText, int textStartPos, int fragNum)
+ {
+ this.markedUpText = markedUpText;
+ this.textStartPos = textStartPos;
+ this.fragNum = fragNum;
+ }
+
+ public float Score
+ {
+ get { return score; }
+ set { this.score = value; }
+ }
+
+ /// <summary></summary>
+ /// <param name="frag2">Fragment to be merged into this one</param>
+ public void Merge(TextFragment frag2)
+ {
+ textEndPos = frag2.textEndPos;
+ score = Math.Max(score, frag2.score);
+ }
+
+ /// <summary>
+ /// true if this fragment follows the one passed
+ /// </summary>
+ public bool Follows(TextFragment fragment)
+ {
+ return textStartPos == fragment.textEndPos;
+ }
+
+ /// <summary>
+ /// the fragment sequence number
+ /// </summary>
+ public int FragNum
+ {
+ get { return fragNum; }
+ }
+
+ /// <summary>
+ /// Returns the marked-up text for this text fragment
+ /// </summary>
+ public override string ToString()
+ {
+ return markedUpText.ToString(textStartPos, textEndPos - textStartPos);
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/45ba8d83/src/contrib/Highlighter/Highlight/TokenGroup.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Highlighter/Highlight/TokenGroup.cs b/src/contrib/Highlighter/Highlight/TokenGroup.cs
new file mode 100644
index 0000000..2c2a57c
--- /dev/null
+++ b/src/contrib/Highlighter/Highlight/TokenGroup.cs
@@ -0,0 +1,141 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Tokenattributes;
+
+namespace Lucene.Net.Search.Highlight
+{
+ /// <summary> One, or several overlapping tokens, along with the score(s) and the
+ /// scope of the original text
+ /// </summary>
+ public class TokenGroup
+ {
+ private static readonly int MAX_NUM_TOKENS_PER_GROUP = 50;
+
+ private Token[] tokens = new Token[MAX_NUM_TOKENS_PER_GROUP];
+ private float[] scores = new float[MAX_NUM_TOKENS_PER_GROUP];
+ private int numTokens = 0;
+ private int startOffset = 0;
+ private int endOffset = 0;
+ private float tot;
+ private int matchStartOffset, matchEndOffset;
+
+ private IOffsetAttribute offsetAtt;
+ private ICharTermAttribute termAtt;
+
+ public TokenGroup(TokenStream tokenStream)
+ {
+ offsetAtt = tokenStream.AddAttribute<IOffsetAttribute>();
+ termAtt = tokenStream.AddAttribute<ICharTermAttribute>();
+ }
+
+ protected internal void AddToken(float score)
+ {
+ if (numTokens < MAX_NUM_TOKENS_PER_GROUP)
+ {
+ int termStartOffset = offsetAtt.StartOffset;
+ int termEndOffset = offsetAtt.EndOffset;
+ if (numTokens == 0)
+ {
+ startOffset = matchStartOffset = termStartOffset;
+ endOffset = matchEndOffset = termEndOffset;
+ tot += score;
+ }
+ else
+ {
+ startOffset = Math.Min(startOffset, termStartOffset);
+ endOffset = Math.Max(endOffset, termEndOffset);
+ if (score > 0)
+ {
+ if (tot == 0)
+ {
+ matchStartOffset = offsetAtt.StartOffset;
+ matchEndOffset = offsetAtt.EndOffset;
+ }
+ else
+ {
+ matchStartOffset = Math.Min(matchStartOffset, termStartOffset);
+ matchEndOffset = Math.Max(matchEndOffset, termEndOffset);
+ }
+ tot += score;
+ }
+ }
+ Token token = new Token(termStartOffset, termEndOffset);
+ token.SetEmpty().Append(termAtt);
+ tokens[numTokens] = token;
+ scores[numTokens] = score;
+ numTokens++;
+ }
+ }
+
+ protected internal bool IsDistinct()
+ {
+ return offsetAtt.StartOffset >= endOffset;
+ }
+
+ protected internal void Clear()
+ {
+ numTokens = 0;
+ tot = 0;
+ }
+
+
+ /// <summary>
+ /// the "n"th token
+ /// </summary>
+ /// <param name="index">a value between 0 and numTokens -1</param>
+ public Token GetToken(int index)
+ {
+ return tokens[index];
+ }
+
+ /// <summary>
+ /// the "n"th score
+ /// </summary>
+ /// <param name="index">a value between 0 and numTokens -1</param>
+ public float GetScore(int index)
+ {
+ return scores[index];
+ }
+
+ /// <summary>
+ /// the end position in the original text
+ /// </summary>
+ public int EndOffset
+ {
+ get { return endOffset; }
+ }
+
+ /// <summary>
+ /// The start position in the original text
+ /// </summary>
+ public int StartOffset
+ {
+ get { return startOffset; }
+ }
+
+ /// <summary>
+ /// All tokens' scores summed up
+ /// </summary>
+ public float TotalScore
+ {
+ get { return tot; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/45ba8d83/src/contrib/Highlighter/Highlight/TokenSources.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Highlighter/Highlight/TokenSources.cs b/src/contrib/Highlighter/Highlight/TokenSources.cs
new file mode 100644
index 0000000..8805f6e
--- /dev/null
+++ b/src/contrib/Highlighter/Highlight/TokenSources.cs
@@ -0,0 +1,310 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/*
+* Created on 28-Oct-2004
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Tokenattributes;
+using Lucene.Net.Documents;
+using Lucene.Net.Index;
+using Lucene.Net.Util;
+
+namespace Lucene.Net.Search.Highlight
+{
+
+ /// <summary> Hides implementation issues associated with obtaining a TokenStream for use with
+ /// the higlighter - can obtain from TermFreqVectors with offsets and (optionally) positions or
+ /// from Analyzer class reparsing the stored content.
+ /// </summary>
+ public class TokenSources
+ {
+ public class StoredTokenStream : TokenStream
+ {
+ protected internal Token[] tokens;
+ protected internal int currentToken = 0;
+ protected internal ICharTermAttribute termAtt;
+ protected internal IOffsetAttribute offsetAtt;
+ protected internal IPositionIncrementAttribute posincAtt;
+
+ protected internal StoredTokenStream(Token[] tokens)
+ {
+ this.tokens = tokens;
+ termAtt = AddAttribute<ICharTermAttribute>();
+ offsetAtt = AddAttribute<IOffsetAttribute>();
+ posincAtt = AddAttribute<IPositionIncrementAttribute>();
+ }
+
+ public override bool IncrementToken()
+ {
+ if (currentToken >= tokens.Length)
+ {
+ return false;
+ }
+ Token token = tokens[currentToken++];
+ ClearAttributes();
+ termAtt.SetEmpty().Append(token);
+ offsetAtt.SetOffset(token.StartOffset, token.EndOffset);
+ posincAtt.PositionIncrement =
+ (currentToken <= 1 || tokens[currentToken - 1].StartOffset > tokens[currentToken - 2].StartOffset)
+ ? 1
+ : 0;
+ return true;
+ }
+
+ protected override void Dispose(bool disposing)
+ {
+ // do nothing
+ }
+ }
+
+ /// <summary>
+ /// A convenience method that tries to first get a TermPositionVector for the specified docId, then, falls back to
+ /// using the passed in {@link org.apache.lucene.document.Document} to retrieve the TokenStream. This is useful when
+ /// you already have the document, but would prefer to use the vector first.
+ /// </summary>
+ /// <param name="reader">The <see cref="IndexReader"/> to use to try and get the vector from</param>
+ /// <param name="docId">The docId to retrieve.</param>
+ /// <param name="field">The field to retrieve on the document</param>
+ /// <param name="doc">The document to fall back on</param>
+ /// <param name="analyzer">The analyzer to use for creating the TokenStream if the vector doesn't exist</param>
+ /// <returns>The <see cref="TokenStream"/> for the <see cref="IFieldable"/> on the <see cref="Document"/></returns>
+ /// <exception cref="IOException">if there was an error loading</exception>
+ public static TokenStream GetAnyTokenStream(IndexReader reader, int docId, string field, Document doc,
+ Analyzer analyzer)
+ {
+ TokenStream ts = null;
+
+ Fields vectors = reader.GetTermVectors(docId);
+ if (vectors != null)
+ {
+ Terms vector = vectors.Terms(field);
+ if (vector != null)
+ {
+ ts = GetTokenStream(vector);
+ }
+ }
+
+ // No token info stored so fall back to analyzing raw content
+ if (ts == null)
+ {
+ ts = GetTokenStream(doc, field, analyzer);
+ }
+ return ts;
+ }
+
+ /// <summary>
+ /// A convenience method that tries a number of approaches to getting a token stream.
+ /// The cost of finding there are no termVectors in the index is minimal (1000 invocations still
+ /// registers 0 ms). So this "lazy" (flexible?) approach to coding is probably acceptable
+ /// </summary>
+ /// <returns>null if field not stored correctly</returns>
+ public static TokenStream GetAnyTokenStream(IndexReader reader, int docId, string field, Analyzer analyzer)
+ {
+ TokenStream ts = null;
+
+ Fields vectors = reader.GetTermVectors(docId);
+ if (vectors != null)
+ {
+ Terms vector = vectors.Terms(field);
+ if (vector != null)
+ {
+ ts = GetTokenStream(vector);
+ }
+ }
+
+ // No token info stored so fall back to analyzing raw content
+ if (ts == null)
+ {
+ ts = GetTokenStream(reader, docId, field, analyzer);
+ }
+ return ts;
+ }
+
+ public static TokenStream GetTokenStream(Terms vector)
+ {
+ //assumes the worst and makes no assumptions about token position sequences.
+ return GetTokenStream(vector, false);
+ }
+
+ /// <summary>
+ /// Low level api.
+ /// Returns a token stream or null if no offset info available in index.
+ /// This can be used to feed the highlighter with a pre-parsed token stream
+ ///
+ /// In my tests the speeds to recreate 1000 token streams using this method are:
+ /// - with TermVector offset only data stored - 420 milliseconds
+ /// - with TermVector offset AND position data stored - 271 milliseconds
+ /// (nb timings for TermVector with position data are based on a tokenizer with contiguous
+ /// positions - no overlaps or gaps)
+ /// The cost of not using TermPositionVector to store
+ /// pre-parsed content and using an analyzer to re-parse the original content:
+ /// - reanalyzing the original content - 980 milliseconds
+ ///
+ /// The re-analyze timings will typically vary depending on -
+ /// 1) The complexity of the analyzer code (timings above were using a
+ /// stemmer/lowercaser/stopword combo)
+ /// 2) The number of other fields (Lucene reads ALL fields off the disk
+ /// when accessing just one document field - can cost dear!)
+ /// 3) Use of compression on field storage - could be faster due to compression (less disk IO)
+ /// or slower (more CPU burn) depending on the content.
+ /// </summary>
+ /// <param name="tpv"/>
+ /// <param name="tokenPositionsGuaranteedContiguous">true if the token position numbers have no overlaps or gaps. If looking
+ /// to eek out the last drops of performance, set to true. If in doubt, set to false.</param>
+ public static TokenStream GetTokenStream(Terms tpv, bool tokenPositionsGuaranteedContiguous)
+ {
+ if (!tpv.HasOffsets)
+ {
+ throw new ArgumentException("Cannot create TokenStream from Terms without offsets");
+ }
+
+ if (!tokenPositionsGuaranteedContiguous && tpv.HasPositions)
+ {
+ return new TokenStreamFromTermPositionVector(tpv);
+ }
+ // code to reconstruct the original sequence of Tokens
+ TermsEnum termsEnum = tpv.Iterator(null);
+ int totalTokens = 0;
+ while (termsEnum.Next() != null)
+ {
+ totalTokens += (int)termsEnum.TotalTermFreq;
+ }
+ Token[] tokensInOriginalOrder = new Token[totalTokens];
+ List<Token> unsortedTokens = null;
+ termsEnum = tpv.Iterator(null);
+ BytesRef text;
+ DocsAndPositionsEnum dpEnum = null;
+ while ((text = termsEnum.Next()) != null)
+ {
+
+ dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
+ if (dpEnum == null)
+ {
+ throw new ArgumentException(
+ "Required TermVector Offset information was not found");
+ }
+ string term = text.Utf8ToString();
+
+ dpEnum.NextDoc();
+ int freq = dpEnum.Freq;
+ for (int posUpto = 0; posUpto < freq; posUpto++)
+ {
+ int pos = dpEnum.NextPosition();
+ if (dpEnum.StartOffset < 0)
+ {
+ throw new ArgumentException(
+ "Required TermVector Offset information was not found");
+ }
+ Token token = new Token(term,
+ dpEnum.StartOffset,
+ dpEnum.EndOffset);
+ if (tokenPositionsGuaranteedContiguous && pos != -1)
+ {
+ // We have positions stored and a guarantee that the token position
+ // information is contiguous
+
+ // This may be fast BUT wont work if Tokenizers used which create >1
+ // token in same position or
+ // creates jumps in position numbers - this code would fail under those
+ // circumstances
+
+ // tokens stored with positions - can use this to index straight into
+ // sorted array
+ tokensInOriginalOrder[pos] = token;
+ }
+ else
+ {
+ // tokens NOT stored with positions or not guaranteed contiguous - must
+ // add to list and sort later
+ if (unsortedTokens == null)
+ {
+ unsortedTokens = new List<Token>();
+ }
+ unsortedTokens.Add(token);
+ }
+ }
+ }
+ //If the field has been stored without position data we must perform a sort
+ if (unsortedTokens != null)
+ {
+ tokensInOriginalOrder = unsortedTokens.ToArray();
+ ArrayUtil.MergeSort(tokensInOriginalOrder, new AnonymousTokenComparator());
+ }
+ return new StoredTokenStream(tokensInOriginalOrder);
+ }
+
+ private sealed class AnonymousTokenComparator : IComparer<Token>
+ {
+ public int Compare(Token t1, Token t2)
+ {
+ if (t1.StartOffset == t2.StartOffset) return t1.EndOffset - t2.EndOffset;
+ else return t1.StartOffset - t2.StartOffset;
+ }
+ }
+
+ public static TokenStream GetTokenStreamWithOffsets(IndexReader reader, int docId, string field)
+ {
+ Fields vectors = reader.GetTermVectors(docId);
+ if (vectors == null)
+ {
+ return null;
+ }
+
+ Terms vector = vectors.Terms(field);
+ if (vector == null)
+ {
+ return null;
+ }
+
+ if (!vector.HasPositions || !vector.HasOffsets)
+ {
+ return null;
+ }
+
+ return GetTokenStream(vector);
+ }
+
+ //convenience method
+ public static TokenStream GetTokenStream(IndexReader reader, int docId, string field, Analyzer analyzer)
+ {
+ Document doc = reader.Document(docId);
+ return GetTokenStream(doc, field, analyzer);
+ }
+
+ public static TokenStream GetTokenStream(Document doc, string field, Analyzer analyzer)
+ {
+ string contents = doc.Get(field);
+ if (contents == null)
+ {
+ throw new ArgumentException("Field " + field + " in document is not stored and cannot be analyzed");
+ }
+ return GetTokenStream(field, contents, analyzer);
+ }
+
+ //convenience method
+ public static TokenStream GetTokenStream(string field, string contents, Analyzer analyzer)
+ {
+ return analyzer.TokenStream(field, new StringReader(contents));
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/45ba8d83/src/contrib/Highlighter/Highlight/TokenStreamFromTermPositionVector.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Highlighter/Highlight/TokenStreamFromTermPositionVector.cs b/src/contrib/Highlighter/Highlight/TokenStreamFromTermPositionVector.cs
new file mode 100644
index 0000000..15f1c0f
--- /dev/null
+++ b/src/contrib/Highlighter/Highlight/TokenStreamFromTermPositionVector.cs
@@ -0,0 +1,96 @@
+using Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Tokenattributes;
+using Lucene.Net.Index;
+using Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Lucene.Net.Search.Highlight
+{
+ public sealed class TokenStreamFromTermPositionVector : TokenStream
+ {
+ private readonly IList<Token> positionedTokens = new List<Token>();
+ private IEnumerator<Token> tokensAtCurrentPosition;
+ private ICharTermAttribute termAttribute;
+ private IPositionIncrementAttribute positionIncrementAttribute;
+ private IOffsetAttribute offsetAttribute;
+
+ public TokenStreamFromTermPositionVector(Terms vector)
+ {
+ termAttribute = AddAttribute<ICharTermAttribute>();
+ positionIncrementAttribute = AddAttribute<IPositionIncrementAttribute>();
+ offsetAttribute = AddAttribute<IOffsetAttribute>();
+ bool hasOffsets = vector.HasOffsets;
+ TermsEnum termsEnum = vector.Iterator(null);
+ BytesRef text;
+ DocsAndPositionsEnum dpEnum = null;
+ while ((text = termsEnum.Next()) != null)
+ {
+ dpEnum = termsEnum.DocsAndPositions(null, dpEnum);
+ ;
+ dpEnum.NextDoc();
+ int freq = dpEnum.Freq;
+ for (int j = 0; j < freq; j++)
+ {
+ int pos = dpEnum.NextPosition();
+ Token token;
+ if (hasOffsets)
+ {
+ token = new Token(text.Utf8ToString(), dpEnum.StartOffset, dpEnum.EndOffset);
+ }
+ else
+ {
+ token = new Token();
+ token.SetEmpty().Append(text.Utf8ToString());
+ }
+
+ token.PositionIncrement = pos;
+ this.positionedTokens.Add(token);
+ }
+ }
+
+ CollectionUtil.MergeSort(this.positionedTokens, tokenComparator);
+ int lastPosition = -1;
+ foreach (Token token in this.positionedTokens)
+ {
+ int thisPosition = token.PositionIncrement;
+ token.PositionIncrement = thisPosition - lastPosition;
+ lastPosition = thisPosition;
+ }
+
+ this.tokensAtCurrentPosition = this.positionedTokens.GetEnumerator();
+ }
+
+ private static readonly IComparer<Token> tokenComparator = new AnonymousTokenComparator();
+
+ private sealed class AnonymousTokenComparator : IComparer<Token>
+ {
+ public int Compare(Token o1, Token o2)
+ {
+ return o1.PositionIncrement - o2.PositionIncrement;
+ }
+ }
+
+ public override bool IncrementToken()
+ {
+ if (this.tokensAtCurrentPosition.MoveNext())
+ {
+ Token next = this.tokensAtCurrentPosition.Current;
+ ClearAttributes();
+ termAttribute.SetEmpty().Append(next);
+ positionIncrementAttribute.PositionIncrement = next.PositionIncrement;
+ offsetAttribute.SetOffset(next.StartOffset, next.EndOffset);
+ return true;
+ }
+
+ return false;
+ }
+
+ public override void Reset()
+ {
+ this.tokensAtCurrentPosition = this.positionedTokens.GetEnumerator();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/45ba8d83/src/contrib/Highlighter/Highlight/WeightedSpanTerm.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Highlighter/Highlight/WeightedSpanTerm.cs b/src/contrib/Highlighter/Highlight/WeightedSpanTerm.cs
new file mode 100644
index 0000000..6fa70ea
--- /dev/null
+++ b/src/contrib/Highlighter/Highlight/WeightedSpanTerm.cs
@@ -0,0 +1,88 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace Lucene.Net.Search.Highlight
+{
+ /// <summary>
+ /// Lightweight class to hold term, Weight, and positions used for scoring this term.
+ /// </summary>
+ public class WeightedSpanTerm : WeightedTerm
+ {
+ private bool _positionSensitive;
+ private readonly List<PositionSpan> _positionSpans = new List<PositionSpan>();
+
+ public WeightedSpanTerm(float weight, string term)
+ : base(weight, term)
+ {
+ this._positionSpans = new List<PositionSpan>();
+ }
+
+ public WeightedSpanTerm(float weight, string term, bool positionSensitive)
+ : base(weight, term)
+ {
+ this._positionSensitive = positionSensitive;
+ }
+
+ /// <summary>
+ /// Checks to see if this term is valid at <c>position</c>.
+ /// </summary>
+ /// <param name="position">to check against valid term postions</param>
+ /// <returns>true iff this term is a hit at this position</returns>
+ public bool CheckPosition(int position)
+ {
+ // There would probably be a slight speed improvement if PositionSpans
+ // where kept in some sort of priority queue - that way this method
+ // could
+ // bail early without checking each PositionSpan.
+
+ foreach (var positionSpan in _positionSpans)
+ {
+ if (((position >= positionSpan.start) && (position <= positionSpan.end)))
+ {
+ return true;
+ }
+ }
+
+ return false;
+ }
+
+ public void AddPositionSpans(IList<PositionSpan> positionSpans)
+ {
+ this._positionSpans.AddRange(positionSpans);
+ }
+
+ public bool IsPositionSensitive()
+ {
+ return _positionSensitive;
+ }
+
+ public void SetPositionSensitive(bool positionSensitive)
+ {
+ this._positionSensitive = positionSensitive;
+ }
+
+ public List<PositionSpan> GetPositionSpans()
+ {
+ return _positionSpans;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/45ba8d83/src/contrib/Highlighter/Highlight/WeightedSpanTermExtractor.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Highlighter/Highlight/WeightedSpanTermExtractor.cs b/src/contrib/Highlighter/Highlight/WeightedSpanTermExtractor.cs
new file mode 100644
index 0000000..ff22481
--- /dev/null
+++ b/src/contrib/Highlighter/Highlight/WeightedSpanTermExtractor.cs
@@ -0,0 +1,726 @@
+/*
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+*/
+
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using Lucene.Net.Analysis;
+using Lucene.Net.Index;
+using Lucene.Net.Index.Memory;
+using Lucene.Net.Search.Spans;
+using Lucene.Net.Store;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+
+namespace Lucene.Net.Search.Highlight
+{
+ /// <summary>
+ /// Class used to extract <see cref="WeightedSpanTerm"/>s from a <see cref="Query"/> based on whether
+ /// <see cref="Term"/>s from the <see cref="Query"/> are contained in a supplied <see cref="Analysis.TokenStream"/>.
+ /// </summary>
+ public class WeightedSpanTermExtractor
+ {
+ private string fieldName;
+ private TokenStream tokenStream;
+ private string defaultField;
+ private bool expandMultiTermQuery;
+ private bool cachedTokenStream;
+ private bool wrapToCaching = true;
+ private int maxDocCharsToAnalyze;
+ private AtomicReader internalReader = null;
+
+ public WeightedSpanTermExtractor()
+ {
+ }
+
+ public WeightedSpanTermExtractor(string defaultField)
+ {
+ if (defaultField != null)
+ {
+ this.defaultField = string.Intern(defaultField);
+ }
+ }
+
+ /// <summary>
+ /// Fills a <c>Map</c> with <see cref="WeightedSpanTerm"/>s using the terms from the supplied <c>Query</c>.
+ /// </summary>
+ /// <param name="query">Query to extract Terms from</param>
+ /// <param name="terms">Map to place created WeightedSpanTerms in</param>
+ private void Extract(Query query, IDictionary<string, WeightedSpanTerm> terms)
+ {
+ if (query is BooleanQuery)
+ {
+ BooleanClause[] queryClauses = ((BooleanQuery)query).Clauses;
+
+ for (int i = 0; i < queryClauses.Length; i++)
+ {
+ if (!queryClauses[i].IsProhibited)
+ {
+ Extract(queryClauses[i].Query, terms);
+ }
+ }
+ }
+ else if (query is PhraseQuery)
+ {
+ PhraseQuery phraseQuery = ((PhraseQuery)query);
+ Term[] phraseQueryTerms = phraseQuery.GetTerms();
+ SpanQuery[] clauses = new SpanQuery[phraseQueryTerms.Length];
+ for (int i = 0; i < phraseQueryTerms.Length; i++)
+ {
+ clauses[i] = new SpanTermQuery(phraseQueryTerms[i]);
+ }
+ int slop = phraseQuery.Slop;
+ int[] positions = phraseQuery.GetPositions();
+ // add largest position increment to slop
+ if (positions.Length > 0)
+ {
+ int lastPos = positions[0];
+ int largestInc = 0;
+ int sz = positions.Length;
+ for (int i = 1; i < sz; i++)
+ {
+ int pos = positions[i];
+ int inc = pos - lastPos;
+ if (inc > largestInc)
+ {
+ largestInc = inc;
+ }
+ lastPos = pos;
+ }
+ if (largestInc > 1)
+ {
+ slop += largestInc;
+ }
+ }
+
+ bool inorder = false;
+
+ if (slop == 0)
+ {
+ inorder = true;
+ }
+
+ SpanNearQuery sp = new SpanNearQuery(clauses, slop, inorder);
+ sp.Boost = query.Boost;
+ ExtractWeightedSpanTerms(terms, sp);
+ }
+ else if (query is TermQuery)
+ {
+ ExtractWeightedTerms(terms, query);
+ }
+ else if (query is SpanQuery)
+ {
+ ExtractWeightedSpanTerms(terms, (SpanQuery)query);
+ }
+ else if (query is FilteredQuery)
+ {
+ Extract(((FilteredQuery)query).Query, terms);
+ }
+ else if (query is ConstantScoreQuery)
+ {
+ Query q = ((ConstantScoreQuery)query).Query;
+ if (q != null)
+ {
+ Extract(q, terms);
+ }
+ }
+ else if (query is CommonTermsQuery)
+ {
+ // specialized since rewriting would change the result query
+ // this query is TermContext sensitive.
+ ExtractWeightedTerms(terms, query);
+ }
+ else if (query is DisjunctionMaxQuery)
+ {
+ foreach (var q in ((DisjunctionMaxQuery)query))
+ {
+ Extract(q, terms);
+ }
+ }
+ else if (query is MultiPhraseQuery)
+ {
+ MultiPhraseQuery mpq = (MultiPhraseQuery)query;
+ IList<Term[]> termArrays = mpq.GetTermArrays();
+ int[] positions = mpq.GetPositions();
+ if (positions.Length > 0)
+ {
+
+ int maxPosition = positions[positions.Length - 1];
+ for (int i = 0; i < positions.Length - 1; ++i)
+ {
+ if (positions[i] > maxPosition)
+ {
+ maxPosition = positions[i];
+ }
+ }
+
+ var disjunctLists = new List<SpanQuery>[maxPosition + 1];
+ int distinctPositions = 0;
+
+ for (int i = 0; i < termArrays.Count; ++i)
+ {
+ Term[] termArray = termArrays[i];
+ List<SpanQuery> disjuncts = disjunctLists[positions[i]];
+ if (disjuncts == null)
+ {
+ disjuncts = (disjunctLists[positions[i]] = new List<SpanQuery>(termArray.Length));
+ ++distinctPositions;
+ }
+ for (int j = 0; j < termArray.Length; ++j)
+ {
+ disjuncts.Add(new SpanTermQuery(termArray[j]));
+ }
+ }
+
+ int positionGaps = 0;
+ int position = 0;
+ SpanQuery[] clauses = new SpanQuery[distinctPositions];
+ for (int i = 0; i < disjunctLists.Length; ++i)
+ {
+ List<SpanQuery> disjuncts = disjunctLists[i];
+ if (disjuncts != null)
+ {
+ clauses[position++] = new SpanOrQuery(disjuncts.ToArray());
+ }
+ else
+ {
+ ++positionGaps;
+ }
+ }
+
+ int slop = mpq.Slop;
+ bool inorder = (slop == 0);
+
+ SpanNearQuery sp = new SpanNearQuery(clauses, slop + positionGaps, inorder);
+ sp.Boost = query.Boost;
+ ExtractWeightedSpanTerms(terms, sp);
+ }
+ }
+ else
+ {
+ Query origQuery = query;
+ if (query is MultiTermQuery)
+ {
+ if (!expandMultiTermQuery)
+ {
+ return;
+ }
+ MultiTermQuery copy = (MultiTermQuery)query.Clone();
+ copy.SetRewriteMethod(MultiTermQuery.SCORING_BOOLEAN_QUERY_REWRITE);
+ origQuery = copy;
+ }
+ IndexReader reader = GetLeafContext().Reader;
+ Query rewritten = origQuery.Rewrite(reader);
+ if (rewritten != origQuery)
+ {
+ // only rewrite once and then flatten again - the rewritten query could have a speacial treatment
+ // if this method is overwritten in a subclass or above in the next recursion
+ Extract(rewritten, terms);
+ }
+ }
+
+ ExtractUnknownQuery(query, terms);
+ }
+
+ protected virtual void ExtractUnknownQuery(Query query, IDictionary<string, WeightedSpanTerm> terms)
+ {
+ // for sub-classing to extract custom queries
+ }
+
+ /// <summary>
+ /// Fills a <c>Map</c> with <see cref="WeightedSpanTerm"/>s using the terms from the supplied <c>SpanQuery</c>.
+ /// </summary>
+ /// <param name="terms">Map to place created WeightedSpanTerms in</param>
+ /// <param name="spanQuery">SpanQuery to extract Terms from</param>
+ private void ExtractWeightedSpanTerms(IDictionary<string, WeightedSpanTerm> terms, SpanQuery spanQuery)
+ {
+ ISet<String> fieldNames;
+
+ if (fieldName == null)
+ {
+ fieldNames = new HashSet<String>();
+ CollectSpanQueryFields(spanQuery, fieldNames);
+ }
+ else
+ {
+ fieldNames = new HashSet<String>();
+ fieldNames.Add(fieldName);
+ }
+ // To support the use of the default field name
+ if (defaultField != null)
+ {
+ fieldNames.Add(defaultField);
+ }
+
+ IDictionary<string, SpanQuery> queries = new HashMap<string, SpanQuery>();
+
+ var nonWeightedTerms = new HashSet<Term>();
+ bool mustRewriteQuery = MustRewriteQuery(spanQuery);
+ if (mustRewriteQuery)
+ {
+ foreach (String field in fieldNames)
+ {
+ SpanQuery rewrittenQuery = (SpanQuery)spanQuery.Rewrite(GetLeafContext().Reader);
+ queries[field] = rewrittenQuery;
+ rewrittenQuery.ExtractTerms(nonWeightedTerms);
+ }
+ }
+ else
+ {
+ spanQuery.ExtractTerms(nonWeightedTerms);
+ }
+
+ IList<PositionSpan> spanPositions = new List<PositionSpan>();
+
+ foreach (String field in fieldNames)
+ {
+ SpanQuery q;
+ if (mustRewriteQuery)
+ {
+ q = queries[field];
+ }
+ else
+ {
+ q = spanQuery;
+ }
+ AtomicReaderContext context = GetLeafContext();
+ IDictionary<Term, TermContext> termContexts = new HashMap<Term, TermContext>();
+ ISet<Term> extractedTerms = new SortedSet<Term>();
+ q.ExtractTerms(extractedTerms);
+ foreach (Term term in extractedTerms)
+ {
+ termContexts[term] = TermContext.Build(context, term, true);
+ }
+ IBits acceptDocs = context.AtomicReader.LiveDocs;
+ SpansBase spans = q.GetSpans(context, acceptDocs, termContexts);
+
+ // collect span positions
+ while (spans.Next())
+ {
+ spanPositions.Add(new PositionSpan(spans.Start, spans.End - 1));
+ }
+ }
+
+ if (spanPositions.Count == 0)
+ {
+ // no spans found
+ return;
+ }
+
+ foreach (Term queryTerm in nonWeightedTerms)
+ {
+ if (FieldNameComparator(queryTerm.Field))
+ {
+ WeightedSpanTerm weightedSpanTerm = terms[queryTerm.Text];
+
+ if (weightedSpanTerm == null)
+ {
+ weightedSpanTerm = new WeightedSpanTerm(spanQuery.Boost, queryTerm.Text);
+ weightedSpanTerm.AddPositionSpans(spanPositions);
+ weightedSpanTerm.SetPositionSensitive(true);
+ terms[queryTerm.Text] = weightedSpanTerm;
+ }
+ else
+ {
+ if (spanPositions.Count > 0)
+ {
+ weightedSpanTerm.AddPositionSpans(spanPositions);
+ }
+ }
+ }
+ }
+ }
+
+ /// <summary>
+ /// Fills a <c>Map</c> with <see cref="WeightedSpanTerm"/>s using the terms from the supplied <c>Query</c>.
+ /// </summary>
+ /// <param name="terms"></param>
+ /// <param name="query"></param>
+ private void ExtractWeightedTerms(IDictionary<string, WeightedSpanTerm> terms, Query query)
+ {
+ var nonWeightedTerms = new HashSet<Term>();
+ query.ExtractTerms(nonWeightedTerms);
+
+ foreach (Term queryTerm in nonWeightedTerms)
+ {
+ if (FieldNameComparator(queryTerm.Field))
+ {
+ WeightedSpanTerm weightedSpanTerm = new WeightedSpanTerm(query.Boost, queryTerm.Text);
+ terms[queryTerm.Text] = weightedSpanTerm;
+ }
+ }
+ }
+
+ /// <summary>
+ /// Necessary to implement matches for queries against <c>defaultField</c>
+ /// </summary>
+ private bool FieldNameComparator(string fieldNameToCheck)
+ {
+ bool rv = fieldName == null || fieldNameToCheck == fieldName
+ || (defaultField != null && defaultField.Equals(fieldNameToCheck));
+ return rv;
+ }
+
+ protected AtomicReaderContext GetLeafContext()
+ {
+ if (internalReader == null)
+ {
+ if (wrapToCaching && !(tokenStream is CachingTokenFilter))
+ {
+ //assert !cachedTokenStream;
+ tokenStream = new CachingTokenFilter(new OffsetLimitTokenFilter(tokenStream, maxDocCharsToAnalyze));
+ cachedTokenStream = true;
+ }
+ MemoryIndex indexer = new MemoryIndex(true);
+ indexer.AddField(DelegatingAtomicReader.FIELD_NAME, tokenStream);
+ tokenStream.Reset();
+ IndexSearcher searcher = indexer.CreateSearcher();
+ // MEM index has only atomic ctx
+ internalReader = new DelegatingAtomicReader(((AtomicReaderContext)searcher.TopReaderContext).AtomicReader);
+ }
+ return internalReader.AtomicContext;
+ }
+
+ private sealed class DelegatingAtomicReader : FilterAtomicReader
+ {
+ internal const string FIELD_NAME = "shadowed_field";
+
+ internal DelegatingAtomicReader(AtomicReader input)
+ : base(input)
+ {
+ }
+
+ public override FieldInfos FieldInfos
+ {
+ get
+ {
+ throw new NotSupportedException();
+ }
+ }
+
+ public override Fields Fields
+ {
+ get
+ {
+ return new AnonymousFilterFields();
+ }
+ }
+
+ public override NumericDocValues GetNumericDocValues(string field)
+ {
+ return base.GetNumericDocValues(FIELD_NAME);
+ }
+
+ public override BinaryDocValues GetBinaryDocValues(string field)
+ {
+ return base.GetBinaryDocValues(FIELD_NAME);
+ }
+
+ public override SortedDocValues GetSortedDocValues(string field)
+ {
+ return base.GetSortedDocValues(FIELD_NAME);
+ }
+
+ public override NumericDocValues GetNormValues(string field)
+ {
+ return base.GetNormValues(FIELD_NAME);
+ }
+ }
+
+ private sealed class AnonymousFilterFields : FilterAtomicReader.FilterFields
+ {
+ public override Terms Terms(string field)
+ {
+ return base.Terms(DelegatingAtomicReader.FIELD_NAME);
+ }
+
+ public override IEnumerator<string> GetEnumerator()
+ {
+ return (new List<string> { DelegatingAtomicReader.FIELD_NAME }).GetEnumerator();
+ }
+
+ public override int Size
+ {
+ get
+ {
+ return 1;
+ }
+ }
+ }
+
+ /// <summary>
+ /// Creates a Map of <c>WeightedSpanTerms</c> from the given <c>Query</c> and <c>TokenStream</c>.
+ /// </summary>
+ /// <param name="query">query that caused hit</param>
+ /// <param name="tokenStream">TokenStream of text to be highlighted</param>
+ /// <returns>Map containing WeightedSpanTerms</returns>
+ public IDictionary<String, WeightedSpanTerm> GetWeightedSpanTerms(Query query, TokenStream tokenStream)
+ {
+ return GetWeightedSpanTerms(query, tokenStream, null);
+ }
+
+
+ /// <summary>
+ /// Creates a Map of <c>WeightedSpanTerms</c> from the given <c>Query</c> and <c>TokenStream</c>.
+ /// </summary>
+ /// <param name="query">query that caused hit</param>
+ /// <param name="tokenStream">tokenStream of text to be highlighted</param>
+ /// <param name="fieldName">restricts Term's used based on field name</param>
+ /// <returns>Map containing WeightedSpanTerms</returns>
+ public IDictionary<String, WeightedSpanTerm> GetWeightedSpanTerms(Query query, TokenStream tokenStream,
+ string fieldName)
+ {
+ if (fieldName != null)
+ {
+ this.fieldName = string.Intern(fieldName);
+ }
+ else
+ {
+ this.fieldName = null;
+ }
+
+ IDictionary<String, WeightedSpanTerm> terms = new PositionCheckingMap<String>();
+ this.tokenStream = tokenStream;
+ try
+ {
+ Extract(query, terms);
+ }
+ finally
+ {
+ IOUtils.Close(internalReader);
+ }
+
+ return terms;
+ }
+
+ /// <summary>
+ /// Creates a Map of <c>WeightedSpanTerms</c> from the given <c>Query</c> and <c>TokenStream</c>. Uses a supplied
+ /// <c>IndexReader</c> to properly Weight terms (for gradient highlighting).
+ /// </summary>
+ /// <param name="query">Query that caused hit</param>
+ /// <param name="tokenStream">Tokenstream of text to be highlighted</param>
+ /// <param name="fieldName">restricts Term's used based on field name</param>
+ /// <param name="reader">to use for scoring</param>
+ /// <returns>Map of WeightedSpanTerms with quasi tf/idf scores</returns>
+ public IDictionary<String, WeightedSpanTerm> GetWeightedSpanTermsWithScores(Query query, TokenStream tokenStream,
+ string fieldName, IndexReader reader)
+ {
+ if (fieldName != null)
+ {
+ this.fieldName = string.Intern(fieldName);
+ }
+ else
+ {
+ this.fieldName = null;
+ }
+ this.tokenStream = tokenStream;
+
+ IDictionary<String, WeightedSpanTerm> terms = new PositionCheckingMap<String>();
+ Extract(query, terms);
+
+ int totalNumDocs = reader.MaxDoc;
+ var weightedTerms = terms.Keys;
+
+ try
+ {
+ foreach (var wt in weightedTerms)
+ {
+ WeightedSpanTerm weightedSpanTerm = terms[wt];
+ int docFreq = reader.DocFreq(new Term(fieldName, weightedSpanTerm.Term));
+
+ // IDF algorithm taken from DefaultSimilarity class
+ float idf = (float)(Math.Log((float)totalNumDocs / (double)(docFreq + 1)) + 1.0);
+ weightedSpanTerm.Weight *= idf;
+ }
+ }
+ finally
+ {
+ IOUtils.Close(internalReader);
+ }
+
+ return terms;
+ }
+
+ private void CollectSpanQueryFields(SpanQuery spanQuery, ISet<String> fieldNames)
+ {
+ if (spanQuery is FieldMaskingSpanQuery)
+ {
+ CollectSpanQueryFields(((FieldMaskingSpanQuery)spanQuery).MaskedQuery, fieldNames);
+ }
+ else if (spanQuery is SpanFirstQuery)
+ {
+ CollectSpanQueryFields(((SpanFirstQuery)spanQuery).Match, fieldNames);
+ }
+ else if (spanQuery is SpanNearQuery)
+ {
+ foreach (SpanQuery clause in ((SpanNearQuery)spanQuery).GetClauses())
+ {
+ CollectSpanQueryFields(clause, fieldNames);
+ }
+ }
+ else if (spanQuery is SpanNotQuery)
+ {
+ CollectSpanQueryFields(((SpanNotQuery)spanQuery).Include, fieldNames);
+ }
+ else if (spanQuery is SpanOrQuery)
+ {
+ foreach (SpanQuery clause in ((SpanOrQuery)spanQuery).GetClauses())
+ {
+ CollectSpanQueryFields(clause, fieldNames);
+ }
+ }
+ else
+ {
+ fieldNames.Add(spanQuery.Field);
+ }
+ }
+
+ private bool MustRewriteQuery(SpanQuery spanQuery)
+ {
+ if (!expandMultiTermQuery)
+ {
+ return false; // Will throw UnsupportedOperationException in case of a SpanRegexQuery.
+ }
+ else if (spanQuery is FieldMaskingSpanQuery)
+ {
+ return MustRewriteQuery(((FieldMaskingSpanQuery)spanQuery).MaskedQuery);
+ }
+ else if (spanQuery is SpanFirstQuery)
+ {
+ return MustRewriteQuery(((SpanFirstQuery)spanQuery).Match);
+ }
+ else if (spanQuery is SpanNearQuery)
+ {
+ foreach (SpanQuery clause in ((SpanNearQuery)spanQuery).GetClauses())
+ {
+ if (MustRewriteQuery(clause))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+ else if (spanQuery is SpanNotQuery)
+ {
+ SpanNotQuery spanNotQuery = (SpanNotQuery)spanQuery;
+ return MustRewriteQuery(spanNotQuery.Include) || MustRewriteQuery(spanNotQuery.Exclude);
+ }
+ else if (spanQuery is SpanOrQuery)
+ {
+ foreach (SpanQuery clause in ((SpanOrQuery)spanQuery).GetClauses())
+ {
+ if (MustRewriteQuery(clause))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+ else if (spanQuery is SpanTermQuery)
+ {
+ return false;
+ }
+ else
+ {
+ return true;
+ }
+ }
+
+
+ /// <summary>
+ /// This class makes sure that if both position sensitive and insensitive
+ /// versions of the same term are added, the position insensitive one wins.
+ /// </summary>
+ /// <typeparam name="K"></typeparam>
+ private class PositionCheckingMap<K> : HashMap<K, WeightedSpanTerm>
+ {
+ public PositionCheckingMap()
+ {
+
+ }
+
+ public PositionCheckingMap(IEnumerable<KeyValuePair<K, WeightedSpanTerm>> m)
+ {
+ PutAll(m);
+ }
+
+ public void PutAll(IEnumerable<KeyValuePair<K, WeightedSpanTerm>> m)
+ {
+ foreach (var entry in m)
+ {
+ Add(entry.Key, entry.Value);
+ }
+ }
+
+ public override void Add(K key, WeightedSpanTerm value)
+ {
+ base.Add(key, value);
+ WeightedSpanTerm prev = this[key];
+
+ if (prev == null) return;
+
+ WeightedSpanTerm prevTerm = prev;
+ WeightedSpanTerm newTerm = value;
+ if (!prevTerm.IsPositionSensitive())
+ {
+ newTerm.SetPositionSensitive(false);
+ }
+ }
+
+ }
+
+ public bool ExpandMultiTermQuery
+ {
+ set { this.expandMultiTermQuery = value; }
+ get { return expandMultiTermQuery; }
+ }
+
+ public bool IsCachedTokenStream
+ {
+ get { return cachedTokenStream; }
+ }
+
+ public TokenStream TokenStream
+ {
+ get { return tokenStream; }
+ }
+
+
+ /// <summary>
+ /// By default, <see cref="Analysis.TokenStream"/>s that are not of the type
+ /// <see cref="CachingTokenFilter"/> are wrapped in a <see cref="CachingTokenFilter"/> to
+ /// <see cref="Analysis.TokenStream"/> impl and you don't want it to be wrapped, set this to
+ /// false.
+ /// </summary>
+ public void SetWrapIfNotCachingTokenFilter(bool wrap)
+ {
+ this.wrapToCaching = wrap;
+ }
+
+ public int MaxDocCharsToAnalyze
+ {
+ get { return maxDocCharsToAnalyze; }
+ set { maxDocCharsToAnalyze = value; }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/45ba8d83/src/contrib/Highlighter/Highlight/WeightedTerm.cs
----------------------------------------------------------------------
diff --git a/src/contrib/Highlighter/Highlight/WeightedTerm.cs b/src/contrib/Highlighter/Highlight/WeightedTerm.cs
new file mode 100644
index 0000000..2ee406a
--- /dev/null
+++ b/src/contrib/Highlighter/Highlight/WeightedTerm.cs
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+using System;
+
+namespace Lucene.Net.Search.Highlight
+{
+ /// <summary>
+ /// Lightweight class to hold term and a Weight value used for scoring this term
+ /// </summary>
+ public class WeightedTerm
+ {
+ public WeightedTerm(float weight, string term)
+ {
+ this.Weight = weight;
+ this.Term = term;
+ }
+
+ /// <summary>
+ /// the term value (stemmed)
+ /// </summary>
+ public string Term { get; set; }
+
+ /// <summary>
+ /// the Weight associated with this term
+ /// </summary>
+ /// <value> </value>
+ public float Weight { get; set; }
+ }
+}
\ No newline at end of file