You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2019/08/09 19:50:02 UTC
[lucenenet] 01/09:
Lucene.Net.ICU.PostingsHighlight.PostingsHighlighter: Changed default
BreakIterator implementation to be the ICU default rather than using a
customized RuleBasedBreakIterator that was built just to make the tests
pass. Renamed ICUPostingsFormatter to reflect the change and be consistent
with other ICU classes. Confirmed against Java that this is the correct
default behavior and created separate ICU-specific tests. Moved the
JdkBreakIterator to Lucene.Net.ICU.Tests because it is just a test [...]
This is an automated email from the ASF dual-hosted git repository.
nightowl888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucenenet.git
commit 9c4a3c38a9f744ae5993a88a9a80933a63aafc0c
Author: Shad Storhaug <sh...@shadstorhaug.com>
AuthorDate: Fri Aug 9 10:49:18 2019 +0700
Lucene.Net.ICU.PostingsHighlight.PostingsHighlighter: Changed default BreakIterator implementation to be the ICU default rather than using a customized RuleBasedBreakIterator that was built just to make the tests pass. Renamed ICUPostingsFormatter to reflect the change and be consistent with other ICU classes. Confirmed against Java that this is the correct default behavior and created separate ICU-specific tests. Moved the JdkBreakIterator to Lucene.Net.ICU.Tests because it is just a [...]
---
.../PostingsHighlight/MultiTermHighlighting.cs | 2 +-
.../PostingsHighlight/PassageFormatter.cs | 2 +-
.../PostingsHighlight/PassageScorer.cs | 2 +-
.../PostingsHighlight/PostingsHighlighter.cs | 27 ++-
.../PostingsHighlight/TestMultiTermHighlighting.cs | 161 +--------------
.../PostingsHighlight/TestPostingsHighlighter.cs | 17 +-
.../TestPostingsHighlighterRanking.cs | 44 +---
.../Lucene.Net.Tests.ICU.csproj | 4 +
.../TestICUMultiTermHighlighting.cs} | 222 ++++-----------------
.../TestICUPostingsHighlighter.cs} | 141 +++++++------
.../TestICUPostingsHighlighterRanking.cs} | 61 ++----
.../Support}/Data/jdk7/jdksent.rbbi | 0
.../Support}/Data/jdk7/jdkword.rbbi | 0
.../Support/JDKBreakIterator.cs | 3 +-
.../Support/PostingsHighlighter.cs | 29 +++
.../Support/TestJDKBreakIterator.cs | 2 +-
.../Support/jdksent.brk | Bin
.../Support/jdkword.brk | Bin
18 files changed, 226 insertions(+), 491 deletions(-)
diff --git a/src/Lucene.Net.Highlighter/PostingsHighlight/MultiTermHighlighting.cs b/src/Lucene.Net.Highlighter/PostingsHighlight/MultiTermHighlighting.cs
index 482716c..4d0c46a 100644
--- a/src/Lucene.Net.Highlighter/PostingsHighlight/MultiTermHighlighting.cs
+++ b/src/Lucene.Net.Highlighter/PostingsHighlight/MultiTermHighlighting.cs
@@ -221,7 +221,7 @@ namespace Lucene.Net.Search.PostingsHighlight
/// Returns a "fake" <see cref="DocsAndPositionsEnum"/> over the tokenstream, returning offsets where <paramref name="matchers"/>
/// matches tokens.
/// <para/>
- /// This is solely used internally by <see cref="PostingsHighlighter"/>: <b>DO NOT USE THIS METHOD!</b>
+ /// This is solely used internally by <see cref="ICUPostingsHighlighter"/>: <b>DO NOT USE THIS METHOD!</b>
/// </summary>
internal static DocsAndPositionsEnum GetDocsEnum(TokenStream ts, CharacterRunAutomaton[] matchers)
{
diff --git a/src/Lucene.Net.Highlighter/PostingsHighlight/PassageFormatter.cs b/src/Lucene.Net.Highlighter/PostingsHighlight/PassageFormatter.cs
index 770a6fa..80f0e09 100644
--- a/src/Lucene.Net.Highlighter/PostingsHighlight/PassageFormatter.cs
+++ b/src/Lucene.Net.Highlighter/PostingsHighlight/PassageFormatter.cs
@@ -36,7 +36,7 @@ namespace Lucene.Net.Search.PostingsHighlight
/// <param name="content">content for the field.</param>
/// <returns>
/// formatted highlight. Note that for the
- /// non-expert APIs in <see cref="PostingsHighlighter"/> that
+ /// non-expert APIs in <see cref="ICUPostingsHighlighter"/> that
/// return <see cref="string"/>, the <see cref="object.ToString()"/> method on the <see cref="object"/>
/// returned by this method is used to compute the string.
/// </returns>
diff --git a/src/Lucene.Net.Highlighter/PostingsHighlight/PassageScorer.cs b/src/Lucene.Net.Highlighter/PostingsHighlight/PassageScorer.cs
index fc64a70..ef23b20 100644
--- a/src/Lucene.Net.Highlighter/PostingsHighlight/PassageScorer.cs
+++ b/src/Lucene.Net.Highlighter/PostingsHighlight/PassageScorer.cs
@@ -21,7 +21,7 @@ namespace Lucene.Net.Search.PostingsHighlight
*/
/// <summary>
- /// Ranks passages found by <see cref="PostingsHighlighter"/>.
+ /// Ranks passages found by <see cref="ICUPostingsHighlighter"/>.
/// <para/>
/// Each passage is scored as a miniature document within the document.
/// The final score is computed as <c>norm</c> * ∑ (<c>weight</c> * <c>tf</c>).
diff --git a/src/Lucene.Net.Highlighter/PostingsHighlight/PostingsHighlighter.cs b/src/Lucene.Net.Highlighter/PostingsHighlight/PostingsHighlighter.cs
index 274b44f..58a5d7e 100644
--- a/src/Lucene.Net.Highlighter/PostingsHighlight/PostingsHighlighter.cs
+++ b/src/Lucene.Net.Highlighter/PostingsHighlight/PostingsHighlighter.cs
@@ -1,7 +1,6 @@
#if FEATURE_BREAKITERATOR
using ICU4N.Text;
using Lucene.Net.Analysis;
-using Lucene.Net.ICU.Support;
using Lucene.Net.Index;
using Lucene.Net.Support;
using Lucene.Net.Util;
@@ -37,10 +36,11 @@ namespace Lucene.Net.Search.PostingsHighlight
/// Simple highlighter that does not analyze fields nor use
/// term vectors. Instead it requires
/// <see cref="IndexOptions.DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS"/>.
- /// <para/>
+ /// </summary>
+ /// <remarks>
/// PostingsHighlighter treats the single original document as the whole corpus, and then scores individual
/// passages as if they were documents in this corpus. It uses a <see cref="BreakIterator"/> to find
- /// passages in the text; by default it breaks using <see cref="JdkBreakIterator.GetSentenceInstance(CultureInfo)"/> (for sentence breaking).
+ /// passages in the text; by default it breaks using <see cref="BreakIterator.GetSentenceInstance(CultureInfo)"/> (for sentence breaking).
/// It then iterates in parallel (merge sorting by offset) through
/// the positions of all terms from the query, coalescing those hits that occur in a single passage
/// into a <see cref="Passage"/>, and then scores each Passage using a separate <see cref="PassageScorer"/>.
@@ -64,16 +64,25 @@ namespace Lucene.Net.Search.PostingsHighlight
/// Field body = new Field("body", "foobar", offsetsType);
///
/// // retrieve highlights at query time
- /// PostingsHighlighter highlighter = new PostingsHighlighter();
+ /// ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
/// Query query = new TermQuery(new Term("body", "highlighting"));
/// TopDocs topDocs = searcher.Search(query, n);
/// string highlights[] = highlighter.Highlight("body", query, searcher, topDocs);
/// </code>
/// <para/>
/// This is thread-safe, and can be used across different readers.
+ /// <para/>
+ /// Note that the .NET implementation differs from the <c>PostingsHighlighter</c> in Lucene in
+ /// that it is backed by an ICU <see cref="RuleBasedBreakIterator"/>, which differs slightly in default behavior
+ /// than the one in the JDK. However, the ICU <see cref="RuleBasedBreakIterator"/> behavior can be customized
+ /// to meet a lot of scenarios that the one in the JDK cannot. See the ICU documentation at
+ /// <a href="http://userguide.icu-project.org/boundaryanalysis/break-rules">http://userguide.icu-project.org/boundaryanalysis/break-rules</a>
+ /// for more information how to pass custom rules to an ICU <see cref="RuleBasedBreakIterator"/>.
+ /// <para/>
/// @lucene.experimental
- /// </summary>
- public class PostingsHighlighter
+ /// </remarks>
+ [ExceptionToClassNameConvention]
+ public class ICUPostingsHighlighter // LUCENENET specific - renamed ICUPostingsHighlighter to reflect the change in default behavior
{
// TODO: maybe allow re-analysis for tiny fields? currently we require offsets,
// but if the analyzer is really fast and the field is tiny, this might really be
@@ -104,7 +113,7 @@ namespace Lucene.Net.Search.PostingsHighlight
/// <summary>
/// Creates a new highlighter with <see cref="DEFAULT_MAX_LENGTH"/>.
/// </summary>
- public PostingsHighlighter()
+ public ICUPostingsHighlighter()
: this(DEFAULT_MAX_LENGTH)
{
}
@@ -114,7 +123,7 @@ namespace Lucene.Net.Search.PostingsHighlight
/// </summary>
/// <param name="maxLength">maximum content size to process.</param>
/// <exception cref="ArgumentException">if <paramref name="maxLength"/> is negative or <c>int.MaxValue</c></exception>
- public PostingsHighlighter(int maxLength)
+ public ICUPostingsHighlighter(int maxLength)
{
if (maxLength < 0 || maxLength == int.MaxValue)
{
@@ -133,7 +142,7 @@ namespace Lucene.Net.Search.PostingsHighlight
/// </summary>
protected virtual BreakIterator GetBreakIterator(string field)
{
- return JdkBreakIterator.GetSentenceInstance(CultureInfo.InvariantCulture);
+ return BreakIterator.GetSentenceInstance(CultureInfo.InvariantCulture);
}
/// <summary>
diff --git a/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestMultiTermHighlighting.cs b/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestMultiTermHighlighting.cs
index eeb6d47..87841f1 100644
--- a/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestMultiTermHighlighting.cs
+++ b/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestMultiTermHighlighting.cs
@@ -38,6 +38,17 @@ namespace Lucene.Net.Search.PostingsHighlight
/// Some tests that override <see cref="PostingsHighlighter.GetIndexAnalyzer(string)"/> to
/// highlight wilcard, fuzzy, etc queries.
/// </summary>
+ /// <remarks>
+ /// LUCENENET specific - These are the original tests from Lucene. They are only here as proof that we
+ /// can customize the <see cref="ICUPostingsHighlighter"/> to act like the PostingsHighlighter in Lucene,
+ /// which has slightly different default behavior than that of ICU because Lucene uses
+ /// the RuleBasedBreakIterator from the JDK, not that of ICU4J.
+ /// <para/>
+ /// These tests use a mock <see cref="PostingsHighlighter"/>, which is backed by an ICU
+ /// <see cref="ICU4N.Text.RuleBasedBreakIterator"/> that is customized a bit to act (sort of)
+ /// like the one in the JDK. However, this customized implementation is not a logical default for
+ /// the <see cref="ICUPostingsHighlighter"/>.
+ /// </remarks>
[SuppressCodecs("MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Lucene3x")]
public class TestMultiTermHighlighting : LuceneTestCase
{
@@ -82,13 +93,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- // PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- // {
- // return analyzer;
- // }
- //};
Query query = new WildcardQuery(new Term("body", "te*"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -138,13 +142,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
Query query = new PrefixQuery(new Term("body", "te"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -194,13 +191,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
Query query = new RegexpQuery(new Term("body", "te.*"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -250,13 +240,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
Query query = new FuzzyQuery(new Term("body", "tets"), 1);
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -315,13 +298,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
Query query = TermRangeQuery.NewStringRange("body", "ta", "tf", true, true);
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -429,13 +405,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
BooleanQuery query = new BooleanQuery();
query.Add(new WildcardQuery(new Term("body", "te*")), Occur.SHOULD);
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
@@ -486,13 +455,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
DisjunctionMaxQuery query = new DisjunctionMaxQuery(0);
query.Add(new WildcardQuery(new Term("body", "te*")));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
@@ -532,13 +494,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
Query query = new SpanMultiTermQueryWrapper<WildcardQuery>(new WildcardQuery(new Term("body", "te*")));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -577,13 +532,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
SpanQuery childQuery = new SpanMultiTermQueryWrapper<WildcardQuery>(new WildcardQuery(new Term("body", "te*")));
Query query = new SpanOrQuery(new SpanQuery[] { childQuery });
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
@@ -623,13 +571,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
SpanQuery childQuery = new SpanMultiTermQueryWrapper<WildcardQuery>(new WildcardQuery(new Term("body", "te*")));
Query query = new SpanNearQuery(new SpanQuery[] { childQuery }, 0, true);
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
@@ -669,13 +610,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
SpanQuery include = new SpanMultiTermQueryWrapper<WildcardQuery>(new WildcardQuery(new Term("body", "te*")));
SpanQuery exclude = new SpanTermQuery(new Term("body", "bogus"));
Query query = new SpanNotQuery(include, exclude);
@@ -716,13 +650,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
SpanQuery childQuery = new SpanMultiTermQueryWrapper<WildcardQuery>(new WildcardQuery(new Term("body", "te*")));
Query query = new SpanFirstQuery(childQuery, 1000000);
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
@@ -823,13 +750,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
BooleanQuery query = new BooleanQuery();
query.Add(new WildcardQuery(new Term("body", "te*")), Occur.SHOULD);
query.Add(new WildcardQuery(new Term("body", "one")), Occur.SHOULD);
@@ -846,65 +766,6 @@ namespace Lucene.Net.Search.PostingsHighlight
// matching term's text into the result:
highlighter = new PostingsHighlighterAnalyzerAndFormatterHelper(analyzer, new PassageFormatterHelper());
- //highlighter = new PostingsHighlighter()
- //{
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
-
- //@Override
- // protected PassageFormatter getFormatter(String field)
- //{
- // return new PassageFormatter() {
-
- // @Override
- // public Object format(Passage passages[], String content)
- //{
- // // Copied from DefaultPassageFormatter, but
- // // tweaked to include the matched term:
- // StringBuilder sb = new StringBuilder();
- // int pos = 0;
- // for (Passage passage : passages)
- // {
- // // don't add ellipsis if its the first one, or if its connected.
- // if (passage.startOffset > pos && pos > 0)
- // {
- // sb.append("... ");
- // }
- // pos = passage.startOffset;
- // for (int i = 0; i < passage.numMatches; i++)
- // {
- // int start = passage.matchStarts[i];
- // int end = passage.matchEnds[i];
- // // its possible to have overlapping terms
- // if (start > pos)
- // {
- // sb.append(content, pos, start);
- // }
- // if (end > pos)
- // {
- // sb.append("<b>");
- // sb.append(content, Math.max(pos, start), end);
- // sb.append('(');
- // sb.append(passage.getMatchTerms()[i].utf8ToString());
- // sb.append(')');
- // sb.append("</b>");
- // pos = end;
- // }
- // }
- // // its possible a "term" from the analyzer could span a sentence boundary.
- // sb.append(content, pos, Math.max(pos, passage.endOffset));
- // pos = passage.endOffset;
- // }
- // return sb.toString();
- //}
- // };
- // }
- // };
-
-
assertEquals(1, topDocs.TotalHits);
snippets = highlighter.Highlight("body", query, searcher, topDocs);
assertEquals(1, snippets.Length);
diff --git a/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighter.cs b/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighter.cs
index b96efea..1b24a8c 100644
--- a/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighter.cs
+++ b/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighter.cs
@@ -33,6 +33,17 @@ namespace Lucene.Net.Search.PostingsHighlight
* limitations under the License.
*/
+ /// <summary>
+ /// LUCENENET specific - These are the original tests from Lucene. They are only here as proof that we
+ /// can customize the <see cref="ICUPostingsHighlighter"/> to act like the PostingsHighlighter in Lucene,
+ /// which has slightly different default behavior than that of ICU because Lucene uses
+ /// the RuleBasedBreakIterator from the JDK, not that of ICU4J.
+ /// <para/>
+ /// These tests use a mock <see cref="PostingsHighlighter"/>, which is backed by an ICU
+ /// <see cref="ICU4N.Text.RuleBasedBreakIterator"/> that is customized a bit to act (sort of)
+ /// like the one in the JDK. However, this customized implementation is not a logical default for
+ /// the <see cref="ICUPostingsHighlighter"/>.
+ /// </summary>
[SuppressCodecs("MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Lucene3x")]
public class TestPostingsHighlighter : LuceneTestCase
{
@@ -746,8 +757,8 @@ namespace Lucene.Net.Search.PostingsHighlight
protected override string[][] LoadFieldValues(IndexSearcher searcher, string[] fields, int[] docids, int maxLength)
{
- Debug.Assert( fields.Length == 1);
- Debug.Assert( docids.Length == 1);
+ Debug.Assert(fields.Length == 1);
+ Debug.Assert(docids.Length == 1);
String[][] contents = RectangularArrays.ReturnRectangularArray<string>(1, 1); //= new String[1][1];
contents[0][0] = text;
return contents;
@@ -1150,7 +1161,7 @@ namespace Lucene.Net.Search.PostingsHighlight
{
protected override char GetMultiValuedSeparator(string field)
{
- Debug.Assert( field.Equals("body", StringComparison.Ordinal));
+ Debug.Assert(field.Equals("body", StringComparison.Ordinal));
return '\u2029';
}
}
diff --git a/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighterRanking.cs b/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighterRanking.cs
index 6d94c24..0243ed6 100644
--- a/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighterRanking.cs
+++ b/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighterRanking.cs
@@ -30,6 +30,17 @@ namespace Lucene.Net.Search.PostingsHighlight
* limitations under the License.
*/
+ /// <summary>
+ /// LUCENENET specific - These are the original tests from Lucene. They are only here as proof that we
+ /// can customize the <see cref="ICUPostingsHighlighter"/> to act like the PostingsHighlighter in Lucene,
+ /// which has slightly different default behavior than that of ICU because Lucene uses
+ /// the RuleBasedBreakIterator from the JDK, not that of ICU4J.
+ /// <para/>
+ /// These tests use a mock <see cref="PostingsHighlighter"/>, which is backed by an ICU
+ /// <see cref="ICU4N.Text.RuleBasedBreakIterator"/> that is customized a bit to act (sort of)
+ /// like the one in the JDK. However, this customized implementation is not a logical default for
+ /// the <see cref="ICUPostingsHighlighter"/>.
+ /// </summary>
[SuppressCodecs("MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Lucene3x")]
public class TestPostingsHighlighterRanking : LuceneTestCase
{
@@ -122,25 +133,6 @@ namespace Lucene.Net.Search.PostingsHighlight
{
CheckQueryPostingsHighlighter p1 = new CheckQueryPostingsHighlighter(int.MaxValue - 1);
CheckQueryPostingsHighlighter p2 = new CheckQueryPostingsHighlighter(int.MaxValue - 1);
- // FakePassageFormatter f1 = new FakePassageFormatter();
- // PostingsHighlighter p1 = new PostingsHighlighter(int.MaxValue - 1) {
- // @Override
- // protected PassageFormatter getFormatter(String field)
- //{
- // assertEquals("body", field);
- // return f1;
- //}
- // };
-
- // FakePassageFormatter f2 = new FakePassageFormatter();
- //PostingsHighlighter p2 = new PostingsHighlighter(Integer.MAX_VALUE - 1) {
- // @Override
- // protected PassageFormatter getFormatter(String field)
- //{
- // assertEquals("body", field);
- // return f2;
- //}
- // };
BooleanQuery bq = new BooleanQuery(false);
bq.Add(query, Occur.MUST);
@@ -300,13 +292,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new CustomBPostingsHighlighter();
- //PostingsHighlighter highlighter = new PostingsHighlighter(10000) {
- // @Override
- // protected PassageScorer getScorer(String field)
- //{
- // return new PassageScorer(1.2f, 0, 87);
- //}
- // };
Query query = new TermQuery(new Term("body", "test"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(1, topDocs.TotalHits);
@@ -351,13 +336,6 @@ namespace Lucene.Net.Search.PostingsHighlight
IndexSearcher searcher = NewSearcher(ir);
PostingsHighlighter highlighter = new CustomK1PostingsHighlighter();
- //PostingsHighlighter highlighter = new PostingsHighlighter(10000) {
- // @Override
- // protected PassageScorer getScorer(String field)
- //{
- // return new PassageScorer(0, 0.75f, 87);
- //}
- // };
BooleanQuery query = new BooleanQuery();
query.Add(new TermQuery(new Term("body", "foo")), Occur.SHOULD);
query.Add(new TermQuery(new Term("body", "bar")), Occur.SHOULD);
diff --git a/src/dotnet/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.csproj b/src/dotnet/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.csproj
index cc2a743..eb5d448 100644
--- a/src/dotnet/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.csproj
+++ b/src/dotnet/Lucene.Net.Tests.ICU/Lucene.Net.Tests.ICU.csproj
@@ -40,7 +40,11 @@
<EmbeddedResource Include="..\..\Lucene.Net.Tests.Analysis.ICU\Analysis\Icu\Segmentation\*.rbbi" LinkBase="Analysis\Icu\Segmentation" />
<Compile Include="..\..\Lucene.Net.Tests.Analysis.ICU\Collation\**\*.cs" LinkBase="Collation" />
<Compile Include="..\..\Lucene.Net.Tests.Highlighter\PostingsHighlight\**\*.cs" LinkBase="Search\PostingsHighlight" />
+ <None Remove="Support\jdksent.brk" />
+ <None Remove="Support\jdkword.brk" />
<EmbeddedResource Include="..\..\Lucene.Net.Tests.Highlighter\PostingsHighlight\CambridgeMA.utf8" Link="Search\PostingsHighlight\CambridgeMA.utf8" />
+ <EmbeddedResource Include="Support\jdksent.brk" />
+ <EmbeddedResource Include="Support\jdkword.brk" />
<Compile Include="..\..\Lucene.Net.Tests.Highlighter\VectorHighlight\BreakIteratorBoundaryScannerTest.cs" Link="Search\VectorHighlight\BreakIteratorBoundaryScannerTest.cs" />
</ItemGroup>
diff --git a/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestMultiTermHighlighting.cs b/src/dotnet/Lucene.Net.Tests.ICU/Search/PostingsHighlight/TestICUMultiTermHighlighting.cs
similarity index 81%
copy from src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestMultiTermHighlighting.cs
copy to src/dotnet/Lucene.Net.Tests.ICU/Search/PostingsHighlight/TestICUMultiTermHighlighting.cs
index eeb6d47..2849b81 100644
--- a/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestMultiTermHighlighting.cs
+++ b/src/dotnet/Lucene.Net.Tests.ICU/Search/PostingsHighlight/TestICUMultiTermHighlighting.cs
@@ -1,18 +1,15 @@
#if FEATURE_BREAKITERATOR
using Lucene.Net.Analysis;
+using Lucene.Net.Attributes;
using Lucene.Net.Documents;
using Lucene.Net.Index;
-using Lucene.Net.Search;
using Lucene.Net.Search.Spans;
using Lucene.Net.Store;
using Lucene.Net.Support;
using Lucene.Net.Util;
using NUnit.Framework;
using System;
-using System.Collections.Generic;
-using System.Linq;
using System.Text;
-using System.Threading.Tasks;
using IndexOptions = Lucene.Net.Index.IndexOptions;
namespace Lucene.Net.Search.PostingsHighlight
@@ -35,13 +32,22 @@ namespace Lucene.Net.Search.PostingsHighlight
*/
/// <summary>
- /// Some tests that override <see cref="PostingsHighlighter.GetIndexAnalyzer(string)"/> to
+ /// Some tests that override <see cref="ICUPostingsHighlighter.GetIndexAnalyzer(string)"/> to
/// highlight wilcard, fuzzy, etc queries.
/// </summary>
+ /// <remarks>
+ /// LUCENENET specific - Modified the behavior of the PostingsHighlighter in Java to return the
+ /// org.ibm.icu.BreakIterator version 60.1 instead of java.text.BreakIterator and modified the original Lucene
+ /// tests to pass, then ported to .NET. There are no changes in this class from that of Lucene 4.8.1.
+ /// <para/>
+ /// Although the ICU <see cref="ICU4N.Text.BreakIterator"/> acts slightly different than the JDK's verision, using the default
+ /// behavior of the ICU <see cref="ICU4N.Text.BreakIterator"/> is the most logical default to use in .NET. It is the same
+ /// default that was chosen in Apache Harmony.
+ /// </remarks>
[SuppressCodecs("MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Lucene3x")]
- public class TestMultiTermHighlighting : LuceneTestCase
+ public class TestICUMultiTermHighlighting : LuceneTestCase
{
- internal class PostingsHighlighterAnalyzerHelper : PostingsHighlighter
+ internal class PostingsHighlighterAnalyzerHelper : ICUPostingsHighlighter
{
private readonly Analyzer analyzer;
@@ -56,7 +62,7 @@ namespace Lucene.Net.Search.PostingsHighlight
}
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestWildcards()
{
Directory dir = NewDirectory();
@@ -81,14 +87,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- // PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- // {
- // return analyzer;
- // }
- //};
+ ICUPostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
Query query = new WildcardQuery(new Term("body", "te*"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -112,7 +111,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestOnePrefix()
{
Directory dir = NewDirectory();
@@ -137,14 +136,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
+ ICUPostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
Query query = new PrefixQuery(new Term("body", "te"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -168,7 +160,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestOneRegexp()
{
Directory dir = NewDirectory();
@@ -193,14 +185,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
+ ICUPostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
Query query = new RegexpQuery(new Term("body", "te.*"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -224,7 +209,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestOneFuzzy()
{
Directory dir = NewDirectory();
@@ -249,14 +234,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
+ ICUPostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
Query query = new FuzzyQuery(new Term("body", "tets"), 1);
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -289,7 +267,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestRanges()
{
Directory dir = NewDirectory();
@@ -314,14 +292,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
+ ICUPostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
Query query = TermRangeQuery.NewStringRange("body", "ta", "tf", true, true);
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -403,7 +374,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestWildcardInBoolean()
{
Directory dir = NewDirectory();
@@ -428,14 +399,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
+ ICUPostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
BooleanQuery query = new BooleanQuery();
query.Add(new WildcardQuery(new Term("body", "te*")), Occur.SHOULD);
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
@@ -460,7 +424,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestWildcardInDisjunctionMax()
{
Directory dir = NewDirectory();
@@ -485,14 +449,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
+ ICUPostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
DisjunctionMaxQuery query = new DisjunctionMaxQuery(0);
query.Add(new WildcardQuery(new Term("body", "te*")));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
@@ -506,7 +463,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestSpanWildcard()
{
Directory dir = NewDirectory();
@@ -531,14 +488,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
+ ICUPostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
Query query = new SpanMultiTermQueryWrapper<WildcardQuery>(new WildcardQuery(new Term("body", "te*")));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -551,7 +501,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestSpanOr()
{
Directory dir = NewDirectory();
@@ -576,14 +526,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
+ ICUPostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
SpanQuery childQuery = new SpanMultiTermQueryWrapper<WildcardQuery>(new WildcardQuery(new Term("body", "te*")));
Query query = new SpanOrQuery(new SpanQuery[] { childQuery });
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
@@ -597,7 +540,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestSpanNear()
{
Directory dir = NewDirectory();
@@ -622,14 +565,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
+ ICUPostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
SpanQuery childQuery = new SpanMultiTermQueryWrapper<WildcardQuery>(new WildcardQuery(new Term("body", "te*")));
Query query = new SpanNearQuery(new SpanQuery[] { childQuery }, 0, true);
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
@@ -643,7 +579,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestSpanNot()
{
Directory dir = NewDirectory();
@@ -668,14 +604,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
+ ICUPostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
SpanQuery include = new SpanMultiTermQueryWrapper<WildcardQuery>(new WildcardQuery(new Term("body", "te*")));
SpanQuery exclude = new SpanTermQuery(new Term("body", "bogus"));
Query query = new SpanNotQuery(include, exclude);
@@ -690,7 +619,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestSpanPositionCheck()
{
Directory dir = NewDirectory();
@@ -715,14 +644,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
+ ICUPostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
SpanQuery childQuery = new SpanMultiTermQueryWrapper<WildcardQuery>(new WildcardQuery(new Term("body", "te*")));
Query query = new SpanFirstQuery(childQuery, 1000000);
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
@@ -799,7 +721,7 @@ namespace Lucene.Net.Search.PostingsHighlight
/** Runs a query with two MTQs and confirms the formatter
* can tell which query matched which hit. */
- [Test]
+ [Test, LuceneNetSpecific]
public void TestWhichMTQMatched()
{
Directory dir = NewDirectory();
@@ -822,14 +744,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
- //PostingsHighlighter highlighter = new PostingsHighlighter() {
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
- // };
+ ICUPostingsHighlighter highlighter = new PostingsHighlighterAnalyzerHelper(analyzer);
BooleanQuery query = new BooleanQuery();
query.Add(new WildcardQuery(new Term("body", "te*")), Occur.SHOULD);
query.Add(new WildcardQuery(new Term("body", "one")), Occur.SHOULD);
@@ -846,65 +761,6 @@ namespace Lucene.Net.Search.PostingsHighlight
// matching term's text into the result:
highlighter = new PostingsHighlighterAnalyzerAndFormatterHelper(analyzer, new PassageFormatterHelper());
- //highlighter = new PostingsHighlighter()
- //{
- // @Override
- // protected Analyzer getIndexAnalyzer(String field)
- //{
- // return analyzer;
- //}
-
- //@Override
- // protected PassageFormatter getFormatter(String field)
- //{
- // return new PassageFormatter() {
-
- // @Override
- // public Object format(Passage passages[], String content)
- //{
- // // Copied from DefaultPassageFormatter, but
- // // tweaked to include the matched term:
- // StringBuilder sb = new StringBuilder();
- // int pos = 0;
- // for (Passage passage : passages)
- // {
- // // don't add ellipsis if its the first one, or if its connected.
- // if (passage.startOffset > pos && pos > 0)
- // {
- // sb.append("... ");
- // }
- // pos = passage.startOffset;
- // for (int i = 0; i < passage.numMatches; i++)
- // {
- // int start = passage.matchStarts[i];
- // int end = passage.matchEnds[i];
- // // its possible to have overlapping terms
- // if (start > pos)
- // {
- // sb.append(content, pos, start);
- // }
- // if (end > pos)
- // {
- // sb.append("<b>");
- // sb.append(content, Math.max(pos, start), end);
- // sb.append('(');
- // sb.append(passage.getMatchTerms()[i].utf8ToString());
- // sb.append(')');
- // sb.append("</b>");
- // pos = end;
- // }
- // }
- // // its possible a "term" from the analyzer could span a sentence boundary.
- // sb.append(content, pos, Math.max(pos, passage.endOffset));
- // pos = passage.endOffset;
- // }
- // return sb.toString();
- //}
- // };
- // }
- // };
-
-
assertEquals(1, topDocs.TotalHits);
snippets = highlighter.Highlight("body", query, searcher, topDocs);
assertEquals(1, snippets.Length);
diff --git a/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighter.cs b/src/dotnet/Lucene.Net.Tests.ICU/Search/PostingsHighlight/TestICUPostingsHighlighter.cs
similarity index 91%
copy from src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighter.cs
copy to src/dotnet/Lucene.Net.Tests.ICU/Search/PostingsHighlight/TestICUPostingsHighlighter.cs
index b96efea..b5e1f47 100644
--- a/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighter.cs
+++ b/src/dotnet/Lucene.Net.Tests.ICU/Search/PostingsHighlight/TestICUPostingsHighlighter.cs
@@ -1,6 +1,7 @@
#if FEATURE_BREAKITERATOR
using ICU4N.Text;
using Lucene.Net.Analysis;
+using Lucene.Net.Attributes;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.Support;
@@ -33,10 +34,20 @@ namespace Lucene.Net.Search.PostingsHighlight
* limitations under the License.
*/
+ /// <summary>
+ /// LUCENENET specific - Modified the behavior of the PostingsHighlighter in Java to return the
+ /// org.ibm.icu.BreakIterator version 60.1 instead of java.text.BreakIterator and modified the original Lucene
+ /// tests to pass, then ported to .NET. The only change required was that of the TestEmptyHighlights method
+ /// which breaks the sentence in a different place than in the JDK.
+ /// <para/>
+ /// Although the ICU <see cref="BreakIterator"/> acts slightly different than the JDK's verision, using the default
+ /// behavior of the ICU <see cref="BreakIterator"/> is the most logical default to use in .NET. It is the same
+ /// default that was chosen in Apache Harmony.
+ /// </summary>
[SuppressCodecs("MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Lucene3x")]
- public class TestPostingsHighlighter : LuceneTestCase
+ public class TestICUPostingsHighlighter : LuceneTestCase
{
- [Test]
+ [Test, LuceneNetSpecific]
public void TestBasics()
{
Directory dir = NewDirectory();
@@ -59,7 +70,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
Query query = new TermQuery(new Term("body", "highlighting"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -72,7 +83,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestFormatWithMatchExceedingContentLength2()
{
@@ -85,7 +96,7 @@ namespace Lucene.Net.Search.PostingsHighlight
assertEquals("123 <b>TEST</b> 01234 TE", snippets[0]);
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestFormatWithMatchExceedingContentLength3()
{
@@ -98,7 +109,7 @@ namespace Lucene.Net.Search.PostingsHighlight
assertEquals("123 5678 01234 TE", snippets[0]);
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestFormatWithMatchExceedingContentLength()
{
@@ -143,7 +154,7 @@ namespace Lucene.Net.Search.PostingsHighlight
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(1, topDocs.TotalHits);
- PostingsHighlighter highlighter = new PostingsHighlighter(maxLength);
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter(maxLength);
String[] snippets = highlighter.Highlight("body", query, searcher, topDocs);
@@ -153,7 +164,7 @@ namespace Lucene.Net.Search.PostingsHighlight
}
// simple test highlighting last word.
- [Test]
+ [Test, LuceneNetSpecific]
public void TestHighlightLastWord()
{
Directory dir = NewDirectory();
@@ -174,7 +185,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
Query query = new TermQuery(new Term("body", "test"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(1, topDocs.TotalHits);
@@ -187,7 +198,7 @@ namespace Lucene.Net.Search.PostingsHighlight
}
// simple test with one sentence documents.
- [Test]
+ [Test, LuceneNetSpecific]
public void TestOneSentence()
{
Directory dir = NewDirectory();
@@ -211,7 +222,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
Query query = new TermQuery(new Term("body", "test"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -225,7 +236,7 @@ namespace Lucene.Net.Search.PostingsHighlight
}
// simple test with multiple values that make a result longer than maxLength.
- [Test]
+ [Test, LuceneNetSpecific]
public void TestMaxLengthWithMultivalue()
{
Directory dir = NewDirectory();
@@ -251,7 +262,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter(40);
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter(40);
Query query = new TermQuery(new Term("body", "field"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(1, topDocs.TotalHits);
@@ -264,7 +275,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestMultipleFields()
{
Directory dir = NewDirectory();
@@ -291,7 +302,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
BooleanQuery query = new BooleanQuery();
query.Add(new TermQuery(new Term("body", "highlighting")), Occur.SHOULD);
query.Add(new TermQuery(new Term("title", "best")), Occur.SHOULD);
@@ -307,7 +318,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestMultipleTerms()
{
Directory dir = NewDirectory();
@@ -330,7 +341,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
BooleanQuery query = new BooleanQuery();
query.Add(new TermQuery(new Term("body", "highlighting")), Occur.SHOULD);
query.Add(new TermQuery(new Term("body", "just")), Occur.SHOULD);
@@ -346,7 +357,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestMultiplePassages()
{
Directory dir = NewDirectory();
@@ -369,7 +380,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
Query query = new TermQuery(new Term("body", "test"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -382,7 +393,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestUserFailedToIndexOffsets()
{
Directory dir = NewDirectory();
@@ -409,7 +420,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
Query query = new TermQuery(new Term("body", "test"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -440,7 +451,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestBuddhism()
{
String text = "This eight-volume set brings together seminal papers in Buddhist studies from a vast " +
@@ -470,7 +481,7 @@ namespace Lucene.Net.Search.PostingsHighlight
query.Add(new Term("body", "origins"));
TopDocs topDocs = searcher.Search(query, 10);
assertEquals(1, topDocs.TotalHits);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
String[] snippets = highlighter.Highlight("body", query, searcher, topDocs, 2);
assertEquals(1, snippets.Length);
assertTrue(snippets[0].Contains("<b>Buddhist</b> <b>origins</b>"));
@@ -478,7 +489,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestCuriousGeorge()
{
String text = "It’s the formula for success for preschoolers—Curious George and fire trucks! " +
@@ -502,7 +513,7 @@ namespace Lucene.Net.Search.PostingsHighlight
query.Add(new Term("body", "george"));
TopDocs topDocs = searcher.Search(query, 10);
assertEquals(1, topDocs.TotalHits);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
String[] snippets = highlighter.Highlight("body", query, searcher, topDocs, 2);
assertEquals(1, snippets.Length);
assertFalse(snippets[0].Contains("<b>Curious</b>Curious"));
@@ -510,7 +521,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestCambridgeMA()
{
String text;
@@ -537,7 +548,7 @@ namespace Lucene.Net.Search.PostingsHighlight
query.Add(new TermQuery(new Term("body", "massachusetts")), Occur.SHOULD);
TopDocs topDocs = searcher.Search(query, 10);
assertEquals(1, topDocs.TotalHits);
- PostingsHighlighter highlighter = new PostingsHighlighter(int.MaxValue - 1);
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter(int.MaxValue - 1);
String[] snippets = highlighter.Highlight("body", query, searcher, topDocs, 2);
assertEquals(1, snippets.Length);
assertTrue(snippets[0].Contains("<b>Square</b>"));
@@ -546,7 +557,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestPassageRanking()
{
Directory dir = NewDirectory();
@@ -567,7 +578,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
Query query = new TermQuery(new Term("body", "test"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(1, topDocs.TotalHits);
@@ -579,7 +590,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestBooleanMustNot()
{
Directory dir = NewDirectory();
@@ -601,7 +612,7 @@ namespace Lucene.Net.Search.PostingsHighlight
query2.Add(new TermQuery(new Term("body", "both")), Occur.MUST_NOT);
TopDocs topDocs = searcher.Search(query, 10);
assertEquals(1, topDocs.TotalHits);
- PostingsHighlighter highlighter = new PostingsHighlighter(int.MaxValue - 1);
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter(int.MaxValue - 1);
String[] snippets = highlighter.Highlight("body", query, searcher, topDocs, 2);
assertEquals(1, snippets.Length);
assertFalse(snippets[0].Contains("<b>both</b>"));
@@ -609,7 +620,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestHighlightAllText()
{
Directory dir = NewDirectory();
@@ -630,7 +641,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new WholeBreakIteratorPostingsHighlighter(10000);
+ ICUPostingsHighlighter highlighter = new WholeBreakIteratorPostingsHighlighter(10000);
Query query = new TermQuery(new Term("body", "test"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(1, topDocs.TotalHits);
@@ -642,7 +653,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- internal class WholeBreakIteratorPostingsHighlighter : PostingsHighlighter
+ internal class WholeBreakIteratorPostingsHighlighter : ICUPostingsHighlighter
{
public WholeBreakIteratorPostingsHighlighter()
: base()
@@ -660,7 +671,7 @@ namespace Lucene.Net.Search.PostingsHighlight
}
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestSpecificDocIDs()
{
Directory dir = NewDirectory();
@@ -683,7 +694,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
Query query = new TermQuery(new Term("body", "highlighting"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(2, topDocs.TotalHits);
@@ -700,7 +711,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestCustomFieldValueSource()
{
Directory dir = NewDirectory();
@@ -721,7 +732,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new LoadFieldValuesPostingsHighlighter(10000, text);
+ ICUPostingsHighlighter highlighter = new LoadFieldValuesPostingsHighlighter(10000, text);
Query query = new TermQuery(new Term("body", "test"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
@@ -756,7 +767,7 @@ namespace Lucene.Net.Search.PostingsHighlight
/** Make sure highlighter returns first N sentences if
* there were no hits. */
- [Test]
+ [Test, LuceneNetSpecific]
public void TestEmptyHighlights()
{
Directory dir = NewDirectory();
@@ -776,12 +787,12 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
Query query = new TermQuery(new Term("body", "highlighting"));
int[] docIDs = new int[] { 0 };
String[] snippets = highlighter.HighlightFields(new String[] { "body" }, query, searcher, docIDs, new int[] { 2 })["body"];
assertEquals(1, snippets.Length);
- assertEquals("test this is. another sentence this test has. ", snippets[0]);
+ assertEquals("test this is. another sentence this test has. far away is that planet.", snippets[0]);
ir.Dispose();
dir.Dispose();
@@ -789,7 +800,7 @@ namespace Lucene.Net.Search.PostingsHighlight
/** Make sure highlighter we can customize how emtpy
* highlight is returned. */
- [Test]
+ [Test, LuceneNetSpecific]
public void TestCustomEmptyHighlights()
{
Directory dir = NewDirectory();
@@ -809,7 +820,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new GetEmptyHighlightPostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new GetEmptyHighlightPostingsHighlighter();
Query query = new TermQuery(new Term("body", "highlighting"));
int[] docIDs = new int[] { 0 };
String[] snippets = highlighter.HighlightFields(new String[] { "body" }, query, searcher, docIDs, new int[] { 2 })["body"];
@@ -820,7 +831,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- internal class GetEmptyHighlightPostingsHighlighter : PostingsHighlighter
+ internal class GetEmptyHighlightPostingsHighlighter : ICUPostingsHighlighter
{
protected override Passage[] GetEmptyHighlight(string fieldName, BreakIterator bi, int maxPassages)
{
@@ -830,7 +841,7 @@ namespace Lucene.Net.Search.PostingsHighlight
/** Make sure highlighter returns whole text when there
* are no hits and BreakIterator is null. */
- [Test]
+ [Test, LuceneNetSpecific]
public void TestEmptyHighlightsWhole()
{
Directory dir = NewDirectory();
@@ -850,7 +861,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new WholeBreakIteratorPostingsHighlighter(10000);
+ ICUPostingsHighlighter highlighter = new WholeBreakIteratorPostingsHighlighter(10000);
Query query = new TermQuery(new Term("body", "highlighting"));
int[] docIDs = new int[] { 0 };
String[] snippets = highlighter.HighlightFields(new String[] { "body" }, query, searcher, docIDs, new int[] { 2 })["body"];
@@ -863,7 +874,7 @@ namespace Lucene.Net.Search.PostingsHighlight
/** Make sure highlighter is OK with entirely missing
* field. */
- [Test]
+ [Test, LuceneNetSpecific]
public void TestFieldIsMissing()
{
Directory dir = NewDirectory();
@@ -883,7 +894,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
Query query = new TermQuery(new Term("bogus", "highlighting"));
int[] docIDs = new int[] { 0 };
String[] snippets = highlighter.HighlightFields(new String[] { "bogus" }, query, searcher, docIDs, new int[] { 2 })["bogus"];
@@ -894,7 +905,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestFieldIsJustSpace()
{
Directory dir = NewDirectory();
@@ -918,7 +929,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
int docID = searcher.Search(new TermQuery(new Term("id", "id")), 1).ScoreDocs[0].Doc;
Query query = new TermQuery(new Term("body", "highlighting"));
@@ -932,7 +943,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestFieldIsEmptyString()
{
Directory dir = NewDirectory();
@@ -956,7 +967,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
int docID = searcher.Search(new TermQuery(new Term("id", "id")), 1).ScoreDocs[0].Doc;
Query query = new TermQuery(new Term("body", "highlighting"));
@@ -970,7 +981,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestMultipleDocs()
{
Directory dir = NewDirectory();
@@ -1004,7 +1015,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
Query query = new TermQuery(new Term("body", "answer"));
TopDocs hits = searcher.Search(query, numDocs);
assertEquals(numDocs, hits.TotalHits);
@@ -1027,7 +1038,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestMultipleSnippetSizes()
{
Directory dir = NewDirectory();
@@ -1051,7 +1062,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new PostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ICUPostingsHighlighter();
BooleanQuery query = new BooleanQuery();
query.Add(new TermQuery(new Term("body", "test")), Occur.SHOULD);
query.Add(new TermQuery(new Term("title", "test")), Occur.SHOULD);
@@ -1064,7 +1075,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- [Test]
+ [Test, LuceneNetSpecific]
public void TestEncode()
{
Directory dir = NewDirectory();
@@ -1085,7 +1096,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new GetFormatterPostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new GetFormatterPostingsHighlighter();
Query query = new TermQuery(new Term("body", "highlighting"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(1, topDocs.TotalHits);
@@ -1097,7 +1108,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- internal class GetFormatterPostingsHighlighter : PostingsHighlighter
+ internal class GetFormatterPostingsHighlighter : ICUPostingsHighlighter
{
protected override PassageFormatter GetFormatter(string field)
{
@@ -1106,7 +1117,7 @@ namespace Lucene.Net.Search.PostingsHighlight
}
/** customizing the gap separator to force a sentence break */
- [Test]
+ [Test, LuceneNetSpecific]
public void TestGapSeparator()
{
Directory dir = NewDirectory();
@@ -1133,7 +1144,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new GetMultiValuedSeparatorPostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new GetMultiValuedSeparatorPostingsHighlighter();
Query query = new TermQuery(new Term("body", "field"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
@@ -1146,7 +1157,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- internal class GetMultiValuedSeparatorPostingsHighlighter : PostingsHighlighter
+ internal class GetMultiValuedSeparatorPostingsHighlighter : ICUPostingsHighlighter
{
protected override char GetMultiValuedSeparator(string field)
{
@@ -1156,7 +1167,7 @@ namespace Lucene.Net.Search.PostingsHighlight
}
// LUCENE-4906
- [Test]
+ [Test, LuceneNetSpecific]
public void TestObjectFormatter()
{
Directory dir = NewDirectory();
@@ -1177,7 +1188,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new ObjectFormatterPostingsHighlighter();
+ ICUPostingsHighlighter highlighter = new ObjectFormatterPostingsHighlighter();
Query query = new TermQuery(new Term("body", "highlighting"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
@@ -1193,7 +1204,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- internal class ObjectFormatterPostingsHighlighter : PostingsHighlighter
+ internal class ObjectFormatterPostingsHighlighter : ICUPostingsHighlighter
{
protected override PassageFormatter GetFormatter(string field)
{
diff --git a/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighterRanking.cs b/src/dotnet/Lucene.Net.Tests.ICU/Search/PostingsHighlight/TestICUPostingsHighlighterRanking.cs
similarity index 87%
copy from src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighterRanking.cs
copy to src/dotnet/Lucene.Net.Tests.ICU/Search/PostingsHighlight/TestICUPostingsHighlighterRanking.cs
index 6d94c24..1f73a12 100644
--- a/src/Lucene.Net.Tests.Highlighter/PostingsHighlight/TestPostingsHighlighterRanking.cs
+++ b/src/dotnet/Lucene.Net.Tests.ICU/Search/PostingsHighlight/TestICUPostingsHighlighterRanking.cs
@@ -1,5 +1,6 @@
#if FEATURE_BREAKITERATOR
using Lucene.Net.Analysis;
+using Lucene.Net.Attributes;
using Lucene.Net.Documents;
using Lucene.Net.Index;
using Lucene.Net.Store;
@@ -30,15 +31,24 @@ namespace Lucene.Net.Search.PostingsHighlight
* limitations under the License.
*/
+ /// <summary>
+ /// LUCENENET specific - Modified the behavior of the PostingsHighlighter in Java to return the
+ /// org.ibm.icu.BreakIterator version 60.1 instead of java.text.BreakIterator and modified the original Lucene
+ /// tests to pass, then ported to .NET. There are no changes in this class from that of Lucene 4.8.1.
+ /// <para/>
+ /// Although the ICU <see cref="ICU4N.Text.BreakIterator"/> acts slightly different than the JDK's verision, using the default
+ /// behavior of the ICU <see cref="ICU4N.Text.BreakIterator"/> is the most logical default to use in .NET. It is the same
+ /// default that was chosen in Apache Harmony.
+ /// </summary>
[SuppressCodecs("MockFixedIntBlock", "MockVariableIntBlock", "MockSep", "MockRandom", "Lucene3x")]
- public class TestPostingsHighlighterRanking : LuceneTestCase
+ public class TestICUPostingsHighlighterRanking : LuceneTestCase
{
/// <summary>
/// indexes a bunch of gibberish, and then highlights top(n).
/// asserts that top(n) highlights is a subset of top(n+1) up to some max N
/// </summary>
// TODO: this only tests single-valued fields. we should also index multiple values per field!
- [Test]
+ [Test, LuceneNetSpecific]
public void TestRanking()
{
// number of documents: we will check each one
@@ -100,7 +110,7 @@ namespace Lucene.Net.Search.PostingsHighlight
}
}
- internal class CheckQueryPostingsHighlighter : PostingsHighlighter
+ internal class CheckQueryPostingsHighlighter : ICUPostingsHighlighter
{
internal FakePassageFormatter f = new FakePassageFormatter();
@@ -122,25 +132,6 @@ namespace Lucene.Net.Search.PostingsHighlight
{
CheckQueryPostingsHighlighter p1 = new CheckQueryPostingsHighlighter(int.MaxValue - 1);
CheckQueryPostingsHighlighter p2 = new CheckQueryPostingsHighlighter(int.MaxValue - 1);
- // FakePassageFormatter f1 = new FakePassageFormatter();
- // PostingsHighlighter p1 = new PostingsHighlighter(int.MaxValue - 1) {
- // @Override
- // protected PassageFormatter getFormatter(String field)
- //{
- // assertEquals("body", field);
- // return f1;
- //}
- // };
-
- // FakePassageFormatter f2 = new FakePassageFormatter();
- //PostingsHighlighter p2 = new PostingsHighlighter(Integer.MAX_VALUE - 1) {
- // @Override
- // protected PassageFormatter getFormatter(String field)
- //{
- // assertEquals("body", field);
- // return f2;
- //}
- // };
BooleanQuery bq = new BooleanQuery(false);
bq.Add(query, Occur.MUST);
@@ -277,7 +268,7 @@ namespace Lucene.Net.Search.PostingsHighlight
}
/** sets b=0 to disable passage length normalization */
- [Test]
+ [Test, LuceneNetSpecific]
public void TestCustomB()
{
Directory dir = NewDirectory();
@@ -299,14 +290,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new CustomBPostingsHighlighter();
- //PostingsHighlighter highlighter = new PostingsHighlighter(10000) {
- // @Override
- // protected PassageScorer getScorer(String field)
- //{
- // return new PassageScorer(1.2f, 0, 87);
- //}
- // };
+ ICUPostingsHighlighter highlighter = new CustomBPostingsHighlighter();
Query query = new TermQuery(new Term("body", "test"));
TopDocs topDocs = searcher.Search(query, null, 10, Sort.INDEXORDER);
assertEquals(1, topDocs.TotalHits);
@@ -318,7 +302,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- internal class CustomBPostingsHighlighter : PostingsHighlighter
+ internal class CustomBPostingsHighlighter : ICUPostingsHighlighter
{
protected override PassageScorer GetScorer(string field)
{
@@ -327,7 +311,7 @@ namespace Lucene.Net.Search.PostingsHighlight
}
/** sets k1=0 for simple coordinate-level match (# of query terms present) */
- [Test]
+ [Test, LuceneNetSpecific]
public void TestCustomK1()
{
Directory dir = NewDirectory();
@@ -350,14 +334,7 @@ namespace Lucene.Net.Search.PostingsHighlight
iw.Dispose();
IndexSearcher searcher = NewSearcher(ir);
- PostingsHighlighter highlighter = new CustomK1PostingsHighlighter();
- //PostingsHighlighter highlighter = new PostingsHighlighter(10000) {
- // @Override
- // protected PassageScorer getScorer(String field)
- //{
- // return new PassageScorer(0, 0.75f, 87);
- //}
- // };
+ ICUPostingsHighlighter highlighter = new CustomK1PostingsHighlighter();
BooleanQuery query = new BooleanQuery();
query.Add(new TermQuery(new Term("body", "foo")), Occur.SHOULD);
query.Add(new TermQuery(new Term("body", "bar")), Occur.SHOULD);
@@ -371,7 +348,7 @@ namespace Lucene.Net.Search.PostingsHighlight
dir.Dispose();
}
- internal class CustomK1PostingsHighlighter : PostingsHighlighter
+ internal class CustomK1PostingsHighlighter : ICUPostingsHighlighter
{
public CustomK1PostingsHighlighter()
: base(10000)
diff --git a/src/dotnet/Lucene.Net.ICU/Data/jdk7/jdksent.rbbi b/src/dotnet/Lucene.Net.Tests.ICU/Support/Data/jdk7/jdksent.rbbi
similarity index 100%
rename from src/dotnet/Lucene.Net.ICU/Data/jdk7/jdksent.rbbi
rename to src/dotnet/Lucene.Net.Tests.ICU/Support/Data/jdk7/jdksent.rbbi
diff --git a/src/dotnet/Lucene.Net.ICU/Data/jdk7/jdkword.rbbi b/src/dotnet/Lucene.Net.Tests.ICU/Support/Data/jdk7/jdkword.rbbi
similarity index 100%
rename from src/dotnet/Lucene.Net.ICU/Data/jdk7/jdkword.rbbi
rename to src/dotnet/Lucene.Net.Tests.ICU/Support/Data/jdk7/jdkword.rbbi
diff --git a/src/dotnet/Lucene.Net.ICU/Support/JDKBreakIterator.cs b/src/dotnet/Lucene.Net.Tests.ICU/Support/JDKBreakIterator.cs
similarity index 98%
rename from src/dotnet/Lucene.Net.ICU/Support/JDKBreakIterator.cs
rename to src/dotnet/Lucene.Net.Tests.ICU/Support/JDKBreakIterator.cs
index 266b36a..169b04c 100644
--- a/src/dotnet/Lucene.Net.ICU/Support/JDKBreakIterator.cs
+++ b/src/dotnet/Lucene.Net.Tests.ICU/Support/JDKBreakIterator.cs
@@ -1,10 +1,9 @@
using ICU4N.Text;
-using Lucene.Net.Support;
using System.Globalization;
using System.IO;
using System.Reflection;
-namespace Lucene.Net.ICU.Support
+namespace Lucene.Net.Support
{
/// <summary>
/// Static methods to create <see cref="BreakIterator"/> instances that behave (somewhat) like the JDK.
diff --git a/src/dotnet/Lucene.Net.Tests.ICU/Support/PostingsHighlighter.cs b/src/dotnet/Lucene.Net.Tests.ICU/Support/PostingsHighlighter.cs
new file mode 100644
index 0000000..71bf5c3
--- /dev/null
+++ b/src/dotnet/Lucene.Net.Tests.ICU/Support/PostingsHighlighter.cs
@@ -0,0 +1,29 @@
+using ICU4N.Text;
+using Lucene.Net.Support;
+using System.Globalization;
+
+namespace Lucene.Net.Search.PostingsHighlight
+{
+ /// <summary>
+ /// Mock of the original Lucene <see cref="PostingsHighlighter"/> that is backed
+ /// by a <see cref="JdkBreakIterator"/> with custom rules to act
+ /// (sort of) like the JDK. This is just to verify we can make the behavior work
+ /// similar to the implementation in Lucene by customizing the <see cref="BreakIterator"/>.
+ /// </summary>
+ public class PostingsHighlighter : ICUPostingsHighlighter
+ {
+ public PostingsHighlighter()
+ : base()
+ {
+ }
+
+ public PostingsHighlighter(int maxLength)
+ : base(maxLength)
+ {
+ }
+ protected override BreakIterator GetBreakIterator(string field)
+ {
+ return JdkBreakIterator.GetSentenceInstance(CultureInfo.InvariantCulture);
+ }
+ }
+}
diff --git a/src/dotnet/Lucene.Net.Tests.ICU/Support/TestJDKBreakIterator.cs b/src/dotnet/Lucene.Net.Tests.ICU/Support/TestJDKBreakIterator.cs
index 04a09b7..ea8651c 100644
--- a/src/dotnet/Lucene.Net.Tests.ICU/Support/TestJDKBreakIterator.cs
+++ b/src/dotnet/Lucene.Net.Tests.ICU/Support/TestJDKBreakIterator.cs
@@ -1,6 +1,6 @@
using ICU4N.Text;
using Lucene.Net.Attributes;
-using Lucene.Net.ICU.Support;
+using Lucene.Net.Support;
using NUnit.Framework;
using System;
diff --git a/src/dotnet/Lucene.Net.ICU/Support/jdksent.brk b/src/dotnet/Lucene.Net.Tests.ICU/Support/jdksent.brk
similarity index 100%
rename from src/dotnet/Lucene.Net.ICU/Support/jdksent.brk
rename to src/dotnet/Lucene.Net.Tests.ICU/Support/jdksent.brk
diff --git a/src/dotnet/Lucene.Net.ICU/Support/jdkword.brk b/src/dotnet/Lucene.Net.Tests.ICU/Support/jdkword.brk
similarity index 100%
rename from src/dotnet/Lucene.Net.ICU/Support/jdkword.brk
rename to src/dotnet/Lucene.Net.Tests.ICU/Support/jdkword.brk