You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2016/12/06 15:11:38 UTC
[03/58] lucenenet git commit: WIP on QueryParsers.Flexible
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.Tests.QueryParser/Flexible/Precedence/TestPrecedenceQueryParser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.QueryParser/Flexible/Precedence/TestPrecedenceQueryParser.cs b/src/Lucene.Net.Tests.QueryParser/Flexible/Precedence/TestPrecedenceQueryParser.cs
new file mode 100644
index 0000000..9c418a8
--- /dev/null
+++ b/src/Lucene.Net.Tests.QueryParser/Flexible/Precedence/TestPrecedenceQueryParser.cs
@@ -0,0 +1,727 @@
+\ufeffusing Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Tokenattributes;
+using Lucene.Net.Documents;
+using Lucene.Net.QueryParsers.Flexible.Core;
+using Lucene.Net.QueryParsers.Flexible.Standard.Config;
+using Lucene.Net.QueryParsers.Flexible.Standard.Parser;
+using Lucene.Net.Search;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using Lucene.Net.Util.Automaton;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Precedence
+{
+ /// <summary>
+ /// This test case tests {@link PrecedenceQueryParser}.
+ /// <para/>
+ /// It contains all tests from {@link QueryParserTestBase}
+ /// with some adjusted to fit the precedence requirement, plus some precedence test cases.
+ /// </summary>
+ /// <seealso cref="QueryParserTestBase"/>
+ //TODO: refactor this to actually extend that class, overriding the tests
+ //that it adjusts to fit the precedence requirement, adding its extra tests.
+ public class TestPrecedenceQueryParser : LuceneTestCase
+ {
+ public static Analyzer qpAnalyzer;
+
+ [TestFixtureSetUp]
+ public static void beforeClass()
+ {
+ qpAnalyzer = new QPTestAnalyzer();
+ }
+
+ [TestFixtureTearDown]
+ public static void afterClass()
+ {
+ qpAnalyzer = null;
+ }
+
+ public sealed class QPTestFilter : TokenFilter
+ {
+ /**
+ * Filter which discards the token 'stop' and which expands the token
+ * 'phrase' into 'phrase1 phrase2'
+ */
+ public QPTestFilter(TokenStream @in)
+ : base(@in)
+ {
+ termAtt = AddAttribute<ICharTermAttribute>();
+ offsetAtt = AddAttribute<IOffsetAttribute>();
+ }
+
+ private bool inPhrase = false;
+
+ private int savedStart = 0;
+ private int savedEnd = 0;
+
+ private readonly ICharTermAttribute termAtt;
+
+ private readonly IOffsetAttribute offsetAtt;
+
+
+ public override bool IncrementToken()
+ {
+ if (inPhrase)
+ {
+ inPhrase = false;
+ termAtt.SetEmpty().Append("phrase2");
+ offsetAtt.SetOffset(savedStart, savedEnd);
+ return true;
+ }
+ else
+ while (input.IncrementToken())
+ if (termAtt.toString().equals("phrase"))
+ {
+ inPhrase = true;
+ savedStart = offsetAtt.StartOffset();
+ savedEnd = offsetAtt.EndOffset();
+ termAtt.SetEmpty().Append("phrase1");
+ offsetAtt.SetOffset(savedStart, savedEnd);
+ return true;
+ }
+ else if (!termAtt.toString().equals("stop"))
+ return true;
+ return false;
+ }
+
+
+ public override void Reset()
+ {
+ base.Reset();
+ this.inPhrase = false;
+ this.savedStart = 0;
+ this.savedEnd = 0;
+ }
+ }
+
+ public sealed class QPTestAnalyzer : Analyzer
+ {
+
+ /** Filters MockTokenizer with StopFilter. */
+
+ public override sealed TokenStreamComponents CreateComponents(string fieldName, TextReader reader)
+ {
+ Tokenizer tokenizer = new MockTokenizer(reader, MockTokenizer.SIMPLE, true);
+ return new TokenStreamComponents(tokenizer, new QPTestFilter(tokenizer));
+ }
+ }
+
+ private int originalMaxClauses;
+
+
+ public override void SetUp()
+ {
+ base.SetUp();
+ originalMaxClauses = BooleanQuery.MaxClauseCount;
+ }
+
+ public PrecedenceQueryParser GetParser(Analyzer a)
+ {
+ if (a == null)
+ a = new MockAnalyzer(Random(), MockTokenizer.SIMPLE, true);
+ PrecedenceQueryParser qp = new PrecedenceQueryParser();
+ qp.Analyzer = (a);
+ qp.SetDefaultOperator(/*StandardQueryConfigHandler.*/Operator.OR); // LUCENENET TODO: Change API back to the way it was..?
+ return qp;
+ }
+
+ public Query GetQuery(string query, Analyzer a)
+ {
+ return (Query)GetParser(a).Parse(query, "field"); // LUCENENET TODO: There was no cast here in the original - perhaps object is the wrong type on the interface
+ }
+
+ public void assertQueryEquals(string query, Analyzer a, string result)
+ {
+ Query q = GetQuery(query, a);
+ String s = q.ToString("field");
+ if (!s.equals(result))
+ {
+ fail("Query /" + query + "/ yielded /" + s + "/, expecting /" + result
+ + "/");
+ }
+ }
+
+ public void assertWildcardQueryEquals(String query, bool lowercase,
+ String result)
+ {
+ PrecedenceQueryParser qp = GetParser(null);
+ qp.LowercaseExpandedTerms = (lowercase);
+ Query q = (Query)qp.Parse(query, "field");
+ String s = q.ToString("field");
+ if (!s.equals(result))
+ {
+ fail("WildcardQuery /" + query + "/ yielded /" + s + "/, expecting /"
+ + result + "/");
+ }
+ }
+
+ public void assertWildcardQueryEquals(String query, String result)
+ {
+ PrecedenceQueryParser qp = GetParser(null);
+ Query q = (Query)qp.Parse(query, "field");
+ String s = q.ToString("field");
+ if (!s.equals(result))
+ {
+ fail("WildcardQuery /" + query + "/ yielded /" + s + "/, expecting /"
+ + result + "/");
+ }
+ }
+
+ public Query getQueryDOA(String query, Analyzer a)
+ {
+ if (a == null)
+ a = new MockAnalyzer(Random(), MockTokenizer.SIMPLE, true);
+ PrecedenceQueryParser qp = new PrecedenceQueryParser();
+ qp.Analyzer = (a);
+ qp.SetDefaultOperator(/*StandardQueryConfigHandler.*/Operator.AND);
+ return (Query)qp.Parse(query, "field");
+ }
+
+ public void assertQueryEqualsDOA(String query, Analyzer a, String result)
+ {
+ Query q = getQueryDOA(query, a);
+ String s = q.ToString("field");
+ if (!s.equals(result))
+ {
+ fail("Query /" + query + "/ yielded /" + s + "/, expecting /" + result
+ + "/");
+ }
+ }
+
+ [Test]
+ public void testSimple()
+ {
+ assertQueryEquals("term term term", null, "term term term");
+ assertQueryEquals("t�rm term term", null, "t�rm term term");
+ assertQueryEquals("�mlaut", null, "�mlaut");
+
+ assertQueryEquals("a AND b", null, "+a +b");
+ assertQueryEquals("(a AND b)", null, "+a +b");
+ assertQueryEquals("c OR (a AND b)", null, "c (+a +b)");
+ assertQueryEquals("a AND NOT b", null, "+a -b");
+ assertQueryEquals("a AND -b", null, "+a -b");
+ assertQueryEquals("a AND !b", null, "+a -b");
+ assertQueryEquals("a && b", null, "+a +b");
+ assertQueryEquals("a && ! b", null, "+a -b");
+
+ assertQueryEquals("a OR b", null, "a b");
+ assertQueryEquals("a || b", null, "a b");
+
+ assertQueryEquals("+term -term term", null, "+term -term term");
+ assertQueryEquals("foo:term AND field:anotherTerm", null,
+ "+foo:term +anotherterm");
+ assertQueryEquals("term AND \"phrase phrase\"", null,
+ "+term +\"phrase phrase\"");
+ assertQueryEquals("\"hello there\"", null, "\"hello there\"");
+ assertTrue(GetQuery("a AND b", null) is BooleanQuery);
+ assertTrue(GetQuery("hello", null) is TermQuery);
+ assertTrue(GetQuery("\"hello there\"", null) is PhraseQuery);
+
+ assertQueryEquals("germ term^2.0", null, "germ term^2.0");
+ assertQueryEquals("(term)^2.0", null, "term^2.0");
+ assertQueryEquals("(germ term)^2.0", null, "(germ term)^2.0");
+ assertQueryEquals("term^2.0", null, "term^2.0");
+ assertQueryEquals("term^2", null, "term^2.0");
+ assertQueryEquals("\"germ term\"^2.0", null, "\"germ term\"^2.0");
+ assertQueryEquals("\"term germ\"^2", null, "\"term germ\"^2.0");
+
+ assertQueryEquals("(foo OR bar) AND (baz OR boo)", null,
+ "+(foo bar) +(baz boo)");
+ assertQueryEquals("((a OR b) AND NOT c) OR d", null, "(+(a b) -c) d");
+ assertQueryEquals("+(apple \"steve jobs\") -(foo bar baz)", null,
+ "+(apple \"steve jobs\") -(foo bar baz)");
+ assertQueryEquals("+title:(dog OR cat) -author:\"bob dole\"", null,
+ "+(title:dog title:cat) -author:\"bob dole\"");
+
+ PrecedenceQueryParser qp = new PrecedenceQueryParser();
+ qp.Analyzer = (new MockAnalyzer(Random()));
+ // make sure OR is the default:
+ assertEquals(/*StandardQueryConfigHandler.*/Operator.OR, qp.GetDefaultOperator());
+ qp.SetDefaultOperator(/*StandardQueryConfigHandler.*/ Operator.AND);
+ assertEquals(/*StandardQueryConfigHandler.*/ Operator.AND, qp.GetDefaultOperator());
+ qp.SetDefaultOperator(/*StandardQueryConfigHandler.*/ Operator.OR);
+ assertEquals(/*StandardQueryConfigHandler.*/ Operator.OR, qp.GetDefaultOperator());
+
+ assertQueryEquals("a OR !b", null, "a -b");
+ assertQueryEquals("a OR ! b", null, "a -b");
+ assertQueryEquals("a OR -b", null, "a -b");
+ }
+
+ [Test]
+ public void testPunct()
+ {
+ Analyzer a = new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false);
+ assertQueryEquals("a&b", a, "a&b");
+ assertQueryEquals("a&&b", a, "a&&b");
+ assertQueryEquals(".NET", a, ".NET");
+ }
+
+ [Test]
+ public void testSlop()
+ {
+ assertQueryEquals("\"term germ\"~2", null, "\"term germ\"~2");
+ assertQueryEquals("\"term germ\"~2 flork", null, "\"term germ\"~2 flork");
+ assertQueryEquals("\"term\"~2", null, "term");
+ assertQueryEquals("\" \"~2 germ", null, "germ");
+ assertQueryEquals("\"term germ\"~2^2", null, "\"term germ\"~2^2.0");
+ }
+
+ [Test]
+ public void testNumber()
+ {
+ // The numbers go away because SimpleAnalzyer ignores them
+ assertQueryEquals("3", null, "");
+ assertQueryEquals("term 1.0 1 2", null, "term");
+ assertQueryEquals("term term1 term2", null, "term term term");
+
+ Analyzer a = new MockAnalyzer(Random());
+ assertQueryEquals("3", a, "3");
+ assertQueryEquals("term 1.0 1 2", a, "term 1.0 1 2");
+ assertQueryEquals("term term1 term2", a, "term term1 term2");
+ }
+
+ [Test]
+ public void testWildcard()
+ {
+ assertQueryEquals("term*", null, "term*");
+ assertQueryEquals("term*^2", null, "term*^2.0");
+ assertQueryEquals("term~", null, "term~2");
+ assertQueryEquals("term~0.7", null, "term~1");
+ assertQueryEquals("term~^3", null, "term~2^3.0");
+ assertQueryEquals("term^3~", null, "term~2^3.0");
+ assertQueryEquals("term*germ", null, "term*germ");
+ assertQueryEquals("term*germ^3", null, "term*germ^3.0");
+
+ assertTrue(GetQuery("term*", null) is PrefixQuery);
+ assertTrue(GetQuery("term*^2", null) is PrefixQuery);
+ assertTrue(GetQuery("term~", null) is FuzzyQuery);
+ assertTrue(GetQuery("term~0.7", null) is FuzzyQuery);
+ FuzzyQuery fq = (FuzzyQuery)GetQuery("term~0.7", null);
+ assertEquals(1, fq.MaxEdits);
+ assertEquals(FuzzyQuery.DefaultPrefixLength, fq.PrefixLength);
+ fq = (FuzzyQuery)GetQuery("term~", null);
+ assertEquals(2, fq.MaxEdits);
+ assertEquals(FuzzyQuery.DefaultPrefixLength, fq.PrefixLength);
+ try
+ {
+ GetQuery("term~1.1", null); // value > 1, throws exception
+ fail();
+ }
+ catch (ParseException pe)
+ {
+ // expected exception
+ }
+ assertTrue(GetQuery("term*germ", null) is WildcardQuery);
+
+ /*
+ * Tests to see that wild card terms are (or are not) properly lower-cased
+ * with propery parser configuration
+ */
+ // First prefix queries:
+ // by default, convert to lowercase:
+ assertWildcardQueryEquals("Term*", true, "term*");
+ // explicitly set lowercase:
+ assertWildcardQueryEquals("term*", true, "term*");
+ assertWildcardQueryEquals("Term*", true, "term*");
+ assertWildcardQueryEquals("TERM*", true, "term*");
+ // explicitly disable lowercase conversion:
+ assertWildcardQueryEquals("term*", false, "term*");
+ assertWildcardQueryEquals("Term*", false, "Term*");
+ assertWildcardQueryEquals("TERM*", false, "TERM*");
+ // Then 'full' wildcard queries:
+ // by default, convert to lowercase:
+ assertWildcardQueryEquals("Te?m", "te?m");
+ // explicitly set lowercase:
+ assertWildcardQueryEquals("te?m", true, "te?m");
+ assertWildcardQueryEquals("Te?m", true, "te?m");
+ assertWildcardQueryEquals("TE?M", true, "te?m");
+ assertWildcardQueryEquals("Te?m*gerM", true, "te?m*germ");
+ // explicitly disable lowercase conversion:
+ assertWildcardQueryEquals("te?m", false, "te?m");
+ assertWildcardQueryEquals("Te?m", false, "Te?m");
+ assertWildcardQueryEquals("TE?M", false, "TE?M");
+ assertWildcardQueryEquals("Te?m*gerM", false, "Te?m*gerM");
+ // Fuzzy queries:
+ assertWildcardQueryEquals("Term~", "term~2");
+ assertWildcardQueryEquals("Term~", true, "term~2");
+ assertWildcardQueryEquals("Term~", false, "Term~2");
+ // Range queries:
+ assertWildcardQueryEquals("[A TO C]", "[a TO c]");
+ assertWildcardQueryEquals("[A TO C]", true, "[a TO c]");
+ assertWildcardQueryEquals("[A TO C]", false, "[A TO C]");
+ }
+
+ [Test]
+ public void testQPA()
+ {
+ assertQueryEquals("term term term", qpAnalyzer, "term term term");
+ assertQueryEquals("term +stop term", qpAnalyzer, "term term");
+ assertQueryEquals("term -stop term", qpAnalyzer, "term term");
+ assertQueryEquals("drop AND stop AND roll", qpAnalyzer, "+drop +roll");
+ assertQueryEquals("term phrase term", qpAnalyzer,
+ "term (phrase1 phrase2) term");
+ // note the parens in this next assertion differ from the original
+ // QueryParser behavior
+ assertQueryEquals("term AND NOT phrase term", qpAnalyzer,
+ "(+term -(phrase1 phrase2)) term");
+ assertQueryEquals("stop", qpAnalyzer, "");
+ assertQueryEquals("stop OR stop AND stop", qpAnalyzer, "");
+ assertTrue(GetQuery("term term term", qpAnalyzer) is BooleanQuery);
+ assertTrue(GetQuery("term +stop", qpAnalyzer) is TermQuery);
+ }
+
+ [Test]
+ public void testRange()
+ {
+ assertQueryEquals("[ a TO z]", null, "[a TO z]");
+ assertTrue(GetQuery("[ a TO z]", null) is TermRangeQuery);
+ assertQueryEquals("[ a TO z ]", null, "[a TO z]");
+ assertQueryEquals("{ a TO z}", null, "{a TO z}");
+ assertQueryEquals("{ a TO z }", null, "{a TO z}");
+ assertQueryEquals("{ a TO z }^2.0", null, "{a TO z}^2.0");
+ assertQueryEquals("[ a TO z] OR bar", null, "[a TO z] bar");
+ assertQueryEquals("[ a TO z] AND bar", null, "+[a TO z] +bar");
+ assertQueryEquals("( bar blar { a TO z}) ", null, "bar blar {a TO z}");
+ assertQueryEquals("gack ( bar blar { a TO z}) ", null,
+ "gack (bar blar {a TO z})");
+ }
+
+ private String escapeDateString(String s)
+ {
+ if (s.Contains(" "))
+ {
+ return "\"" + s + "\"";
+ }
+ else
+ {
+ return s;
+ }
+ }
+
+ public String getDate(String s)
+ {
+ // we use the default Locale since LuceneTestCase randomizes it
+ //DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT, Locale.getDefault());
+ //return DateTools.DateToString(df.parse(s), DateTools.Resolution.DAY);
+
+ //DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT, Locale.getDefault());
+ return DateTools.DateToString(DateTime.Parse(s), DateTools.Resolution.DAY);
+ }
+
+ private String getLocalizedDate(int year, int month, int day,
+ bool extendLastDate)
+ {
+ //// we use the default Locale/TZ since LuceneTestCase randomizes it
+ //DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT, Locale.getDefault());
+ //Calendar calendar = new GregorianCalendar(TimeZone.getDefault(), Locale.getDefault());
+ //calendar.set(year, month, day);
+ //if (extendLastDate)
+ //{
+ // calendar.set(Calendar.HOUR_OF_DAY, 23);
+ // calendar.set(Calendar.MINUTE, 59);
+ // calendar.set(Calendar.SECOND, 59);
+ // calendar.set(Calendar.MILLISECOND, 999);
+ //}
+ //return df.format(calendar.getTime());
+
+
+ var calendar = CultureInfo.CurrentCulture.Calendar;
+ DateTime lastDate = new DateTime(year, month, day, calendar);
+
+ if (extendLastDate)
+ {
+ lastDate = calendar.AddHours(lastDate, 23);
+ lastDate = calendar.AddMinutes(lastDate, 59);
+ lastDate = calendar.AddSeconds(lastDate, 59);
+ lastDate = calendar.AddMilliseconds(lastDate, 999);
+ }
+
+ return lastDate.ToShortDateString();
+ }
+
+ [Test]
+ public void testDateRange()
+ {
+ String startDate = getLocalizedDate(2002, 1, 1, false);
+ String endDate = getLocalizedDate(2002, 1, 4, false);
+ // we use the default Locale/TZ since LuceneTestCase randomizes it
+ //Calendar endDateExpected = new GregorianCalendar(TimeZone.getDefault(), Locale.getDefault());
+ //endDateExpected.set(2002, 1, 4, 23, 59, 59);
+ //endDateExpected.set(Calendar.MILLISECOND, 999);
+ DateTime endDateExpected = new DateTime(2002, 1, 4, 23, 59, 59, 999, new GregorianCalendar());
+
+
+ String defaultField = "default";
+ String monthField = "month";
+ String hourField = "hour";
+ PrecedenceQueryParser qp = new PrecedenceQueryParser(new MockAnalyzer(Random()));
+
+ // LUCENENET TODO: Can we eliminate this nullable??
+ IDictionary<string, DateTools.Resolution?> fieldMap = new HashMap<string, DateTools.Resolution?>();
+ // set a field specific date resolution
+ fieldMap.Put(monthField, DateTools.Resolution.MONTH);
+ qp.SetDateResolution(fieldMap);
+
+ // set default date resolution to MILLISECOND
+ qp.SetDateResolution(DateTools.Resolution.MILLISECOND);
+
+ // set second field specific date resolution
+ fieldMap.Put(hourField, DateTools.Resolution.HOUR);
+ qp.SetDateResolution(fieldMap);
+
+ // for this field no field specific date resolution has been set,
+ // so verify if the default resolution is used
+ assertDateRangeQueryEquals(qp, defaultField, startDate, endDate,
+ endDateExpected, DateTools.Resolution.MILLISECOND);
+
+ // verify if field specific date resolutions are used for these two fields
+ assertDateRangeQueryEquals(qp, monthField, startDate, endDate,
+ endDateExpected, DateTools.Resolution.MONTH);
+
+ assertDateRangeQueryEquals(qp, hourField, startDate, endDate,
+ endDateExpected, DateTools.Resolution.HOUR);
+ }
+
+ /** for testing DateTools support */
+ private String getDate(String s, DateTools.Resolution resolution)
+ {
+ // we use the default Locale since LuceneTestCase randomizes it
+ //DateFormat df = DateFormat.getDateInstance(DateFormat.SHORT, Locale.getDefault());
+ //return getDate(df.parse(s), resolution);
+ return getDate(DateTime.Parse(s), resolution);
+ }
+
+ /** for testing DateTools support */
+ private String getDate(DateTime d, DateTools.Resolution resolution)
+ {
+ return DateTools.DateToString(d, resolution);
+ }
+
+ public void assertQueryEquals(PrecedenceQueryParser qp, String field, String query,
+ String result)
+ {
+ Query q = (Query)qp.Parse(query, field);
+ String s = q.ToString(field);
+ if (!s.equals(result))
+ {
+ fail("Query /" + query + "/ yielded /" + s + "/, expecting /" + result
+ + "/");
+ }
+ }
+
+ public void assertDateRangeQueryEquals(PrecedenceQueryParser qp, String field,
+ String startDate, String endDate, DateTime endDateInclusive,
+ DateTools.Resolution resolution)
+ {
+ assertQueryEquals(qp, field, field + ":[" + escapeDateString(startDate)
+ + " TO " + escapeDateString(endDate) + "]", "["
+ + getDate(startDate, resolution) + " TO "
+ + getDate(endDateInclusive, resolution) + "]");
+ assertQueryEquals(qp, field, field + ":{" + escapeDateString(startDate)
+ + " TO " + escapeDateString(endDate) + "}", "{"
+ + getDate(startDate, resolution) + " TO "
+ + getDate(endDate, resolution) + "}");
+ }
+
+ [Test]
+ public void testEscaped()
+ {
+ Analyzer a = new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false);
+
+ assertQueryEquals("a\\-b:c", a, "a-b:c");
+ assertQueryEquals("a\\+b:c", a, "a+b:c");
+ assertQueryEquals("a\\:b:c", a, "a:b:c");
+ assertQueryEquals("a\\\\b:c", a, "a\\b:c");
+
+ assertQueryEquals("a:b\\-c", a, "a:b-c");
+ assertQueryEquals("a:b\\+c", a, "a:b+c");
+ assertQueryEquals("a:b\\:c", a, "a:b:c");
+ assertQueryEquals("a:b\\\\c", a, "a:b\\c");
+
+ assertQueryEquals("a:b\\-c*", a, "a:b-c*");
+ assertQueryEquals("a:b\\+c*", a, "a:b+c*");
+ assertQueryEquals("a:b\\:c*", a, "a:b:c*");
+
+ assertQueryEquals("a:b\\\\c*", a, "a:b\\c*");
+
+ assertQueryEquals("a:b\\-?c", a, "a:b-?c");
+ assertQueryEquals("a:b\\+?c", a, "a:b+?c");
+ assertQueryEquals("a:b\\:?c", a, "a:b:?c");
+
+ assertQueryEquals("a:b\\\\?c", a, "a:b\\?c");
+
+ assertQueryEquals("a:b\\-c~", a, "a:b-c~2");
+ assertQueryEquals("a:b\\+c~", a, "a:b+c~2");
+ assertQueryEquals("a:b\\:c~", a, "a:b:c~2");
+ assertQueryEquals("a:b\\\\c~", a, "a:b\\c~2");
+
+ assertQueryEquals("[ a\\- TO a\\+ ]", null, "[a- TO a+]");
+ assertQueryEquals("[ a\\: TO a\\~ ]", null, "[a: TO a~]");
+ assertQueryEquals("[ a\\\\ TO a\\* ]", null, "[a\\ TO a*]");
+ }
+
+ [Test]
+ public void testTabNewlineCarriageReturn()
+ {
+ assertQueryEqualsDOA("+weltbank +worlbank", null, "+weltbank +worlbank");
+
+ assertQueryEqualsDOA("+weltbank\n+worlbank", null, "+weltbank +worlbank");
+ assertQueryEqualsDOA("weltbank \n+worlbank", null, "+weltbank +worlbank");
+ assertQueryEqualsDOA("weltbank \n +worlbank", null, "+weltbank +worlbank");
+
+ assertQueryEqualsDOA("+weltbank\r+worlbank", null, "+weltbank +worlbank");
+ assertQueryEqualsDOA("weltbank \r+worlbank", null, "+weltbank +worlbank");
+ assertQueryEqualsDOA("weltbank \r +worlbank", null, "+weltbank +worlbank");
+
+ assertQueryEqualsDOA("+weltbank\r\n+worlbank", null, "+weltbank +worlbank");
+ assertQueryEqualsDOA("weltbank \r\n+worlbank", null, "+weltbank +worlbank");
+ assertQueryEqualsDOA("weltbank \r\n +worlbank", null, "+weltbank +worlbank");
+ assertQueryEqualsDOA("weltbank \r \n +worlbank", null,
+ "+weltbank +worlbank");
+
+ assertQueryEqualsDOA("+weltbank\t+worlbank", null, "+weltbank +worlbank");
+ assertQueryEqualsDOA("weltbank \t+worlbank", null, "+weltbank +worlbank");
+ assertQueryEqualsDOA("weltbank \t +worlbank", null, "+weltbank +worlbank");
+ }
+
+ [Test]
+ public void testSimpleDAO()
+ {
+ assertQueryEqualsDOA("term term term", null, "+term +term +term");
+ assertQueryEqualsDOA("term +term term", null, "+term +term +term");
+ assertQueryEqualsDOA("term term +term", null, "+term +term +term");
+ assertQueryEqualsDOA("term +term +term", null, "+term +term +term");
+ assertQueryEqualsDOA("-term term term", null, "-term +term +term");
+ }
+
+ [Test]
+ public void testBoost()
+ {
+ CharacterRunAutomaton stopSet = new CharacterRunAutomaton(BasicAutomata.MakeString("on"));
+ Analyzer oneStopAnalyzer = new MockAnalyzer(Random(), MockTokenizer.SIMPLE, true, stopSet);
+
+ PrecedenceQueryParser qp = new PrecedenceQueryParser();
+ qp.Analyzer = (oneStopAnalyzer);
+ Query q = (Query)qp.Parse("on^1.0", "field");
+ assertNotNull(q);
+ q = (Query)qp.Parse("\"hello\"^2.0", "field");
+ assertNotNull(q);
+ assertEquals(q.Boost, (float)2.0, (float)0.5);
+ q = (Query)qp.Parse("hello^2.0", "field");
+ assertNotNull(q);
+ assertEquals(q.Boost, (float)2.0, (float)0.5);
+ q = (Query)qp.Parse("\"on\"^1.0", "field");
+ assertNotNull(q);
+
+ q = (Query)GetParser(new MockAnalyzer(Random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)).Parse("the^3",
+ "field");
+ assertNotNull(q);
+ }
+
+ [Test]
+ public void testException()
+ {
+ try
+ {
+ assertQueryEquals("\"some phrase", null, "abc");
+ fail("ParseException expected, not thrown");
+ }
+ catch (QueryNodeParseException expected)
+ {
+ }
+ }
+
+ [Test]
+ public void testBooleanQuery()
+ {
+ BooleanQuery.MaxClauseCount = (2);
+ try
+ {
+ GetParser(new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false)).Parse("one two three", "field");
+ fail("ParseException expected due to too many boolean clauses");
+ }
+ catch (QueryNodeException expected)
+ {
+ // too many boolean clauses, so ParseException is expected
+ }
+ }
+
+ // LUCENE-792
+ [Test]
+ public void testNOT()
+ {
+ Analyzer a = new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false);
+ assertQueryEquals("NOT foo AND bar", a, "-foo +bar");
+ }
+
+ /**
+ * This test differs from the original QueryParser, showing how the precedence
+ * issue has been corrected.
+ */
+ [Test]
+ public void testPrecedence()
+ {
+ PrecedenceQueryParser parser = GetParser(new MockAnalyzer(Random(), MockTokenizer.WHITESPACE, false));
+ Query query1 = (Query)parser.Parse("A AND B OR C AND D", "field");
+ Query query2 = (Query)parser.Parse("(A AND B) OR (C AND D)", "field");
+ assertEquals(query1, query2);
+
+ query1 = (Query)parser.Parse("A OR B C", "field");
+ query2 = (Query)parser.Parse("(A B) C", "field");
+ assertEquals(query1, query2);
+
+ query1 = (Query)parser.Parse("A AND B C", "field");
+ query2 = (Query)parser.Parse("(+A +B) C", "field");
+ assertEquals(query1, query2);
+
+ query1 = (Query)parser.Parse("A AND NOT B", "field");
+ query2 = (Query)parser.Parse("+A -B", "field");
+ assertEquals(query1, query2);
+
+ query1 = (Query)parser.Parse("A OR NOT B", "field");
+ query2 = (Query)parser.Parse("A -B", "field");
+ assertEquals(query1, query2);
+
+ query1 = (Query)parser.Parse("A OR NOT B AND C", "field");
+ query2 = (Query)parser.Parse("A (-B +C)", "field");
+ assertEquals(query1, query2);
+
+ parser.SetDefaultOperator(/*StandardQueryConfigHandler.*/Operator.AND);
+ query1 = (Query)parser.Parse("A AND B OR C AND D", "field");
+ query2 = (Query)parser.Parse("(A AND B) OR (C AND D)", "field");
+ assertEquals(query1, query2);
+
+ query1 = (Query)parser.Parse("A AND B C", "field");
+ query2 = (Query)parser.Parse("(A B) C", "field");
+ assertEquals(query1, query2);
+
+ query1 = (Query)parser.Parse("A AND B C", "field");
+ query2 = (Query)parser.Parse("(+A +B) C", "field");
+ assertEquals(query1, query2);
+
+ query1 = (Query)parser.Parse("A AND NOT B", "field");
+ query2 = (Query)parser.Parse("+A -B", "field");
+ assertEquals(query1, query2);
+
+ query1 = (Query)parser.Parse("A AND NOT B OR C", "field");
+ query2 = (Query)parser.Parse("(+A -B) OR C", "field");
+ assertEquals(query1, query2);
+
+ }
+
+
+ public override void TearDown()
+ {
+ BooleanQuery.MaxClauseCount = (originalMaxClauses);
+ base.TearDown();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpanOrQueryNodeBuilder.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpanOrQueryNodeBuilder.cs b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpanOrQueryNodeBuilder.cs
new file mode 100644
index 0000000..c93c267
--- /dev/null
+++ b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpanOrQueryNodeBuilder.cs
@@ -0,0 +1,50 @@
+\ufeffusing Lucene.Net.QueryParsers.Flexible.Core.Builders;
+using Lucene.Net.QueryParsers.Flexible.Core.Nodes;
+using Lucene.Net.QueryParsers.Flexible.Standard.Builders;
+using Lucene.Net.Search;
+using Lucene.Net.Search.Spans;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Spans
+{
+ /// <summary>
+ /// This builder creates {@link SpanOrQuery}s from a {@link BooleanQueryNode}.
+ /// <para/>
+ /// It assumes that the {@link BooleanQueryNode} instance has at least one child.
+ /// </summary>
+ public class SpanOrQueryNodeBuilder : IStandardQueryBuilder
+ {
+ public virtual Query Build(IQueryNode node)
+ {
+
+ // validates node
+ BooleanQueryNode booleanNode = (BooleanQueryNode)node;
+
+ IList<IQueryNode> children = booleanNode.GetChildren();
+ SpanQuery[]
+ spanQueries = new SpanQuery[children.size()];
+
+ int i = 0;
+ foreach (IQueryNode child in children)
+ {
+ spanQueries[i++] = (SpanQuery)child
+ .GetTag(QueryTreeBuilder.QUERY_TREE_BUILDER_TAGID);
+ }
+
+ return new SpanOrQuery(spanQueries);
+
+ }
+
+ /// <summary>
+ /// LUCENENET specific overload for supporting IQueryBuilder
+ /// </summary>
+ object IQueryBuilder.Build(IQueryNode queryNode)
+ {
+ return Build(queryNode);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpanTermQueryNodeBuilder.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpanTermQueryNodeBuilder.cs b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpanTermQueryNodeBuilder.cs
new file mode 100644
index 0000000..30946d2
--- /dev/null
+++ b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpanTermQueryNodeBuilder.cs
@@ -0,0 +1,40 @@
+\ufeffusing Lucene.Net.Index;
+using Lucene.Net.QueryParsers.Flexible.Core.Builders;
+using Lucene.Net.QueryParsers.Flexible.Core.Nodes;
+using Lucene.Net.QueryParsers.Flexible.Standard.Builders;
+using Lucene.Net.Search;
+using Lucene.Net.Search.Spans;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Spans
+{
+ /// <summary>
+ /// This builder creates {@link SpanTermQuery}s from a {@link FieldQueryNode}
+ /// object.
+ /// </summary>
+ public class SpanTermQueryNodeBuilder : IStandardQueryBuilder
+ {
+
+ public Query Build(IQueryNode node)
+ {
+ FieldQueryNode fieldQueryNode = (FieldQueryNode)node;
+
+ return new SpanTermQuery(new Term(fieldQueryNode.GetFieldAsString(),
+ fieldQueryNode.GetTextAsString()));
+
+ }
+
+ /// <summary>
+ /// LUCENENET specific overload for supporting IQueryBuilder
+ /// </summary>
+ object IQueryBuilder.Build(IQueryNode queryNode)
+ {
+ return Build(queryNode);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpansQueryConfigHandler.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpansQueryConfigHandler.cs b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpansQueryConfigHandler.cs
new file mode 100644
index 0000000..a5b8f7a
--- /dev/null
+++ b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpansQueryConfigHandler.cs
@@ -0,0 +1,33 @@
+\ufeffusing Lucene.Net.QueryParsers.Flexible.Core.Config;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Spans
+{
+ /// <summary>
+ /// This query config handler only adds the {@link UniqueFieldAttribute} to it.
+ /// <para/>
+ /// It does not return any configuration for a field in specific.
+ /// </summary>
+ public class SpansQueryConfigHandler : QueryConfigHandler
+ {
+ public readonly static ConfigurationKey<string> UNIQUE_FIELD = ConfigurationKey.NewInstance<string>();
+
+ public SpansQueryConfigHandler()
+ {
+ // empty constructor
+ }
+
+
+ public override FieldConfig GetFieldConfig(string fieldName)
+ {
+
+ // there is no field configuration, always return null
+ return null;
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpansQueryTreeBuilder.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpansQueryTreeBuilder.cs b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpansQueryTreeBuilder.cs
new file mode 100644
index 0000000..8810367
--- /dev/null
+++ b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpansQueryTreeBuilder.cs
@@ -0,0 +1,44 @@
+\ufeffusing Lucene.Net.QueryParsers.Flexible.Core.Builders;
+using Lucene.Net.QueryParsers.Flexible.Core.Nodes;
+using Lucene.Net.QueryParsers.Flexible.Standard.Builders;
+using Lucene.Net.Search;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Spans
+{
+ /// <summary>
+ /// Sets up a query tree builder to build a span query tree from a query node
+ /// tree.
+ /// <para/>
+ /// The defined map is:
+ /// - every BooleanQueryNode instance is delegated to the SpanOrQueryNodeBuilder
+ /// - every FieldQueryNode instance is delegated to the SpanTermQueryNodeBuilder
+ /// </summary>
+ public class SpansQueryTreeBuilder : QueryTreeBuilder, IStandardQueryBuilder
+ {
+ public SpansQueryTreeBuilder()
+ {
+ SetBuilder(typeof(BooleanQueryNode), new SpanOrQueryNodeBuilder());
+ SetBuilder(typeof(FieldQueryNode), new SpanTermQueryNodeBuilder());
+
+ }
+
+
+ Query IStandardQueryBuilder.Build(IQueryNode queryTree)
+ {
+ return (Query)base.Build(queryTree);
+ }
+
+ /// <summary>
+ /// LUCENENET specific overload for supporting IQueryBuilder
+ /// </summary>
+ public override object Build(IQueryNode queryNode)
+ {
+ return Build(queryNode);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpansValidatorQueryNodeProcessor.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpansValidatorQueryNodeProcessor.cs b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpansValidatorQueryNodeProcessor.cs
new file mode 100644
index 0000000..c44c1f5
--- /dev/null
+++ b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/SpansValidatorQueryNodeProcessor.cs
@@ -0,0 +1,55 @@
+\ufeffusing Lucene.Net.QueryParsers.Flexible.Core;
+using Lucene.Net.QueryParsers.Flexible.Core.Messages;
+using Lucene.Net.QueryParsers.Flexible.Core.Nodes;
+using Lucene.Net.QueryParsers.Flexible.Core.Processors;
+using Lucene.Net.QueryParsers.Flexible.Messages;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Spans
+{
+ /// <summary>
+ /// Validates every query node in a query node tree. This processor will pass
+ /// fine if the query nodes are only {@link BooleanQueryNode}s,
+ /// {@link OrQueryNode}s or {@link FieldQueryNode}s, otherwise an exception will
+ /// be thrown.
+ /// <para/>
+ /// If they are {@link AndQueryNode} or an instance of anything else that
+ /// implements {@link FieldQueryNode} the exception will also be thrown.
+ /// </summary>
+ public class SpansValidatorQueryNodeProcessor : QueryNodeProcessorImpl
+ {
+ protected override IQueryNode PostProcessNode(IQueryNode node)
+ {
+
+ return node;
+
+ }
+
+
+ protected override IQueryNode PreProcessNode(IQueryNode node)
+ {
+
+ if (!((node is BooleanQueryNode && !(node is AndQueryNode)) || node
+ .GetType() == typeof(FieldQueryNode)))
+ {
+ throw new QueryNodeException(new MessageImpl(
+ QueryParserMessages.NODE_ACTION_NOT_SUPPORTED));
+ }
+
+ return node;
+
+ }
+
+
+ protected override IList<IQueryNode> SetChildrenOrder(IList<IQueryNode> children)
+ {
+
+ return children;
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/TestSpanQueryParser.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/TestSpanQueryParser.cs b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/TestSpanQueryParser.cs
new file mode 100644
index 0000000..eff0c6e
--- /dev/null
+++ b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/TestSpanQueryParser.cs
@@ -0,0 +1,238 @@
+\ufeffusing Lucene.Net.QueryParsers.Flexible.Core;
+using Lucene.Net.QueryParsers.Flexible.Core.Nodes;
+using Lucene.Net.QueryParsers.Flexible.Core.Parser;
+using Lucene.Net.QueryParsers.Flexible.Core.Processors;
+using Lucene.Net.QueryParsers.Flexible.Standard.Parser;
+using Lucene.Net.QueryParsers.Flexible.Standard.Processors;
+using Lucene.Net.Search.Spans;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Spans
+{
+ /// <summary>
+ /// This test case demonstrates how the new query parser can be used.
+ /// <para/>
+ /// It tests queries likes "term", "field:term" "term1 term2" "term1 OR term2",
+ /// which are all already supported by the current syntax parser (
+ /// {@link StandardSyntaxParser}).
+ /// <para/>
+ /// The goals is to create a new query parser that supports only the pair
+ /// "field:term" or a list of pairs separated or not by an OR operator, and from
+ /// this query generate {@link SpanQuery} objects instead of the regular
+ /// {@link Query} objects. Basically, every pair will be converted to a
+ /// {@link SpanTermQuery} object and if there are more than one pair they will be
+ /// grouped by an {@link OrQueryNode}.
+ /// <para/>
+ /// Another functionality that will be added is the ability to convert every
+ /// field defined in the query to an unique specific field.
+ /// <para/>
+ /// The query generation is divided in three different steps: parsing (syntax),
+ /// processing (semantic) and building.
+ /// <para/>
+ /// The parsing phase, as already mentioned will be performed by the current
+ /// query parser: {@link StandardSyntaxParser}.
+ /// <para/>
+ /// The processing phase will be performed by a processor pipeline which is
+ /// compound by 2 processors: {@link SpansValidatorQueryNodeProcessor} and
+ /// {@link UniqueFieldQueryNodeProcessor}.
+ /// <para/>
+ ///
+ /// {@link SpansValidatorQueryNodeProcessor}: as it's going to use the current
+ /// query parser to parse the syntax, it will support more features than we want,
+ /// this processor basically validates the query node tree generated by the parser
+ /// and just let got through the elements we want, all the other elements as
+ /// wildcards, range queries, etc...if found, an exception is thrown.
+ ///
+ /// {@link UniqueFieldQueryNodeProcessor}: this processor will take care of reading
+ /// what is the "unique field" from the configuration and convert every field defined
+ /// in every pair to this "unique field". For that, a {@link SpansQueryConfigHandler} is
+ /// used, which has the {@link UniqueFieldAttribute} defined in it.
+ ///
+ /// <para/>
+ /// The building phase is performed by the {@link SpansQueryTreeBuilder}, which
+ /// basically contains a map that defines which builder will be used to generate
+ /// {@link SpanQuery} objects from {@link QueryNode} objects.
+ /// </summary>
+ /// <seealso cref="SpansQueryConfigHandler"/>
+ /// <seealso cref="SpansQueryTreeBuilder"/>
+ /// <seealso cref="SpansValidatorQueryNodeProcessor"/>
+ /// <seealso cref="SpanOrQueryNodeBuilder"/>
+ /// <seealso cref="SpanTermQueryNodeBuilder"/>
+ /// <seealso cref="StandardSyntaxParser"/>
+ /// <seealso cref="UniqueFieldQueryNodeProcessor"/>
+ /// <seealso cref="IUniqueFieldAttribute"/>
+ public class TestSpanQueryParser : LuceneTestCase
+ {
+ private QueryNodeProcessorPipeline spanProcessorPipeline;
+
+ private SpansQueryConfigHandler spanQueryConfigHandler;
+
+ private SpansQueryTreeBuilder spansQueryTreeBuilder;
+
+ private ISyntaxParser queryParser = new StandardSyntaxParser();
+
+ public TestSpanQueryParser()
+ {
+ // empty constructor
+ }
+
+
+ public override void SetUp()
+ {
+ base.SetUp();
+
+ this.spanProcessorPipeline = new QueryNodeProcessorPipeline();
+ this.spanQueryConfigHandler = new SpansQueryConfigHandler();
+ this.spansQueryTreeBuilder = new SpansQueryTreeBuilder();
+
+ // set up the processor pipeline
+ this.spanProcessorPipeline
+ .SetQueryConfigHandler(this.spanQueryConfigHandler);
+
+ this.spanProcessorPipeline.Add(new WildcardQueryNodeProcessor());
+ this.spanProcessorPipeline.Add(new SpansValidatorQueryNodeProcessor());
+ this.spanProcessorPipeline.Add(new UniqueFieldQueryNodeProcessor());
+
+ }
+
+ public SpanQuery GetSpanQuery(/*CharSequence*/string query)
+ {
+ return GetSpanQuery("", query);
+ }
+
+ public SpanQuery GetSpanQuery(String uniqueField, /*CharSequence*/string query)
+ {
+
+ this.spanQueryConfigHandler.Set(SpansQueryConfigHandler.UNIQUE_FIELD, uniqueField);
+
+ IQueryNode queryTree = this.queryParser.Parse(query, "defaultField");
+ queryTree = this.spanProcessorPipeline.Process(queryTree);
+
+ return (SpanQuery)this.spansQueryTreeBuilder.Build(queryTree); // LUCENENET TODO: Find way to remove cast
+
+ }
+
+ [Test]
+ public void testTermSpans()
+ {
+ assertEquals(GetSpanQuery("field:term").toString(), "term");
+ assertEquals(GetSpanQuery("term").toString(), "term");
+
+ assertTrue(GetSpanQuery("field:term") is SpanTermQuery);
+ assertTrue(GetSpanQuery("term") is SpanTermQuery);
+
+ }
+
+ [Test]
+ public void testUniqueField()
+ {
+ assertEquals(GetSpanQuery("field", "term").toString(), "field:term");
+ assertEquals(GetSpanQuery("field", "field:term").toString(), "field:term");
+ assertEquals(GetSpanQuery("field", "anotherField:term").toString(),
+ "field:term");
+
+ }
+
+ [Test]
+ public void testOrSpans()
+ {
+ assertEquals(GetSpanQuery("term1 term2").toString(),
+ "spanOr([term1, term2])");
+ assertEquals(GetSpanQuery("term1 OR term2").toString(),
+ "spanOr([term1, term2])");
+
+ assertTrue(GetSpanQuery("term1 term2") is SpanOrQuery);
+ assertTrue(GetSpanQuery("term1 term2") is SpanOrQuery);
+
+ }
+
+ [Test]
+ public void testQueryValidator()
+ {
+
+ try
+ {
+ GetSpanQuery("term*");
+ fail("QueryNodeException was expected, wildcard queries should not be supported");
+
+ }
+ catch (QueryNodeException ex)
+ {
+ // expected exception
+ }
+
+ try
+ {
+ GetSpanQuery("[a TO z]");
+ fail("QueryNodeException was expected, range queries should not be supported");
+
+ }
+ catch (QueryNodeException ex)
+ {
+ // expected exception
+ }
+
+ try
+ {
+ GetSpanQuery("a~0.5");
+ fail("QueryNodeException was expected, boost queries should not be supported");
+
+ }
+ catch (QueryNodeException ex)
+ {
+ // expected exception
+ }
+
+ try
+ {
+ GetSpanQuery("a^0.5");
+ fail("QueryNodeException was expected, fuzzy queries should not be supported");
+
+ }
+ catch (QueryNodeException ex)
+ {
+ // expected exception
+ }
+
+ try
+ {
+ GetSpanQuery("\"a b\"");
+ fail("QueryNodeException was expected, quoted queries should not be supported");
+
+ }
+ catch (QueryNodeException ex)
+ {
+ // expected exception
+ }
+
+ try
+ {
+ GetSpanQuery("(a b)");
+ fail("QueryNodeException was expected, parenthesized queries should not be supported");
+
+ }
+ catch (QueryNodeException ex)
+ {
+ // expected exception
+ }
+
+ try
+ {
+ GetSpanQuery("a AND b");
+ fail("QueryNodeException was expected, and queries should not be supported");
+
+ }
+ catch (QueryNodeException ex)
+ {
+ // expected exception
+ }
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/TestSpanQueryParserSimpleSample.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/TestSpanQueryParserSimpleSample.cs b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/TestSpanQueryParserSimpleSample.cs
new file mode 100644
index 0000000..1f8f8e4
--- /dev/null
+++ b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/TestSpanQueryParserSimpleSample.cs
@@ -0,0 +1,113 @@
+\ufeffusing Lucene.Net.QueryParsers.Flexible.Core.Config;
+using Lucene.Net.QueryParsers.Flexible.Core.Nodes;
+using Lucene.Net.QueryParsers.Flexible.Core.Parser;
+using Lucene.Net.QueryParsers.Flexible.Core.Processors;
+using Lucene.Net.QueryParsers.Flexible.Standard.Parser;
+using Lucene.Net.Search.Spans;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Spans
+{
+ /// <summary>
+ /// This test case demonstrates how the new query parser can be used.
+ /// <para/>
+ /// It tests queries likes "term", "field:term" "term1 term2" "term1 OR term2",
+ /// which are all already supported by the current syntax parser (
+ /// {@link StandardSyntaxParser}).
+ /// <para/>
+ /// The goals is to create a new query parser that supports only the pair
+ /// "field:term" or a list of pairs separated or not by an OR operator, and from
+ /// this query generate {@link SpanQuery} objects instead of the regular
+ /// {@link Query} objects. Basically, every pair will be converted to a
+ /// {@link SpanTermQuery} object and if there are more than one pair they will be
+ /// grouped by an {@link OrQueryNode}.
+ /// <para/>
+ /// Another functionality that will be added is the ability to convert every
+ /// field defined in the query to an unique specific field.
+ /// <para/>
+ /// The query generation is divided in three different steps: parsing (syntax),
+ /// processing (semantic) and building.
+ /// <para/>
+ /// The parsing phase, as already mentioned will be performed by the current
+ /// query parser: {@link StandardSyntaxParser}.
+ /// <para/>
+ /// The processing phase will be performed by a processor pipeline which is
+ /// compound by 2 processors: {@link SpansValidatorQueryNodeProcessor} and
+ /// {@link UniqueFieldQueryNodeProcessor}.
+ /// <para/>
+ /// {@link SpansValidatorQueryNodeProcessor}: as it's going to use the current
+ /// query parser to parse the syntax, it will support more features than we want,
+ /// this processor basically validates the query node tree generated by the parser
+ /// and just let got through the elements we want, all the other elements as
+ /// wildcards, range queries, etc...if found, an exception is thrown.
+ /// <para/>
+ /// {@link UniqueFieldQueryNodeProcessor}: this processor will take care of reading
+ /// what is the "unique field" from the configuration and convert every field defined
+ /// in every pair to this "unique field". For that, a {@link SpansQueryConfigHandler} is
+ /// used, which has the {@link UniqueFieldAttribute} defined in it.
+ /// <para/>
+ /// The building phase is performed by the {@link SpansQueryTreeBuilder}, which
+ /// basically contains a map that defines which builder will be used to generate
+ /// {@link SpanQuery} objects from {@link QueryNode} objects.
+ /// </summary>
+ /// <seealso cref="TestSpanQueryParser">for a more advanced example</seealso>
+ ///
+ /// <seealso cref="SpansQueryConfigHandler"/>
+ /// <seealso cref="SpansQueryTreeBuilder"/>
+ /// <seealso cref="SpansValidatorQueryNodeProcessor"/>
+ /// <seealso cref="SpanOrQueryNodeBuilder"/>
+ /// <seealso cref="SpanTermQueryNodeBuilder"/>
+ /// <seealso cref="StandardSyntaxParser"/>
+ /// <seealso cref="UniqueFieldQueryNodeProcessor"/>
+ /// <seealso cref="IUniqueFieldAttribute"/>
+ public class TestSpanQueryParserSimpleSample : LuceneTestCase
+ {
+ [Test]
+ public void testBasicDemo()
+ {
+ ISyntaxParser queryParser = new StandardSyntaxParser();
+
+ // convert the CharSequence into a QueryNode tree
+ IQueryNode queryTree = queryParser.Parse("body:text", null);
+
+ // create a config handler with a attribute used in
+ // UniqueFieldQueryNodeProcessor
+ QueryConfigHandler spanQueryConfigHandler = new SpansQueryConfigHandler();
+ spanQueryConfigHandler.Set(SpansQueryConfigHandler.UNIQUE_FIELD, "index");
+
+ // set up the processor pipeline with the ConfigHandler
+ // and create the pipeline for this simple demo
+ QueryNodeProcessorPipeline spanProcessorPipeline = new QueryNodeProcessorPipeline(
+ spanQueryConfigHandler);
+ // @see SpansValidatorQueryNodeProcessor
+ spanProcessorPipeline.Add(new SpansValidatorQueryNodeProcessor());
+ // @see UniqueFieldQueryNodeProcessor
+ spanProcessorPipeline.Add(new UniqueFieldQueryNodeProcessor());
+
+ // print to show out the QueryNode tree before being processed
+ if (VERBOSE) Console.WriteLine(queryTree);
+
+ // Process the QueryTree using our new Processors
+ queryTree = spanProcessorPipeline.Process(queryTree);
+
+ // print to show out the QueryNode tree after being processed
+ if (VERBOSE) Console.WriteLine(queryTree);
+
+ // create a instance off the Builder
+ SpansQueryTreeBuilder spansQueryTreeBuilder = new SpansQueryTreeBuilder();
+
+ // convert QueryNode tree to span query Objects
+ SpanQuery spanquery = (SpanQuery)spansQueryTreeBuilder.Build(queryTree); // LUCENENET TODO: Find a way to remove the cast
+
+ assertTrue(spanquery is SpanTermQuery);
+ assertEquals(spanquery.toString(), "index:text");
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/UniqueFieldAttribute.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/UniqueFieldAttribute.cs b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/UniqueFieldAttribute.cs
new file mode 100644
index 0000000..5cc2628
--- /dev/null
+++ b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/UniqueFieldAttribute.cs
@@ -0,0 +1,22 @@
+\ufeffusing Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Spans
+{
+ /// <summary>
+ /// This attribute is used by the {@link UniqueFieldQueryNodeProcessor}
+ /// processor. It holds a value that defines which is the unique field name that
+ /// should be set in every {@link FieldableNode}.
+ /// </summary>
+ /// <seealso cref="UniqueFieldQueryNodeProcessor"/>
+ public interface IUniqueFieldAttribute : IAttribute
+ {
+ void SetUniqueField(string uniqueField);
+
+ string GetUniqueField();
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/UniqueFieldAttributeImpl.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/UniqueFieldAttributeImpl.cs b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/UniqueFieldAttributeImpl.cs
new file mode 100644
index 0000000..1fc4fc6
--- /dev/null
+++ b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/UniqueFieldAttributeImpl.cs
@@ -0,0 +1,88 @@
+\ufeff//using System;
+using Lucene.Net.Util;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Spans
+{
+ /// <summary>
+ /// This attribute is used by the {@link UniqueFieldQueryNodeProcessor}
+ /// processor. It holds a value that defines which is the unique field name that
+ /// should be set in every {@link FieldableNode}.
+ /// </summary>
+ /// <seealso cref="UniqueFieldQueryNodeProcessor"/>
+ public class UniqueFieldAttributeImpl : AttributeImpl, IUniqueFieldAttribute
+ {
+ private string uniqueField;
+
+ public UniqueFieldAttributeImpl()
+ {
+ Clear();
+ }
+
+
+ public override void Clear()
+ {
+ this.uniqueField = "";
+ }
+
+
+ public void SetUniqueField(string uniqueField)
+ {
+ this.uniqueField = uniqueField;
+ }
+
+
+ public string GetUniqueField()
+ {
+ return this.uniqueField;
+ }
+
+
+ public override void CopyTo(Attribute target)
+ {
+
+ if (!(target is UniqueFieldAttributeImpl))
+ {
+ throw new System.ArgumentException(
+ "cannot copy the values from attribute UniqueFieldAttribute to an instance of "
+ + target.GetType().Name);
+ }
+
+ UniqueFieldAttributeImpl uniqueFieldAttr = (UniqueFieldAttributeImpl)target;
+ uniqueFieldAttr.uniqueField = uniqueField.toString();
+
+ }
+
+
+ public override bool Equals(object other)
+ {
+
+ if (other is UniqueFieldAttributeImpl)
+ {
+
+ return ((UniqueFieldAttributeImpl)other).uniqueField
+ .equals(this.uniqueField);
+
+ }
+
+ return false;
+
+ }
+
+
+ public override int GetHashCode()
+ {
+ return this.uniqueField.GetHashCode();
+ }
+
+
+ public override string ToString()
+ {
+ return "<uniqueField uniqueField='" + this.uniqueField + "'/>";
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/UniqueFieldQueryNodeProcessor.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/UniqueFieldQueryNodeProcessor.cs b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/UniqueFieldQueryNodeProcessor.cs
new file mode 100644
index 0000000..38ad997
--- /dev/null
+++ b/src/Lucene.Net.Tests.QueryParser/Flexible/Spans/UniqueFieldQueryNodeProcessor.cs
@@ -0,0 +1,68 @@
+\ufeffusing Lucene.Net.QueryParsers.Flexible.Core.Config;
+using Lucene.Net.QueryParsers.Flexible.Core.Nodes;
+using Lucene.Net.QueryParsers.Flexible.Core.Processors;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Spans
+{
+ /// <summary>
+ /// This processor changes every field name of each {@link FieldableNode} query
+ /// node contained in the query tree to the field name defined in the
+ /// {@link UniqueFieldAttribute}. So, the {@link UniqueFieldAttribute} must be
+ /// defined in the {@link QueryConfigHandler} object set in this processor,
+ /// otherwise it throws an exception.
+ /// </summary>
+ /// <seealso cref="UniqueFieldAttribute"/>
+ public class UniqueFieldQueryNodeProcessor : QueryNodeProcessorImpl
+ {
+ protected override IQueryNode PostProcessNode(IQueryNode node)
+ {
+
+ return node;
+
+ }
+
+
+ protected override IQueryNode PreProcessNode(IQueryNode node)
+ {
+
+ if (node is IFieldableNode)
+ {
+ IFieldableNode fieldNode = (IFieldableNode)node;
+
+ QueryConfigHandler queryConfig = GetQueryConfigHandler();
+
+ if (queryConfig == null)
+ {
+ throw new ArgumentException(
+ "A config handler is expected by the processor UniqueFieldQueryNodeProcessor!");
+ }
+
+ if (!queryConfig.has(SpansQueryConfigHandler.UNIQUE_FIELD))
+ {
+ throw new ArgumentException(
+ "UniqueFieldAttribute should be defined in the config handler!");
+ }
+
+ String uniqueField = queryConfig.Get(SpansQueryConfigHandler.UNIQUE_FIELD);
+ fieldNode.Field = (uniqueField);
+
+ }
+
+ return node;
+
+ }
+
+
+ protected override IList<IQueryNode> SetChildrenOrder(IList<IQueryNode> children)
+ {
+
+ return children;
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.Tests.QueryParser/Flexible/Standard/TestMultiAnalyzerQPHelper.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.Tests.QueryParser/Flexible/Standard/TestMultiAnalyzerQPHelper.cs b/src/Lucene.Net.Tests.QueryParser/Flexible/Standard/TestMultiAnalyzerQPHelper.cs
new file mode 100644
index 0000000..d67bcb7
--- /dev/null
+++ b/src/Lucene.Net.Tests.QueryParser/Flexible/Standard/TestMultiAnalyzerQPHelper.cs
@@ -0,0 +1,267 @@
+\ufeffusing Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Tokenattributes;
+using Lucene.Net.QueryParsers.Flexible.Standard.Config;
+using Lucene.Net.Util;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Standard
+{
+ /// <summary>
+ /// This test case is a copy of the core Lucene query parser test, it was adapted
+ /// to use new QueryParserHelper instead of the old query parser.
+ ///
+ /// Test QueryParser's ability to deal with Analyzers that return more than one
+ /// token per position or that return tokens with a position increment > 1.
+ /// </summary>
+ public class TestMultiAnalyzerQPHelper : LuceneTestCase
+ {
+ private static int multiToken = 0;
+
+ [Test]
+ public void testMultiAnalyzer()
+ {
+
+ StandardQueryParser qp = new StandardQueryParser();
+ qp.Analyzer = (new MultiAnalyzer());
+
+ // trivial, no multiple tokens:
+ assertEquals("foo", qp.Parse("foo", "").toString());
+ assertEquals("foo", qp.Parse("\"foo\"", "").toString());
+ assertEquals("foo foobar", qp.Parse("foo foobar", "").toString());
+ assertEquals("\"foo foobar\"", qp.Parse("\"foo foobar\"", "").toString());
+ assertEquals("\"foo foobar blah\"", qp.Parse("\"foo foobar blah\"", "")
+ .toString());
+
+ // two tokens at the same position:
+ assertEquals("(multi multi2) foo", qp.Parse("multi foo", "").toString());
+ assertEquals("foo (multi multi2)", qp.Parse("foo multi", "").toString());
+ assertEquals("(multi multi2) (multi multi2)", qp.Parse("multi multi", "")
+ .toString());
+ assertEquals("+(foo (multi multi2)) +(bar (multi multi2))", qp.Parse(
+ "+(foo multi) +(bar multi)", "").toString());
+ assertEquals("+(foo (multi multi2)) field:\"bar (multi multi2)\"", qp
+ .Parse("+(foo multi) field:\"bar multi\"", "").toString());
+
+ // phrases:
+ assertEquals("\"(multi multi2) foo\"", qp.Parse("\"multi foo\"", "")
+ .toString());
+ assertEquals("\"foo (multi multi2)\"", qp.Parse("\"foo multi\"", "")
+ .toString());
+ assertEquals("\"foo (multi multi2) foobar (multi multi2)\"", qp.Parse(
+ "\"foo multi foobar multi\"", "").toString());
+
+ // fields:
+ assertEquals("(field:multi field:multi2) field:foo", qp.Parse(
+ "field:multi field:foo", "").toString());
+ assertEquals("field:\"(multi multi2) foo\"", qp.Parse(
+ "field:\"multi foo\"", "").toString());
+
+ // three tokens at one position:
+ assertEquals("triplemulti multi3 multi2", qp.Parse("triplemulti", "")
+ .toString());
+ assertEquals("foo (triplemulti multi3 multi2) foobar", qp.Parse(
+ "foo triplemulti foobar", "").toString());
+
+ // phrase with non-default slop:
+ assertEquals("\"(multi multi2) foo\"~10", qp.Parse("\"multi foo\"~10", "")
+ .toString());
+
+ // phrase with non-default boost:
+ assertEquals("\"(multi multi2) foo\"^2.0", qp.Parse("\"multi foo\"^2", "")
+ .toString());
+
+ // phrase after changing default slop
+ qp.SetDefaultPhraseSlop(99);
+ assertEquals("\"(multi multi2) foo\"~99 bar", qp.Parse("\"multi foo\" bar",
+ "").toString());
+ assertEquals("\"(multi multi2) foo\"~99 \"foo bar\"~2", qp.Parse(
+ "\"multi foo\" \"foo bar\"~2", "").toString());
+ qp.SetDefaultPhraseSlop(0);
+
+ // non-default operator:
+ qp.SetDefaultOperator(/*StandardQueryConfigHandler.*/Operator.AND);
+ assertEquals("+(multi multi2) +foo", qp.Parse("multi foo", "").toString());
+
+ }
+
+ // public void testMultiAnalyzerWithSubclassOfQueryParser() throws
+ // ParseException {
+ // this test doesn't make sense when using the new QueryParser API
+ // DumbQueryParser qp = new DumbQueryParser("", new MultiAnalyzer());
+ // qp.setPhraseSlop(99); // modified default slop
+ //
+ // // direct call to (super's) getFieldQuery to demonstrate differnce
+ // // between phrase and multiphrase with modified default slop
+ // assertEquals("\"foo bar\"~99",
+ // qp.getSuperFieldQuery("","foo bar").toString());
+ // assertEquals("\"(multi multi2) bar\"~99",
+ // qp.getSuperFieldQuery("","multi bar").toString());
+ //
+ //
+ // // ask sublcass to parse phrase with modified default slop
+ // assertEquals("\"(multi multi2) foo\"~99 bar",
+ // qp.parse("\"multi foo\" bar").toString());
+ //
+ // }
+
+ [Test]
+ public void testPosIncrementAnalyzer()
+ {
+ StandardQueryParser qp = new StandardQueryParser();
+ qp.Analyzer = (new PosIncrementAnalyzer());
+
+ assertEquals("quick brown", qp.Parse("the quick brown", "").toString());
+ assertEquals("\"? quick brown\"", qp.Parse("\"the quick brown\"", "")
+ .toString());
+ assertEquals("quick brown fox", qp.Parse("the quick brown fox", "")
+ .toString());
+ assertEquals("\"? quick brown fox\"", qp.Parse("\"the quick brown fox\"", "")
+ .toString());
+ }
+
+ /**
+ * Expands "multi" to "multi" and "multi2", both at the same position, and
+ * expands "triplemulti" to "triplemulti", "multi3", and "multi2".
+ */
+ private class MultiAnalyzer : Analyzer
+ {
+
+
+ public override TokenStreamComponents CreateComponents(String fieldName, TextReader reader)
+ {
+ Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ return new TokenStreamComponents(result, new TestFilter(result));
+ }
+ }
+
+ private sealed class TestFilter : TokenFilter
+ {
+
+ private String prevType;
+ private int prevStartOffset;
+ private int prevEndOffset;
+
+ private readonly ICharTermAttribute termAtt;
+ private readonly IPositionIncrementAttribute posIncrAtt;
+ private readonly IOffsetAttribute offsetAtt;
+ private readonly ITypeAttribute typeAtt;
+
+ public TestFilter(TokenStream @in)
+ : base(@in)
+ {
+ termAtt = AddAttribute<ICharTermAttribute>();
+ posIncrAtt = AddAttribute<IPositionIncrementAttribute>();
+ offsetAtt = AddAttribute<IOffsetAttribute>();
+ typeAtt = AddAttribute<ITypeAttribute>();
+ }
+
+
+ public override sealed bool IncrementToken()
+ {
+ if (multiToken > 0)
+ {
+ termAtt.SetEmpty().Append("multi" + (multiToken + 1));
+ offsetAtt.SetOffset(prevStartOffset, prevEndOffset);
+ typeAtt.Type = (prevType);
+ posIncrAtt.PositionIncrement = (0);
+ multiToken--;
+ return true;
+ }
+ else
+ {
+ bool next = input.IncrementToken();
+ if (!next)
+ {
+ return false;
+ }
+ prevType = typeAtt.Type;
+ prevStartOffset = offsetAtt.StartOffset();
+ prevEndOffset = offsetAtt.EndOffset();
+ String text = termAtt.toString();
+ if (text.equals("triplemulti"))
+ {
+ multiToken = 2;
+ return true;
+ }
+ else if (text.equals("multi"))
+ {
+ multiToken = 1;
+ return true;
+ }
+ else
+ {
+ return true;
+ }
+ }
+ }
+
+
+ public override void Reset()
+ {
+ base.Reset();
+ this.prevType = null;
+ this.prevStartOffset = 0;
+ this.prevEndOffset = 0;
+ }
+ }
+
+ /**
+ * Analyzes "the quick brown" as: quick(incr=2) brown(incr=1). Does not work
+ * correctly for input other than "the quick brown ...".
+ */
+ private class PosIncrementAnalyzer : Analyzer
+ {
+
+
+ public override TokenStreamComponents CreateComponents(String fieldName, TextReader reader)
+ {
+ Tokenizer result = new MockTokenizer(reader, MockTokenizer.WHITESPACE, true);
+ return new TokenStreamComponents(result, new TestPosIncrementFilter(result));
+ }
+ }
+
+ private class TestPosIncrementFilter : TokenFilter
+ {
+
+ private readonly ICharTermAttribute termAtt;
+ private readonly IPositionIncrementAttribute posIncrAtt;
+
+ public TestPosIncrementFilter(TokenStream @in)
+ : base(@in)
+ {
+ termAtt = AddAttribute<ICharTermAttribute>();
+ posIncrAtt = AddAttribute<IPositionIncrementAttribute>();
+ }
+
+
+ public override sealed bool IncrementToken()
+ {
+ while (input.IncrementToken())
+ {
+ if (termAtt.toString().equals("the"))
+ {
+ // stopword, do nothing
+ }
+ else if (termAtt.toString().equals("quick"))
+ {
+ posIncrAtt.PositionIncrement = (2);
+ return true;
+ }
+ else
+ {
+ posIncrAtt.PositionIncrement = (1);
+ return true;
+ }
+ }
+ return false;
+ }
+
+ }
+ }
+}