You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucenenet.apache.org by ni...@apache.org on 2016/12/06 15:11:41 UTC
[06/58] lucenenet git commit: WIP on QueryParsers.Flexible
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.QueryParser/Flexible/Standard/Parser/StandardSyntaxParserTokenManager.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.QueryParser/Flexible/Standard/Parser/StandardSyntaxParserTokenManager.cs b/src/Lucene.Net.QueryParser/Flexible/Standard/Parser/StandardSyntaxParserTokenManager.cs
new file mode 100644
index 0000000..a6d8952
--- /dev/null
+++ b/src/Lucene.Net.QueryParser/Flexible/Standard/Parser/StandardSyntaxParserTokenManager.cs
@@ -0,0 +1,959 @@
+\ufeffusing System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Standard.Parser
+{
+ /// <summary>
+ /// Token Manager.
+ /// </summary>
+ public class StandardSyntaxParserTokenManager /*: StandardSyntaxParserConstants*/
+ {
+ /** Debug output. */
+ public TextWriter debugStream = Console.Out;
+ /** Set debug output. */
+ public void setDebugStream(TextWriter ds) { debugStream = ds; }
+ private int jjStopStringLiteralDfa_2(int pos, long active0)
+ {
+ switch (pos)
+ {
+ default:
+ return -1;
+ }
+ }
+ private int jjStartNfa_2(int pos, long active0)
+ {
+ return jjMoveNfa_2(jjStopStringLiteralDfa_2(pos, active0), pos + 1);
+ }
+ private int jjStopAtPos(int pos, int kind)
+ {
+ jjmatchedKind = kind;
+ jjmatchedPos = pos;
+ return pos + 1;
+ }
+ private int jjMoveStringLiteralDfa0_2()
+ {
+ switch ((int)curChar)
+ {
+ case 40:
+ return jjStopAtPos(0, 13);
+ case 41:
+ return jjStopAtPos(0, 14);
+ case 43:
+ return jjStopAtPos(0, 11);
+ case 45:
+ return jjStopAtPos(0, 12);
+ case 58:
+ return jjStopAtPos(0, 15);
+ case 60:
+ jjmatchedKind = 17;
+ return jjMoveStringLiteralDfa1_2(0x40000L);
+ case 61:
+ return jjStopAtPos(0, 16);
+ case 62:
+ jjmatchedKind = 19;
+ return jjMoveStringLiteralDfa1_2(0x100000L);
+ case 91:
+ return jjStopAtPos(0, 26);
+ case 94:
+ return jjStopAtPos(0, 21);
+ case 123:
+ return jjStopAtPos(0, 27);
+ default:
+ return jjMoveNfa_2(0, 0);
+ }
+ }
+ private int jjMoveStringLiteralDfa1_2(long active0)
+ {
+ try { curChar = input_stream.ReadChar(); }
+ catch (IOException e)
+ {
+ jjStopStringLiteralDfa_2(0, active0);
+ return 1;
+ }
+ switch ((int)curChar)
+ {
+ case 61:
+ if ((active0 & 0x40000L) != 0L)
+ return jjStopAtPos(1, 18);
+ else if ((active0 & 0x100000L) != 0L)
+ return jjStopAtPos(1, 20);
+ break;
+ default:
+ break;
+ }
+ return jjStartNfa_2(0, active0);
+ }
+ internal static readonly ulong[] jjbitVec0 = {
+ 0x1L, 0x0L, 0x0L, 0x0L
+ };
+ internal static readonly ulong[] jjbitVec1 = {
+ 0xfffffffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL
+ };
+ internal static readonly ulong[] jjbitVec3 = {
+ 0x0L, 0x0L, 0xffffffffffffffffL, 0xffffffffffffffffL
+ };
+ internal static readonly ulong[] jjbitVec4 = {
+ 0xfffefffffffffffeL, 0xffffffffffffffffL, 0xffffffffffffffffL, 0xffffffffffffffffL
+ };
+ private int jjMoveNfa_2(int startState, int curPos)
+ {
+ int startsAt = 0;
+ jjnewStateCnt = 33;
+ int i = 1;
+ jjstateSet[0] = startState;
+ int kind = 0x7fffffff;
+ for (;;)
+ {
+ if (++jjround == 0x7fffffff)
+ ReInitRounds();
+ if (curChar < 64)
+ {
+ ulong l = (ulong)(1L << curChar);
+ do
+ {
+ switch (jjstateSet[--i])
+ {
+ case 0:
+ if ((0x8bff54f8ffffd9ffL & l) != 0L)
+ {
+ if (kind > 23)
+ kind = 23;
+ jjCheckNAddTwoStates(20, 21);
+ }
+ else if ((0x100002600L & l) != 0L)
+ {
+ if (kind > 7)
+ kind = 7;
+ }
+ else if (curChar == 47)
+ jjCheckNAddStates(0, 2);
+ else if (curChar == 34)
+ jjCheckNAddStates(3, 5);
+ else if (curChar == 33)
+ {
+ if (kind > 10)
+ kind = 10;
+ }
+ if (curChar == 38)
+ jjstateSet[jjnewStateCnt++] = 4;
+ break;
+ case 4:
+ if (curChar == 38 && kind > 8)
+ kind = 8;
+ break;
+ case 5:
+ if (curChar == 38)
+ jjstateSet[jjnewStateCnt++] = 4;
+ break;
+ case 13:
+ if (curChar == 33 && kind > 10)
+ kind = 10;
+ break;
+ case 14:
+ if (curChar == 34)
+ jjCheckNAddStates(3, 5);
+ break;
+ case 15:
+ if ((0xfffffffbffffffffL & l) != 0L)
+ jjCheckNAddStates(3, 5);
+ break;
+ case 17:
+ jjCheckNAddStates(3, 5);
+ break;
+ case 18:
+ if (curChar == 34 && kind > 22)
+ kind = 22;
+ break;
+ case 19:
+ if ((0x8bff54f8ffffd9ffL & l) == 0L)
+ break;
+ if (kind > 23)
+ kind = 23;
+ jjCheckNAddTwoStates(20, 21);
+ break;
+ case 20:
+ if ((0x8bff7cf8ffffd9ffL & l) == 0L)
+ break;
+ if (kind > 23)
+ kind = 23;
+ jjCheckNAddTwoStates(20, 21);
+ break;
+ case 22:
+ if (kind > 23)
+ kind = 23;
+ jjCheckNAddTwoStates(20, 21);
+ break;
+ case 25:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 24)
+ kind = 24;
+ jjAddStates(6, 7);
+ break;
+ case 26:
+ if (curChar == 46)
+ jjCheckNAdd(27);
+ break;
+ case 27:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 24)
+ kind = 24;
+ jjCheckNAdd(27);
+ break;
+ case 28:
+ case 30:
+ if (curChar == 47)
+ jjCheckNAddStates(0, 2);
+ break;
+ case 29:
+ if ((0xffff7fffffffffffL & l) != 0L)
+ jjCheckNAddStates(0, 2);
+ break;
+ case 32:
+ if (curChar == 47 && kind > 25)
+ kind = 25;
+ break;
+ default: break;
+ }
+ } while (i != startsAt);
+ }
+ else if (curChar < 128)
+ {
+ //long l = 1L << (curChar & 077);
+ ulong l = (ulong)(1L << (curChar & 63));
+ do
+ {
+ switch (jjstateSet[--i])
+ {
+ case 0:
+ if ((0x97ffffff87ffffffL & l) != 0L)
+ {
+ if (kind > 23)
+ kind = 23;
+ jjCheckNAddTwoStates(20, 21);
+ }
+ else if (curChar == 126)
+ {
+ if (kind > 24)
+ kind = 24;
+ jjstateSet[jjnewStateCnt++] = 25;
+ }
+ else if (curChar == 92)
+ jjCheckNAdd(22);
+ if (curChar == 78)
+ jjstateSet[jjnewStateCnt++] = 11;
+ else if (curChar == 124)
+ jjstateSet[jjnewStateCnt++] = 8;
+ else if (curChar == 79)
+ jjstateSet[jjnewStateCnt++] = 6;
+ else if (curChar == 65)
+ jjstateSet[jjnewStateCnt++] = 2;
+ break;
+ case 1:
+ if (curChar == 68 && kind > 8)
+ kind = 8;
+ break;
+ case 2:
+ if (curChar == 78)
+ jjstateSet[jjnewStateCnt++] = 1;
+ break;
+ case 3:
+ if (curChar == 65)
+ jjstateSet[jjnewStateCnt++] = 2;
+ break;
+ case 6:
+ if (curChar == 82 && kind > 9)
+ kind = 9;
+ break;
+ case 7:
+ if (curChar == 79)
+ jjstateSet[jjnewStateCnt++] = 6;
+ break;
+ case 8:
+ if (curChar == 124 && kind > 9)
+ kind = 9;
+ break;
+ case 9:
+ if (curChar == 124)
+ jjstateSet[jjnewStateCnt++] = 8;
+ break;
+ case 10:
+ if (curChar == 84 && kind > 10)
+ kind = 10;
+ break;
+ case 11:
+ if (curChar == 79)
+ jjstateSet[jjnewStateCnt++] = 10;
+ break;
+ case 12:
+ if (curChar == 78)
+ jjstateSet[jjnewStateCnt++] = 11;
+ break;
+ case 15:
+ if ((0xffffffffefffffffL & l) != 0L)
+ jjCheckNAddStates(3, 5);
+ break;
+ case 16:
+ if (curChar == 92)
+ jjstateSet[jjnewStateCnt++] = 17;
+ break;
+ case 17:
+ jjCheckNAddStates(3, 5);
+ break;
+ case 19:
+ case 20:
+ if ((0x97ffffff87ffffffL & l) == 0L)
+ break;
+ if (kind > 23)
+ kind = 23;
+ jjCheckNAddTwoStates(20, 21);
+ break;
+ case 21:
+ if (curChar == 92)
+ jjCheckNAddTwoStates(22, 22);
+ break;
+ case 22:
+ if (kind > 23)
+ kind = 23;
+ jjCheckNAddTwoStates(20, 21);
+ break;
+ case 23:
+ if (curChar == 92)
+ jjCheckNAdd(22);
+ break;
+ case 24:
+ if (curChar != 126)
+ break;
+ if (kind > 24)
+ kind = 24;
+ jjstateSet[jjnewStateCnt++] = 25;
+ break;
+ case 29:
+ jjAddStates(0, 2);
+ break;
+ case 31:
+ if (curChar == 92)
+ jjstateSet[jjnewStateCnt++] = 30;
+ break;
+ default: break;
+ }
+ } while (i != startsAt);
+ }
+ else
+ {
+ int hiByte = (int)(curChar >> 8);
+ int i1 = hiByte >> 6;
+ //long l1 = 1L << (hiByte & 077);
+ ulong l1 = (ulong)(1L << (hiByte & 63));
+ int i2 = (curChar & 0xff) >> 6;
+ //long l2 = 1L << (curChar & 077);
+ ulong l2 = (ulong)(1L << (curChar & 63));
+ do
+ {
+ switch (jjstateSet[--i])
+ {
+ case 0:
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+ {
+ if (kind > 7)
+ kind = 7;
+ }
+ if (jjCanMove_2(hiByte, i1, i2, l1, l2))
+ {
+ if (kind > 23)
+ kind = 23;
+ jjCheckNAddTwoStates(20, 21);
+ }
+ break;
+ case 15:
+ case 17:
+ if (jjCanMove_1(hiByte, i1, i2, l1, l2))
+ jjCheckNAddStates(3, 5);
+ break;
+ case 19:
+ case 20:
+ if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
+ break;
+ if (kind > 23)
+ kind = 23;
+ jjCheckNAddTwoStates(20, 21);
+ break;
+ case 22:
+ if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+ break;
+ if (kind > 23)
+ kind = 23;
+ jjCheckNAddTwoStates(20, 21);
+ break;
+ case 29:
+ if (jjCanMove_1(hiByte, i1, i2, l1, l2))
+ jjAddStates(0, 2);
+ break;
+ default: break;
+ }
+ } while (i != startsAt);
+ }
+ if (kind != 0x7fffffff)
+ {
+ jjmatchedKind = kind;
+ jjmatchedPos = curPos;
+ kind = 0x7fffffff;
+ }
+ ++curPos;
+ if ((i = jjnewStateCnt) == (startsAt = 33 - (jjnewStateCnt = startsAt)))
+ return curPos;
+ try { curChar = input_stream.ReadChar(); }
+ catch (IOException e) { return curPos; }
+ }
+ }
+ private int jjMoveStringLiteralDfa0_0()
+ {
+ return jjMoveNfa_0(0, 0);
+ }
+ private int jjMoveNfa_0(int startState, int curPos)
+ {
+ int startsAt = 0;
+ jjnewStateCnt = 3;
+ int i = 1;
+ jjstateSet[0] = startState;
+ int kind = 0x7fffffff;
+ for (;;)
+ {
+ if (++jjround == 0x7fffffff)
+ ReInitRounds();
+ if (curChar < 64)
+ {
+ long l = 1L << curChar;
+ do
+ {
+ switch (jjstateSet[--i])
+ {
+ case 0:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 28)
+ kind = 28;
+ jjAddStates(8, 9);
+ break;
+ case 1:
+ if (curChar == 46)
+ jjCheckNAdd(2);
+ break;
+ case 2:
+ if ((0x3ff000000000000L & l) == 0L)
+ break;
+ if (kind > 28)
+ kind = 28;
+ jjCheckNAdd(2);
+ break;
+ default: break;
+ }
+ } while (i != startsAt);
+ }
+ else if (curChar < 128)
+ {
+ //long l = 1L << (curChar & 077);
+ long l = 1L << (curChar & 63);
+ do
+ {
+ switch (jjstateSet[--i])
+ {
+ default: break;
+ }
+ } while (i != startsAt);
+ }
+ else
+ {
+ int hiByte = (int)(curChar >> 8);
+ int i1 = hiByte >> 6;
+ //long l1 = 1L << (hiByte & 077);
+ long l1 = 1L << (hiByte & 63);
+ int i2 = (curChar & 0xff) >> 6;
+ //long l2 = 1L << (curChar & 077);
+ long l2 = 1L << (curChar & 63);
+ do
+ {
+ switch (jjstateSet[--i])
+ {
+ default: break;
+ }
+ } while (i != startsAt);
+ }
+ if (kind != 0x7fffffff)
+ {
+ jjmatchedKind = kind;
+ jjmatchedPos = curPos;
+ kind = 0x7fffffff;
+ }
+ ++curPos;
+ if ((i = jjnewStateCnt) == (startsAt = 3 - (jjnewStateCnt = startsAt)))
+ return curPos;
+ try { curChar = input_stream.ReadChar(); }
+ catch (IOException e) { return curPos; }
+ }
+ }
+ private int jjStopStringLiteralDfa_1(int pos, long active0)
+ {
+ switch (pos)
+ {
+ case 0:
+ if ((active0 & 0x20000000L) != 0L)
+ {
+ jjmatchedKind = 33;
+ return 6;
+ }
+ return -1;
+ default:
+ return -1;
+ }
+ }
+ private int jjStartNfa_1(int pos, long active0)
+ {
+ return jjMoveNfa_1(jjStopStringLiteralDfa_1(pos, active0), pos + 1);
+ }
+ private int jjMoveStringLiteralDfa0_1()
+ {
+ switch ((int)curChar)
+ {
+ case 84:
+ return jjMoveStringLiteralDfa1_1(0x20000000L);
+ case 93:
+ return jjStopAtPos(0, 30);
+ case 125:
+ return jjStopAtPos(0, 31);
+ default:
+ return jjMoveNfa_1(0, 0);
+ }
+ }
+ private int jjMoveStringLiteralDfa1_1(long active0)
+ {
+ try { curChar = input_stream.ReadChar(); }
+ catch (IOException e)
+ {
+ jjStopStringLiteralDfa_1(0, active0);
+ return 1;
+ }
+ switch ((int)curChar)
+ {
+ case 79:
+ if ((active0 & 0x20000000L) != 0L)
+ return jjStartNfaWithStates_1(1, 29, 6);
+ break;
+ default:
+ break;
+ }
+ return jjStartNfa_1(0, active0);
+ }
+ private int jjStartNfaWithStates_1(int pos, int kind, int state)
+ {
+ jjmatchedKind = kind;
+ jjmatchedPos = pos;
+ try { curChar = input_stream.ReadChar(); }
+ catch (IOException e) { return pos + 1; }
+ return jjMoveNfa_1(state, pos + 1);
+ }
+ private int jjMoveNfa_1(int startState, int curPos)
+ {
+ int startsAt = 0;
+ jjnewStateCnt = 7;
+ int i = 1;
+ jjstateSet[0] = startState;
+ int kind = 0x7fffffff;
+ for (;;)
+ {
+ if (++jjround == 0x7fffffff)
+ ReInitRounds();
+ if (curChar < 64)
+ {
+ ulong l = (ulong)(1L << curChar);
+ do
+ {
+ switch (jjstateSet[--i])
+ {
+ case 0:
+ if ((0xfffffffeffffffffL & l) != 0L)
+ {
+ if (kind > 33)
+ kind = 33;
+ jjCheckNAdd(6);
+ }
+ if ((0x100002600L & l) != 0L)
+ {
+ if (kind > 7)
+ kind = 7;
+ }
+ else if (curChar == 34)
+ jjCheckNAddTwoStates(2, 4);
+ break;
+ case 1:
+ if (curChar == 34)
+ jjCheckNAddTwoStates(2, 4);
+ break;
+ case 2:
+ if ((0xfffffffbffffffffL & l) != 0L)
+ jjCheckNAddStates(10, 12);
+ break;
+ case 3:
+ if (curChar == 34)
+ jjCheckNAddStates(10, 12);
+ break;
+ case 5:
+ if (curChar == 34 && kind > 32)
+ kind = 32;
+ break;
+ case 6:
+ if ((0xfffffffeffffffffL & l) == 0L)
+ break;
+ if (kind > 33)
+ kind = 33;
+ jjCheckNAdd(6);
+ break;
+ default: break;
+ }
+ } while (i != startsAt);
+ }
+ else if (curChar < 128)
+ {
+ //long l = 1L << (curChar & 077);
+ ulong l = (ulong)(1L << (curChar & 63));
+ do
+ {
+ switch (jjstateSet[--i])
+ {
+ case 0:
+ case 6:
+ if ((0xdfffffffdfffffffL & l) == 0L)
+ break;
+ if (kind > 33)
+ kind = 33;
+ jjCheckNAdd(6);
+ break;
+ case 2:
+ jjAddStates(10, 12);
+ break;
+ case 4:
+ if (curChar == 92)
+ jjstateSet[jjnewStateCnt++] = 3;
+ break;
+ default: break;
+ }
+ } while (i != startsAt);
+ }
+ else
+ {
+ int hiByte = (int)(curChar >> 8);
+ int i1 = hiByte >> 6;
+ //long l1 = 1L << (hiByte & 077);
+ ulong l1 = (ulong)(1L << (hiByte & 63));
+ int i2 = (curChar & 0xff) >> 6;
+ //long l2 = 1L << (curChar & 077);
+ ulong l2 = (ulong)(1L << (curChar & 63));
+ do
+ {
+ switch (jjstateSet[--i])
+ {
+ case 0:
+ if (jjCanMove_0(hiByte, i1, i2, l1, l2))
+ {
+ if (kind > 7)
+ kind = 7;
+ }
+ if (jjCanMove_1(hiByte, i1, i2, l1, l2))
+ {
+ if (kind > 33)
+ kind = 33;
+ jjCheckNAdd(6);
+ }
+ break;
+ case 2:
+ if (jjCanMove_1(hiByte, i1, i2, l1, l2))
+ jjAddStates(10, 12);
+ break;
+ case 6:
+ if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+ break;
+ if (kind > 33)
+ kind = 33;
+ jjCheckNAdd(6);
+ break;
+ default: break;
+ }
+ } while (i != startsAt);
+ }
+ if (kind != 0x7fffffff)
+ {
+ jjmatchedKind = kind;
+ jjmatchedPos = curPos;
+ kind = 0x7fffffff;
+ }
+ ++curPos;
+ if ((i = jjnewStateCnt) == (startsAt = 7 - (jjnewStateCnt = startsAt)))
+ return curPos;
+ try { curChar = input_stream.ReadChar(); }
+ catch (IOException e) { return curPos; }
+ }
+ }
+ internal static readonly int[] jjnextStates = {
+ 29, 31, 32, 15, 16, 18, 25, 26, 0, 1, 2, 4, 5,
+ };
+ private static bool jjCanMove_0(int hiByte, int i1, int i2, ulong l1, ulong l2)
+ {
+ switch (hiByte)
+ {
+ case 48:
+ return ((jjbitVec0[i2] & l2) != 0L);
+ default:
+ return false;
+ }
+ }
+ private static bool jjCanMove_1(int hiByte, int i1, int i2, ulong l1, ulong l2)
+ {
+ switch (hiByte)
+ {
+ case 0:
+ return ((jjbitVec3[i2] & l2) != 0L);
+ default:
+ if ((jjbitVec1[i1] & l1) != 0L)
+ return true;
+ return false;
+ }
+ }
+ private static bool jjCanMove_2(int hiByte, int i1, int i2, ulong l1, ulong l2)
+ {
+ switch (hiByte)
+ {
+ case 0:
+ return ((jjbitVec3[i2] & l2) != 0L);
+ case 48:
+ return ((jjbitVec1[i2] & l2) != 0L);
+ default:
+ if ((jjbitVec4[i1] & l1) != 0L)
+ return true;
+ return false;
+ }
+ }
+
+ /** Token literal values. */
+ public static readonly string[] jjstrLiteralImages = {
+ "", null, null, null, null, null, null, null, null, null, null, "\x002B", "\x002D",
+ "\x0028", "\x0029", "\x003A", "\x003D", "\x003C", "\x003C\x003D", "\x003E", "\x003E\x003D", "\x005E", null, null,
+ null, null, "\x005B", "\x007B", null, "\x0054\x004F", "\x005D", "\x007D", null, null,
+ };
+
+ /** Lexer state names. */
+ public static readonly string[] lexStateNames = {
+ "Boost",
+ "Range",
+ "DEFAULT",
+ };
+
+ /** Lex State array. */
+ public static readonly int[] jjnewLexState = {
+ -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, -1, -1, -1,
+ -1, 1, 1, 2, -1, 2, 2, -1, -1,
+ };
+ static readonly long[] jjtoToken = {
+ 0x3ffffff01L,
+ };
+ static readonly long[] jjtoSkip = {
+ 0x80L,
+ };
+ protected ICharStream input_stream;
+ private readonly uint[] jjrounds = new uint[33];
+ private readonly int[] jjstateSet = new int[66];
+ protected char curChar;
+ /** Constructor. */
+ public StandardSyntaxParserTokenManager(ICharStream stream)
+ {
+ input_stream = stream;
+ }
+
+ /** Constructor. */
+ public StandardSyntaxParserTokenManager(ICharStream stream, int lexState)
+ : this(stream)
+ {
+ SwitchTo(lexState);
+ }
+
+ /** Reinitialise parser. */
+ public void ReInit(ICharStream stream)
+ {
+ jjmatchedPos = jjnewStateCnt = 0;
+ curLexState = defaultLexState;
+ input_stream = stream;
+ ReInitRounds();
+ }
+ private void ReInitRounds()
+ {
+ int i;
+ jjround = 0x80000001;
+ for (i = 33; i-- > 0;)
+ jjrounds[i] = 0x80000000;
+ }
+
+ /** Reinitialise parser. */
+ public void ReInit(ICharStream stream, int lexState)
+ {
+ ReInit(stream);
+ SwitchTo(lexState);
+ }
+
+ /** Switch to specified lex state. */
+ public void SwitchTo(int lexState)
+ {
+ if (lexState >= 3 || lexState < 0)
+ throw new TokenMgrError("Error: Ignoring invalid lexical state : " + lexState + ". State unchanged.", TokenMgrError.INVALID_LEXICAL_STATE);
+ else
+ curLexState = lexState;
+ }
+
+ protected Token jjFillToken()
+ {
+ Token t;
+ string curTokenImage;
+ int beginLine;
+ int endLine;
+ int beginColumn;
+ int endColumn;
+ string im = jjstrLiteralImages[jjmatchedKind];
+ curTokenImage = (im == null) ? input_stream.GetImage() : im;
+ beginLine = input_stream.GetBeginLine();
+ beginColumn = input_stream.GetBeginColumn();
+ endLine = input_stream.GetEndLine();
+ endColumn = input_stream.GetEndColumn();
+ t = Token.NewToken(jjmatchedKind, curTokenImage);
+
+ t.beginLine = beginLine;
+ t.endLine = endLine;
+ t.beginColumn = beginColumn;
+ t.endColumn = endColumn;
+
+ return t;
+ }
+
+ internal int curLexState = 2;
+ internal int defaultLexState = 2;
+ internal int jjnewStateCnt;
+ internal uint jjround;
+ internal int jjmatchedPos;
+ internal int jjmatchedKind;
+
+ /** Get the next Token. */
+ public Token getNextToken()
+ {
+ Token matchedToken;
+ int curPos = 0;
+
+
+ for (;;)
+ {
+ try
+ {
+ curChar = input_stream.BeginToken();
+ }
+ catch (IOException e)
+ {
+ jjmatchedKind = 0;
+ matchedToken = jjFillToken();
+ return matchedToken;
+ }
+
+ switch (curLexState)
+ {
+ case 0:
+ jjmatchedKind = 0x7fffffff;
+ jjmatchedPos = 0;
+ curPos = jjMoveStringLiteralDfa0_0();
+ break;
+ case 1:
+ jjmatchedKind = 0x7fffffff;
+ jjmatchedPos = 0;
+ curPos = jjMoveStringLiteralDfa0_1();
+ break;
+ case 2:
+ jjmatchedKind = 0x7fffffff;
+ jjmatchedPos = 0;
+ curPos = jjMoveStringLiteralDfa0_2();
+ break;
+ }
+ if (jjmatchedKind != 0x7fffffff)
+ {
+ if (jjmatchedPos + 1 < curPos)
+ input_stream.BackUp(curPos - jjmatchedPos - 1);
+ if ((jjtoToken[jjmatchedKind >> 6] & (1L << (jjmatchedKind & 63 /*077*/))) != 0L)
+ {
+ matchedToken = jjFillToken();
+ if (jjnewLexState[jjmatchedKind] != -1)
+ curLexState = jjnewLexState[jjmatchedKind];
+ return matchedToken;
+ }
+ else
+ {
+ if (jjnewLexState[jjmatchedKind] != -1)
+ curLexState = jjnewLexState[jjmatchedKind];
+ goto EOFLoop_continue;
+ }
+ }
+ int error_line = input_stream.GetEndLine();
+ int error_column = input_stream.GetEndColumn();
+ string error_after = null;
+ bool EOFSeen = false;
+ try { input_stream.ReadChar(); input_stream.BackUp(1); }
+ catch (IOException e1)
+ {
+ EOFSeen = true;
+ error_after = curPos <= 1 ? "" : input_stream.GetImage();
+ if (curChar == '\n' || curChar == '\r')
+ {
+ error_line++;
+ error_column = 0;
+ }
+ else
+ error_column++;
+ }
+ if (!EOFSeen)
+ {
+ input_stream.BackUp(1);
+ error_after = curPos <= 1 ? "" : input_stream.GetImage();
+ }
+ throw new TokenMgrError(EOFSeen, curLexState, error_line, error_column, error_after, curChar, TokenMgrError.LEXICAL_ERROR);
+
+ EOFLoop_continue: { }
+ }
+ }
+
+ private void jjCheckNAdd(int state)
+ {
+ if (jjrounds[state] != jjround)
+ {
+ jjstateSet[jjnewStateCnt++] = state;
+ jjrounds[state] = jjround;
+ }
+ }
+ private void jjAddStates(int start, int end)
+ {
+ do
+ {
+ jjstateSet[jjnewStateCnt++] = jjnextStates[start];
+ } while (start++ != end);
+ }
+ private void jjCheckNAddTwoStates(int state1, int state2)
+ {
+ jjCheckNAdd(state1);
+ jjCheckNAdd(state2);
+ }
+
+ private void jjCheckNAddStates(int start, int end)
+ {
+ do
+ {
+ jjCheckNAdd(jjnextStates[start]);
+ } while (start++ != end);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.QueryParser/Flexible/Standard/Parser/Token.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.QueryParser/Flexible/Standard/Parser/Token.cs b/src/Lucene.Net.QueryParser/Flexible/Standard/Parser/Token.cs
new file mode 100644
index 0000000..03a4bea
--- /dev/null
+++ b/src/Lucene.Net.QueryParser/Flexible/Standard/Parser/Token.cs
@@ -0,0 +1,135 @@
+\ufeffusing System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Standard.Parser
+{
+ /// <summary>
+ /// Describes the input token stream.
+ /// </summary>
+ [Serializable]
+ public class Token
+ {
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static readonly long serialVersionUID = 1L;
+
+ /**
+ * An integer that describes the kind of this token. This numbering
+ * system is determined by JavaCCParser, and a table of these numbers is
+ * stored in the file ...Constants.java.
+ */
+ public int kind;
+
+ /** The line number of the first character of this Token. */
+ public int beginLine;
+ /** The column number of the first character of this Token. */
+ public int beginColumn;
+ /** The line number of the last character of this Token. */
+ public int endLine;
+ /** The column number of the last character of this Token. */
+ public int endColumn;
+
+ /**
+ * The string image of the token.
+ */
+ public string image;
+
+ /**
+ * A reference to the next regular (non-special) token from the input
+ * stream. If this is the last token from the input stream, or if the
+ * token manager has not read tokens beyond this one, this field is
+ * set to null. This is true only if this token is also a regular
+ * token. Otherwise, see below for a description of the contents of
+ * this field.
+ */
+ public Token next;
+
+ /**
+ * This field is used to access special tokens that occur prior to this
+ * token, but after the immediately preceding regular (non-special) token.
+ * If there are no such special tokens, this field is set to null.
+ * When there are more than one such special token, this field refers
+ * to the last of these special tokens, which in turn refers to the next
+ * previous special token through its specialToken field, and so on
+ * until the first special token (whose specialToken field is null).
+ * The next fields of special tokens refer to other special tokens that
+ * immediately follow it (without an intervening regular token). If there
+ * is no such token, this field is null.
+ */
+ public Token specialToken;
+
+ /**
+ * An optional attribute value of the Token.
+ * Tokens which are not used as syntactic sugar will often contain
+ * meaningful values that will be used later on by the compiler or
+ * interpreter. This attribute value is often different from the image.
+ * Any subclass of Token that actually wants to return a non-null value can
+ * override this method as appropriate.
+ */
+ public object GetValue()
+ {
+ return null;
+ }
+
+ /**
+ * No-argument constructor
+ */
+ public Token() { }
+
+ /**
+ * Constructs a new token for the specified Image.
+ */
+ public Token(int kind)
+ : this(kind, null)
+ {
+ }
+
+ /**
+ * Constructs a new token for the specified Image and Kind.
+ */
+ public Token(int kind, string image)
+ {
+ this.kind = kind;
+ this.image = image;
+ }
+
+ /**
+ * Returns the image.
+ */
+ public override string ToString()
+ {
+ return image;
+ }
+
+ /**
+ * Returns a new Token object, by default. However, if you want, you
+ * can create and return subclass objects based on the value of ofKind.
+ * Simply add the cases to the switch for all those special cases.
+ * For example, if you have a subclass of Token called IDToken that
+ * you want to create if ofKind is ID, simply add something like :
+ *
+ * case MyParserConstants.ID : return new IDToken(ofKind, image);
+ *
+ * to the following switch statement. Then you can cast matchedToken
+ * variable to the appropriate type and use sit in your lexical actions.
+ */
+ public static Token NewToken(int ofKind, string image)
+ {
+ switch (ofKind)
+ {
+ default: return new Token(ofKind, image);
+ }
+ }
+
+ public static Token NewToken(int ofKind)
+ {
+ return NewToken(ofKind, null);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.QueryParser/Flexible/Standard/Parser/TokenMgrError.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.QueryParser/Flexible/Standard/Parser/TokenMgrError.cs b/src/Lucene.Net.QueryParser/Flexible/Standard/Parser/TokenMgrError.cs
new file mode 100644
index 0000000..0485601
--- /dev/null
+++ b/src/Lucene.Net.QueryParser/Flexible/Standard/Parser/TokenMgrError.cs
@@ -0,0 +1,164 @@
+\ufeffusing System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Standard.Parser
+{
+ /// <summary>
+ /// Token Manager Error.
+ /// </summary>
+ [Serializable]
+ public class TokenMgrError : Exception
+ {
+ /**
+ * The version identifier for this Serializable class.
+ * Increment only if the <i>serialized</i> form of the
+ * class changes.
+ */
+ private static readonly long serialVersionUID = 1L;
+
+ /*
+ * Ordinals for various reasons why an Error of this type can be thrown.
+ */
+
+ /**
+ * Lexical error occurred.
+ */
+ internal static readonly int LEXICAL_ERROR = 0;
+
+ /**
+ * An attempt was made to create a second instance of a static token manager.
+ */
+ internal static readonly int STATIC_LEXER_ERROR = 1;
+
+ /**
+ * Tried to change to an invalid lexical state.
+ */
+ internal static readonly int INVALID_LEXICAL_STATE = 2;
+
+ /**
+ * Detected (and bailed out of) an infinite loop in the token manager.
+ */
+ internal static readonly int LOOP_DETECTED = 3;
+
+ /**
+ * Indicates the reason why the exception is thrown. It will have
+ * one of the above 4 values.
+ */
+ internal int errorCode;
+
+ /**
+ * Replaces unprintable characters by their escaped (or unicode escaped)
+ * equivalents in the given string
+ */
+ protected static string AddEscapes(string str)
+ {
+ StringBuilder retval = new StringBuilder();
+ char ch;
+ for (int i = 0; i < str.Length; i++)
+ {
+ switch (str[i])
+ {
+ case (char)0:
+ continue;
+ case '\b':
+ retval.Append("\\b");
+ continue;
+ case '\t':
+ retval.Append("\\t");
+ continue;
+ case '\n':
+ retval.Append("\\n");
+ continue;
+ case '\f':
+ retval.Append("\\f");
+ continue;
+ case '\r':
+ retval.Append("\\r");
+ continue;
+ case '\"':
+ retval.Append("\\\"");
+ continue;
+ case '\'':
+ retval.Append("\\\'");
+ continue;
+ case '\\':
+ retval.Append("\\\\");
+ continue;
+ default:
+ if ((ch = str[i]) < 0x20 || ch > 0x7e)
+ {
+ //string s = "0000" + Integer.toString(ch, 16);
+ string s = "0000" + Convert.ToString(ch, 16);
+ retval.Append("\\u" + s.Substring(s.Length - 4, /*s.Length - (s.Length - 4)*/ 4));
+ }
+ else
+ {
+ retval.Append(ch);
+ }
+ continue;
+ }
+ }
+ return retval.ToString();
+ }
+
+ /**
+ * Returns a detailed message for the Error when it is thrown by the
+ * token manager to indicate a lexical error.
+ * Parameters :
+ * EOFSeen : indicates if EOF caused the lexical error
+ * curLexState : lexical state in which this error occurred
+ * errorLine : line number when the error occurred
+ * errorColumn : column number when the error occurred
+ * errorAfter : prefix that was seen before this error occurred
+ * curchar : the offending character
+ * Note: You can customize the lexical error message by modifying this method.
+ */
+ protected static string LexicalError(bool EOFSeen, int lexState, int errorLine, int errorColumn, string errorAfter, char curChar)
+ {
+ return ("Lexical error at line " +
+ errorLine + ", column " +
+ errorColumn + ". Encountered: " +
+ (EOFSeen ? "<EOF> " : ("\"" + AddEscapes(curChar.ToString()) + "\"") + " (" + (int)curChar + "), ") +
+ "after : \"" + AddEscapes(errorAfter) + "\"");
+ }
+
+ /**
+ * You can also modify the body of this method to customize your error messages.
+ * For example, cases like LOOP_DETECTED and INVALID_LEXICAL_STATE are not
+ * of end-users concern, so you can return something like :
+ *
+ * "Internal Error : Please file a bug report .... "
+ *
+ * from this method for such cases in the release version of your parser.
+ */
+ public override string Message
+ {
+ get { return base.Message; }
+ }
+
+ /*
+ * Constructors of various flavors follow.
+ */
+
+ /** No arg constructor. */
+ public TokenMgrError()
+ {
+ }
+
+ /** Constructor with message and reason. */
+ public TokenMgrError(string message, int reason)
+ : base(message)
+ {
+ errorCode = reason;
+ }
+
+ /** Full Constructor. */
+ public TokenMgrError(bool EOFSeen, int lexState, int errorLine, int errorColumn, string errorAfter, char curChar, int reason)
+ : this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason)
+ {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/AllowLeadingWildcardProcessor.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/AllowLeadingWildcardProcessor.cs b/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/AllowLeadingWildcardProcessor.cs
new file mode 100644
index 0000000..a672fa4
--- /dev/null
+++ b/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/AllowLeadingWildcardProcessor.cs
@@ -0,0 +1,100 @@
+\ufeffusing Lucene.Net.QueryParsers.Flexible.Core;
+using Lucene.Net.QueryParsers.Flexible.Core.Messages;
+using Lucene.Net.QueryParsers.Flexible.Core.Nodes;
+using Lucene.Net.QueryParsers.Flexible.Core.Processors;
+using Lucene.Net.QueryParsers.Flexible.Core.Util;
+using Lucene.Net.QueryParsers.Flexible.Messages;
+using Lucene.Net.QueryParsers.Flexible.Standard.Config;
+using Lucene.Net.QueryParsers.Flexible.Standard.Nodes;
+using Lucene.Net.QueryParsers.Flexible.Standard.Parser;
+using Lucene.Net.Support;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Standard.Processors
+{
+ /// <summary>
+ /// This processor verifies if
+ /// {@link ConfigurationKeys#ALLOW_LEADING_WILDCARD} is defined in the
+ /// {@link QueryConfigHandler}. If it is and leading wildcard is not allowed, it
+ /// looks for every {@link WildcardQueryNode} contained in the query node tree
+ /// and throws an exception if any of them has a leading wildcard ('*' or '?').
+ /// </summary>
+ /// <seealso cref="ConfigurationKeys#ALLOW_LEADING_WILDCARD"/>
+ public class AllowLeadingWildcardProcessor : QueryNodeProcessorImpl
+ {
+ public AllowLeadingWildcardProcessor()
+ {
+ // empty constructor
+ }
+
+
+ public override IQueryNode Process(IQueryNode queryTree)
+ {
+ bool? allowsLeadingWildcard = GetQueryConfigHandler().Get(ConfigurationKeys.ALLOW_LEADING_WILDCARD);
+
+ if (allowsLeadingWildcard != null)
+ {
+
+ if (!allowsLeadingWildcard.Value)
+ {
+ return base.Process(queryTree);
+ }
+
+ }
+
+ return queryTree;
+ }
+
+
+ protected override IQueryNode PostProcessNode(IQueryNode node)
+ {
+
+ if (node is WildcardQueryNode)
+ {
+ WildcardQueryNode wildcardNode = (WildcardQueryNode)node;
+
+ if (wildcardNode.Text.Length > 0)
+ {
+
+ // Validate if the wildcard was escaped
+ if (UnescapedCharSequence.WasEscaped(wildcardNode.Text, 0))
+ return node;
+
+ switch (wildcardNode.Text[0])
+ {
+ case '*':
+ case '?':
+ throw new QueryNodeException(new MessageImpl(
+ QueryParserMessages.LEADING_WILDCARD_NOT_ALLOWED, node
+ .ToQueryString(new EscapeQuerySyntaxImpl())));
+ }
+ }
+
+ }
+
+ return node;
+
+ }
+
+
+ protected override IQueryNode PreProcessNode(IQueryNode node)
+ {
+
+ return node;
+
+ }
+
+
+ protected override IList<IQueryNode> SetChildrenOrder(IList<IQueryNode> children)
+ {
+
+ return children;
+
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/AnalyzerQueryNodeProcessor.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/AnalyzerQueryNodeProcessor.cs b/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/AnalyzerQueryNodeProcessor.cs
new file mode 100644
index 0000000..4510388
--- /dev/null
+++ b/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/AnalyzerQueryNodeProcessor.cs
@@ -0,0 +1,435 @@
+\ufeffusing Lucene.Net.Analysis;
+using Lucene.Net.Analysis.Tokenattributes;
+using Lucene.Net.QueryParsers.Flexible.Core.Nodes;
+using Lucene.Net.QueryParsers.Flexible.Core.Processors;
+using Lucene.Net.QueryParsers.Flexible.Standard.Config;
+using Lucene.Net.QueryParsers.Flexible.Standard.Nodes;
+using Lucene.Net.Support;
+using Lucene.Net.Util;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using static Lucene.Net.QueryParsers.Classic.QueryParserBase;
+
+namespace Lucene.Net.QueryParsers.Flexible.Standard.Processors
+{
+ /// <summary>
+ /// This processor verifies if {@link ConfigurationKeys#ANALYZER}
+ /// is defined in the {@link QueryConfigHandler}. If it is and the analyzer is
+ /// not <code>null</code>, it looks for every {@link FieldQueryNode} that is not
+ /// {@link WildcardQueryNode}, {@link FuzzyQueryNode} or
+ /// {@link RangeQueryNode} contained in the query node tree, then it applies
+ /// the analyzer to that {@link FieldQueryNode} object.
+ /// <para/>
+ /// If the analyzer return only one term, the returned term is set to the
+ /// {@link FieldQueryNode} and it's returned.
+ /// <para/>
+ /// If the analyzer return more than one term, a {@link TokenizedPhraseQueryNode}
+ /// or {@link MultiPhraseQueryNode} is created, whether there is one or more
+ /// terms at the same position, and it's returned.
+ /// <para/>
+ /// If no term is returned by the analyzer a {@link NoTokenFoundQueryNode} object
+ /// is returned.
+ /// </summary>
+ /// <seealso cref="ConfigurationKeys#ANALYZER"/>
+ /// <seealso cref="Analyzer"/>
+ /// <seealso cref="TokenStream"/>
+ public class AnalyzerQueryNodeProcessor : QueryNodeProcessorImpl
+ {
+ private Analyzer analyzer;
+
+ private bool positionIncrementsEnabled;
+
+ private Config.Operator defaultOperator;
+
+ public AnalyzerQueryNodeProcessor()
+ {
+ // empty constructor
+ }
+
+
+ public override IQueryNode Process(IQueryNode queryTree)
+ {
+ Analyzer analyzer = GetQueryConfigHandler().Get(ConfigurationKeys.ANALYZER);
+
+ if (analyzer != null)
+ {
+ this.analyzer = analyzer;
+ this.positionIncrementsEnabled = false;
+ bool? positionIncrementsEnabled = GetQueryConfigHandler().Get(ConfigurationKeys.ENABLE_POSITION_INCREMENTS);
+ var defaultOperator = GetQueryConfigHandler().Get(ConfigurationKeys.DEFAULT_OPERATOR);
+ this.defaultOperator = defaultOperator != null ? defaultOperator.Value : Config.Operator.OR;
+
+ if (positionIncrementsEnabled != null)
+ {
+ this.positionIncrementsEnabled = positionIncrementsEnabled.Value;
+ }
+
+ if (this.analyzer != null)
+ {
+ return base.Process(queryTree);
+ }
+
+ }
+
+ return queryTree;
+
+ }
+
+
+ protected override IQueryNode PostProcessNode(IQueryNode node)
+ {
+
+ if (node is ITextableQueryNode
+ && !(node is WildcardQueryNode)
+ && !(node is FuzzyQueryNode)
+ && !(node is RegexpQueryNode)
+ && !(node.GetParent() is IRangeQueryNode))
+ {
+
+ FieldQueryNode fieldNode = ((FieldQueryNode)node);
+ string text = fieldNode.GetTextAsString();
+ string field = fieldNode.GetFieldAsString();
+
+ CachingTokenFilter buffer = null;
+ IPositionIncrementAttribute posIncrAtt = null;
+ int numTokens = 0;
+ int positionCount = 0;
+ bool severalTokensAtSamePosition = false;
+
+ TokenStream source = null;
+ try
+ {
+ source = this.analyzer.TokenStream(field, text);
+ source.Reset();
+ buffer = new CachingTokenFilter(source);
+
+ if (buffer.HasAttribute<IPositionIncrementAttribute>())
+ {
+ posIncrAtt = buffer.GetAttribute<IPositionIncrementAttribute>();
+ }
+
+ try
+ {
+
+ while (buffer.IncrementToken())
+ {
+ numTokens++;
+ int positionIncrement = (posIncrAtt != null) ? posIncrAtt
+ .PositionIncrement : 1;
+ if (positionIncrement != 0)
+ {
+ positionCount += positionIncrement;
+
+ }
+ else
+ {
+ severalTokensAtSamePosition = true;
+ }
+
+ }
+
+ }
+ catch (IOException e)
+ {
+ // ignore
+ }
+ }
+ catch (IOException e)
+ {
+ throw new Exception(e.Message, e);
+ }
+ finally
+ {
+ IOUtils.CloseWhileHandlingException(source);
+ }
+
+ // rewind the buffer stream
+ buffer.Reset();
+
+ if (!buffer.HasAttribute<ICharTermAttribute>())
+ {
+ return new NoTokenFoundQueryNode();
+ }
+
+ ICharTermAttribute termAtt = buffer.GetAttribute<ICharTermAttribute>();
+
+ if (numTokens == 0)
+ {
+ return new NoTokenFoundQueryNode();
+
+ }
+ else if (numTokens == 1)
+ {
+ string term = null;
+ try
+ {
+ bool hasNext;
+ hasNext = buffer.IncrementToken();
+ Debug.Assert(hasNext == true);
+ term = termAtt.ToString();
+
+ }
+ catch (IOException e)
+ {
+ // safe to ignore, because we know the number of tokens
+ }
+
+ fieldNode.Text = term.ToCharSequence();
+
+ return fieldNode;
+
+ }
+ else if (severalTokensAtSamePosition || !(node is QuotedFieldQueryNode))
+ {
+ if (positionCount == 1 || !(node is QuotedFieldQueryNode))
+ {
+ // no phrase query:
+
+ if (positionCount == 1)
+ {
+ // simple case: only one position, with synonyms
+ List<IQueryNode> children = new List<IQueryNode>();
+
+ for (int i = 0; i < numTokens; i++)
+ {
+ string term = null;
+ try
+ {
+ bool hasNext = buffer.IncrementToken();
+ Debug.Assert(hasNext == true);
+ term = termAtt.ToString();
+
+ }
+ catch (IOException e)
+ {
+ // safe to ignore, because we know the number of tokens
+ }
+
+ children.Add(new FieldQueryNode(field, term, -1, -1));
+
+ }
+ return new GroupQueryNode(
+ new StandardBooleanQueryNode(children, positionCount == 1));
+
+ }
+ else
+ {
+ // multiple positions
+ IQueryNode q = new StandardBooleanQueryNode(new List<IQueryNode>(), false);
+ IQueryNode currentQuery = null;
+ for (int i = 0; i < numTokens; i++)
+ {
+ string term = null;
+ try
+ {
+ bool hasNext = buffer.IncrementToken();
+ Debug.Assert(hasNext == true);
+ term = termAtt.ToString();
+ }
+ catch (IOException e)
+ {
+ // safe to ignore, because we know the number of tokens
+ }
+ if (posIncrAtt != null && posIncrAtt.PositionIncrement == 0)
+ {
+ if (!(currentQuery is BooleanQueryNode))
+ {
+ IQueryNode t = currentQuery;
+ currentQuery = new StandardBooleanQueryNode(new List<IQueryNode>(), true);
+ ((BooleanQueryNode)currentQuery).Add(t);
+ }
+ ((BooleanQueryNode)currentQuery).Add(new FieldQueryNode(field, term, -1, -1));
+ }
+ else
+ {
+ if (currentQuery != null)
+ {
+ if (this.defaultOperator == Config.Operator.OR)
+ {
+ q.Add(currentQuery);
+ }
+ else
+ {
+ q.Add(new ModifierQueryNode(currentQuery, Modifier.MOD_REQ));
+ }
+ }
+ currentQuery = new FieldQueryNode(field, term, -1, -1);
+ }
+ }
+ if (this.defaultOperator == Config.Operator.OR)
+ {
+ q.Add(currentQuery);
+ }
+ else
+ {
+ q.Add(new ModifierQueryNode(currentQuery, Modifier.MOD_REQ));
+ }
+
+ if (q is BooleanQueryNode)
+ {
+ q = new GroupQueryNode(q);
+ }
+ return q;
+ }
+ }
+ else
+ {
+ // phrase query:
+ MultiPhraseQueryNode mpq = new MultiPhraseQueryNode();
+
+ List<FieldQueryNode> multiTerms = new List<FieldQueryNode>();
+ int position = -1;
+ int i = 0;
+ int termGroupCount = 0;
+ for (; i < numTokens; i++)
+ {
+ string term = null;
+ int positionIncrement = 1;
+ try
+ {
+ bool hasNext = buffer.IncrementToken();
+ Debug.Assert(hasNext == true);
+ term = termAtt.ToString();
+ if (posIncrAtt != null)
+ {
+ positionIncrement = posIncrAtt.PositionIncrement;
+ }
+
+ }
+ catch (IOException e)
+ {
+ // safe to ignore, because we know the number of tokens
+ }
+
+ if (positionIncrement > 0 && multiTerms.Count > 0)
+ {
+
+ foreach (FieldQueryNode termNode in multiTerms)
+ {
+
+ if (this.positionIncrementsEnabled)
+ {
+ termNode.SetPositionIncrement(position);
+ }
+ else
+ {
+ termNode.SetPositionIncrement(termGroupCount);
+ }
+
+ mpq.Add(termNode);
+
+ }
+
+ // Only increment once for each "group" of
+ // terms that were in the same position:
+ termGroupCount++;
+
+ multiTerms.Clear();
+
+ }
+
+ position += positionIncrement;
+ multiTerms.Add(new FieldQueryNode(field, term, -1, -1));
+
+ }
+
+ foreach (FieldQueryNode termNode in multiTerms)
+ {
+
+ if (this.positionIncrementsEnabled)
+ {
+ termNode.SetPositionIncrement(position);
+
+ }
+ else
+ {
+ termNode.SetPositionIncrement(termGroupCount);
+ }
+
+ mpq.Add(termNode);
+
+ }
+
+ return mpq;
+
+ }
+
+ }
+ else
+ {
+
+ TokenizedPhraseQueryNode pq = new TokenizedPhraseQueryNode();
+
+ int position = -1;
+
+ for (int i = 0; i < numTokens; i++)
+ {
+ string term = null;
+ int positionIncrement = 1;
+
+ try
+ {
+ bool hasNext = buffer.IncrementToken();
+ Debug.Assert(hasNext == true);
+ term = termAtt.ToString();
+
+ if (posIncrAtt != null)
+ {
+ positionIncrement = posIncrAtt.PositionIncrement;
+ }
+
+ }
+ catch (IOException e)
+ {
+ // safe to ignore, because we know the number of tokens
+ }
+
+ FieldQueryNode newFieldNode = new FieldQueryNode(field, term, -1, -1);
+
+ if (this.positionIncrementsEnabled)
+ {
+ position += positionIncrement;
+ newFieldNode.SetPositionIncrement(position);
+
+ }
+ else
+ {
+ newFieldNode.SetPositionIncrement(i);
+ }
+
+ pq.Add(newFieldNode);
+
+ }
+
+ return pq;
+
+ }
+
+ }
+
+ return node;
+
+ }
+
+
+ protected override IQueryNode PreProcessNode(IQueryNode node)
+ {
+
+ return node;
+
+ }
+
+
+ protected override IList<IQueryNode> SetChildrenOrder(IList<IQueryNode> children)
+ {
+
+ return children;
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/BooleanQuery2ModifierNodeProcessor.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/BooleanQuery2ModifierNodeProcessor.cs b/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/BooleanQuery2ModifierNodeProcessor.cs
new file mode 100644
index 0000000..8b02976
--- /dev/null
+++ b/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/BooleanQuery2ModifierNodeProcessor.cs
@@ -0,0 +1,210 @@
+\ufeffusing Lucene.Net.QueryParsers.Flexible.Core.Config;
+using Lucene.Net.QueryParsers.Flexible.Core.Nodes;
+using Lucene.Net.QueryParsers.Flexible.Core.Processors;
+using Lucene.Net.QueryParsers.Flexible.Standard.Config;
+using Lucene.Net.QueryParsers.Flexible.Standard.Nodes;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Standard.Processors
+{
+ /// <summary>
+ /// This processor is used to apply the correct {@link ModifierQueryNode} to
+ /// {@link BooleanQueryNode}s children. This is a variant of
+ /// {@link BooleanModifiersQueryNodeProcessor} which ignores precedence.
+ /// <para/>
+ /// The {@link StandardSyntaxParser} knows the rules of precedence, but lucene
+ /// does not. e.g. <code>(A AND B OR C AND D)</code> ist treated like
+ /// <code>(+A +B +C +D)</code>.
+ /// <para/>
+ /// This processor walks through the query node tree looking for
+ /// {@link BooleanQueryNode}s. If an {@link AndQueryNode} is found, every child,
+ /// which is not a {@link ModifierQueryNode} or the {@link ModifierQueryNode} is
+ /// {@link Modifier#MOD_NONE}, becomes a {@link Modifier#MOD_REQ}. For default
+ /// {@link BooleanQueryNode}, it checks the default operator is
+ /// {@link Operator#AND}, if it is, the same operation when an
+ /// {@link AndQueryNode} is found is applied to it. Each {@link BooleanQueryNode}
+ /// which direct parent is also a {@link BooleanQueryNode} is removed (to ignore
+ /// the rules of precedence).
+ /// </summary>
+ /// <seealso cref="ConfigurationKeys#DEFAULT_OPERATOR"/>
+ /// <seealso cref="BooleanModifiersQueryNodeProcessor"/>
+ public class BooleanQuery2ModifierNodeProcessor : IQueryNodeProcessor
+ {
+ internal readonly static string TAG_REMOVE = "remove";
+ internal readonly static string TAG_MODIFIER = "wrapWithModifier";
+ internal readonly static string TAG_BOOLEAN_ROOT = "booleanRoot";
+
+ QueryConfigHandler queryConfigHandler;
+
+ private readonly List<IQueryNode> childrenBuffer = new List<IQueryNode>();
+
+ private bool usingAnd = false;
+
+ public BooleanQuery2ModifierNodeProcessor()
+ {
+ // empty constructor
+ }
+
+
+ public virtual IQueryNode Process(IQueryNode queryTree)
+ {
+ Operator? op = GetQueryConfigHandler().Get(
+ ConfigurationKeys.DEFAULT_OPERATOR);
+
+ if (op == null)
+ {
+ throw new ArgumentException(
+ "StandardQueryConfigHandler.ConfigurationKeys.DEFAULT_OPERATOR should be set on the QueryConfigHandler");
+ }
+
+ this.usingAnd = Operator.AND == op;
+
+ return ProcessIteration(queryTree);
+
+ }
+
+
+ protected void ProcessChildren(IQueryNode queryTree)
+ {
+ IList<IQueryNode> children = queryTree.GetChildren();
+ if (children != null && children.Count > 0)
+ {
+ foreach (IQueryNode child in children)
+ {
+ /*child = */
+ ProcessIteration(child);
+ }
+ }
+ }
+
+ private IQueryNode ProcessIteration(IQueryNode queryTree)
+ {
+ queryTree = PreProcessNode(queryTree);
+
+ ProcessChildren(queryTree);
+
+ queryTree = PostProcessNode(queryTree);
+
+ return queryTree;
+
+ }
+
+ protected virtual void FillChildrenBufferAndApplyModifiery(IQueryNode parent)
+ {
+ foreach (IQueryNode node in parent.GetChildren())
+ {
+ if (node.ContainsTag(TAG_REMOVE))
+ {
+ FillChildrenBufferAndApplyModifiery(node);
+ }
+ else if (node.ContainsTag(TAG_MODIFIER))
+ {
+ childrenBuffer.Add(ApplyModifier(node,
+ (Modifier)node.GetTag(TAG_MODIFIER)));
+ }
+ else
+ {
+ childrenBuffer.Add(node);
+ }
+ }
+ }
+
+ protected IQueryNode PostProcessNode(IQueryNode node)
+ {
+ if (node.ContainsTag(TAG_BOOLEAN_ROOT))
+ {
+ this.childrenBuffer.Clear();
+ FillChildrenBufferAndApplyModifiery(node);
+ node.Set(childrenBuffer);
+ }
+ return node;
+
+ }
+
+ protected IQueryNode PreProcessNode(IQueryNode node)
+ {
+ IQueryNode parent = node.GetParent();
+ if (node is BooleanQueryNode)
+ {
+ if (parent is BooleanQueryNode)
+ {
+ node.SetTag(TAG_REMOVE, true); // no precedence
+ }
+ else
+ {
+ node.SetTag(TAG_BOOLEAN_ROOT, true);
+ }
+ }
+ else if (parent is BooleanQueryNode)
+ {
+ if ((parent is AndQueryNode)
+ || (usingAnd && IsDefaultBooleanQueryNode(parent)))
+ {
+ TagModifierButDoNotOverride(node, Modifier.MOD_REQ);
+ }
+ }
+ return node;
+ }
+
+ protected bool IsDefaultBooleanQueryNode(IQueryNode toTest)
+ {
+ return toTest != null && typeof(BooleanQueryNode).Equals(toTest.GetType());
+ }
+
+ private IQueryNode ApplyModifier(IQueryNode node, Modifier mod)
+ {
+
+ // check if modifier is not already defined and is default
+ if (!(node is ModifierQueryNode))
+ {
+ return new BooleanModifierNode(node, mod);
+
+ }
+ else
+ {
+ ModifierQueryNode modNode = (ModifierQueryNode)node;
+
+ if (modNode.GetModifier() == Modifier.MOD_NONE)
+ {
+ return new ModifierQueryNode(modNode.GetChild(), mod);
+ }
+
+ }
+
+ return node;
+
+ }
+
+ protected void TagModifierButDoNotOverride(IQueryNode node, Modifier mod)
+ {
+ if (node is ModifierQueryNode)
+ {
+ ModifierQueryNode modNode = (ModifierQueryNode)node;
+ if (modNode.GetModifier() == Modifier.MOD_NONE)
+ {
+ node.SetTag(TAG_MODIFIER, mod);
+ }
+ }
+ else
+ {
+ node.SetTag(TAG_MODIFIER, Modifier.MOD_REQ);
+ }
+ }
+
+ public virtual void SetQueryConfigHandler(QueryConfigHandler queryConfigHandler)
+ {
+ this.queryConfigHandler = queryConfigHandler;
+
+ }
+
+
+ public virtual QueryConfigHandler GetQueryConfigHandler()
+ {
+ return queryConfigHandler;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/BooleanSingleChildOptimizationQueryNodeProcessor.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/BooleanSingleChildOptimizationQueryNodeProcessor.cs b/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/BooleanSingleChildOptimizationQueryNodeProcessor.cs
new file mode 100644
index 0000000..36637e6
--- /dev/null
+++ b/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/BooleanSingleChildOptimizationQueryNodeProcessor.cs
@@ -0,0 +1,79 @@
+\ufeffusing Lucene.Net.QueryParsers.Flexible.Core.Nodes;
+using Lucene.Net.QueryParsers.Flexible.Core.Processors;
+using Lucene.Net.QueryParsers.Flexible.Standard.Nodes;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Standard.Processors
+{
+ /// <summary>
+ /// This processor removes every {@link BooleanQueryNode} that contains only one
+ /// child and returns this child. If this child is {@link ModifierQueryNode} that
+ /// was defined by the user. A modifier is not defined by the user when it's a
+ /// {@link BooleanModifierNode}
+ /// </summary>
+ /// <seealso cref="ModifierQueryNode"/>
+ public class BooleanSingleChildOptimizationQueryNodeProcessor : QueryNodeProcessorImpl
+ {
+ public BooleanSingleChildOptimizationQueryNodeProcessor()
+ {
+ // empty constructor
+ }
+
+
+ protected override IQueryNode PostProcessNode(IQueryNode node)
+ {
+
+ if (node is BooleanQueryNode)
+ {
+ IList<IQueryNode> children = node.GetChildren();
+
+ if (children != null && children.Count == 1)
+ {
+ IQueryNode child = children[0];
+
+ if (child is ModifierQueryNode)
+ {
+ ModifierQueryNode modNode = (ModifierQueryNode)child;
+
+ if (modNode is BooleanModifierNode
+ || modNode.GetModifier() == Modifier.MOD_NONE)
+ {
+
+ return child;
+
+ }
+
+ }
+ else
+ {
+ return child;
+ }
+
+ }
+
+ }
+
+ return node;
+
+ }
+
+
+ protected override IQueryNode PreProcessNode(IQueryNode node)
+ {
+
+ return node;
+
+ }
+
+ protected override IList<IQueryNode> SetChildrenOrder(IList<IQueryNode> children)
+ {
+
+ return children;
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/BoostQueryNodeProcessor.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/BoostQueryNodeProcessor.cs b/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/BoostQueryNodeProcessor.cs
new file mode 100644
index 0000000..03776d2
--- /dev/null
+++ b/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/BoostQueryNodeProcessor.cs
@@ -0,0 +1,74 @@
+\ufeffusing Lucene.Net.QueryParsers.Flexible.Core.Config;
+using Lucene.Net.QueryParsers.Flexible.Core.Nodes;
+using Lucene.Net.QueryParsers.Flexible.Core.Processors;
+using Lucene.Net.QueryParsers.Flexible.Core.Util;
+using Lucene.Net.QueryParsers.Flexible.Standard.Config;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Standard.Processors
+{
+ /// <summary>
+ /// This processor iterates the query node tree looking for every
+ /// {@link FieldableNode} that has {@link ConfigurationKeys#BOOST} in its
+ /// config. If there is, the boost is applied to that {@link FieldableNode}.
+ /// </summary>
+ /// <seealso cref="ConfigurationKeys#BOOST"/>
+ /// <seealso cref="QueryConfigHandler"/>
+ /// <seealso cref="IFieldableNode"/>
+ public class BoostQueryNodeProcessor : QueryNodeProcessorImpl
+ {
+ protected override IQueryNode PostProcessNode(IQueryNode node)
+ {
+
+ if (node is IFieldableNode &&
+ (node.GetParent() == null || !(node.GetParent() is IFieldableNode)))
+ {
+
+ IFieldableNode fieldNode = (IFieldableNode)node;
+ QueryConfigHandler config = GetQueryConfigHandler();
+
+ if (config != null)
+ {
+ string field = fieldNode.Field;
+ FieldConfig fieldConfig = config.GetFieldConfig(StringUtils.ToString(field));
+
+ if (fieldConfig != null)
+ {
+ float? boost = fieldConfig.Get(ConfigurationKeys.BOOST);
+
+ if (boost != null)
+ {
+ return new BoostQueryNode(node, boost.Value);
+ }
+
+ }
+
+ }
+
+ }
+
+ return node;
+
+ }
+
+
+ protected override IQueryNode PreProcessNode(IQueryNode node)
+ {
+
+ return node;
+
+ }
+
+
+ protected override IList<IQueryNode> SetChildrenOrder(IList<IQueryNode> children)
+ {
+
+ return children;
+
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucenenet/blob/c83be6be/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/DefaultPhraseSlopQueryNodeProcessor.cs
----------------------------------------------------------------------
diff --git a/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/DefaultPhraseSlopQueryNodeProcessor.cs b/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/DefaultPhraseSlopQueryNodeProcessor.cs
new file mode 100644
index 0000000..f52ebba
--- /dev/null
+++ b/src/Lucene.Net.QueryParser/Flexible/Standard/Processors/DefaultPhraseSlopQueryNodeProcessor.cs
@@ -0,0 +1,112 @@
+\ufeffusing Lucene.Net.QueryParsers.Flexible.Core.Config;
+using Lucene.Net.QueryParsers.Flexible.Core.Nodes;
+using Lucene.Net.QueryParsers.Flexible.Core.Processors;
+using Lucene.Net.QueryParsers.Flexible.Standard.Config;
+using Lucene.Net.QueryParsers.Flexible.Standard.Nodes;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Lucene.Net.QueryParsers.Flexible.Standard.Processors
+{
+ /// <summary>
+ /// This processor verifies if {@link ConfigurationKeys#PHRASE_SLOP}
+ /// is defined in the {@link QueryConfigHandler}. If it is, it looks for every
+ /// {@link TokenizedPhraseQueryNode} and {@link MultiPhraseQueryNode} that does
+ /// not have any {@link SlopQueryNode} applied to it and creates an
+ /// {@link SlopQueryNode} and apply to it. The new {@link SlopQueryNode} has the
+ /// same slop value defined in the configuration.
+ /// </summary>
+ /// <seealso cref="SlopQueryNode"/>
+ /// <seealso cref="ConfigurationKeys#PHRASE_SLOP"/>
+ public class DefaultPhraseSlopQueryNodeProcessor : QueryNodeProcessorImpl
+ {
+ private bool processChildren = true;
+
+ private int defaultPhraseSlop;
+
+ public DefaultPhraseSlopQueryNodeProcessor()
+ {
+ // empty constructor
+ }
+
+
+ public override IQueryNode Process(IQueryNode queryTree)
+ {
+ QueryConfigHandler queryConfig = GetQueryConfigHandler();
+
+ if (queryConfig != null)
+ {
+ int? defaultPhraseSlop = queryConfig.Get(ConfigurationKeys.PHRASE_SLOP);
+
+ if (defaultPhraseSlop != null)
+ {
+ this.defaultPhraseSlop = defaultPhraseSlop.Value;
+
+ return base.Process(queryTree);
+
+ }
+
+ }
+
+ return queryTree;
+
+ }
+
+
+ protected override IQueryNode PostProcessNode(IQueryNode node)
+ {
+
+ if (node is TokenizedPhraseQueryNode
+ || node is MultiPhraseQueryNode)
+ {
+
+ return new SlopQueryNode(node, this.defaultPhraseSlop);
+
+ }
+
+ return node;
+
+ }
+
+
+ protected override IQueryNode PreProcessNode(IQueryNode node)
+ {
+
+ if (node is SlopQueryNode)
+ {
+ this.processChildren = false;
+
+ }
+
+ return node;
+
+ }
+
+
+ protected override void ProcessChildren(IQueryNode queryTree)
+ {
+
+ if (this.processChildren)
+ {
+ base.ProcessChildren(queryTree);
+
+ }
+ else
+ {
+ this.processChildren = true;
+ }
+
+ }
+
+
+ protected override IList<IQueryNode> SetChildrenOrder(IList<IQueryNode> children)
+ {
+
+ return children;
+
+ }
+ }
+}