You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/03/10 16:05:21 UTC

svn commit: r1454862 [28/49] - in /lucene/dev/branches/lucene4258: ./ dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/ dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/demo/ dev-tools/idea/lucene/facet/ dev-tools/idea/lucene/highlighter/ dev-to...

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java Sun Mar 10 15:04:57 2013
@@ -18,29 +18,20 @@ package org.apache.lucene.queries.functi
  */
 
 import java.io.IOException;
-import java.util.Date;
 
 import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.ByteDocValuesField;
-import org.apache.lucene.document.DerefBytesDocValuesField;
+import org.apache.lucene.document.BinaryDocValuesField;
 import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleDocValuesField;
 import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.document.IntDocValuesField;
-import org.apache.lucene.document.LongDocValuesField;
-import org.apache.lucene.document.PackedLongDocValuesField;
-import org.apache.lucene.document.ShortDocValuesField;
-import org.apache.lucene.document.SortedBytesDocValuesField;
-import org.apache.lucene.document.StraightBytesDocValuesField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
 import org.apache.lucene.index.AtomicReaderContext;
 import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.queries.function.valuesource.DateDocValuesFieldSource;
-import org.apache.lucene.queries.function.valuesource.NumericDocValuesFieldSource;
-import org.apache.lucene.queries.function.valuesource.StrDocValuesFieldSource;
+import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
+import org.apache.lucene.queries.function.valuesource.LongFieldSource;
 import org.apache.lucene.store.Directory;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.LuceneTestCase;
@@ -49,53 +40,24 @@ import org.apache.lucene.util.packed.Pac
 
 import com.carrotsearch.randomizedtesting.generators.RandomInts;
 
+
 public class TestDocValuesFieldSources extends LuceneTestCase {
 
-  public void test(DocValues.Type type) throws IOException {
+  public void test(DocValuesType type) throws IOException {
     Directory d = newDirectory();
     IndexWriterConfig iwConfig = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
     final int nDocs = atLeast(50);
-    final Field id = new IntDocValuesField("id", 0);
+    final Field id = new NumericDocValuesField("id", 0);
     final Field f;
     switch (type) {
-      case BYTES_FIXED_DEREF:
-        f = new DerefBytesDocValuesField("dv", new BytesRef(), true);
-        break;
-      case BYTES_FIXED_SORTED:
-        f = new SortedBytesDocValuesField("dv", new BytesRef(), true);
-        break;
-      case BYTES_FIXED_STRAIGHT:
-        f = new StraightBytesDocValuesField("dv", new BytesRef(), true);
-        break;
-      case BYTES_VAR_DEREF:
-        f = new DerefBytesDocValuesField("dv", new BytesRef(), false);
-        break;
-      case BYTES_VAR_SORTED:
-        f = new SortedBytesDocValuesField("dv", new BytesRef(), false);
-        break;
-      case BYTES_VAR_STRAIGHT:
-        f = new StraightBytesDocValuesField("dv", new BytesRef(), false);
-        break;
-      case FIXED_INTS_8:
-        f = new ByteDocValuesField("dv", (byte) 0);
-        break;
-      case FIXED_INTS_16:
-        f = new ShortDocValuesField("dv", (short) 0);
-        break;
-      case FIXED_INTS_32:
-        f = new IntDocValuesField("dv", 0);
-        break;
-      case FIXED_INTS_64:
-        f = new LongDocValuesField("dv", 0L);
-        break;
-      case VAR_INTS:
-        f = new PackedLongDocValuesField("dv", 0L);
+      case BINARY:
+        f = new BinaryDocValuesField("dv", new BytesRef());
         break;
-      case FLOAT_32:
-        f = new FloatDocValuesField("dv", 0f);
+      case SORTED:
+        f = new SortedDocValuesField("dv", new BytesRef());
         break;
-      case FLOAT_64:
-        f = new DoubleDocValuesField("dv", 0d);
+      case NUMERIC:
+        f = new NumericDocValuesField("dv", 0);
         break;
       default:
         throw new AssertionError();
@@ -108,46 +70,20 @@ public class TestDocValuesFieldSources e
 
     RandomIndexWriter iw = new RandomIndexWriter(random(), d, iwConfig);
     for (int i = 0; i < nDocs; ++i) {
-      id.setIntValue(i);
+      id.setLongValue(i);
       switch (type) {
-        case BYTES_FIXED_DEREF:
-        case BYTES_FIXED_SORTED:
-        case BYTES_FIXED_STRAIGHT:
-          vals[i] = _TestUtil.randomFixedByteLengthUnicodeString(random(), 10);
+        case SORTED:
+        case BINARY:
+          do {
+            vals[i] = _TestUtil.randomSimpleString(random(), 20);
+          } while (((String) vals[i]).isEmpty());
           f.setBytesValue(new BytesRef((String) vals[i]));
           break;
-        case BYTES_VAR_DEREF:
-        case BYTES_VAR_SORTED:
-        case BYTES_VAR_STRAIGHT:
-          vals[i] = _TestUtil.randomSimpleString(random(), 20);
-          f.setBytesValue(new BytesRef((String) vals[i]));
-          break;
-        case FIXED_INTS_8:
-          vals[i] = (byte) random().nextInt(256);
-          f.setByteValue((Byte) vals[i]);
-          break;
-        case FIXED_INTS_16:
-          vals[i] = (short) random().nextInt(1 << 16);
-          f.setShortValue((Short) vals[i]);
-          break;
-        case FIXED_INTS_32:
-          vals[i] = random().nextInt();
-          f.setIntValue((Integer) vals[i]);
-          break;
-        case FIXED_INTS_64:
-        case VAR_INTS:
+        case NUMERIC:
           final int bitsPerValue = RandomInts.randomIntBetween(random(), 1, 31); // keep it an int
           vals[i] = (long) random().nextInt((int) PackedInts.maxValue(bitsPerValue));
           f.setLongValue((Long) vals[i]);
           break;
-        case FLOAT_32:
-          vals[i] = random().nextFloat();
-          f.setFloatValue((Float) vals[i]);
-          break;
-        case FLOAT_64:
-          vals[i] = random().nextDouble();
-          f.setDoubleValue((Double) vals[i]);
-          break;
       }
       iw.addDocument(document);
       if (random().nextBoolean() && i % 10 == 9) {
@@ -158,32 +94,15 @@ public class TestDocValuesFieldSources e
 
     DirectoryReader rd = DirectoryReader.open(d);
     for (AtomicReaderContext leave : rd.leaves()) {
-      final FunctionValues ids = new NumericDocValuesFieldSource("id", false).getValues(null, leave);
+      final FunctionValues ids = new LongFieldSource("id").getValues(null, leave);
       final ValueSource vs;
-      final boolean direct = random().nextBoolean();
       switch (type) {
-        case BYTES_FIXED_DEREF:
-        case BYTES_FIXED_SORTED:
-        case BYTES_FIXED_STRAIGHT:
-        case BYTES_VAR_DEREF:
-        case BYTES_VAR_SORTED:
-        case BYTES_VAR_STRAIGHT:
-          vs = new StrDocValuesFieldSource("dv", direct);
-          break;
-        case FLOAT_32:
-        case FLOAT_64:
-        case FIXED_INTS_8:
-        case FIXED_INTS_16:
-        case FIXED_INTS_32:
-          vs = new NumericDocValuesFieldSource("dv", direct);
+        case BINARY:
+        case SORTED:
+          vs = new BytesRefFieldSource("dv");
           break;
-        case FIXED_INTS_64:
-        case VAR_INTS:
-          if (random().nextBoolean()) {
-            vs = new NumericDocValuesFieldSource("dv", direct);
-          } else {
-            vs = new DateDocValuesFieldSource("dv", direct);
-          }
+        case NUMERIC:
+          vs = new LongFieldSource("dv");
           break;
         default:
           throw new AssertionError();
@@ -192,61 +111,21 @@ public class TestDocValuesFieldSources e
       BytesRef bytes = new BytesRef();
       for (int i = 0; i < leave.reader().maxDoc(); ++i) {
         assertTrue(values.exists(i));
-        if (vs instanceof StrDocValuesFieldSource) {
+        if (vs instanceof BytesRefFieldSource) {
           assertTrue(values.objectVal(i) instanceof String);
-        } else if (vs instanceof NumericDocValuesFieldSource) {
-          assertTrue(values.objectVal(i) instanceof Number);
-          switch (type) {
-            case FIXED_INTS_8:
-              assertTrue(values.objectVal(i) instanceof Byte);
-              assertTrue(values.bytesVal(i, bytes));
-              assertEquals(1, bytes.length);
-              break;
-            case FIXED_INTS_16:
-              assertTrue(values.objectVal(i) instanceof Short);
-              assertTrue(values.bytesVal(i, bytes));
-              assertEquals(2, bytes.length);
-              break;
-            case FIXED_INTS_32:
-              assertTrue(values.objectVal(i) instanceof Integer);
-              assertTrue(values.bytesVal(i, bytes));
-              assertEquals(4, bytes.length);
-              break;
-            case FIXED_INTS_64:
-            case VAR_INTS:
-              assertTrue(values.objectVal(i) instanceof Long);
-              assertTrue(values.bytesVal(i, bytes));
-              assertEquals(8, bytes.length);
-              break;
-            case FLOAT_32:
-              assertTrue(values.objectVal(i) instanceof Float);
-              assertTrue(values.bytesVal(i, bytes));
-              assertEquals(4, bytes.length);
-              break;
-            case FLOAT_64:
-              assertTrue(values.objectVal(i) instanceof Double);
-              assertTrue(values.bytesVal(i, bytes));
-              assertEquals(8, bytes.length);
-              break;
-            default:
-              throw new AssertionError();
-          }
-        } else if (vs instanceof DateDocValuesFieldSource) {
-          assertTrue(values.objectVal(i) instanceof Date);
+        } else if (vs instanceof LongFieldSource) {
+          assertTrue(values.objectVal(i) instanceof Long);
+          assertTrue(values.bytesVal(i, bytes));
         } else {
           throw new AssertionError();
         }
         
         Object expected = vals[ids.intVal(i)];
         switch (type) {
-          case BYTES_VAR_SORTED:
-          case BYTES_FIXED_SORTED:
+          case SORTED:
             values.ordVal(i); // no exception
             assertTrue(values.numOrd() >= 1);
-          case BYTES_FIXED_DEREF:
-          case BYTES_FIXED_STRAIGHT:
-          case BYTES_VAR_DEREF:
-          case BYTES_VAR_STRAIGHT:
+          case BINARY:
             assertEquals(expected, values.objectVal(i));
             assertEquals(expected, values.strVal(i));
             assertEquals(expected, values.objectVal(i));
@@ -254,17 +133,7 @@ public class TestDocValuesFieldSources e
             assertTrue(values.bytesVal(i, bytes));
             assertEquals(new BytesRef((String) expected), bytes);
             break;
-          case FLOAT_32:
-            assertEquals(((Number) expected).floatValue(), values.floatVal(i), 0.001);
-            break;
-          case FLOAT_64:
-            assertEquals(((Number) expected).doubleValue(), values.doubleVal(i), 0.001d);
-            break;
-          case FIXED_INTS_8:
-          case FIXED_INTS_16:
-          case FIXED_INTS_32:
-          case FIXED_INTS_64:
-          case VAR_INTS:
+          case NUMERIC:
             assertEquals(((Number) expected).longValue(), values.longVal(i));
             break;
         }
@@ -275,8 +144,10 @@ public class TestDocValuesFieldSources e
   }
 
   public void test() throws IOException {
-    for (DocValues.Type type : DocValues.Type.values()) {
-      test(type);
+    for (DocValuesType type : DocValuesType.values()) {
+      if (type != DocValuesType.SORTED_SET) {
+        test(type);
+      }
     }
   }
 

Modified: lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java Sun Mar 10 15:04:57 2013
@@ -133,7 +133,7 @@ public class TestOrdValues extends Funct
       String id = s.getIndexReader().document(sd[i].doc).get(ID_FIELD);
       log("-------- " + i + ". Explain doc " + id);
       log(s.explain(q, sd[i].doc));
-      float expectedScore = N_DOCS - i;
+      float expectedScore = N_DOCS - i - 1;
       assertEquals("score of result " + i + " shuould be " + expectedScore + " != " + score, expectedScore, score, TEST_SCORE_TOLERANCE_DELTA);
       String expectedId = inOrder
               ? id2String(N_DOCS - i) // in-order ==> larger  values first

Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj Sun Mar 10 15:04:57 2013
@@ -162,7 +162,7 @@ PARSER_END(QueryParser)
 | <CARAT:     "^" > : Boost
 | <QUOTED:     "\"" (<_QUOTED_CHAR>)* "\"">
 | <TERM:      <_TERM_START_CHAR> (<_TERM_CHAR>)*  >
-| <FUZZY_SLOP:     "~" ( (<_NUM_CHAR>)+ ( "." (<_NUM_CHAR>)+ )? )? >
+| <FUZZY_SLOP:     "~" ((<_NUM_CHAR>)+ (( "." (<_NUM_CHAR>)+ )? (<_TERM_CHAR>)*) | (<_TERM_CHAR>)*)  >
 | <PREFIXTERM:  ("*") | ( <_TERM_START_CHAR> (<_TERM_CHAR>)* "*" ) >
 | <WILDTERM:  (<_TERM_START_CHAR> | [ "*", "?" ]) (<_TERM_CHAR> | ( [ "*", "?" ] ))* >
 | <REGEXPTERM: "/" (~[ "/" ] | "\\/" )* "/" >

Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java Sun Mar 10 15:04:57 2013
@@ -32,6 +32,7 @@ import org.apache.lucene.index.Term;
 import org.apache.lucene.queryparser.classic.QueryParser.Operator;
 import org.apache.lucene.queryparser.flexible.standard.CommonQueryParserConfiguration;
 import org.apache.lucene.search.*;
+import org.apache.lucene.search.BooleanQuery.TooManyClauses;
 import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.Version;
 
@@ -310,11 +311,11 @@ public abstract class QueryParserBase im
 
   /**
    * By default QueryParser uses {@link org.apache.lucene.search.MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}
-   * when creating a PrefixQuery, WildcardQuery or RangeQuery. This implementation is generally preferable because it
+   * when creating a {@link PrefixQuery}, {@link WildcardQuery} or {@link TermRangeQuery}. This implementation is generally preferable because it
    * a) Runs faster b) Does not have the scarcity of terms unduly influence score
-   * c) avoids any "TooManyBooleanClauses" exception.
+   * c) avoids any {@link TooManyClauses} exception.
    * However, if your application really needs to use the
-   * old-fashioned BooleanQuery expansion rewriting and the above
+   * old-fashioned {@link BooleanQuery} expansion rewriting and the above
    * points are not relevant then use this to change
    * the rewrite method.
    */
@@ -416,9 +417,9 @@ public abstract class QueryParserBase im
   }
 
   /**
-   * Set whether or not to analyze range terms when constructing RangeQuerys.
+   * Set whether or not to analyze range terms when constructing {@link TermRangeQuery}s.
    * For example, setting this to true can enable analyzing terms into 
-   * collation keys for locale-sensitive RangeQuery.
+   * collation keys for locale-sensitive {@link TermRangeQuery}.
    * 
    * @param analyzeRangeTerms whether or not terms should be analyzed for RangeQuerys
    */
@@ -427,7 +428,7 @@ public abstract class QueryParserBase im
   }
 
   /**
-   * @return whether or not to analyze range terms when constructing RangeQuerys.
+   * @return whether or not to analyze range terms when constructing {@link TermRangeQuery}s.
    */
   public boolean getAnalyzeRangeTerms() {
     return analyzeRangeTerms;
@@ -844,13 +845,13 @@ public abstract class QueryParserBase im
   }
 
   /**
-   * Builds a new TermRangeQuery instance
+   * Builds a new {@link TermRangeQuery} instance
    * @param field Field
    * @param part1 min
    * @param part2 max
    * @param startInclusive true if the start of the range is inclusive
    * @param endInclusive true if the end of the range is inclusive
-   * @return new TermRangeQuery instance
+   * @return new {@link TermRangeQuery} instance
    */
   protected Query newRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
     final BytesRef start;
@@ -1070,22 +1071,29 @@ public abstract class QueryParserBase im
     } else if (regexp) {
       q = getRegexpQuery(qfield, term.image.substring(1, term.image.length()-1));
     } else if (fuzzy) {
-      float fms = fuzzyMinSim;
-      try {
-        fms = Float.valueOf(fuzzySlop.image.substring(1)).floatValue();
-      } catch (Exception ignored) { }
-      if(fms < 0.0f){
-        throw new ParseException("Minimum similarity for a FuzzyQuery has to be between 0.0f and 1.0f !");
-      } else if (fms >= 1.0f && fms != (int) fms) {
-        throw new ParseException("Fractional edit distances are not allowed!");
-      }
-      q = getFuzzyQuery(qfield, termImage, fms);
+      q = handleBareFuzzy(qfield, fuzzySlop, termImage);
     } else {
       q = getFieldQuery(qfield, termImage, false);
     }
     return q;
   }
 
+  Query handleBareFuzzy(String qfield, Token fuzzySlop, String termImage)
+      throws ParseException {
+    Query q;
+    float fms = fuzzyMinSim;
+    try {
+      fms = Float.valueOf(fuzzySlop.image.substring(1)).floatValue();
+    } catch (Exception ignored) { }
+    if(fms < 0.0f){
+      throw new ParseException("Minimum similarity for a FuzzyQuery has to be between 0.0f and 1.0f !");
+    } else if (fms >= 1.0f && fms != (int) fms) {
+      throw new ParseException("Fractional edit distances are not allowed!");
+    }
+    q = getFuzzyQuery(qfield, termImage, fms);
+    return q;
+  }
+
   // extracted from the .jj grammar
   Query handleQuotedTerm(String qfield, Token term, Token fuzzySlop) throws ParseException {
     int s = phraseSlop;  // default

Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java Sun Mar 10 15:04:57 2013
@@ -47,7 +47,7 @@ private int jjMoveStringLiteralDfa0_2()
       case 41:
          return jjStopAtPos(0, 15);
       case 42:
-         return jjStartNfaWithStates_2(0, 17, 43);
+         return jjStartNfaWithStates_2(0, 17, 49);
       case 43:
          return jjStartNfaWithStates_2(0, 11, 15);
       case 45:
@@ -87,7 +87,7 @@ static final long[] jjbitVec4 = {
 private int jjMoveNfa_2(int startState, int curPos)
 {
    int startsAt = 0;
-   jjnewStateCnt = 43;
+   jjnewStateCnt = 49;
    int i = 1;
    jjstateSet[0] = startState;
    int kind = 0x7fffffff;
@@ -102,12 +102,20 @@ private int jjMoveNfa_2(int startState, 
          {
             switch(jjstateSet[--i])
             {
+               case 49:
+               case 33:
+                  if ((0xfbff7cf8ffffd9ffL & l) == 0L)
+                     break;
+                  if (kind > 23)
+                     kind = 23;
+                  jjCheckNAddTwoStates(33, 34);
+                  break;
                case 0:
                   if ((0xfbff54f8ffffd9ffL & l) != 0L)
                   {
                      if (kind > 23)
                         kind = 23;
-                     jjCheckNAddTwoStates(27, 28);
+                     jjCheckNAddTwoStates(33, 34);
                   }
                   else if ((0x100002600L & l) != 0L)
                   {
@@ -139,14 +147,6 @@ private int jjMoveNfa_2(int startState, 
                   if (curChar == 38)
                      jjstateSet[jjnewStateCnt++] = 4;
                   break;
-               case 43:
-               case 27:
-                  if ((0xfbff7cf8ffffd9ffL & l) == 0L)
-                     break;
-                  if (kind > 23)
-                     kind = 23;
-                  jjCheckNAddTwoStates(27, 28);
-                  break;
                case 4:
                   if (curChar == 38 && kind > 8)
                      kind = 8;
@@ -187,7 +187,7 @@ private int jjMoveNfa_2(int startState, 
                      break;
                   if (kind > 21)
                      kind = 21;
-                  jjAddStates(11, 12);
+                  jjCheckNAddStates(11, 14);
                   break;
                case 23:
                   if (curChar == 46)
@@ -198,62 +198,86 @@ private int jjMoveNfa_2(int startState, 
                      break;
                   if (kind > 21)
                      kind = 21;
-                  jjCheckNAdd(24);
+                  jjCheckNAddStates(15, 17);
                   break;
                case 25:
+                  if ((0x7bff78f8ffffd9ffL & l) == 0L)
+                     break;
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(25, 26);
+                  break;
+               case 27:
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(25, 26);
+                  break;
+               case 28:
+                  if ((0x7bff78f8ffffd9ffL & l) == 0L)
+                     break;
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(28, 29);
+                  break;
+               case 30:
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(28, 29);
+                  break;
+               case 31:
                   if (curChar == 42 && kind > 22)
                      kind = 22;
                   break;
-               case 26:
+               case 32:
                   if ((0xfbff54f8ffffd9ffL & l) == 0L)
                      break;
                   if (kind > 23)
                      kind = 23;
-                  jjCheckNAddTwoStates(27, 28);
+                  jjCheckNAddTwoStates(33, 34);
                   break;
-               case 29:
+               case 35:
                   if (kind > 23)
                      kind = 23;
-                  jjCheckNAddTwoStates(27, 28);
+                  jjCheckNAddTwoStates(33, 34);
                   break;
-               case 30:
-               case 32:
+               case 36:
+               case 38:
                   if (curChar == 47)
                      jjCheckNAddStates(0, 2);
                   break;
-               case 31:
+               case 37:
                   if ((0xffff7fffffffffffL & l) != 0L)
                      jjCheckNAddStates(0, 2);
                   break;
-               case 34:
+               case 40:
                   if (curChar == 47 && kind > 24)
                      kind = 24;
                   break;
-               case 35:
+               case 41:
                   if ((0x7bff50f8ffffd9ffL & l) == 0L)
                      break;
                   if (kind > 20)
                      kind = 20;
                   jjCheckNAddStates(6, 10);
                   break;
-               case 36:
+               case 42:
                   if ((0x7bff78f8ffffd9ffL & l) == 0L)
                      break;
                   if (kind > 20)
                      kind = 20;
-                  jjCheckNAddTwoStates(36, 37);
+                  jjCheckNAddTwoStates(42, 43);
                   break;
-               case 38:
+               case 44:
                   if (kind > 20)
                      kind = 20;
-                  jjCheckNAddTwoStates(36, 37);
+                  jjCheckNAddTwoStates(42, 43);
                   break;
-               case 39:
+               case 45:
                   if ((0x7bff78f8ffffd9ffL & l) != 0L)
-                     jjCheckNAddStates(13, 15);
+                     jjCheckNAddStates(18, 20);
                   break;
-               case 41:
-                  jjCheckNAddStates(13, 15);
+               case 47:
+                  jjCheckNAddStates(18, 20);
                   break;
                default : break;
             }
@@ -266,6 +290,16 @@ private int jjMoveNfa_2(int startState, 
          {
             switch(jjstateSet[--i])
             {
+               case 49:
+                  if ((0x97ffffff87ffffffL & l) != 0L)
+                  {
+                     if (kind > 23)
+                        kind = 23;
+                     jjCheckNAddTwoStates(33, 34);
+                  }
+                  else if (curChar == 92)
+                     jjCheckNAddTwoStates(35, 35);
+                  break;
                case 0:
                   if ((0x97ffffff87ffffffL & l) != 0L)
                   {
@@ -274,18 +308,18 @@ private int jjMoveNfa_2(int startState, 
                      jjCheckNAddStates(6, 10);
                   }
                   else if (curChar == 92)
-                     jjCheckNAddStates(16, 18);
+                     jjCheckNAddStates(21, 23);
                   else if (curChar == 126)
                   {
                      if (kind > 21)
                         kind = 21;
-                     jjstateSet[jjnewStateCnt++] = 22;
+                     jjCheckNAddStates(24, 26);
                   }
                   if ((0x97ffffff87ffffffL & l) != 0L)
                   {
                      if (kind > 23)
                         kind = 23;
-                     jjCheckNAddTwoStates(27, 28);
+                     jjCheckNAddTwoStates(33, 34);
                   }
                   if (curChar == 78)
                      jjstateSet[jjnewStateCnt++] = 11;
@@ -296,16 +330,6 @@ private int jjMoveNfa_2(int startState, 
                   else if (curChar == 65)
                      jjstateSet[jjnewStateCnt++] = 2;
                   break;
-               case 43:
-                  if ((0x97ffffff87ffffffL & l) != 0L)
-                  {
-                     if (kind > 23)
-                        kind = 23;
-                     jjCheckNAddTwoStates(27, 28);
-                  }
-                  else if (curChar == 92)
-                     jjCheckNAddTwoStates(29, 29);
-                  break;
                case 1:
                   if (curChar == 68 && kind > 8)
                      kind = 8;
@@ -362,75 +386,107 @@ private int jjMoveNfa_2(int startState, 
                      break;
                   if (kind > 21)
                      kind = 21;
-                  jjstateSet[jjnewStateCnt++] = 22;
+                  jjCheckNAddStates(24, 26);
+                  break;
+               case 25:
+                  if ((0x97ffffff87ffffffL & l) == 0L)
+                     break;
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(25, 26);
                   break;
                case 26:
+                  if (curChar == 92)
+                     jjAddStates(27, 28);
+                  break;
+               case 27:
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(25, 26);
+                  break;
+               case 28:
+                  if ((0x97ffffff87ffffffL & l) == 0L)
+                     break;
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(28, 29);
+                  break;
+               case 29:
+                  if (curChar == 92)
+                     jjAddStates(29, 30);
+                  break;
+               case 30:
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(28, 29);
+                  break;
+               case 32:
                   if ((0x97ffffff87ffffffL & l) == 0L)
                      break;
                   if (kind > 23)
                      kind = 23;
-                  jjCheckNAddTwoStates(27, 28);
+                  jjCheckNAddTwoStates(33, 34);
                   break;
-               case 27:
+               case 33:
                   if ((0x97ffffff87ffffffL & l) == 0L)
                      break;
                   if (kind > 23)
                      kind = 23;
-                  jjCheckNAddTwoStates(27, 28);
+                  jjCheckNAddTwoStates(33, 34);
                   break;
-               case 28:
+               case 34:
                   if (curChar == 92)
-                     jjCheckNAddTwoStates(29, 29);
+                     jjCheckNAddTwoStates(35, 35);
                   break;
-               case 29:
+               case 35:
                   if (kind > 23)
                      kind = 23;
-                  jjCheckNAddTwoStates(27, 28);
+                  jjCheckNAddTwoStates(33, 34);
                   break;
-               case 31:
+               case 37:
                   jjAddStates(0, 2);
                   break;
-               case 33:
+               case 39:
                   if (curChar == 92)
-                     jjstateSet[jjnewStateCnt++] = 32;
+                     jjstateSet[jjnewStateCnt++] = 38;
                   break;
-               case 35:
+               case 41:
                   if ((0x97ffffff87ffffffL & l) == 0L)
                      break;
                   if (kind > 20)
                      kind = 20;
                   jjCheckNAddStates(6, 10);
                   break;
-               case 36:
+               case 42:
                   if ((0x97ffffff87ffffffL & l) == 0L)
                      break;
                   if (kind > 20)
                      kind = 20;
-                  jjCheckNAddTwoStates(36, 37);
+                  jjCheckNAddTwoStates(42, 43);
                   break;
-               case 37:
+               case 43:
                   if (curChar == 92)
-                     jjCheckNAddTwoStates(38, 38);
+                     jjCheckNAddTwoStates(44, 44);
                   break;
-               case 38:
+               case 44:
                   if (kind > 20)
                      kind = 20;
-                  jjCheckNAddTwoStates(36, 37);
+                  jjCheckNAddTwoStates(42, 43);
                   break;
-               case 39:
+               case 45:
                   if ((0x97ffffff87ffffffL & l) != 0L)
-                     jjCheckNAddStates(13, 15);
+                     jjCheckNAddStates(18, 20);
                   break;
-               case 40:
+               case 46:
                   if (curChar == 92)
-                     jjCheckNAddTwoStates(41, 41);
+                     jjCheckNAddTwoStates(47, 47);
                   break;
-               case 41:
-                  jjCheckNAddStates(13, 15);
+               case 47:
+                  jjCheckNAddStates(18, 20);
                   break;
-               case 42:
+               case 48:
                   if (curChar == 92)
-                     jjCheckNAddStates(16, 18);
+                     jjCheckNAddStates(21, 23);
                   break;
                default : break;
             }
@@ -447,6 +503,14 @@ private int jjMoveNfa_2(int startState, 
          {
             switch(jjstateSet[--i])
             {
+               case 49:
+               case 33:
+                  if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 23)
+                     kind = 23;
+                  jjCheckNAddTwoStates(33, 34);
+                  break;
                case 0:
                   if (jjCanMove_0(hiByte, i1, i2, l1, l2))
                   {
@@ -457,7 +521,7 @@ private int jjMoveNfa_2(int startState, 
                   {
                      if (kind > 23)
                         kind = 23;
-                     jjCheckNAddTwoStates(27, 28);
+                     jjCheckNAddTwoStates(33, 34);
                   }
                   if (jjCanMove_2(hiByte, i1, i2, l1, l2))
                   {
@@ -466,14 +530,6 @@ private int jjMoveNfa_2(int startState, 
                      jjCheckNAddStates(6, 10);
                   }
                   break;
-               case 43:
-               case 27:
-                  if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
-                     break;
-                  if (kind > 23)
-                     kind = 23;
-                  jjCheckNAddTwoStates(27, 28);
-                  break;
                case 15:
                   if (jjCanMove_0(hiByte, i1, i2, l1, l2) && kind > 13)
                      kind = 13;
@@ -483,52 +539,80 @@ private int jjMoveNfa_2(int startState, 
                   if (jjCanMove_1(hiByte, i1, i2, l1, l2))
                      jjCheckNAddStates(3, 5);
                   break;
-               case 26:
+               case 25:
+                  if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(25, 26);
+                  break;
+               case 27:
+                  if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(25, 26);
+                  break;
+               case 28:
+                  if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(28, 29);
+                  break;
+               case 30:
+                  if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+                     break;
+                  if (kind > 21)
+                     kind = 21;
+                  jjCheckNAddTwoStates(28, 29);
+                  break;
+               case 32:
                   if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 23)
                      kind = 23;
-                  jjCheckNAddTwoStates(27, 28);
+                  jjCheckNAddTwoStates(33, 34);
                   break;
-               case 29:
+               case 35:
                   if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 23)
                      kind = 23;
-                  jjCheckNAddTwoStates(27, 28);
+                  jjCheckNAddTwoStates(33, 34);
                   break;
-               case 31:
+               case 37:
                   if (jjCanMove_1(hiByte, i1, i2, l1, l2))
                      jjAddStates(0, 2);
                   break;
-               case 35:
+               case 41:
                   if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 20)
                      kind = 20;
                   jjCheckNAddStates(6, 10);
                   break;
-               case 36:
+               case 42:
                   if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 20)
                      kind = 20;
-                  jjCheckNAddTwoStates(36, 37);
+                  jjCheckNAddTwoStates(42, 43);
                   break;
-               case 38:
+               case 44:
                   if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
                      break;
                   if (kind > 20)
                      kind = 20;
-                  jjCheckNAddTwoStates(36, 37);
+                  jjCheckNAddTwoStates(42, 43);
                   break;
-               case 39:
+               case 45:
                   if (jjCanMove_2(hiByte, i1, i2, l1, l2))
-                     jjCheckNAddStates(13, 15);
+                     jjCheckNAddStates(18, 20);
                   break;
-               case 41:
+               case 47:
                   if (jjCanMove_1(hiByte, i1, i2, l1, l2))
-                     jjCheckNAddStates(13, 15);
+                     jjCheckNAddStates(18, 20);
                   break;
                default : break;
             }
@@ -541,7 +625,7 @@ private int jjMoveNfa_2(int startState, 
          kind = 0x7fffffff;
       }
       ++curPos;
-      if ((i = jjnewStateCnt) == (startsAt = 43 - (jjnewStateCnt = startsAt)))
+      if ((i = jjnewStateCnt) == (startsAt = 49 - (jjnewStateCnt = startsAt)))
          return curPos;
       try { curChar = input_stream.readChar(); }
       catch(java.io.IOException e) { return curPos; }
@@ -574,7 +658,7 @@ private int jjMoveNfa_0(int startState, 
                      break;
                   if (kind > 27)
                      kind = 27;
-                  jjAddStates(19, 20);
+                  jjAddStates(31, 32);
                   break;
                case 1:
                   if (curChar == 46)
@@ -728,11 +812,11 @@ private int jjMoveNfa_1(int startState, 
                   break;
                case 2:
                   if ((0xfffffffbffffffffL & l) != 0L)
-                     jjCheckNAddStates(21, 23);
+                     jjCheckNAddStates(33, 35);
                   break;
                case 3:
                   if (curChar == 34)
-                     jjCheckNAddStates(21, 23);
+                     jjCheckNAddStates(33, 35);
                   break;
                case 5:
                   if (curChar == 34 && kind > 31)
@@ -765,7 +849,7 @@ private int jjMoveNfa_1(int startState, 
                   jjCheckNAdd(6);
                   break;
                case 2:
-                  jjAddStates(21, 23);
+                  jjAddStates(33, 35);
                   break;
                case 4:
                   if (curChar == 92)
@@ -801,7 +885,7 @@ private int jjMoveNfa_1(int startState, 
                   break;
                case 2:
                   if (jjCanMove_1(hiByte, i1, i2, l1, l2))
-                     jjAddStates(21, 23);
+                     jjAddStates(33, 35);
                   break;
                case 6:
                   if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
@@ -828,8 +912,9 @@ private int jjMoveNfa_1(int startState, 
    }
 }
 static final int[] jjnextStates = {
-   31, 33, 34, 17, 18, 20, 36, 39, 25, 40, 37, 22, 23, 39, 25, 40, 
-   38, 41, 29, 0, 1, 2, 4, 5, 
+   37, 39, 40, 17, 18, 20, 42, 45, 31, 46, 43, 22, 23, 25, 26, 24, 
+   25, 26, 45, 31, 46, 44, 47, 35, 22, 28, 29, 27, 27, 30, 30, 0, 
+   1, 2, 4, 5, 
 };
 private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2)
 {
@@ -893,8 +978,8 @@ static final long[] jjtoSkip = {
    0x80L, 
 };
 protected CharStream input_stream;
-private final int[] jjrounds = new int[43];
-private final int[] jjstateSet = new int[86];
+private final int[] jjrounds = new int[49];
+private final int[] jjstateSet = new int[98];
 protected char curChar;
 /** Constructor. */
 public QueryParserTokenManager(CharStream stream){
@@ -919,7 +1004,7 @@ private void ReInitRounds()
 {
    int i;
    jjround = 0x80000001;
-   for (i = 43; i-- > 0;)
+   for (i = 49; i-- > 0;)
       jjrounds[i] = 0x80000000;
 }
 

Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/package.html?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/package.html (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/package.html Sun Mar 10 15:04:57 2013
@@ -167,11 +167,11 @@ enabling substantial customization to ho
         If you want to find the document entitled "The Right Way" which contains the text "don't go this way", you can enter: </p>
 <pre class="code">title:"The Right Way" AND text:go</pre>
 <p>or</p>
-<pre class="code">title:"Do it right" AND right</pre>
+<pre class="code">title:"The Right Way" AND go</pre>
 <p>Since text is the default field, the field indicator is not required.</p>
 <p>Note: The field is only valid for the term that it directly precedes, so the query</p>
-<pre class="code">title:Do it right</pre>
-<p>Will only find "Do" in the title field. It will find "it" and "right" in the default field (in this case the text field). </p>
+<pre class="code">title:The Right Way</pre>
+<p>Will only find "The" in the title field. It will find "Right" and "Way" in the default field (in this case the text field). </p>
 </div>
 
         

Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BooleanQuery2ModifierNodeProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BooleanQuery2ModifierNodeProcessor.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BooleanQuery2ModifierNodeProcessor.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BooleanQuery2ModifierNodeProcessor.java Sun Mar 10 15:04:57 2013
@@ -55,7 +55,7 @@ import org.apache.lucene.queryparser.fle
  * {@link Operator#AND}, if it is, the same operation when an
  * {@link AndQueryNode} is found is applied to it. Each {@link BooleanQueryNode}
  * which direct parent is also a {@link BooleanQueryNode} is removed (to ignore
- * the rules of precidence).
+ * the rules of precedence).
  * </p>
  * 
  * @see ConfigurationKeys#DEFAULT_OPERATOR

Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java Sun Mar 10 15:04:57 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.queryparser.cl
  */
 
 import java.io.Reader;
+import java.io.StringReader;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -33,6 +34,7 @@ import org.apache.lucene.search.IndexSea
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 
 /**
@@ -300,7 +302,7 @@ public class TestMultiFieldQueryParser e
   }
   
   /**
-   * Return empty tokens for field "f1".
+   * Return no tokens for field "f1".
    */
   private static class AnalyzerReturningNull extends Analyzer {
     MockAnalyzer stdAnalyzer = new MockAnalyzer(random());
@@ -310,13 +312,21 @@ public class TestMultiFieldQueryParser e
     }
 
     @Override
-    public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+    protected Reader initReader(String fieldName, Reader reader) {
       if ("f1".equals(fieldName)) {
-        return new TokenStreamComponents(new EmptyTokenizer(reader));
+        // we don't use the reader, so close it:
+        IOUtils.closeWhileHandlingException(reader);
+        // return empty reader, so MockTokenizer returns no tokens:
+        return new StringReader("");
       } else {
-        return stdAnalyzer.createComponents(fieldName, reader);
+        return super.initReader(fieldName, reader);
       }
     }
+
+    @Override
+    public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+      return stdAnalyzer.createComponents(fieldName, reader);
+    }
   }
 
 }

Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java Sun Mar 10 15:04:57 2013
@@ -160,6 +160,28 @@ public class TestQueryParser extends Que
     }
   }
   
+  public void testFuzzySlopeExtendability() throws ParseException {
+    QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "a",  new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)) {
+
+      @Override
+      Query handleBareFuzzy(String qfield, Token fuzzySlop, String termImage)
+          throws ParseException {
+        
+        if(fuzzySlop.image.endsWith("€")) {
+          float fms = fuzzyMinSim;
+          try {
+            fms = Float.valueOf(fuzzySlop.image.substring(1, fuzzySlop.image.length()-1)).floatValue();
+          } catch (Exception ignored) { }
+          float value = Float.parseFloat(termImage);
+          return getRangeQuery(qfield, Float.toString(value-fms/2.f), Float.toString(value+fms/2.f), true, true);
+        }
+        return super.handleBareFuzzy(qfield, fuzzySlop, termImage);
+      }
+      
+    };
+    assertEquals(qp.parse("a:[11.95 TO 12.95]"), qp.parse("12.45~1€"));
+  }
+  
   @Override
   public void testStarParsing() throws Exception {
     final int[] type = new int[1];

Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java Sun Mar 10 15:04:57 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.queryparser.fl
  */
 
 import java.io.Reader;
+import java.io.StringReader;
 import java.util.HashMap;
 import java.util.Map;
 
@@ -35,6 +36,7 @@ import org.apache.lucene.search.IndexSea
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.ScoreDoc;
 import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
 import org.apache.lucene.util.LuceneTestCase;
 
 /**
@@ -339,9 +341,9 @@ public class TestMultiFieldQPHelper exte
   }
 
   /**
-   * Return empty tokens for field "f1".
+   * Return no tokens for field "f1".
    */
-  private static final class AnalyzerReturningNull extends Analyzer {
+  private static class AnalyzerReturningNull extends Analyzer {
     MockAnalyzer stdAnalyzer = new MockAnalyzer(random());
 
     public AnalyzerReturningNull() {
@@ -349,13 +351,21 @@ public class TestMultiFieldQPHelper exte
     }
 
     @Override
-    public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+    protected Reader initReader(String fieldName, Reader reader) {
       if ("f1".equals(fieldName)) {
-        return new TokenStreamComponents(new EmptyTokenizer(reader));
+        // we don't use the reader, so close it:
+        IOUtils.closeWhileHandlingException(reader);
+        // return empty reader, so MockTokenizer returns no tokens:
+        return new StringReader("");
       } else {
-        return stdAnalyzer.createComponents(fieldName, reader);
+        return super.initReader(fieldName, reader);
       }
     }
+
+    @Override
+    public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+      return stdAnalyzer.createComponents(fieldName, reader);
+    }
   }
 
 }

Modified: lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java Sun Mar 10 15:04:57 2013
@@ -30,6 +30,7 @@ import org.apache.lucene.analysis.tokena
 import org.apache.lucene.index.IndexReader;
 import org.apache.lucene.index.MultiFields;
 import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
 import org.apache.lucene.search.*;
 import org.apache.lucene.search.similarities.TFIDFSimilarity;
 import org.apache.lucene.search.similarities.DefaultSimilarity;
@@ -196,6 +197,10 @@ public class FuzzyLikeThisQuery extends 
     int corpusNumDocs = reader.numDocs();
     HashSet<String> processedTerms = new HashSet<String>();
     ts.reset();
+    final Terms terms = MultiFields.getTerms(reader, f.fieldName);
+    if (terms == null) {
+      return;
+    }
     while (ts.incrementToken()) {
       String term = termAtt.toString();
       if (!processedTerms.contains(term)) {
@@ -206,7 +211,7 @@ public class FuzzyLikeThisQuery extends 
         AttributeSource atts = new AttributeSource();
         MaxNonCompetitiveBoostAttribute maxBoostAtt =
             atts.addAttribute(MaxNonCompetitiveBoostAttribute.class);
-        SlowFuzzyTermsEnum fe = new SlowFuzzyTermsEnum(MultiFields.getTerms(reader, startTerm.field()), atts, startTerm, f.minSimilarity, f.prefixLength);
+        SlowFuzzyTermsEnum fe = new SlowFuzzyTermsEnum(terms, atts, startTerm, f.minSimilarity, f.prefixLength);
         //store the df so all variants use same idf
         int df = reader.docFreq(startTerm);
         int numVariants = 0;

Modified: lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java (original)
+++ lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java Sun Mar 10 15:04:57 2013
@@ -21,8 +21,8 @@ import java.io.IOException;
 import java.text.Collator;
 
 import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.BinaryDocValues;
 import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.search.FieldCache.DocTerms;
 import org.apache.lucene.search.FieldComparator;
 import org.apache.lucene.util.BytesRef;
 
@@ -38,7 +38,7 @@ import org.apache.lucene.util.BytesRef;
 public final class SlowCollatedStringComparator extends FieldComparator<String> {
 
   private final String[] values;
-  private DocTerms currentDocTerms;
+  private BinaryDocValues currentDocTerms;
   private final String field;
   final Collator collator;
   private String bottom;
@@ -67,7 +67,8 @@ public final class SlowCollatedStringCom
 
   @Override
   public int compareBottom(int doc) {
-    final String val2 = currentDocTerms.getTerm(doc, tempBR).utf8ToString();
+    currentDocTerms.get(doc, tempBR);
+    final String val2 = tempBR.bytes == BinaryDocValues.MISSING ? null : tempBR.utf8ToString();
     if (bottom == null) {
       if (val2 == null) {
         return 0;
@@ -81,11 +82,11 @@ public final class SlowCollatedStringCom
 
   @Override
   public void copy(int slot, int doc) {
-    final BytesRef br = currentDocTerms.getTerm(doc, tempBR);
-    if (br == null) {
+    currentDocTerms.get(doc, tempBR);
+    if (tempBR.bytes == BinaryDocValues.MISSING) {
       values[slot] = null;
     } else {
-      values[slot] = br.utf8ToString();
+      values[slot] = tempBR.utf8ToString();
     }
   }
 
@@ -121,12 +122,12 @@ public final class SlowCollatedStringCom
 
   @Override
   public int compareDocToValue(int doc, String value) {
-    final BytesRef br = currentDocTerms.getTerm(doc, tempBR);
+    currentDocTerms.get(doc, tempBR);
     final String docValue;
-    if (br == null) {
+    if (tempBR.bytes == BinaryDocValues.MISSING) {
       docValue = null;
     } else {
-      docValue = br.utf8ToString();
+      docValue = tempBR.utf8ToString();
     }
     return compareValues(docValue, value);
   }

Modified: lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java (original)
+++ lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java Sun Mar 10 15:04:57 2013
@@ -108,6 +108,25 @@ public class FuzzyLikeThisQueryTest exte
     StoredDocument doc = searcher.doc(sd[0].doc);
     assertEquals("Should match most similar when using 2 words", "2", doc.get("id"));
   }
+  
+  // LUCENE-4809
+  public void testNonExistingField() throws Throwable {
+    FuzzyLikeThisQuery flt = new FuzzyLikeThisQuery(10, analyzer);
+    flt.addTerms("jonathin smoth", "name", 0.3f, 1);
+    flt.addTerms("jonathin smoth", "this field does not exist", 0.3f, 1);
+    // don't fail here just because the field doesn't exits
+    Query q = flt.rewrite(searcher.getIndexReader());
+    HashSet<Term> queryTerms = new HashSet<Term>();
+    q.extractTerms(queryTerms);
+    assertTrue("Should have variant jonathan", queryTerms.contains(new Term("name", "jonathan")));
+    assertTrue("Should have variant smith", queryTerms.contains(new Term("name", "smith")));
+    TopDocs topDocs = searcher.search(flt, 1);
+    ScoreDoc[] sd = topDocs.scoreDocs;
+    assertTrue("score docs must match 1 doc", (sd != null) && (sd.length > 0));
+    StoredDocument doc = searcher.doc(sd[0].doc);
+    assertEquals("Should match most similar when using 2 words", "2", doc.get("id"));
+  }
+
 
   //Test bug found when first query word does not match anything
   public void testNoMatchFirstWordBug() throws Throwable {

Modified: lucene/dev/branches/lucene4258/lucene/site/xsl/index.xsl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/site/xsl/index.xsl?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/site/xsl/index.xsl (original)
+++ lucene/dev/branches/lucene4258/lucene/site/xsl/index.xsl Sun Mar 10 15:04:57 2013
@@ -58,7 +58,8 @@
         <li><a href="core/overview-summary.html#overview_description">Introduction to Lucene's APIs</a>:
         High-level summary of the different Lucene packages. </li>
         <li><a href="core/org/apache/lucene/analysis/package-summary.html#package_description">Analysis overview</a>:
-        Introduction to Lucene's analysis API. </li>
+        Introduction to Lucene's analysis API.  See also the
+        <a href="core/org/apache/lucene/analysis/TokenStream.html">TokenStream consumer workflow</a>.</li>
         </ul>
         <h2>Reference Documents</h2>
           <ul>
@@ -66,7 +67,7 @@
             <li><a href="SYSTEM_REQUIREMENTS.html">System Requirements</a>: Minimum and supported Java versions.</li>
             <li><a href="MIGRATE.html">Migration Guide</a>: What changed in Lucene 4; how to migrate code from Lucene 3.x.</li>
             <li><a href="JRE_VERSION_MIGRATION.html">JRE Version Migration</a>: Information about upgrading between major JRE versions.</li>
-            <li><a href="core/org/apache/lucene/codecs/lucene41/package-summary.html#package_description">File Formats</a>: Guide to the supported index format used by Lucene.  This can be customized by using <a href="core/org/apache/lucene/codecs/package-summary.html#package_description">an alternate codec</a>.</li>
+            <li><a href="core/org/apache/lucene/codecs/lucene42/package-summary.html#package_description">File Formats</a>: Guide to the supported index format used by Lucene.  This can be customized by using <a href="core/org/apache/lucene/codecs/package-summary.html#package_description">an alternate codec</a>.</li>
             <li><a href="core/org/apache/lucene/search/package-summary.html#package_description">Search and Scoring in Lucene</a>: Introduction to how Lucene scores documents.</li>
             <li><a href="core/org/apache/lucene/search/similarities/TFIDFSimilarity.html">Classic Scoring Formula</a>: Formula of Lucene's classic <a href="http://en.wikipedia.org/wiki/Vector_Space_Model">Vector Space</a> implementation. (look <a href="core/org/apache/lucene/search/similarities/package-summary.html#package_description">here</a> for other models)</li>
             <li><a href="queryparser/org/apache/lucene/queryparser/classic/package-summary.html#package_description">Classic QueryParser Syntax</a>: Overview of the Classic QueryParser's syntax and features.</li>

Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java Sun Mar 10 15:04:57 2013
@@ -41,8 +41,8 @@ import org.apache.lucene.spatial.query.S
  *   <li>What types of query shapes can be used?</li>
  *   <li>What types of query operations are supported?
  *   This might vary per shape.</li>
- *   <li>Does it use the {@link org.apache.lucene.search.FieldCache}, {@link
- *   org.apache.lucene.index.DocValues} or some other type of cache?  When?
+ *   <li>Does it use the {@link org.apache.lucene.search.FieldCache},
+ *   or some other type of cache?  When?
  * </ul>
  * If a strategy only supports certain shapes at index or query time, then in
  * general it will throw an exception if given an incompatible one.  It will not

Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java Sun Mar 10 15:04:57 2013
@@ -64,10 +64,10 @@ public class BBoxSimilarityValueSource e
   @Override
   public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
     AtomicReader reader = readerContext.reader();
-    final double[] minX = FieldCache.DEFAULT.getDoubles(reader, strategy.field_minX, true);
-    final double[] minY = FieldCache.DEFAULT.getDoubles(reader, strategy.field_minY, true);
-    final double[] maxX = FieldCache.DEFAULT.getDoubles(reader, strategy.field_maxX, true);
-    final double[] maxY = FieldCache.DEFAULT.getDoubles(reader, strategy.field_maxY, true);
+    final FieldCache.Doubles minX = FieldCache.DEFAULT.getDoubles(reader, strategy.field_minX, true);
+    final FieldCache.Doubles minY = FieldCache.DEFAULT.getDoubles(reader, strategy.field_minY, true);
+    final FieldCache.Doubles maxX = FieldCache.DEFAULT.getDoubles(reader, strategy.field_maxX, true);
+    final FieldCache.Doubles maxY = FieldCache.DEFAULT.getDoubles(reader, strategy.field_maxY, true);
 
     final Bits validMinX = FieldCache.DEFAULT.getDocsWithField(reader, strategy.field_minX);
     final Bits validMaxX = FieldCache.DEFAULT.getDocsWithField(reader, strategy.field_maxX);
@@ -81,8 +81,8 @@ public class BBoxSimilarityValueSource e
         // make sure it has minX and area
         if (validMinX.get(doc) && validMaxX.get(doc)) {
           rect.reset(
-              minX[doc], maxX[doc],
-              minY[doc], maxY[doc]);
+              minX.get(doc), maxX.get(doc),
+              minY.get(doc), maxY.get(doc));
           return (float) similarity.score(rect, null);
         } else {
           return (float) similarity.score(null, null);
@@ -94,8 +94,8 @@ public class BBoxSimilarityValueSource e
         // make sure it has minX and area
         if (validMinX.get(doc) && validMaxX.get(doc)) {
           rect.reset(
-              minX[doc], maxX[doc],
-              minY[doc], maxY[doc]);
+              minX.get(doc), maxX.get(doc),
+              minY.get(doc), maxY.get(doc));
           Explanation exp = new Explanation();
           similarity.score(rect, exp);
           return exp;

Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java Sun Mar 10 15:04:57 2013
@@ -78,12 +78,14 @@ import java.util.concurrent.ConcurrentHa
 public abstract class PrefixTreeStrategy extends SpatialStrategy {
   protected final SpatialPrefixTree grid;
   private final Map<String, PointPrefixTreeFieldCacheProvider> provider = new ConcurrentHashMap<String, PointPrefixTreeFieldCacheProvider>();
+  protected final boolean simplifyIndexedCells;
   protected int defaultFieldValuesArrayLen = 2;
   protected double distErrPct = SpatialArgs.DEFAULT_DISTERRPCT;// [ 0 TO 0.5 ]
 
-  public PrefixTreeStrategy(SpatialPrefixTree grid, String fieldName) {
+  public PrefixTreeStrategy(SpatialPrefixTree grid, String fieldName, boolean simplifyIndexedCells) {
     super(grid.getSpatialContext(), fieldName);
     this.grid = grid;
+    this.simplifyIndexedCells = simplifyIndexedCells;
   }
 
   /**
@@ -123,7 +125,7 @@ public abstract class PrefixTreeStrategy
 
   public Field[] createIndexableFields(Shape shape, double distErr) {
     int detailLevel = grid.getLevelForDistance(distErr);
-    List<Node> cells = grid.getNodes(shape, detailLevel, true);//true=intermediates cells
+    List<Node> cells = grid.getNodes(shape, detailLevel, true, simplifyIndexedCells);//intermediates cells
 
     //TODO is CellTokenStream supposed to be re-used somehow? see Uwe's comments:
     //  http://code.google.com/p/lucene-spatial-playground/issues/detail?id=4

Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java Sun Mar 10 15:04:57 2013
@@ -25,7 +25,7 @@ import org.apache.lucene.spatial.query.S
 import org.apache.lucene.spatial.query.UnsupportedSpatialOperation;
 
 /**
- * A {@link PrefixTreeStrategy} which uses {@link RecursivePrefixTreeFilter}.
+ * A {@link PrefixTreeStrategy} which uses {@link AbstractVisitingPrefixTreeFilter}.
  * This strategy has support for searching non-point shapes (note: not tested).
  * Even a query shape with distErrPct=0 (fully precise to the grid) should have
  * good performance for typical data, unless there is a lot of indexed data
@@ -38,7 +38,8 @@ public class RecursivePrefixTreeStrategy
   private int prefixGridScanLevel;
 
   public RecursivePrefixTreeStrategy(SpatialPrefixTree grid, String fieldName) {
-    super(grid, fieldName);
+    super(grid, fieldName,
+        true);//simplify indexed cells
     prefixGridScanLevel = grid.getMaxLevels() - 4;//TODO this default constant is dependent on the prefix grid size
   }
 
@@ -69,8 +70,9 @@ public class RecursivePrefixTreeStrategy
 
     int detailLevel = grid.getLevelForDistance(args.resolveDistErr(ctx, distErrPct));
 
-    return new RecursivePrefixTreeFilter(
-        getFieldName(), grid, shape, prefixGridScanLevel, detailLevel);
+    return new IntersectsPrefixTreeFilter(
+        shape, getFieldName(), grid, detailLevel, prefixGridScanLevel,
+        true);//hasIndexedLeaves
   }
 }
 

Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java Sun Mar 10 15:04:57 2013
@@ -32,7 +32,7 @@ import java.util.List;
 /**
  * A basic implementation of {@link PrefixTreeStrategy} using a large {@link
  * TermsFilter} of all the nodes from {@link SpatialPrefixTree#getNodes(com.spatial4j.core.shape.Shape,
- * int, boolean)}. It only supports the search of indexed Point shapes.
+ * int, boolean, boolean)}. It only supports the search of indexed Point shapes.
  * <p/>
  * The precision of query shapes (distErrPct) is an important factor in using
  * this Strategy. If the precision is too precise then it will result in many
@@ -43,7 +43,8 @@ import java.util.List;
 public class TermQueryPrefixTreeStrategy extends PrefixTreeStrategy {
 
   public TermQueryPrefixTreeStrategy(SpatialPrefixTree grid, String fieldName) {
-    super(grid, fieldName);
+    super(grid, fieldName,
+        false);//do not simplify indexed cells
   }
 
   @Override
@@ -54,7 +55,9 @@ public class TermQueryPrefixTreeStrategy
 
     Shape shape = args.getShape();
     int detailLevel = grid.getLevelForDistance(args.resolveDistErr(ctx, distErrPct));
-    List<Node> cells = grid.getNodes(shape, detailLevel, false);
+    List<Node> cells = grid.getNodes(shape, detailLevel,
+        false,//no parents
+        true);//simplify
     BytesRef[] terms = new BytesRef[cells.size()];
     int i = 0;
     for (Node cell : cells) {

Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java Sun Mar 10 15:04:57 2013
@@ -93,12 +93,6 @@ public class GeohashPrefixTree extends S
     return new GhCell(bytes, offset, len);
   }
 
-  @Override
-  public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents) {
-    return shape instanceof Point ? super.getNodesAltPoint((Point) shape, detailLevel, inclParents) :
-        super.getNodes(shape, detailLevel, inclParents);
-  }
-
   class GhCell extends Node {
     GhCell(String token) {
       super(token);

Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java Sun Mar 10 15:04:57 2013
@@ -1,3 +1,5 @@
+package org.apache.lucene.spatial.prefix.tree;
+
 /*
  * Licensed to the Apache Software Foundation (ASF) under one or more
  * contributor license agreements.  See the NOTICE file distributed with
@@ -15,8 +17,6 @@
  * limitations under the License.
  */
 
-package org.apache.lucene.spatial.prefix.tree;
-
 import com.spatial4j.core.shape.Point;
 import com.spatial4j.core.shape.Shape;
 import com.spatial4j.core.shape.SpatialRelation;
@@ -27,7 +27,8 @@ import java.util.Collections;
 import java.util.List;
 
 /**
- * Represents a grid cell. These are not necessarily thread-safe, although new Cell("") (world cell) must be.
+ * Represents a grid cell. These are not necessarily thread-safe, although new
+ * Cell("") (world cell) must be.
  *
  * @lucene.experimental
  */
@@ -44,13 +45,19 @@ public abstract class Node implements Co
 
   private String token;//this is the only part of equality
 
-  /** When set via getSubCells(filter), it is the relationship between this
-   * cell and the given shape filter.  If set via setLeaf() (to WITHIN), it is
-   * meant to indicate no further sub-cells are going to be provided because
-   * maxLevels or a detailLevel is hit. It's always null for points.
+  /**
+   * When set via getSubCells(filter), it is the relationship between this cell
+   * and the given shape filter.
    */
   protected SpatialRelation shapeRel;
 
+  /**
+   * Always false for points. Otherwise, indicate no further sub-cells are going
+   * to be provided because shapeRel is WITHIN or maxLevels or a detailLevel is
+   * hit.
+   */
+  protected boolean leaf;
+
   protected Node(String token) {
     this.token = token;
     if (token.length() > 0 && token.charAt(token.length() - 1) == (char) LEAF_BYTE) {
@@ -96,12 +103,13 @@ public abstract class Node implements Co
    * further cells with this prefix for the shape (always true at maxLevels).
    */
   public boolean isLeaf() {
-    return shapeRel == SpatialRelation.WITHIN;
+    return leaf;
   }
 
+  /** Note: not supported at level 0. */
   public void setLeaf() {
     assert getLevel() != 0;
-    shapeRel = SpatialRelation.WITHIN;
+    leaf = true;
   }
 
   /**
@@ -139,12 +147,11 @@ public abstract class Node implements Co
 
   /**
    * Like {@link #getSubCells()} but with the results filtered by a shape. If
-   * that shape is a {@link com.spatial4j.core.shape.Point} then it
-   * must call {@link #getSubCell(com.spatial4j.core.shape.Point)}.
-   * The returned cells should have their {@link Node#shapeRel} set to their
-   * relation with {@code shapeFilter} for non-point. As such,
-   * {@link org.apache.lucene.spatial.prefix.tree.Node#isLeaf()} should be
-   * accurate.
+   * that shape is a {@link com.spatial4j.core.shape.Point} then it must call
+   * {@link #getSubCell(com.spatial4j.core.shape.Point)}. The returned cells
+   * should have {@link Node#getShapeRel()} set to their relation with {@code
+   * shapeFilter}. In addition, {@link org.apache.lucene.spatial.prefix.tree.Node#isLeaf()}
+   * must be true when that relation is WITHIN.
    * <p/>
    * Precondition: Never called when getLevel() == maxLevel.
    *
@@ -154,29 +161,35 @@ public abstract class Node implements Co
   public Collection<Node> getSubCells(Shape shapeFilter) {
     //Note: Higher-performing subclasses might override to consider the shape filter to generate fewer cells.
     if (shapeFilter instanceof Point) {
-      return Collections.singleton(getSubCell((Point) shapeFilter));
+      Node subCell = getSubCell((Point) shapeFilter);
+      subCell.shapeRel = SpatialRelation.CONTAINS;
+      return Collections.singletonList(subCell);
     }
     Collection<Node> cells = getSubCells();
 
     if (shapeFilter == null) {
       return cells;
     }
-    List<Node> copy = new ArrayList<Node>(cells.size());//copy since cells contractually isn't modifiable
+
+    //TODO change API to return a filtering iterator
+    List<Node> copy = new ArrayList<Node>(cells.size());
     for (Node cell : cells) {
       SpatialRelation rel = cell.getShape().relate(shapeFilter);
       if (rel == SpatialRelation.DISJOINT)
         continue;
       cell.shapeRel = rel;
+      if (rel == SpatialRelation.WITHIN)
+        cell.setLeaf();
       copy.add(cell);
     }
-    cells = copy;
-    return cells;
+    return copy;
   }
 
   /**
-   * Performant implementations are expected to implement this efficiently by considering the current
-   * cell's boundary.
-   * Precondition: Never called when getLevel() == maxLevel.
+   * Performant implementations are expected to implement this efficiently by
+   * considering the current cell's boundary. Precondition: Never called when
+   * getLevel() == maxLevel.
+   * <p/>
    * Precondition: this.getShape().relate(p) != DISJOINT.
    */
   public abstract Node getSubCell(Point p);
@@ -187,7 +200,7 @@ public abstract class Node implements Co
    * Gets the cells at the next grid cell level that cover this cell.
    * Precondition: Never called when getLevel() == maxLevel.
    *
-   * @return A set of cells (no dups), sorted. Not Modifiable.
+   * @return A set of cells (no dups), sorted, modifiable, not empty, not null.
    */
   protected abstract Collection<Node> getSubCells();
 

Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java Sun Mar 10 15:04:57 2013
@@ -156,14 +156,6 @@ public class QuadPrefixTree extends Spat
     return new QuadCell(bytes, offset, len);
   }
 
-  @Override //for performance
-  public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents) {
-    if (shape instanceof Point)
-      return super.getNodesAltPoint((Point) shape, detailLevel, inclParents);
-    else
-      return super.getNodes(shape, detailLevel, inclParents);
-  }
-
   private void build(
       double x,
       double y,

Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java Sun Mar 10 15:04:57 2013
@@ -116,88 +116,86 @@ public abstract class SpatialPrefixTree 
   }
 
   /**
-   * Gets the intersecting & including cells for the specified shape, without exceeding detail level.
-   * The result is a set of cells (no dups), sorted. Unmodifiable.
+   * Gets the intersecting cells for the specified shape, without exceeding
+   * detail level. If a cell is within the query shape then it's marked as a
+   * leaf and none of its children are added.
    * <p/>
-   * This implementation checks if shape is a Point and if so uses an implementation that
-   * recursively calls {@link Node#getSubCell(com.spatial4j.core.shape.Point)}. Cell subclasses
-   * ideally implement that method with a quick implementation, otherwise, subclasses should
-   * override this method to invoke {@link #getNodesAltPoint(com.spatial4j.core.shape.Point, int, boolean)}.
-   * TODO consider another approach returning an iterator -- won't build up all cells in memory.
+   * This implementation checks if shape is a Point and if so returns {@link
+   * #getNodes(com.spatial4j.core.shape.Point, int, boolean)}.
+   *
+   * @param shape       the shape; non-null
+   * @param detailLevel the maximum detail level to get cells for
+   * @param inclParents if true then all parent cells of leaves are returned
+   *                    too. The top world cell is never returned.
+   * @param simplify    for non-point shapes, this will simply/aggregate sets of
+   *                    complete leaves in a cell to its parent, resulting in
+   *                    ~20-25% fewer cells.
+   * @return a set of cells (no dups), sorted, immutable, non-null
    */
-  public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents) {
+  public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents,
+                             boolean simplify) {
+    //TODO consider an on-demand iterator -- it won't build up all cells in memory.
     if (detailLevel > maxLevels) {
       throw new IllegalArgumentException("detailLevel > maxLevels");
     }
-
-    List<Node> cells;
     if (shape instanceof Point) {
-      //optimized point algorithm
-      final int initialCapacity = inclParents ? 1 + detailLevel : 1;
-      cells = new ArrayList<Node>(initialCapacity);
-      recursiveGetNodes(getWorldNode(), (Point) shape, detailLevel, true, cells);
-      assert cells.size() == initialCapacity;
-    } else {
-      cells = new ArrayList<Node>(inclParents ? 1024 : 512);
-      recursiveGetNodes(getWorldNode(), shape, detailLevel, inclParents, cells);
-    }
-    if (inclParents) {
-      Node c = cells.remove(0);//remove getWorldNode()
-      assert c.getLevel() == 0;
+      return getNodes((Point) shape, detailLevel, inclParents);
     }
+    List<Node> cells = new ArrayList<Node>(inclParents ? 4096 : 2048);
+    recursiveGetNodes(getWorldNode(), shape, detailLevel, inclParents, simplify, cells);
     return cells;
   }
 
-  private void recursiveGetNodes(Node node, Shape shape, int detailLevel, boolean inclParents,
-                                 Collection<Node> result) {
-    if (node.isLeaf()) {//cell is within shape
-      result.add(node);
-      return;
-    }
-    final Collection<Node> subCells = node.getSubCells(shape);
-    if (node.getLevel() == detailLevel - 1) {
-      if (subCells.size() < node.getSubCellsSize() || node.getLevel() == 0) {
-        if (inclParents)
-          result.add(node);
-        for (Node subCell : subCells) {
-          subCell.setLeaf();
-        }
-        result.addAll(subCells);
-      } else {//a bottom level (i.e. detail level) optimization where all boxes intersect, so use parent cell.
-        node.setLeaf();//the cell may not be strictly within but its close
-        result.add(node);
-      }
-    } else {
-      if (inclParents) {
-        result.add(node);
-      }
-      for (Node subCell : subCells) {
-        recursiveGetNodes(subCell, shape, detailLevel, inclParents, result);//tail call
-      }
+  /**
+   * Returns true if node was added as a leaf. If it wasn't it recursively
+   * descends.
+   */
+  private boolean recursiveGetNodes(Node node, Shape shape, int detailLevel,
+                                    boolean inclParents, boolean simplify,
+                                    List<Node> result) {
+    if (node.getLevel() == detailLevel) {
+      node.setLeaf();//FYI might already be a leaf
     }
-  }
-
-  private void recursiveGetNodes(Node node, Point point, int detailLevel, boolean inclParents,
-                                 Collection<Node> result) {
-    if (inclParents) {
+    if (node.isLeaf()) {
       result.add(node);
+      return true;
     }
-    final Node pCell = node.getSubCell(point);
-    if (node.getLevel() == detailLevel - 1) {
-      pCell.setLeaf();
-      result.add(pCell);
-    } else {
-      recursiveGetNodes(pCell, point, detailLevel, inclParents, result);//tail call
+    if (inclParents && node.getLevel() != 0)
+      result.add(node);
+
+    Collection<Node> subCells = node.getSubCells(shape);
+    int leaves = 0;
+    for (Node subCell : subCells) {
+      if (recursiveGetNodes(subCell, shape, detailLevel, inclParents, simplify, result))
+        leaves++;
+    }
+    //can we simplify?
+    if (simplify && leaves == node.getSubCellsSize() && node.getLevel() != 0) {
+      //Optimization: substitute the parent as a leaf instead of adding all
+      // children as leaves
+
+      //remove the leaves
+      do {
+        result.remove(result.size() - 1);//remove last
+      } while (--leaves > 0);
+      //add node as the leaf
+      node.setLeaf();
+      if (!inclParents) // otherwise it was already added up above
+        result.add(node);
+      return true;
     }
+    return false;
   }
 
   /**
-   * Subclasses might override {@link #getNodes(com.spatial4j.core.shape.Shape, int, boolean)}
-   * and check if the argument is a shape and if so, delegate
-   * to this implementation, which calls {@link #getNode(com.spatial4j.core.shape.Point, int)} and
-   * then calls {@link #getNode(String)} repeatedly if inclParents is true.
+   * A Point-optimized implementation of
+   * {@link #getNodes(com.spatial4j.core.shape.Shape, int, boolean, boolean)}. That
+   * method in facts calls this for points.
+   * <p/>
+   * This implementation depends on {@link #getNode(String)} being fast, as its
+   * called repeatedly when incPlarents is true.
    */
-  protected final List<Node> getNodesAltPoint(Point p, int detailLevel, boolean inclParents) {
+  public List<Node> getNodes(Point p, int detailLevel, boolean inclParents) {
     Node cell = getNode(p, detailLevel);
     if (!inclParents) {
       return Collections.singletonList(cell);

Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java Sun Mar 10 15:04:57 2013
@@ -47,7 +47,7 @@ public class SpatialArgs {
   /**
    * Computes the distance given a shape and the {@code distErrPct}.  The
    * algorithm is the fraction of the distance from the center of the query
-   * shape to its furthest bounding box corner.
+   * shape to its closest bounding box corner.
    *
    * @param shape Mandatory.
    * @param distErrPct 0 to 0.5
@@ -62,11 +62,13 @@ public class SpatialArgs {
       return 0;
     }
     Rectangle bbox = shape.getBoundingBox();
-    //The diagonal distance should be the same computed from any opposite corner,
-    // and this is the longest distance that might be occurring within the shape.
-    double diagonalDist = ctx.getDistCalc().distance(
-        ctx.makePoint(bbox.getMinX(), bbox.getMinY()), bbox.getMaxX(), bbox.getMaxY());
-    return diagonalDist * 0.5 * distErrPct;
+    //Compute the distance from the center to a corner.  Because the distance
+    // to a bottom corner vs a top corner can vary in a geospatial scenario,
+    // take the closest one (greater precision).
+    Point ctr = bbox.getCenter();
+    double y = (ctr.getY() >= 0 ? bbox.getMaxY() : bbox.getMinY());
+    double diagonalDist = ctx.getDistCalc().distance(ctr, bbox.getMaxX(), y);
+    return diagonalDist * distErrPct;
   }
 
   /**