You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/03/10 16:05:21 UTC
svn commit: r1454862 [28/49] - in /lucene/dev/branches/lucene4258: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/
dev-tools/idea/.idea/libraries/ dev-tools/idea/lucene/demo/
dev-tools/idea/lucene/facet/ dev-tools/idea/lucene/highlighter/ dev-to...
Modified: lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestDocValuesFieldSources.java Sun Mar 10 15:04:57 2013
@@ -18,29 +18,20 @@ package org.apache.lucene.queries.functi
*/
import java.io.IOException;
-import java.util.Date;
import org.apache.lucene.analysis.MockAnalyzer;
-import org.apache.lucene.document.ByteDocValuesField;
-import org.apache.lucene.document.DerefBytesDocValuesField;
+import org.apache.lucene.document.BinaryDocValuesField;
import org.apache.lucene.document.Document;
-import org.apache.lucene.document.DoubleDocValuesField;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.FloatDocValuesField;
-import org.apache.lucene.document.IntDocValuesField;
-import org.apache.lucene.document.LongDocValuesField;
-import org.apache.lucene.document.PackedLongDocValuesField;
-import org.apache.lucene.document.ShortDocValuesField;
-import org.apache.lucene.document.SortedBytesDocValuesField;
-import org.apache.lucene.document.StraightBytesDocValuesField;
+import org.apache.lucene.document.NumericDocValuesField;
+import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.index.AtomicReaderContext;
import org.apache.lucene.index.DirectoryReader;
-import org.apache.lucene.index.DocValues;
+import org.apache.lucene.index.FieldInfo.DocValuesType;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.queries.function.valuesource.DateDocValuesFieldSource;
-import org.apache.lucene.queries.function.valuesource.NumericDocValuesFieldSource;
-import org.apache.lucene.queries.function.valuesource.StrDocValuesFieldSource;
+import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
+import org.apache.lucene.queries.function.valuesource.LongFieldSource;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -49,53 +40,24 @@ import org.apache.lucene.util.packed.Pac
import com.carrotsearch.randomizedtesting.generators.RandomInts;
+
public class TestDocValuesFieldSources extends LuceneTestCase {
- public void test(DocValues.Type type) throws IOException {
+ public void test(DocValuesType type) throws IOException {
Directory d = newDirectory();
IndexWriterConfig iwConfig = newIndexWriterConfig(TEST_VERSION_CURRENT, new MockAnalyzer(random()));
final int nDocs = atLeast(50);
- final Field id = new IntDocValuesField("id", 0);
+ final Field id = new NumericDocValuesField("id", 0);
final Field f;
switch (type) {
- case BYTES_FIXED_DEREF:
- f = new DerefBytesDocValuesField("dv", new BytesRef(), true);
- break;
- case BYTES_FIXED_SORTED:
- f = new SortedBytesDocValuesField("dv", new BytesRef(), true);
- break;
- case BYTES_FIXED_STRAIGHT:
- f = new StraightBytesDocValuesField("dv", new BytesRef(), true);
- break;
- case BYTES_VAR_DEREF:
- f = new DerefBytesDocValuesField("dv", new BytesRef(), false);
- break;
- case BYTES_VAR_SORTED:
- f = new SortedBytesDocValuesField("dv", new BytesRef(), false);
- break;
- case BYTES_VAR_STRAIGHT:
- f = new StraightBytesDocValuesField("dv", new BytesRef(), false);
- break;
- case FIXED_INTS_8:
- f = new ByteDocValuesField("dv", (byte) 0);
- break;
- case FIXED_INTS_16:
- f = new ShortDocValuesField("dv", (short) 0);
- break;
- case FIXED_INTS_32:
- f = new IntDocValuesField("dv", 0);
- break;
- case FIXED_INTS_64:
- f = new LongDocValuesField("dv", 0L);
- break;
- case VAR_INTS:
- f = new PackedLongDocValuesField("dv", 0L);
+ case BINARY:
+ f = new BinaryDocValuesField("dv", new BytesRef());
break;
- case FLOAT_32:
- f = new FloatDocValuesField("dv", 0f);
+ case SORTED:
+ f = new SortedDocValuesField("dv", new BytesRef());
break;
- case FLOAT_64:
- f = new DoubleDocValuesField("dv", 0d);
+ case NUMERIC:
+ f = new NumericDocValuesField("dv", 0);
break;
default:
throw new AssertionError();
@@ -108,46 +70,20 @@ public class TestDocValuesFieldSources e
RandomIndexWriter iw = new RandomIndexWriter(random(), d, iwConfig);
for (int i = 0; i < nDocs; ++i) {
- id.setIntValue(i);
+ id.setLongValue(i);
switch (type) {
- case BYTES_FIXED_DEREF:
- case BYTES_FIXED_SORTED:
- case BYTES_FIXED_STRAIGHT:
- vals[i] = _TestUtil.randomFixedByteLengthUnicodeString(random(), 10);
+ case SORTED:
+ case BINARY:
+ do {
+ vals[i] = _TestUtil.randomSimpleString(random(), 20);
+ } while (((String) vals[i]).isEmpty());
f.setBytesValue(new BytesRef((String) vals[i]));
break;
- case BYTES_VAR_DEREF:
- case BYTES_VAR_SORTED:
- case BYTES_VAR_STRAIGHT:
- vals[i] = _TestUtil.randomSimpleString(random(), 20);
- f.setBytesValue(new BytesRef((String) vals[i]));
- break;
- case FIXED_INTS_8:
- vals[i] = (byte) random().nextInt(256);
- f.setByteValue((Byte) vals[i]);
- break;
- case FIXED_INTS_16:
- vals[i] = (short) random().nextInt(1 << 16);
- f.setShortValue((Short) vals[i]);
- break;
- case FIXED_INTS_32:
- vals[i] = random().nextInt();
- f.setIntValue((Integer) vals[i]);
- break;
- case FIXED_INTS_64:
- case VAR_INTS:
+ case NUMERIC:
final int bitsPerValue = RandomInts.randomIntBetween(random(), 1, 31); // keep it an int
vals[i] = (long) random().nextInt((int) PackedInts.maxValue(bitsPerValue));
f.setLongValue((Long) vals[i]);
break;
- case FLOAT_32:
- vals[i] = random().nextFloat();
- f.setFloatValue((Float) vals[i]);
- break;
- case FLOAT_64:
- vals[i] = random().nextDouble();
- f.setDoubleValue((Double) vals[i]);
- break;
}
iw.addDocument(document);
if (random().nextBoolean() && i % 10 == 9) {
@@ -158,32 +94,15 @@ public class TestDocValuesFieldSources e
DirectoryReader rd = DirectoryReader.open(d);
for (AtomicReaderContext leave : rd.leaves()) {
- final FunctionValues ids = new NumericDocValuesFieldSource("id", false).getValues(null, leave);
+ final FunctionValues ids = new LongFieldSource("id").getValues(null, leave);
final ValueSource vs;
- final boolean direct = random().nextBoolean();
switch (type) {
- case BYTES_FIXED_DEREF:
- case BYTES_FIXED_SORTED:
- case BYTES_FIXED_STRAIGHT:
- case BYTES_VAR_DEREF:
- case BYTES_VAR_SORTED:
- case BYTES_VAR_STRAIGHT:
- vs = new StrDocValuesFieldSource("dv", direct);
- break;
- case FLOAT_32:
- case FLOAT_64:
- case FIXED_INTS_8:
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- vs = new NumericDocValuesFieldSource("dv", direct);
+ case BINARY:
+ case SORTED:
+ vs = new BytesRefFieldSource("dv");
break;
- case FIXED_INTS_64:
- case VAR_INTS:
- if (random().nextBoolean()) {
- vs = new NumericDocValuesFieldSource("dv", direct);
- } else {
- vs = new DateDocValuesFieldSource("dv", direct);
- }
+ case NUMERIC:
+ vs = new LongFieldSource("dv");
break;
default:
throw new AssertionError();
@@ -192,61 +111,21 @@ public class TestDocValuesFieldSources e
BytesRef bytes = new BytesRef();
for (int i = 0; i < leave.reader().maxDoc(); ++i) {
assertTrue(values.exists(i));
- if (vs instanceof StrDocValuesFieldSource) {
+ if (vs instanceof BytesRefFieldSource) {
assertTrue(values.objectVal(i) instanceof String);
- } else if (vs instanceof NumericDocValuesFieldSource) {
- assertTrue(values.objectVal(i) instanceof Number);
- switch (type) {
- case FIXED_INTS_8:
- assertTrue(values.objectVal(i) instanceof Byte);
- assertTrue(values.bytesVal(i, bytes));
- assertEquals(1, bytes.length);
- break;
- case FIXED_INTS_16:
- assertTrue(values.objectVal(i) instanceof Short);
- assertTrue(values.bytesVal(i, bytes));
- assertEquals(2, bytes.length);
- break;
- case FIXED_INTS_32:
- assertTrue(values.objectVal(i) instanceof Integer);
- assertTrue(values.bytesVal(i, bytes));
- assertEquals(4, bytes.length);
- break;
- case FIXED_INTS_64:
- case VAR_INTS:
- assertTrue(values.objectVal(i) instanceof Long);
- assertTrue(values.bytesVal(i, bytes));
- assertEquals(8, bytes.length);
- break;
- case FLOAT_32:
- assertTrue(values.objectVal(i) instanceof Float);
- assertTrue(values.bytesVal(i, bytes));
- assertEquals(4, bytes.length);
- break;
- case FLOAT_64:
- assertTrue(values.objectVal(i) instanceof Double);
- assertTrue(values.bytesVal(i, bytes));
- assertEquals(8, bytes.length);
- break;
- default:
- throw new AssertionError();
- }
- } else if (vs instanceof DateDocValuesFieldSource) {
- assertTrue(values.objectVal(i) instanceof Date);
+ } else if (vs instanceof LongFieldSource) {
+ assertTrue(values.objectVal(i) instanceof Long);
+ assertTrue(values.bytesVal(i, bytes));
} else {
throw new AssertionError();
}
Object expected = vals[ids.intVal(i)];
switch (type) {
- case BYTES_VAR_SORTED:
- case BYTES_FIXED_SORTED:
+ case SORTED:
values.ordVal(i); // no exception
assertTrue(values.numOrd() >= 1);
- case BYTES_FIXED_DEREF:
- case BYTES_FIXED_STRAIGHT:
- case BYTES_VAR_DEREF:
- case BYTES_VAR_STRAIGHT:
+ case BINARY:
assertEquals(expected, values.objectVal(i));
assertEquals(expected, values.strVal(i));
assertEquals(expected, values.objectVal(i));
@@ -254,17 +133,7 @@ public class TestDocValuesFieldSources e
assertTrue(values.bytesVal(i, bytes));
assertEquals(new BytesRef((String) expected), bytes);
break;
- case FLOAT_32:
- assertEquals(((Number) expected).floatValue(), values.floatVal(i), 0.001);
- break;
- case FLOAT_64:
- assertEquals(((Number) expected).doubleValue(), values.doubleVal(i), 0.001d);
- break;
- case FIXED_INTS_8:
- case FIXED_INTS_16:
- case FIXED_INTS_32:
- case FIXED_INTS_64:
- case VAR_INTS:
+ case NUMERIC:
assertEquals(((Number) expected).longValue(), values.longVal(i));
break;
}
@@ -275,8 +144,10 @@ public class TestDocValuesFieldSources e
}
public void test() throws IOException {
- for (DocValues.Type type : DocValues.Type.values()) {
- test(type);
+ for (DocValuesType type : DocValuesType.values()) {
+ if (type != DocValuesType.SORTED_SET) {
+ test(type);
+ }
}
}
Modified: lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queries/src/test/org/apache/lucene/queries/function/TestOrdValues.java Sun Mar 10 15:04:57 2013
@@ -133,7 +133,7 @@ public class TestOrdValues extends Funct
String id = s.getIndexReader().document(sd[i].doc).get(ID_FIELD);
log("-------- " + i + ". Explain doc " + id);
log(s.explain(q, sd[i].doc));
- float expectedScore = N_DOCS - i;
+ float expectedScore = N_DOCS - i - 1;
assertEquals("score of result " + i + " shuould be " + expectedScore + " != " + score, expectedScore, score, TEST_SCORE_TOLERANCE_DELTA);
String expectedId = inOrder
? id2String(N_DOCS - i) // in-order ==> larger values first
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParser.jj Sun Mar 10 15:04:57 2013
@@ -162,7 +162,7 @@ PARSER_END(QueryParser)
| <CARAT: "^" > : Boost
| <QUOTED: "\"" (<_QUOTED_CHAR>)* "\"">
| <TERM: <_TERM_START_CHAR> (<_TERM_CHAR>)* >
-| <FUZZY_SLOP: "~" ( (<_NUM_CHAR>)+ ( "." (<_NUM_CHAR>)+ )? )? >
+| <FUZZY_SLOP: "~" ((<_NUM_CHAR>)+ (( "." (<_NUM_CHAR>)+ )? (<_TERM_CHAR>)*) | (<_TERM_CHAR>)*) >
| <PREFIXTERM: ("*") | ( <_TERM_START_CHAR> (<_TERM_CHAR>)* "*" ) >
| <WILDTERM: (<_TERM_START_CHAR> | [ "*", "?" ]) (<_TERM_CHAR> | ( [ "*", "?" ] ))* >
| <REGEXPTERM: "/" (~[ "/" ] | "\\/" )* "/" >
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java Sun Mar 10 15:04:57 2013
@@ -32,6 +32,7 @@ import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.classic.QueryParser.Operator;
import org.apache.lucene.queryparser.flexible.standard.CommonQueryParserConfiguration;
import org.apache.lucene.search.*;
+import org.apache.lucene.search.BooleanQuery.TooManyClauses;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.Version;
@@ -310,11 +311,11 @@ public abstract class QueryParserBase im
/**
* By default QueryParser uses {@link org.apache.lucene.search.MultiTermQuery#CONSTANT_SCORE_AUTO_REWRITE_DEFAULT}
- * when creating a PrefixQuery, WildcardQuery or RangeQuery. This implementation is generally preferable because it
+ * when creating a {@link PrefixQuery}, {@link WildcardQuery} or {@link TermRangeQuery}. This implementation is generally preferable because it
* a) Runs faster b) Does not have the scarcity of terms unduly influence score
- * c) avoids any "TooManyBooleanClauses" exception.
+ * c) avoids any {@link TooManyClauses} exception.
* However, if your application really needs to use the
- * old-fashioned BooleanQuery expansion rewriting and the above
+ * old-fashioned {@link BooleanQuery} expansion rewriting and the above
* points are not relevant then use this to change
* the rewrite method.
*/
@@ -416,9 +417,9 @@ public abstract class QueryParserBase im
}
/**
- * Set whether or not to analyze range terms when constructing RangeQuerys.
+ * Set whether or not to analyze range terms when constructing {@link TermRangeQuery}s.
* For example, setting this to true can enable analyzing terms into
- * collation keys for locale-sensitive RangeQuery.
+ * collation keys for locale-sensitive {@link TermRangeQuery}.
*
* @param analyzeRangeTerms whether or not terms should be analyzed for RangeQuerys
*/
@@ -427,7 +428,7 @@ public abstract class QueryParserBase im
}
/**
- * @return whether or not to analyze range terms when constructing RangeQuerys.
+ * @return whether or not to analyze range terms when constructing {@link TermRangeQuery}s.
*/
public boolean getAnalyzeRangeTerms() {
return analyzeRangeTerms;
@@ -844,13 +845,13 @@ public abstract class QueryParserBase im
}
/**
- * Builds a new TermRangeQuery instance
+ * Builds a new {@link TermRangeQuery} instance
* @param field Field
* @param part1 min
* @param part2 max
* @param startInclusive true if the start of the range is inclusive
* @param endInclusive true if the end of the range is inclusive
- * @return new TermRangeQuery instance
+ * @return new {@link TermRangeQuery} instance
*/
protected Query newRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) {
final BytesRef start;
@@ -1070,22 +1071,29 @@ public abstract class QueryParserBase im
} else if (regexp) {
q = getRegexpQuery(qfield, term.image.substring(1, term.image.length()-1));
} else if (fuzzy) {
- float fms = fuzzyMinSim;
- try {
- fms = Float.valueOf(fuzzySlop.image.substring(1)).floatValue();
- } catch (Exception ignored) { }
- if(fms < 0.0f){
- throw new ParseException("Minimum similarity for a FuzzyQuery has to be between 0.0f and 1.0f !");
- } else if (fms >= 1.0f && fms != (int) fms) {
- throw new ParseException("Fractional edit distances are not allowed!");
- }
- q = getFuzzyQuery(qfield, termImage, fms);
+ q = handleBareFuzzy(qfield, fuzzySlop, termImage);
} else {
q = getFieldQuery(qfield, termImage, false);
}
return q;
}
+ Query handleBareFuzzy(String qfield, Token fuzzySlop, String termImage)
+ throws ParseException {
+ Query q;
+ float fms = fuzzyMinSim;
+ try {
+ fms = Float.valueOf(fuzzySlop.image.substring(1)).floatValue();
+ } catch (Exception ignored) { }
+ if(fms < 0.0f){
+ throw new ParseException("Minimum similarity for a FuzzyQuery has to be between 0.0f and 1.0f !");
+ } else if (fms >= 1.0f && fms != (int) fms) {
+ throw new ParseException("Fractional edit distances are not allowed!");
+ }
+ q = getFuzzyQuery(qfield, termImage, fms);
+ return q;
+ }
+
// extracted from the .jj grammar
Query handleQuotedTerm(String qfield, Token term, Token fuzzySlop) throws ParseException {
int s = phraseSlop; // default
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserTokenManager.java Sun Mar 10 15:04:57 2013
@@ -47,7 +47,7 @@ private int jjMoveStringLiteralDfa0_2()
case 41:
return jjStopAtPos(0, 15);
case 42:
- return jjStartNfaWithStates_2(0, 17, 43);
+ return jjStartNfaWithStates_2(0, 17, 49);
case 43:
return jjStartNfaWithStates_2(0, 11, 15);
case 45:
@@ -87,7 +87,7 @@ static final long[] jjbitVec4 = {
private int jjMoveNfa_2(int startState, int curPos)
{
int startsAt = 0;
- jjnewStateCnt = 43;
+ jjnewStateCnt = 49;
int i = 1;
jjstateSet[0] = startState;
int kind = 0x7fffffff;
@@ -102,12 +102,20 @@ private int jjMoveNfa_2(int startState,
{
switch(jjstateSet[--i])
{
+ case 49:
+ case 33:
+ if ((0xfbff7cf8ffffd9ffL & l) == 0L)
+ break;
+ if (kind > 23)
+ kind = 23;
+ jjCheckNAddTwoStates(33, 34);
+ break;
case 0:
if ((0xfbff54f8ffffd9ffL & l) != 0L)
{
if (kind > 23)
kind = 23;
- jjCheckNAddTwoStates(27, 28);
+ jjCheckNAddTwoStates(33, 34);
}
else if ((0x100002600L & l) != 0L)
{
@@ -139,14 +147,6 @@ private int jjMoveNfa_2(int startState,
if (curChar == 38)
jjstateSet[jjnewStateCnt++] = 4;
break;
- case 43:
- case 27:
- if ((0xfbff7cf8ffffd9ffL & l) == 0L)
- break;
- if (kind > 23)
- kind = 23;
- jjCheckNAddTwoStates(27, 28);
- break;
case 4:
if (curChar == 38 && kind > 8)
kind = 8;
@@ -187,7 +187,7 @@ private int jjMoveNfa_2(int startState,
break;
if (kind > 21)
kind = 21;
- jjAddStates(11, 12);
+ jjCheckNAddStates(11, 14);
break;
case 23:
if (curChar == 46)
@@ -198,62 +198,86 @@ private int jjMoveNfa_2(int startState,
break;
if (kind > 21)
kind = 21;
- jjCheckNAdd(24);
+ jjCheckNAddStates(15, 17);
break;
case 25:
+ if ((0x7bff78f8ffffd9ffL & l) == 0L)
+ break;
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAddTwoStates(25, 26);
+ break;
+ case 27:
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAddTwoStates(25, 26);
+ break;
+ case 28:
+ if ((0x7bff78f8ffffd9ffL & l) == 0L)
+ break;
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAddTwoStates(28, 29);
+ break;
+ case 30:
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAddTwoStates(28, 29);
+ break;
+ case 31:
if (curChar == 42 && kind > 22)
kind = 22;
break;
- case 26:
+ case 32:
if ((0xfbff54f8ffffd9ffL & l) == 0L)
break;
if (kind > 23)
kind = 23;
- jjCheckNAddTwoStates(27, 28);
+ jjCheckNAddTwoStates(33, 34);
break;
- case 29:
+ case 35:
if (kind > 23)
kind = 23;
- jjCheckNAddTwoStates(27, 28);
+ jjCheckNAddTwoStates(33, 34);
break;
- case 30:
- case 32:
+ case 36:
+ case 38:
if (curChar == 47)
jjCheckNAddStates(0, 2);
break;
- case 31:
+ case 37:
if ((0xffff7fffffffffffL & l) != 0L)
jjCheckNAddStates(0, 2);
break;
- case 34:
+ case 40:
if (curChar == 47 && kind > 24)
kind = 24;
break;
- case 35:
+ case 41:
if ((0x7bff50f8ffffd9ffL & l) == 0L)
break;
if (kind > 20)
kind = 20;
jjCheckNAddStates(6, 10);
break;
- case 36:
+ case 42:
if ((0x7bff78f8ffffd9ffL & l) == 0L)
break;
if (kind > 20)
kind = 20;
- jjCheckNAddTwoStates(36, 37);
+ jjCheckNAddTwoStates(42, 43);
break;
- case 38:
+ case 44:
if (kind > 20)
kind = 20;
- jjCheckNAddTwoStates(36, 37);
+ jjCheckNAddTwoStates(42, 43);
break;
- case 39:
+ case 45:
if ((0x7bff78f8ffffd9ffL & l) != 0L)
- jjCheckNAddStates(13, 15);
+ jjCheckNAddStates(18, 20);
break;
- case 41:
- jjCheckNAddStates(13, 15);
+ case 47:
+ jjCheckNAddStates(18, 20);
break;
default : break;
}
@@ -266,6 +290,16 @@ private int jjMoveNfa_2(int startState,
{
switch(jjstateSet[--i])
{
+ case 49:
+ if ((0x97ffffff87ffffffL & l) != 0L)
+ {
+ if (kind > 23)
+ kind = 23;
+ jjCheckNAddTwoStates(33, 34);
+ }
+ else if (curChar == 92)
+ jjCheckNAddTwoStates(35, 35);
+ break;
case 0:
if ((0x97ffffff87ffffffL & l) != 0L)
{
@@ -274,18 +308,18 @@ private int jjMoveNfa_2(int startState,
jjCheckNAddStates(6, 10);
}
else if (curChar == 92)
- jjCheckNAddStates(16, 18);
+ jjCheckNAddStates(21, 23);
else if (curChar == 126)
{
if (kind > 21)
kind = 21;
- jjstateSet[jjnewStateCnt++] = 22;
+ jjCheckNAddStates(24, 26);
}
if ((0x97ffffff87ffffffL & l) != 0L)
{
if (kind > 23)
kind = 23;
- jjCheckNAddTwoStates(27, 28);
+ jjCheckNAddTwoStates(33, 34);
}
if (curChar == 78)
jjstateSet[jjnewStateCnt++] = 11;
@@ -296,16 +330,6 @@ private int jjMoveNfa_2(int startState,
else if (curChar == 65)
jjstateSet[jjnewStateCnt++] = 2;
break;
- case 43:
- if ((0x97ffffff87ffffffL & l) != 0L)
- {
- if (kind > 23)
- kind = 23;
- jjCheckNAddTwoStates(27, 28);
- }
- else if (curChar == 92)
- jjCheckNAddTwoStates(29, 29);
- break;
case 1:
if (curChar == 68 && kind > 8)
kind = 8;
@@ -362,75 +386,107 @@ private int jjMoveNfa_2(int startState,
break;
if (kind > 21)
kind = 21;
- jjstateSet[jjnewStateCnt++] = 22;
+ jjCheckNAddStates(24, 26);
+ break;
+ case 25:
+ if ((0x97ffffff87ffffffL & l) == 0L)
+ break;
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAddTwoStates(25, 26);
break;
case 26:
+ if (curChar == 92)
+ jjAddStates(27, 28);
+ break;
+ case 27:
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAddTwoStates(25, 26);
+ break;
+ case 28:
+ if ((0x97ffffff87ffffffL & l) == 0L)
+ break;
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAddTwoStates(28, 29);
+ break;
+ case 29:
+ if (curChar == 92)
+ jjAddStates(29, 30);
+ break;
+ case 30:
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAddTwoStates(28, 29);
+ break;
+ case 32:
if ((0x97ffffff87ffffffL & l) == 0L)
break;
if (kind > 23)
kind = 23;
- jjCheckNAddTwoStates(27, 28);
+ jjCheckNAddTwoStates(33, 34);
break;
- case 27:
+ case 33:
if ((0x97ffffff87ffffffL & l) == 0L)
break;
if (kind > 23)
kind = 23;
- jjCheckNAddTwoStates(27, 28);
+ jjCheckNAddTwoStates(33, 34);
break;
- case 28:
+ case 34:
if (curChar == 92)
- jjCheckNAddTwoStates(29, 29);
+ jjCheckNAddTwoStates(35, 35);
break;
- case 29:
+ case 35:
if (kind > 23)
kind = 23;
- jjCheckNAddTwoStates(27, 28);
+ jjCheckNAddTwoStates(33, 34);
break;
- case 31:
+ case 37:
jjAddStates(0, 2);
break;
- case 33:
+ case 39:
if (curChar == 92)
- jjstateSet[jjnewStateCnt++] = 32;
+ jjstateSet[jjnewStateCnt++] = 38;
break;
- case 35:
+ case 41:
if ((0x97ffffff87ffffffL & l) == 0L)
break;
if (kind > 20)
kind = 20;
jjCheckNAddStates(6, 10);
break;
- case 36:
+ case 42:
if ((0x97ffffff87ffffffL & l) == 0L)
break;
if (kind > 20)
kind = 20;
- jjCheckNAddTwoStates(36, 37);
+ jjCheckNAddTwoStates(42, 43);
break;
- case 37:
+ case 43:
if (curChar == 92)
- jjCheckNAddTwoStates(38, 38);
+ jjCheckNAddTwoStates(44, 44);
break;
- case 38:
+ case 44:
if (kind > 20)
kind = 20;
- jjCheckNAddTwoStates(36, 37);
+ jjCheckNAddTwoStates(42, 43);
break;
- case 39:
+ case 45:
if ((0x97ffffff87ffffffL & l) != 0L)
- jjCheckNAddStates(13, 15);
+ jjCheckNAddStates(18, 20);
break;
- case 40:
+ case 46:
if (curChar == 92)
- jjCheckNAddTwoStates(41, 41);
+ jjCheckNAddTwoStates(47, 47);
break;
- case 41:
- jjCheckNAddStates(13, 15);
+ case 47:
+ jjCheckNAddStates(18, 20);
break;
- case 42:
+ case 48:
if (curChar == 92)
- jjCheckNAddStates(16, 18);
+ jjCheckNAddStates(21, 23);
break;
default : break;
}
@@ -447,6 +503,14 @@ private int jjMoveNfa_2(int startState,
{
switch(jjstateSet[--i])
{
+ case 49:
+ case 33:
+ if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
+ break;
+ if (kind > 23)
+ kind = 23;
+ jjCheckNAddTwoStates(33, 34);
+ break;
case 0:
if (jjCanMove_0(hiByte, i1, i2, l1, l2))
{
@@ -457,7 +521,7 @@ private int jjMoveNfa_2(int startState,
{
if (kind > 23)
kind = 23;
- jjCheckNAddTwoStates(27, 28);
+ jjCheckNAddTwoStates(33, 34);
}
if (jjCanMove_2(hiByte, i1, i2, l1, l2))
{
@@ -466,14 +530,6 @@ private int jjMoveNfa_2(int startState,
jjCheckNAddStates(6, 10);
}
break;
- case 43:
- case 27:
- if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
- break;
- if (kind > 23)
- kind = 23;
- jjCheckNAddTwoStates(27, 28);
- break;
case 15:
if (jjCanMove_0(hiByte, i1, i2, l1, l2) && kind > 13)
kind = 13;
@@ -483,52 +539,80 @@ private int jjMoveNfa_2(int startState,
if (jjCanMove_1(hiByte, i1, i2, l1, l2))
jjCheckNAddStates(3, 5);
break;
- case 26:
+ case 25:
+ if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
+ break;
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAddTwoStates(25, 26);
+ break;
+ case 27:
+ if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+ break;
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAddTwoStates(25, 26);
+ break;
+ case 28:
+ if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
+ break;
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAddTwoStates(28, 29);
+ break;
+ case 30:
+ if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
+ break;
+ if (kind > 21)
+ kind = 21;
+ jjCheckNAddTwoStates(28, 29);
+ break;
+ case 32:
if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
break;
if (kind > 23)
kind = 23;
- jjCheckNAddTwoStates(27, 28);
+ jjCheckNAddTwoStates(33, 34);
break;
- case 29:
+ case 35:
if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
break;
if (kind > 23)
kind = 23;
- jjCheckNAddTwoStates(27, 28);
+ jjCheckNAddTwoStates(33, 34);
break;
- case 31:
+ case 37:
if (jjCanMove_1(hiByte, i1, i2, l1, l2))
jjAddStates(0, 2);
break;
- case 35:
+ case 41:
if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
break;
if (kind > 20)
kind = 20;
jjCheckNAddStates(6, 10);
break;
- case 36:
+ case 42:
if (!jjCanMove_2(hiByte, i1, i2, l1, l2))
break;
if (kind > 20)
kind = 20;
- jjCheckNAddTwoStates(36, 37);
+ jjCheckNAddTwoStates(42, 43);
break;
- case 38:
+ case 44:
if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
break;
if (kind > 20)
kind = 20;
- jjCheckNAddTwoStates(36, 37);
+ jjCheckNAddTwoStates(42, 43);
break;
- case 39:
+ case 45:
if (jjCanMove_2(hiByte, i1, i2, l1, l2))
- jjCheckNAddStates(13, 15);
+ jjCheckNAddStates(18, 20);
break;
- case 41:
+ case 47:
if (jjCanMove_1(hiByte, i1, i2, l1, l2))
- jjCheckNAddStates(13, 15);
+ jjCheckNAddStates(18, 20);
break;
default : break;
}
@@ -541,7 +625,7 @@ private int jjMoveNfa_2(int startState,
kind = 0x7fffffff;
}
++curPos;
- if ((i = jjnewStateCnt) == (startsAt = 43 - (jjnewStateCnt = startsAt)))
+ if ((i = jjnewStateCnt) == (startsAt = 49 - (jjnewStateCnt = startsAt)))
return curPos;
try { curChar = input_stream.readChar(); }
catch(java.io.IOException e) { return curPos; }
@@ -574,7 +658,7 @@ private int jjMoveNfa_0(int startState,
break;
if (kind > 27)
kind = 27;
- jjAddStates(19, 20);
+ jjAddStates(31, 32);
break;
case 1:
if (curChar == 46)
@@ -728,11 +812,11 @@ private int jjMoveNfa_1(int startState,
break;
case 2:
if ((0xfffffffbffffffffL & l) != 0L)
- jjCheckNAddStates(21, 23);
+ jjCheckNAddStates(33, 35);
break;
case 3:
if (curChar == 34)
- jjCheckNAddStates(21, 23);
+ jjCheckNAddStates(33, 35);
break;
case 5:
if (curChar == 34 && kind > 31)
@@ -765,7 +849,7 @@ private int jjMoveNfa_1(int startState,
jjCheckNAdd(6);
break;
case 2:
- jjAddStates(21, 23);
+ jjAddStates(33, 35);
break;
case 4:
if (curChar == 92)
@@ -801,7 +885,7 @@ private int jjMoveNfa_1(int startState,
break;
case 2:
if (jjCanMove_1(hiByte, i1, i2, l1, l2))
- jjAddStates(21, 23);
+ jjAddStates(33, 35);
break;
case 6:
if (!jjCanMove_1(hiByte, i1, i2, l1, l2))
@@ -828,8 +912,9 @@ private int jjMoveNfa_1(int startState,
}
}
static final int[] jjnextStates = {
- 31, 33, 34, 17, 18, 20, 36, 39, 25, 40, 37, 22, 23, 39, 25, 40,
- 38, 41, 29, 0, 1, 2, 4, 5,
+ 37, 39, 40, 17, 18, 20, 42, 45, 31, 46, 43, 22, 23, 25, 26, 24,
+ 25, 26, 45, 31, 46, 44, 47, 35, 22, 28, 29, 27, 27, 30, 30, 0,
+ 1, 2, 4, 5,
};
private static final boolean jjCanMove_0(int hiByte, int i1, int i2, long l1, long l2)
{
@@ -893,8 +978,8 @@ static final long[] jjtoSkip = {
0x80L,
};
protected CharStream input_stream;
-private final int[] jjrounds = new int[43];
-private final int[] jjstateSet = new int[86];
+private final int[] jjrounds = new int[49];
+private final int[] jjstateSet = new int[98];
protected char curChar;
/** Constructor. */
public QueryParserTokenManager(CharStream stream){
@@ -919,7 +1004,7 @@ private void ReInitRounds()
{
int i;
jjround = 0x80000001;
- for (i = 43; i-- > 0;)
+ for (i = 49; i-- > 0;)
jjrounds[i] = 0x80000000;
}
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/package.html
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/package.html?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/package.html (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/package.html Sun Mar 10 15:04:57 2013
@@ -167,11 +167,11 @@ enabling substantial customization to ho
If you want to find the document entitled "The Right Way" which contains the text "don't go this way", you can enter: </p>
<pre class="code">title:"The Right Way" AND text:go</pre>
<p>or</p>
-<pre class="code">title:"Do it right" AND right</pre>
+<pre class="code">title:"The Right Way" AND go</pre>
<p>Since text is the default field, the field indicator is not required.</p>
<p>Note: The field is only valid for the term that it directly precedes, so the query</p>
-<pre class="code">title:Do it right</pre>
-<p>Will only find "Do" in the title field. It will find "it" and "right" in the default field (in this case the text field). </p>
+<pre class="code">title:The Right Way</pre>
+<p>Will only find "The" in the title field. It will find "Right" and "Way" in the default field (in this case the text field). </p>
</div>
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BooleanQuery2ModifierNodeProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BooleanQuery2ModifierNodeProcessor.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BooleanQuery2ModifierNodeProcessor.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/BooleanQuery2ModifierNodeProcessor.java Sun Mar 10 15:04:57 2013
@@ -55,7 +55,7 @@ import org.apache.lucene.queryparser.fle
* {@link Operator#AND}, if it is, the same operation when an
* {@link AndQueryNode} is found is applied to it. Each {@link BooleanQueryNode}
* which direct parent is also a {@link BooleanQueryNode} is removed (to ignore
- * the rules of precidence).
+ * the rules of precedence).
* </p>
*
* @see ConfigurationKeys#DEFAULT_OPERATOR
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiFieldQueryParser.java Sun Mar 10 15:04:57 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.queryparser.cl
*/
import java.io.Reader;
+import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
@@ -33,6 +34,7 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
/**
@@ -300,7 +302,7 @@ public class TestMultiFieldQueryParser e
}
/**
- * Return empty tokens for field "f1".
+ * Return no tokens for field "f1".
*/
private static class AnalyzerReturningNull extends Analyzer {
MockAnalyzer stdAnalyzer = new MockAnalyzer(random());
@@ -310,13 +312,21 @@ public class TestMultiFieldQueryParser e
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ protected Reader initReader(String fieldName, Reader reader) {
if ("f1".equals(fieldName)) {
- return new TokenStreamComponents(new EmptyTokenizer(reader));
+ // we don't use the reader, so close it:
+ IOUtils.closeWhileHandlingException(reader);
+ // return empty reader, so MockTokenizer returns no tokens:
+ return new StringReader("");
} else {
- return stdAnalyzer.createComponents(fieldName, reader);
+ return super.initReader(fieldName, reader);
}
}
+
+ @Override
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return stdAnalyzer.createComponents(fieldName, reader);
+ }
}
}
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestQueryParser.java Sun Mar 10 15:04:57 2013
@@ -160,6 +160,28 @@ public class TestQueryParser extends Que
}
}
+ public void testFuzzySlopeExtendability() throws ParseException {
+ QueryParser qp = new QueryParser(TEST_VERSION_CURRENT, "a", new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)) {
+
+ @Override
+ Query handleBareFuzzy(String qfield, Token fuzzySlop, String termImage)
+ throws ParseException {
+
+ if(fuzzySlop.image.endsWith("â¬")) {
+ float fms = fuzzyMinSim;
+ try {
+ fms = Float.valueOf(fuzzySlop.image.substring(1, fuzzySlop.image.length()-1)).floatValue();
+ } catch (Exception ignored) { }
+ float value = Float.parseFloat(termImage);
+ return getRangeQuery(qfield, Float.toString(value-fms/2.f), Float.toString(value+fms/2.f), true, true);
+ }
+ return super.handleBareFuzzy(qfield, fuzzySlop, termImage);
+ }
+
+ };
+ assertEquals(qp.parse("a:[11.95 TO 12.95]"), qp.parse("12.45~1â¬"));
+ }
+
@Override
public void testStarParsing() throws Exception {
final int[] type = new int[1];
Modified: lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java (original)
+++ lucene/dev/branches/lucene4258/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/standard/TestMultiFieldQPHelper.java Sun Mar 10 15:04:57 2013
@@ -18,6 +18,7 @@ package org.apache.lucene.queryparser.fl
*/
import java.io.Reader;
+import java.io.StringReader;
import java.util.HashMap;
import java.util.Map;
@@ -35,6 +36,7 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.IOUtils;
import org.apache.lucene.util.LuceneTestCase;
/**
@@ -339,9 +341,9 @@ public class TestMultiFieldQPHelper exte
}
/**
- * Return empty tokens for field "f1".
+ * Return no tokens for field "f1".
*/
- private static final class AnalyzerReturningNull extends Analyzer {
+ private static class AnalyzerReturningNull extends Analyzer {
MockAnalyzer stdAnalyzer = new MockAnalyzer(random());
public AnalyzerReturningNull() {
@@ -349,13 +351,21 @@ public class TestMultiFieldQPHelper exte
}
@Override
- public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ protected Reader initReader(String fieldName, Reader reader) {
if ("f1".equals(fieldName)) {
- return new TokenStreamComponents(new EmptyTokenizer(reader));
+ // we don't use the reader, so close it:
+ IOUtils.closeWhileHandlingException(reader);
+ // return empty reader, so MockTokenizer returns no tokens:
+ return new StringReader("");
} else {
- return stdAnalyzer.createComponents(fieldName, reader);
+ return super.initReader(fieldName, reader);
}
}
+
+ @Override
+ public TokenStreamComponents createComponents(String fieldName, Reader reader) {
+ return stdAnalyzer.createComponents(fieldName, reader);
+ }
}
}
Modified: lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java Sun Mar 10 15:04:57 2013
@@ -30,6 +30,7 @@ import org.apache.lucene.analysis.tokena
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.MultiFields;
import org.apache.lucene.index.Term;
+import org.apache.lucene.index.Terms;
import org.apache.lucene.search.*;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.search.similarities.DefaultSimilarity;
@@ -196,6 +197,10 @@ public class FuzzyLikeThisQuery extends
int corpusNumDocs = reader.numDocs();
HashSet<String> processedTerms = new HashSet<String>();
ts.reset();
+ final Terms terms = MultiFields.getTerms(reader, f.fieldName);
+ if (terms == null) {
+ return;
+ }
while (ts.incrementToken()) {
String term = termAtt.toString();
if (!processedTerms.contains(term)) {
@@ -206,7 +211,7 @@ public class FuzzyLikeThisQuery extends
AttributeSource atts = new AttributeSource();
MaxNonCompetitiveBoostAttribute maxBoostAtt =
atts.addAttribute(MaxNonCompetitiveBoostAttribute.class);
- SlowFuzzyTermsEnum fe = new SlowFuzzyTermsEnum(MultiFields.getTerms(reader, startTerm.field()), atts, startTerm, f.minSimilarity, f.prefixLength);
+ SlowFuzzyTermsEnum fe = new SlowFuzzyTermsEnum(terms, atts, startTerm, f.minSimilarity, f.prefixLength);
//store the df so all variants use same idf
int df = reader.docFreq(startTerm);
int numVariants = 0;
Modified: lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java (original)
+++ lucene/dev/branches/lucene4258/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowCollatedStringComparator.java Sun Mar 10 15:04:57 2013
@@ -21,8 +21,8 @@ import java.io.IOException;
import java.text.Collator;
import org.apache.lucene.index.AtomicReaderContext;
+import org.apache.lucene.index.BinaryDocValues;
import org.apache.lucene.search.FieldCache;
-import org.apache.lucene.search.FieldCache.DocTerms;
import org.apache.lucene.search.FieldComparator;
import org.apache.lucene.util.BytesRef;
@@ -38,7 +38,7 @@ import org.apache.lucene.util.BytesRef;
public final class SlowCollatedStringComparator extends FieldComparator<String> {
private final String[] values;
- private DocTerms currentDocTerms;
+ private BinaryDocValues currentDocTerms;
private final String field;
final Collator collator;
private String bottom;
@@ -67,7 +67,8 @@ public final class SlowCollatedStringCom
@Override
public int compareBottom(int doc) {
- final String val2 = currentDocTerms.getTerm(doc, tempBR).utf8ToString();
+ currentDocTerms.get(doc, tempBR);
+ final String val2 = tempBR.bytes == BinaryDocValues.MISSING ? null : tempBR.utf8ToString();
if (bottom == null) {
if (val2 == null) {
return 0;
@@ -81,11 +82,11 @@ public final class SlowCollatedStringCom
@Override
public void copy(int slot, int doc) {
- final BytesRef br = currentDocTerms.getTerm(doc, tempBR);
- if (br == null) {
+ currentDocTerms.get(doc, tempBR);
+ if (tempBR.bytes == BinaryDocValues.MISSING) {
values[slot] = null;
} else {
- values[slot] = br.utf8ToString();
+ values[slot] = tempBR.utf8ToString();
}
}
@@ -121,12 +122,12 @@ public final class SlowCollatedStringCom
@Override
public int compareDocToValue(int doc, String value) {
- final BytesRef br = currentDocTerms.getTerm(doc, tempBR);
+ currentDocTerms.get(doc, tempBR);
final String docValue;
- if (br == null) {
+ if (tempBR.bytes == BinaryDocValues.MISSING) {
docValue = null;
} else {
- docValue = br.utf8ToString();
+ docValue = tempBR.utf8ToString();
}
return compareValues(docValue, value);
}
Modified: lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java (original)
+++ lucene/dev/branches/lucene4258/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/FuzzyLikeThisQueryTest.java Sun Mar 10 15:04:57 2013
@@ -108,6 +108,25 @@ public class FuzzyLikeThisQueryTest exte
StoredDocument doc = searcher.doc(sd[0].doc);
assertEquals("Should match most similar when using 2 words", "2", doc.get("id"));
}
+
+ // LUCENE-4809
+ public void testNonExistingField() throws Throwable {
+ FuzzyLikeThisQuery flt = new FuzzyLikeThisQuery(10, analyzer);
+ flt.addTerms("jonathin smoth", "name", 0.3f, 1);
+ flt.addTerms("jonathin smoth", "this field does not exist", 0.3f, 1);
+ // don't fail here just because the field doesn't exits
+ Query q = flt.rewrite(searcher.getIndexReader());
+ HashSet<Term> queryTerms = new HashSet<Term>();
+ q.extractTerms(queryTerms);
+ assertTrue("Should have variant jonathan", queryTerms.contains(new Term("name", "jonathan")));
+ assertTrue("Should have variant smith", queryTerms.contains(new Term("name", "smith")));
+ TopDocs topDocs = searcher.search(flt, 1);
+ ScoreDoc[] sd = topDocs.scoreDocs;
+ assertTrue("score docs must match 1 doc", (sd != null) && (sd.length > 0));
+ StoredDocument doc = searcher.doc(sd[0].doc);
+ assertEquals("Should match most similar when using 2 words", "2", doc.get("id"));
+ }
+
//Test bug found when first query word does not match anything
public void testNoMatchFirstWordBug() throws Throwable {
Modified: lucene/dev/branches/lucene4258/lucene/site/xsl/index.xsl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/site/xsl/index.xsl?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/site/xsl/index.xsl (original)
+++ lucene/dev/branches/lucene4258/lucene/site/xsl/index.xsl Sun Mar 10 15:04:57 2013
@@ -58,7 +58,8 @@
<li><a href="core/overview-summary.html#overview_description">Introduction to Lucene's APIs</a>:
High-level summary of the different Lucene packages. </li>
<li><a href="core/org/apache/lucene/analysis/package-summary.html#package_description">Analysis overview</a>:
- Introduction to Lucene's analysis API. </li>
+ Introduction to Lucene's analysis API. See also the
+ <a href="core/org/apache/lucene/analysis/TokenStream.html">TokenStream consumer workflow</a>.</li>
</ul>
<h2>Reference Documents</h2>
<ul>
@@ -66,7 +67,7 @@
<li><a href="SYSTEM_REQUIREMENTS.html">System Requirements</a>: Minimum and supported Java versions.</li>
<li><a href="MIGRATE.html">Migration Guide</a>: What changed in Lucene 4; how to migrate code from Lucene 3.x.</li>
<li><a href="JRE_VERSION_MIGRATION.html">JRE Version Migration</a>: Information about upgrading between major JRE versions.</li>
- <li><a href="core/org/apache/lucene/codecs/lucene41/package-summary.html#package_description">File Formats</a>: Guide to the supported index format used by Lucene. This can be customized by using <a href="core/org/apache/lucene/codecs/package-summary.html#package_description">an alternate codec</a>.</li>
+ <li><a href="core/org/apache/lucene/codecs/lucene42/package-summary.html#package_description">File Formats</a>: Guide to the supported index format used by Lucene. This can be customized by using <a href="core/org/apache/lucene/codecs/package-summary.html#package_description">an alternate codec</a>.</li>
<li><a href="core/org/apache/lucene/search/package-summary.html#package_description">Search and Scoring in Lucene</a>: Introduction to how Lucene scores documents.</li>
<li><a href="core/org/apache/lucene/search/similarities/TFIDFSimilarity.html">Classic Scoring Formula</a>: Formula of Lucene's classic <a href="http://en.wikipedia.org/wiki/Vector_Space_Model">Vector Space</a> implementation. (look <a href="core/org/apache/lucene/search/similarities/package-summary.html#package_description">here</a> for other models)</li>
<li><a href="queryparser/org/apache/lucene/queryparser/classic/package-summary.html#package_description">Classic QueryParser Syntax</a>: Overview of the Classic QueryParser's syntax and features.</li>
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java Sun Mar 10 15:04:57 2013
@@ -41,8 +41,8 @@ import org.apache.lucene.spatial.query.S
* <li>What types of query shapes can be used?</li>
* <li>What types of query operations are supported?
* This might vary per shape.</li>
- * <li>Does it use the {@link org.apache.lucene.search.FieldCache}, {@link
- * org.apache.lucene.index.DocValues} or some other type of cache? When?
+ * <li>Does it use the {@link org.apache.lucene.search.FieldCache},
+ * or some other type of cache? When?
* </ul>
* If a strategy only supports certain shapes at index or query time, then in
* general it will throw an exception if given an incompatible one. It will not
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxSimilarityValueSource.java Sun Mar 10 15:04:57 2013
@@ -64,10 +64,10 @@ public class BBoxSimilarityValueSource e
@Override
public FunctionValues getValues(Map context, AtomicReaderContext readerContext) throws IOException {
AtomicReader reader = readerContext.reader();
- final double[] minX = FieldCache.DEFAULT.getDoubles(reader, strategy.field_minX, true);
- final double[] minY = FieldCache.DEFAULT.getDoubles(reader, strategy.field_minY, true);
- final double[] maxX = FieldCache.DEFAULT.getDoubles(reader, strategy.field_maxX, true);
- final double[] maxY = FieldCache.DEFAULT.getDoubles(reader, strategy.field_maxY, true);
+ final FieldCache.Doubles minX = FieldCache.DEFAULT.getDoubles(reader, strategy.field_minX, true);
+ final FieldCache.Doubles minY = FieldCache.DEFAULT.getDoubles(reader, strategy.field_minY, true);
+ final FieldCache.Doubles maxX = FieldCache.DEFAULT.getDoubles(reader, strategy.field_maxX, true);
+ final FieldCache.Doubles maxY = FieldCache.DEFAULT.getDoubles(reader, strategy.field_maxY, true);
final Bits validMinX = FieldCache.DEFAULT.getDocsWithField(reader, strategy.field_minX);
final Bits validMaxX = FieldCache.DEFAULT.getDocsWithField(reader, strategy.field_maxX);
@@ -81,8 +81,8 @@ public class BBoxSimilarityValueSource e
// make sure it has minX and area
if (validMinX.get(doc) && validMaxX.get(doc)) {
rect.reset(
- minX[doc], maxX[doc],
- minY[doc], maxY[doc]);
+ minX.get(doc), maxX.get(doc),
+ minY.get(doc), maxY.get(doc));
return (float) similarity.score(rect, null);
} else {
return (float) similarity.score(null, null);
@@ -94,8 +94,8 @@ public class BBoxSimilarityValueSource e
// make sure it has minX and area
if (validMinX.get(doc) && validMaxX.get(doc)) {
rect.reset(
- minX[doc], maxX[doc],
- minY[doc], maxY[doc]);
+ minX.get(doc), maxX.get(doc),
+ minY.get(doc), maxY.get(doc));
Explanation exp = new Explanation();
similarity.score(rect, exp);
return exp;
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/PrefixTreeStrategy.java Sun Mar 10 15:04:57 2013
@@ -78,12 +78,14 @@ import java.util.concurrent.ConcurrentHa
public abstract class PrefixTreeStrategy extends SpatialStrategy {
protected final SpatialPrefixTree grid;
private final Map<String, PointPrefixTreeFieldCacheProvider> provider = new ConcurrentHashMap<String, PointPrefixTreeFieldCacheProvider>();
+ protected final boolean simplifyIndexedCells;
protected int defaultFieldValuesArrayLen = 2;
protected double distErrPct = SpatialArgs.DEFAULT_DISTERRPCT;// [ 0 TO 0.5 ]
- public PrefixTreeStrategy(SpatialPrefixTree grid, String fieldName) {
+ public PrefixTreeStrategy(SpatialPrefixTree grid, String fieldName, boolean simplifyIndexedCells) {
super(grid.getSpatialContext(), fieldName);
this.grid = grid;
+ this.simplifyIndexedCells = simplifyIndexedCells;
}
/**
@@ -123,7 +125,7 @@ public abstract class PrefixTreeStrategy
public Field[] createIndexableFields(Shape shape, double distErr) {
int detailLevel = grid.getLevelForDistance(distErr);
- List<Node> cells = grid.getNodes(shape, detailLevel, true);//true=intermediates cells
+ List<Node> cells = grid.getNodes(shape, detailLevel, true, simplifyIndexedCells);//intermediates cells
//TODO is CellTokenStream supposed to be re-used somehow? see Uwe's comments:
// http://code.google.com/p/lucene-spatial-playground/issues/detail?id=4
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/RecursivePrefixTreeStrategy.java Sun Mar 10 15:04:57 2013
@@ -25,7 +25,7 @@ import org.apache.lucene.spatial.query.S
import org.apache.lucene.spatial.query.UnsupportedSpatialOperation;
/**
- * A {@link PrefixTreeStrategy} which uses {@link RecursivePrefixTreeFilter}.
+ * A {@link PrefixTreeStrategy} which uses {@link AbstractVisitingPrefixTreeFilter}.
* This strategy has support for searching non-point shapes (note: not tested).
* Even a query shape with distErrPct=0 (fully precise to the grid) should have
* good performance for typical data, unless there is a lot of indexed data
@@ -38,7 +38,8 @@ public class RecursivePrefixTreeStrategy
private int prefixGridScanLevel;
public RecursivePrefixTreeStrategy(SpatialPrefixTree grid, String fieldName) {
- super(grid, fieldName);
+ super(grid, fieldName,
+ true);//simplify indexed cells
prefixGridScanLevel = grid.getMaxLevels() - 4;//TODO this default constant is dependent on the prefix grid size
}
@@ -69,8 +70,9 @@ public class RecursivePrefixTreeStrategy
int detailLevel = grid.getLevelForDistance(args.resolveDistErr(ctx, distErrPct));
- return new RecursivePrefixTreeFilter(
- getFieldName(), grid, shape, prefixGridScanLevel, detailLevel);
+ return new IntersectsPrefixTreeFilter(
+ shape, getFieldName(), grid, detailLevel, prefixGridScanLevel,
+ true);//hasIndexedLeaves
}
}
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/TermQueryPrefixTreeStrategy.java Sun Mar 10 15:04:57 2013
@@ -32,7 +32,7 @@ import java.util.List;
/**
* A basic implementation of {@link PrefixTreeStrategy} using a large {@link
* TermsFilter} of all the nodes from {@link SpatialPrefixTree#getNodes(com.spatial4j.core.shape.Shape,
- * int, boolean)}. It only supports the search of indexed Point shapes.
+ * int, boolean, boolean)}. It only supports the search of indexed Point shapes.
* <p/>
* The precision of query shapes (distErrPct) is an important factor in using
* this Strategy. If the precision is too precise then it will result in many
@@ -43,7 +43,8 @@ import java.util.List;
public class TermQueryPrefixTreeStrategy extends PrefixTreeStrategy {
public TermQueryPrefixTreeStrategy(SpatialPrefixTree grid, String fieldName) {
- super(grid, fieldName);
+ super(grid, fieldName,
+ false);//do not simplify indexed cells
}
@Override
@@ -54,7 +55,9 @@ public class TermQueryPrefixTreeStrategy
Shape shape = args.getShape();
int detailLevel = grid.getLevelForDistance(args.resolveDistErr(ctx, distErrPct));
- List<Node> cells = grid.getNodes(shape, detailLevel, false);
+ List<Node> cells = grid.getNodes(shape, detailLevel,
+ false,//no parents
+ true);//simplify
BytesRef[] terms = new BytesRef[cells.size()];
int i = 0;
for (Node cell : cells) {
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/GeohashPrefixTree.java Sun Mar 10 15:04:57 2013
@@ -93,12 +93,6 @@ public class GeohashPrefixTree extends S
return new GhCell(bytes, offset, len);
}
- @Override
- public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents) {
- return shape instanceof Point ? super.getNodesAltPoint((Point) shape, detailLevel, inclParents) :
- super.getNodes(shape, detailLevel, inclParents);
- }
-
class GhCell extends Node {
GhCell(String token) {
super(token);
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/Node.java Sun Mar 10 15:04:57 2013
@@ -1,3 +1,5 @@
+package org.apache.lucene.spatial.prefix.tree;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -15,8 +17,6 @@
* limitations under the License.
*/
-package org.apache.lucene.spatial.prefix.tree;
-
import com.spatial4j.core.shape.Point;
import com.spatial4j.core.shape.Shape;
import com.spatial4j.core.shape.SpatialRelation;
@@ -27,7 +27,8 @@ import java.util.Collections;
import java.util.List;
/**
- * Represents a grid cell. These are not necessarily thread-safe, although new Cell("") (world cell) must be.
+ * Represents a grid cell. These are not necessarily thread-safe, although new
+ * Cell("") (world cell) must be.
*
* @lucene.experimental
*/
@@ -44,13 +45,19 @@ public abstract class Node implements Co
private String token;//this is the only part of equality
- /** When set via getSubCells(filter), it is the relationship between this
- * cell and the given shape filter. If set via setLeaf() (to WITHIN), it is
- * meant to indicate no further sub-cells are going to be provided because
- * maxLevels or a detailLevel is hit. It's always null for points.
+ /**
+ * When set via getSubCells(filter), it is the relationship between this cell
+ * and the given shape filter.
*/
protected SpatialRelation shapeRel;
+ /**
+ * Always false for points. Otherwise, indicate no further sub-cells are going
+ * to be provided because shapeRel is WITHIN or maxLevels or a detailLevel is
+ * hit.
+ */
+ protected boolean leaf;
+
protected Node(String token) {
this.token = token;
if (token.length() > 0 && token.charAt(token.length() - 1) == (char) LEAF_BYTE) {
@@ -96,12 +103,13 @@ public abstract class Node implements Co
* further cells with this prefix for the shape (always true at maxLevels).
*/
public boolean isLeaf() {
- return shapeRel == SpatialRelation.WITHIN;
+ return leaf;
}
+ /** Note: not supported at level 0. */
public void setLeaf() {
assert getLevel() != 0;
- shapeRel = SpatialRelation.WITHIN;
+ leaf = true;
}
/**
@@ -139,12 +147,11 @@ public abstract class Node implements Co
/**
* Like {@link #getSubCells()} but with the results filtered by a shape. If
- * that shape is a {@link com.spatial4j.core.shape.Point} then it
- * must call {@link #getSubCell(com.spatial4j.core.shape.Point)}.
- * The returned cells should have their {@link Node#shapeRel} set to their
- * relation with {@code shapeFilter} for non-point. As such,
- * {@link org.apache.lucene.spatial.prefix.tree.Node#isLeaf()} should be
- * accurate.
+ * that shape is a {@link com.spatial4j.core.shape.Point} then it must call
+ * {@link #getSubCell(com.spatial4j.core.shape.Point)}. The returned cells
+ * should have {@link Node#getShapeRel()} set to their relation with {@code
+ * shapeFilter}. In addition, {@link org.apache.lucene.spatial.prefix.tree.Node#isLeaf()}
+ * must be true when that relation is WITHIN.
* <p/>
* Precondition: Never called when getLevel() == maxLevel.
*
@@ -154,29 +161,35 @@ public abstract class Node implements Co
public Collection<Node> getSubCells(Shape shapeFilter) {
//Note: Higher-performing subclasses might override to consider the shape filter to generate fewer cells.
if (shapeFilter instanceof Point) {
- return Collections.singleton(getSubCell((Point) shapeFilter));
+ Node subCell = getSubCell((Point) shapeFilter);
+ subCell.shapeRel = SpatialRelation.CONTAINS;
+ return Collections.singletonList(subCell);
}
Collection<Node> cells = getSubCells();
if (shapeFilter == null) {
return cells;
}
- List<Node> copy = new ArrayList<Node>(cells.size());//copy since cells contractually isn't modifiable
+
+ //TODO change API to return a filtering iterator
+ List<Node> copy = new ArrayList<Node>(cells.size());
for (Node cell : cells) {
SpatialRelation rel = cell.getShape().relate(shapeFilter);
if (rel == SpatialRelation.DISJOINT)
continue;
cell.shapeRel = rel;
+ if (rel == SpatialRelation.WITHIN)
+ cell.setLeaf();
copy.add(cell);
}
- cells = copy;
- return cells;
+ return copy;
}
/**
- * Performant implementations are expected to implement this efficiently by considering the current
- * cell's boundary.
- * Precondition: Never called when getLevel() == maxLevel.
+ * Performant implementations are expected to implement this efficiently by
+ * considering the current cell's boundary. Precondition: Never called when
+ * getLevel() == maxLevel.
+ * <p/>
* Precondition: this.getShape().relate(p) != DISJOINT.
*/
public abstract Node getSubCell(Point p);
@@ -187,7 +200,7 @@ public abstract class Node implements Co
* Gets the cells at the next grid cell level that cover this cell.
* Precondition: Never called when getLevel() == maxLevel.
*
- * @return A set of cells (no dups), sorted. Not Modifiable.
+ * @return A set of cells (no dups), sorted, modifiable, not empty, not null.
*/
protected abstract Collection<Node> getSubCells();
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/QuadPrefixTree.java Sun Mar 10 15:04:57 2013
@@ -156,14 +156,6 @@ public class QuadPrefixTree extends Spat
return new QuadCell(bytes, offset, len);
}
- @Override //for performance
- public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents) {
- if (shape instanceof Point)
- return super.getNodesAltPoint((Point) shape, detailLevel, inclParents);
- else
- return super.getNodes(shape, detailLevel, inclParents);
- }
-
private void build(
double x,
double y,
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/tree/SpatialPrefixTree.java Sun Mar 10 15:04:57 2013
@@ -116,88 +116,86 @@ public abstract class SpatialPrefixTree
}
/**
- * Gets the intersecting & including cells for the specified shape, without exceeding detail level.
- * The result is a set of cells (no dups), sorted. Unmodifiable.
+ * Gets the intersecting cells for the specified shape, without exceeding
+ * detail level. If a cell is within the query shape then it's marked as a
+ * leaf and none of its children are added.
* <p/>
- * This implementation checks if shape is a Point and if so uses an implementation that
- * recursively calls {@link Node#getSubCell(com.spatial4j.core.shape.Point)}. Cell subclasses
- * ideally implement that method with a quick implementation, otherwise, subclasses should
- * override this method to invoke {@link #getNodesAltPoint(com.spatial4j.core.shape.Point, int, boolean)}.
- * TODO consider another approach returning an iterator -- won't build up all cells in memory.
+ * This implementation checks if shape is a Point and if so returns {@link
+ * #getNodes(com.spatial4j.core.shape.Point, int, boolean)}.
+ *
+ * @param shape the shape; non-null
+ * @param detailLevel the maximum detail level to get cells for
+ * @param inclParents if true then all parent cells of leaves are returned
+ * too. The top world cell is never returned.
+ * @param simplify for non-point shapes, this will simply/aggregate sets of
+ * complete leaves in a cell to its parent, resulting in
+ * ~20-25% fewer cells.
+ * @return a set of cells (no dups), sorted, immutable, non-null
*/
- public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents) {
+ public List<Node> getNodes(Shape shape, int detailLevel, boolean inclParents,
+ boolean simplify) {
+ //TODO consider an on-demand iterator -- it won't build up all cells in memory.
if (detailLevel > maxLevels) {
throw new IllegalArgumentException("detailLevel > maxLevels");
}
-
- List<Node> cells;
if (shape instanceof Point) {
- //optimized point algorithm
- final int initialCapacity = inclParents ? 1 + detailLevel : 1;
- cells = new ArrayList<Node>(initialCapacity);
- recursiveGetNodes(getWorldNode(), (Point) shape, detailLevel, true, cells);
- assert cells.size() == initialCapacity;
- } else {
- cells = new ArrayList<Node>(inclParents ? 1024 : 512);
- recursiveGetNodes(getWorldNode(), shape, detailLevel, inclParents, cells);
- }
- if (inclParents) {
- Node c = cells.remove(0);//remove getWorldNode()
- assert c.getLevel() == 0;
+ return getNodes((Point) shape, detailLevel, inclParents);
}
+ List<Node> cells = new ArrayList<Node>(inclParents ? 4096 : 2048);
+ recursiveGetNodes(getWorldNode(), shape, detailLevel, inclParents, simplify, cells);
return cells;
}
- private void recursiveGetNodes(Node node, Shape shape, int detailLevel, boolean inclParents,
- Collection<Node> result) {
- if (node.isLeaf()) {//cell is within shape
- result.add(node);
- return;
- }
- final Collection<Node> subCells = node.getSubCells(shape);
- if (node.getLevel() == detailLevel - 1) {
- if (subCells.size() < node.getSubCellsSize() || node.getLevel() == 0) {
- if (inclParents)
- result.add(node);
- for (Node subCell : subCells) {
- subCell.setLeaf();
- }
- result.addAll(subCells);
- } else {//a bottom level (i.e. detail level) optimization where all boxes intersect, so use parent cell.
- node.setLeaf();//the cell may not be strictly within but its close
- result.add(node);
- }
- } else {
- if (inclParents) {
- result.add(node);
- }
- for (Node subCell : subCells) {
- recursiveGetNodes(subCell, shape, detailLevel, inclParents, result);//tail call
- }
+ /**
+ * Returns true if node was added as a leaf. If it wasn't it recursively
+ * descends.
+ */
+ private boolean recursiveGetNodes(Node node, Shape shape, int detailLevel,
+ boolean inclParents, boolean simplify,
+ List<Node> result) {
+ if (node.getLevel() == detailLevel) {
+ node.setLeaf();//FYI might already be a leaf
}
- }
-
- private void recursiveGetNodes(Node node, Point point, int detailLevel, boolean inclParents,
- Collection<Node> result) {
- if (inclParents) {
+ if (node.isLeaf()) {
result.add(node);
+ return true;
}
- final Node pCell = node.getSubCell(point);
- if (node.getLevel() == detailLevel - 1) {
- pCell.setLeaf();
- result.add(pCell);
- } else {
- recursiveGetNodes(pCell, point, detailLevel, inclParents, result);//tail call
+ if (inclParents && node.getLevel() != 0)
+ result.add(node);
+
+ Collection<Node> subCells = node.getSubCells(shape);
+ int leaves = 0;
+ for (Node subCell : subCells) {
+ if (recursiveGetNodes(subCell, shape, detailLevel, inclParents, simplify, result))
+ leaves++;
+ }
+ //can we simplify?
+ if (simplify && leaves == node.getSubCellsSize() && node.getLevel() != 0) {
+ //Optimization: substitute the parent as a leaf instead of adding all
+ // children as leaves
+
+ //remove the leaves
+ do {
+ result.remove(result.size() - 1);//remove last
+ } while (--leaves > 0);
+ //add node as the leaf
+ node.setLeaf();
+ if (!inclParents) // otherwise it was already added up above
+ result.add(node);
+ return true;
}
+ return false;
}
/**
- * Subclasses might override {@link #getNodes(com.spatial4j.core.shape.Shape, int, boolean)}
- * and check if the argument is a shape and if so, delegate
- * to this implementation, which calls {@link #getNode(com.spatial4j.core.shape.Point, int)} and
- * then calls {@link #getNode(String)} repeatedly if inclParents is true.
+ * A Point-optimized implementation of
+ * {@link #getNodes(com.spatial4j.core.shape.Shape, int, boolean, boolean)}. That
+ * method in facts calls this for points.
+ * <p/>
+ * This implementation depends on {@link #getNode(String)} being fast, as its
+ * called repeatedly when incPlarents is true.
*/
- protected final List<Node> getNodesAltPoint(Point p, int detailLevel, boolean inclParents) {
+ public List<Node> getNodes(Point p, int detailLevel, boolean inclParents) {
Node cell = getNode(p, detailLevel);
if (!inclParents) {
return Collections.singletonList(cell);
Modified: lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java?rev=1454862&r1=1454861&r2=1454862&view=diff
==============================================================================
--- lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java (original)
+++ lucene/dev/branches/lucene4258/lucene/spatial/src/java/org/apache/lucene/spatial/query/SpatialArgs.java Sun Mar 10 15:04:57 2013
@@ -47,7 +47,7 @@ public class SpatialArgs {
/**
* Computes the distance given a shape and the {@code distErrPct}. The
* algorithm is the fraction of the distance from the center of the query
- * shape to its furthest bounding box corner.
+ * shape to its closest bounding box corner.
*
* @param shape Mandatory.
* @param distErrPct 0 to 0.5
@@ -62,11 +62,13 @@ public class SpatialArgs {
return 0;
}
Rectangle bbox = shape.getBoundingBox();
- //The diagonal distance should be the same computed from any opposite corner,
- // and this is the longest distance that might be occurring within the shape.
- double diagonalDist = ctx.getDistCalc().distance(
- ctx.makePoint(bbox.getMinX(), bbox.getMinY()), bbox.getMaxX(), bbox.getMaxY());
- return diagonalDist * 0.5 * distErrPct;
+ //Compute the distance from the center to a corner. Because the distance
+ // to a bottom corner vs a top corner can vary in a geospatial scenario,
+ // take the closest one (greater precision).
+ Point ctr = bbox.getCenter();
+ double y = (ctr.getY() >= 0 ? bbox.getMaxY() : bbox.getMinY());
+ double diagonalDist = ctx.getDistCalc().distance(ctr, bbox.getMaxX(), y);
+ return diagonalDist * distErrPct;
}
/**