You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cm...@apache.org on 2013/08/11 14:19:39 UTC
svn commit: r1512909 [19/38] - in /lucene/dev/branches/lucene4956: ./
dev-tools/ dev-tools/eclipse/ dev-tools/idea/.idea/libraries/
dev-tools/idea/lucene/suggest/ dev-tools/idea/solr/contrib/dataimporthandler/
dev-tools/idea/solr/core/src/test/ dev-too...
Modified: lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java Sun Aug 11 12:19:13 2013
@@ -29,7 +29,7 @@ import java.io.IOException;
import java.util.Map;
/**
- * Function that returns {@link TFIDFSimilarity#decodeNormValue(byte)}
+ * Function that returns {@link TFIDFSimilarity#decodeNormValue(long)}
* for every document.
* <p>
* Note that the configured Similarity for the field must be
Modified: lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java Sun Aug 11 12:19:13 2013
@@ -29,7 +29,7 @@ import java.io.IOException;
import java.util.Map;
/**
- * Function that returns {@link TFIDFSimilarity#tf(int)}
+ * Function that returns {@link TFIDFSimilarity#tf(float)}
* for every document.
* <p>
* Note that the configured Similarity for the field must be
@@ -67,7 +67,7 @@ public class TFValueSource extends TermF
if (terms != null) {
final TermsEnum termsEnum = terms.iterator(null);
- if (termsEnum.seekExact(indexedBytes, false)) {
+ if (termsEnum.seekExact(indexedBytes)) {
docs = termsEnum.docs(null, null);
} else {
docs = null;
Modified: lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TermFreqValueSource.java Sun Aug 11 12:19:13 2013
@@ -60,7 +60,7 @@ public class TermFreqValueSource extends
if (terms != null) {
final TermsEnum termsEnum = terms.iterator(null);
- if (termsEnum.seekExact(indexedBytes, false)) {
+ if (termsEnum.seekExact(indexedBytes)) {
docs = termsEnum.docs(null, null);
} else {
docs = null;
Modified: lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/CommonTermsQueryTest.java Sun Aug 11 12:19:13 2013
@@ -19,6 +19,7 @@ package org.apache.lucene.queries;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Random;
@@ -146,6 +147,8 @@ public class CommonTermsQueryTest extend
left.add(new Term(_TestUtil.randomRealisticUnicodeString(r), _TestUtil
.randomRealisticUnicodeString(r)));
}
+ left.setHighFreqMinimumNumberShouldMatch(r.nextInt(4));
+ left.setLowFreqMinimumNumberShouldMatch(r.nextInt(4));
r = new Random(seed);
CommonTermsQuery right = new CommonTermsQuery(randomOccur(r),
@@ -155,6 +158,8 @@ public class CommonTermsQueryTest extend
right.add(new Term(_TestUtil.randomRealisticUnicodeString(r), _TestUtil
.randomRealisticUnicodeString(r)));
}
+ right.setHighFreqMinimumNumberShouldMatch(r.nextInt(4));
+ right.setLowFreqMinimumNumberShouldMatch(r.nextInt(4));
QueryUtils.checkEqual(left, right);
}
}
@@ -200,7 +205,7 @@ public class CommonTermsQueryTest extend
query.add(new Term("field", "world"));
query.add(new Term("field", "universe"));
query.add(new Term("field", "right"));
- query.setMinimumNumberShouldMatch(0.5f);
+ query.setLowFreqMinimumNumberShouldMatch(0.5f);
TopDocs search = s.search(query, 10);
assertEquals(search.totalHits, 1);
assertEquals("0", r.document(search.scoreDocs[0].doc).get("id"));
@@ -214,7 +219,7 @@ public class CommonTermsQueryTest extend
query.add(new Term("field", "world"));
query.add(new Term("field", "universe"));
query.add(new Term("field", "right"));
- query.setMinimumNumberShouldMatch(2.0f);
+ query.setLowFreqMinimumNumberShouldMatch(2.0f);
TopDocs search = s.search(query, 10);
assertEquals(search.totalHits, 1);
assertEquals("0", r.document(search.scoreDocs[0].doc).get("id"));
@@ -229,7 +234,7 @@ public class CommonTermsQueryTest extend
query.add(new Term("field", "world"));
query.add(new Term("field", "universe"));
query.add(new Term("field", "right"));
- query.setMinimumNumberShouldMatch(0.49f);
+ query.setLowFreqMinimumNumberShouldMatch(0.49f);
TopDocs search = s.search(query, 10);
assertEquals(search.totalHits, 3);
assertEquals("0", r.document(search.scoreDocs[0].doc).get("id"));
@@ -246,14 +251,37 @@ public class CommonTermsQueryTest extend
query.add(new Term("field", "world"));
query.add(new Term("field", "universe"));
query.add(new Term("field", "right"));
- query.setMinimumNumberShouldMatch(1.0f);
+ query.setLowFreqMinimumNumberShouldMatch(1.0f);
TopDocs search = s.search(query, 10);
assertEquals(search.totalHits, 3);
assertEquals("0", r.document(search.scoreDocs[0].doc).get("id"));
assertEquals("2", r.document(search.scoreDocs[1].doc).get("id"));
assertEquals("3", r.document(search.scoreDocs[2].doc).get("id"));
+ assertTrue(search.scoreDocs[1].score > search.scoreDocs[2].score);
+ }
+
+ {
+ CommonTermsQuery query = new CommonTermsQuery(Occur.SHOULD, Occur.SHOULD,
+ random().nextBoolean() ? 2.0f : 0.5f);
+ query.add(new Term("field", "is"));
+ query.add(new Term("field", "this"));
+ query.add(new Term("field", "end"));
+ query.add(new Term("field", "world"));
+ query.add(new Term("field", "universe"));
+ query.add(new Term("field", "right"));
+ query.setLowFreqMinimumNumberShouldMatch(1.0f);
+ query.setHighFreqMinimumNumberShouldMatch(4.0f);
+ TopDocs search = s.search(query, 10);
+ assertEquals(search.totalHits, 3);
+ assertEquals(search.scoreDocs[1].score, search.scoreDocs[2].score, 0.0f);
+ assertEquals("0", r.document(search.scoreDocs[0].doc).get("id"));
+ // doc 2 and 3 only get a score from low freq terms
+ assertEquals(
+ new HashSet<>(Arrays.asList("2", "3")),
+ new HashSet<>(Arrays.asList(
+ r.document(search.scoreDocs[1].doc).get("id"),
+ r.document(search.scoreDocs[2].doc).get("id"))));
}
-
r.close();
w.close();
dir.close();
Modified: lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/TestCustomScoreQuery.java Sun Aug 11 12:19:13 2013
@@ -53,26 +53,6 @@ public class TestCustomScoreQuery extend
}
/**
- * Test that CustomScoreQuery of Type.BYTE returns the expected scores.
- */
- @Test
- public void testCustomScoreByte() throws Exception {
- // INT field values are small enough to be parsed as byte
- doTestCustomScore(BYTE_VALUESOURCE, 1.0);
- doTestCustomScore(BYTE_VALUESOURCE, 2.0);
- }
-
- /**
- * Test that CustomScoreQuery of Type.SHORT returns the expected scores.
- */
- @Test
- public void testCustomScoreShort() throws Exception {
- // INT field values are small enough to be parsed as short
- doTestCustomScore(SHORT_VALUESOURCE, 1.0);
- doTestCustomScore(SHORT_VALUESOURCE, 3.0);
- }
-
- /**
* Test that CustomScoreQuery of Type.INT returns the expected scores.
*/
@Test
Modified: lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/FunctionTestSetup.java Sun Aug 11 12:19:13 2013
@@ -1,18 +1,25 @@
package org.apache.lucene.queries.function;
+import java.io.IOException;
+
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FieldType;
+import org.apache.lucene.document.FloatField;
+import org.apache.lucene.document.IntField;
import org.apache.lucene.document.TextField;
-import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.IndexWriterConfig;
-import org.apache.lucene.queries.function.valuesource.ByteFieldSource;
+import org.apache.lucene.index.RandomIndexWriter;
+import org.apache.lucene.index.Terms;
+import org.apache.lucene.index.TermsEnum;
import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
import org.apache.lucene.queries.function.valuesource.IntFieldSource;
-import org.apache.lucene.queries.function.valuesource.ShortFieldSource;
+import org.apache.lucene.search.FieldCache;
import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util._TestUtil;
import org.junit.AfterClass;
@@ -53,10 +60,21 @@ public abstract class FunctionTestSetup
protected static final String INT_FIELD = "iii";
protected static final String FLOAT_FIELD = "fff";
- protected ValueSource BYTE_VALUESOURCE = new ByteFieldSource(INT_FIELD);
- protected ValueSource SHORT_VALUESOURCE = new ShortFieldSource(INT_FIELD);
+ private static final FieldCache.FloatParser CUSTOM_FLOAT_PARSER = new FieldCache.FloatParser() {
+
+ @Override
+ public TermsEnum termsEnum(Terms terms) throws IOException {
+ return FieldCache.NUMERIC_UTILS_INT_PARSER.termsEnum(terms);
+ }
+
+ @Override
+ public float parseFloat(BytesRef term) {
+ return (float) FieldCache.NUMERIC_UTILS_INT_PARSER.parseInt(term);
+ }
+ };
+
protected ValueSource INT_VALUESOURCE = new IntFieldSource(INT_FIELD);
- protected ValueSource INT_AS_FLOAT_VALUESOURCE = new FloatFieldSource(INT_FIELD);
+ protected ValueSource INT_AS_FLOAT_VALUESOURCE = new FloatFieldSource(INT_FIELD, CUSTOM_FLOAT_PARSER);
protected ValueSource FLOAT_VALUESOURCE = new FloatFieldSource(FLOAT_FIELD);
private static final String DOC_TEXT_LINES[] = {
@@ -140,10 +158,10 @@ public abstract class FunctionTestSetup
f = newField(TEXT_FIELD, "text of doc" + scoreAndID + textLine(i), customType2); // for regular search
d.add(f);
- f = newField(INT_FIELD, "" + scoreAndID, customType); // for function scoring
+ f = new IntField(INT_FIELD, scoreAndID, Store.YES); // for function scoring
d.add(f);
- f = newField(FLOAT_FIELD, scoreAndID + ".000", customType); // for function scoring
+ f = new FloatField(FLOAT_FIELD, scoreAndID, Store.YES); // for function scoring
d.add(f);
iw.addDocument(d);
Modified: lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/TestFieldScoreQuery.java Sun Aug 11 12:19:13 2013
@@ -19,12 +19,6 @@ package org.apache.lucene.queries.functi
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
-import org.apache.lucene.queries.function.FunctionQuery;
-import org.apache.lucene.queries.function.ValueSource;
-import org.apache.lucene.queries.function.valuesource.ByteFieldSource;
-import org.apache.lucene.queries.function.valuesource.FloatFieldSource;
-import org.apache.lucene.queries.function.valuesource.IntFieldSource;
-import org.apache.lucene.queries.function.valuesource.ShortFieldSource;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.QueryUtils;
import org.apache.lucene.search.ScoreDoc;
@@ -50,20 +44,6 @@ public class TestFieldScoreQuery extends
createIndex(true);
}
- /** Test that FieldScoreQuery of Type.BYTE returns docs in expected order. */
- @Test
- public void testRankByte () throws Exception {
- // INT field values are small enough to be parsed as byte
- doTestRank(BYTE_VALUESOURCE);
- }
-
- /** Test that FieldScoreQuery of Type.SHORT returns docs in expected order. */
- @Test
- public void testRankShort () throws Exception {
- // INT field values are small enough to be parsed as short
- doTestRank(SHORT_VALUESOURCE);
- }
-
/** Test that FieldScoreQuery of Type.INT returns docs in expected order. */
@Test
public void testRankInt () throws Exception {
@@ -99,20 +79,6 @@ public class TestFieldScoreQuery extends
r.close();
}
- /** Test that FieldScoreQuery of Type.BYTE returns the expected scores. */
- @Test
- public void testExactScoreByte () throws Exception {
- // INT field values are small enough to be parsed as byte
- doTestExactScore(BYTE_VALUESOURCE);
- }
-
- /** Test that FieldScoreQuery of Type.SHORT returns the expected scores. */
- @Test
- public void testExactScoreShort () throws Exception {
- // INT field values are small enough to be parsed as short
- doTestExactScore(SHORT_VALUESOURCE);
- }
-
/** Test that FieldScoreQuery of Type.INT returns the expected scores. */
@Test
public void testExactScoreInt () throws Exception {
Modified: lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/TestFunctionQuerySort.java Sun Aug 11 12:19:13 2013
@@ -18,9 +18,10 @@ package org.apache.lucene.queries.functi
*/
import java.io.IOException;
+
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
-import org.apache.lucene.document.StringField;
+import org.apache.lucene.document.IntField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
@@ -46,13 +47,13 @@ public class TestFunctionQuerySort exten
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, iwc);
Document doc = new Document();
- Field field = new StringField("value", "", Field.Store.YES);
+ Field field = new IntField("value", 0, Field.Store.YES);
doc.add(field);
// Save docs unsorted (decreasing value n, n-1, ...)
final int NUM_VALS = 5;
for (int val = NUM_VALS; val > 0; val--) {
- field.setStringValue(Integer.toString(val));
+ field.setIntValue(val);
writer.addDocument(doc);
}
Modified: lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java Sun Aug 11 12:19:13 2013
@@ -22,13 +22,16 @@ import java.util.List;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
+import org.apache.lucene.document.DoubleField;
import org.apache.lucene.document.Field;
+import org.apache.lucene.document.FloatField;
+import org.apache.lucene.document.IntField;
+import org.apache.lucene.document.LongField;
import org.apache.lucene.document.StringField;
import org.apache.lucene.document.TextField;
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.RandomIndexWriter;
-import org.apache.lucene.queries.function.valuesource.ByteFieldSource;
import org.apache.lucene.queries.function.valuesource.BytesRefFieldSource;
import org.apache.lucene.queries.function.valuesource.ConstValueSource;
import org.apache.lucene.queries.function.valuesource.DivFloatFunction;
@@ -54,7 +57,6 @@ import org.apache.lucene.queries.functio
import org.apache.lucene.queries.function.valuesource.RangeMapFloatFunction;
import org.apache.lucene.queries.function.valuesource.ReciprocalFloatFunction;
import org.apache.lucene.queries.function.valuesource.ScaleFloatFunction;
-import org.apache.lucene.queries.function.valuesource.ShortFieldSource;
import org.apache.lucene.queries.function.valuesource.SumFloatFunction;
import org.apache.lucene.queries.function.valuesource.SumTotalTermFreqValueSource;
import org.apache.lucene.queries.function.valuesource.TFValueSource;
@@ -85,9 +87,9 @@ public class TestValueSources extends Lu
static IndexSearcher searcher;
static final List<String[]> documents = Arrays.asList(new String[][] {
- /* id, byte, double, float, int, long, short, string, text */
- new String[] { "0", "5", "3.63", "5.2", "35", "4343", "945", "test", "this is a test test test" },
- new String[] { "1", "12", "5.65", "9.3", "54", "1954", "123", "bar", "second test" },
+ /* id, double, float, int, long, string, text */
+ new String[] { "0", "3.63", "5.2", "35", "4343", "test", "this is a test test test" },
+ new String[] { "1", "5.65", "9.3", "54", "1954", "bar", "second test" },
});
@BeforeClass
@@ -99,18 +101,14 @@ public class TestValueSources extends Lu
Document document = new Document();
Field idField = new StringField("id", "", Field.Store.NO);
document.add(idField);
- Field byteField = new StringField("byte", "", Field.Store.NO);
- document.add(byteField);
- Field doubleField = new StringField("double", "", Field.Store.NO);
+ Field doubleField = new DoubleField("double", 0d, Field.Store.NO);
document.add(doubleField);
- Field floatField = new StringField("float", "", Field.Store.NO);
+ Field floatField = new FloatField("float", 0f, Field.Store.NO);
document.add(floatField);
- Field intField = new StringField("int", "", Field.Store.NO);
+ Field intField = new IntField("int", 0, Field.Store.NO);
document.add(intField);
- Field longField = new StringField("long", "", Field.Store.NO);
+ Field longField = new LongField("long", 0L, Field.Store.NO);
document.add(longField);
- Field shortField = new StringField("short", "", Field.Store.NO);
- document.add(shortField);
Field stringField = new StringField("string", "", Field.Store.NO);
document.add(stringField);
Field textField = new TextField("text", "", Field.Store.NO);
@@ -118,14 +116,12 @@ public class TestValueSources extends Lu
for (String [] doc : documents) {
idField.setStringValue(doc[0]);
- byteField.setStringValue(doc[1]);
- doubleField.setStringValue(doc[2]);
- floatField.setStringValue(doc[3]);
- intField.setStringValue(doc[4]);
- longField.setStringValue(doc[5]);
- shortField.setStringValue(doc[6]);
- stringField.setStringValue(doc[7]);
- textField.setStringValue(doc[8]);
+ doubleField.setDoubleValue(Double.valueOf(doc[1]));
+ floatField.setFloatValue(Float.valueOf(doc[2]));
+ intField.setIntValue(Integer.valueOf(doc[3]));
+ longField.setLongValue(Long.valueOf(doc[4]));
+ stringField.setStringValue(doc[5]);
+ textField.setStringValue(doc[6]);
iw.addDocument(document);
}
@@ -143,11 +139,6 @@ public class TestValueSources extends Lu
dir = null;
}
- public void testByte() throws Exception {
- assertHits(new FunctionQuery(new ByteFieldSource("byte")),
- new float[] { 5f, 12f });
- }
-
public void testConst() throws Exception {
assertHits(new FunctionQuery(new ConstValueSource(0.3f)),
new float[] { 0.3f, 0.3f });
@@ -298,11 +289,6 @@ public class TestValueSources extends Lu
new float[] { 0.0f, 1.0f });
}
- public void testShort() throws Exception {
- assertHits(new FunctionQuery(new ShortFieldSource("short")),
- new float[] { 945f, 123f });
- }
-
public void testSumFloat() throws Exception {
assertHits(new FunctionQuery(new SumFloatFunction(new ValueSource[] {
new ConstValueSource(1f), new ConstValueSource(2f)})),
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/build.xml?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/build.xml (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/build.xml Sun Aug 11 12:19:13 2013
@@ -152,4 +152,6 @@ import org.apache.lucene.queryparser.fle
</sequential>
</macrodef>
+ <target name="regenerate" depends="javacc"/>
+
</project>
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/analyzing/AnalyzingQueryParser.java Sun Aug 11 12:19:13 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.queryparser.an
*/
import java.io.IOException;
-import java.io.StringReader;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -165,7 +164,7 @@ public class AnalyzingQueryParser extend
String analyzed = null;
TokenStream stream = null;
try{
- stream = getAnalyzer().tokenStream(field, new StringReader(chunk));
+ stream = getAnalyzer().tokenStream(field, chunk);
stream.reset();
CharTermAttribute termAtt = stream.getAttribute(CharTermAttribute.class);
// get first and hopefully only output token
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/CharStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/CharStream.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/CharStream.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/CharStream.java Sun Aug 11 12:19:13 2013
@@ -112,4 +112,4 @@ interface CharStream {
void Done();
}
-/* JavaCC - OriginalChecksum=c847dd1920bf7901125a7244125682ad (do not edit this line) */
+/* JavaCC - OriginalChecksum=30b94cad7b10d0d81e3a59a1083939d0 (do not edit this line) */
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/ParseException.java Sun Aug 11 12:19:13 2013
@@ -184,4 +184,4 @@ public class ParseException extends Exce
}
}
-/* JavaCC - OriginalChecksum=61602edcb3a15810cbc58f5593eba40d (do not edit this line) */
+/* JavaCC - OriginalChecksum=b187d97d5bb75c3fc63d642c1c26ac6e (do not edit this line) */
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java Sun Aug 11 12:19:13 2013
@@ -500,7 +500,7 @@ public abstract class QueryParserBase im
TokenStream source;
try {
- source = analyzer.tokenStream(field, new StringReader(queryText));
+ source = analyzer.tokenStream(field, queryText);
source.reset();
} catch (IOException e) {
ParseException p = new ParseException("Unable to initialize TokenStream to analyze query text");
@@ -844,7 +844,7 @@ public abstract class QueryParserBase im
if (analyzerIn == null) analyzerIn = analyzer;
try {
- source = analyzerIn.tokenStream(field, new StringReader(part));
+ source = analyzerIn.tokenStream(field, part);
source.reset();
} catch (IOException e) {
throw new RuntimeException("Unable to initialize TokenStream to analyze multiTerm term: " + part, e);
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/Token.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/Token.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/Token.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/Token.java Sun Aug 11 12:19:13 2013
@@ -128,4 +128,4 @@ public class Token implements java.io.Se
}
}
-/* JavaCC - OriginalChecksum=c1e1418b35aa9e47ef8dc98b87423d70 (do not edit this line) */
+/* JavaCC - OriginalChecksum=405bb5d2fcd84e94ac1c8f0b12c1f914 (do not edit this line) */
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/TokenMgrError.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/TokenMgrError.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/TokenMgrError.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/TokenMgrError.java Sun Aug 11 12:19:13 2013
@@ -144,4 +144,4 @@ public class TokenMgrError extends Error
this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
}
}
-/* JavaCC - OriginalChecksum=0c275864a1972d9a01601ab81426872d (do not edit this line) */
+/* JavaCC - OriginalChecksum=f433e1a52b8eadbf12f3fbbbf87fd140 (do not edit this line) */
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryTreeBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryTreeBuilder.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryTreeBuilder.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/core/builders/QueryTreeBuilder.java Sun Aug 11 12:19:13 2013
@@ -150,7 +150,7 @@ public class QueryTreeBuilder implements
builder = getQueryBuilder(clazz);
if (builder == null) {
- Class<?>[] classes = node.getClass().getInterfaces();
+ Class<?>[] classes = clazz.getInterfaces();
for (Class<?> actualClass : classes) {
builder = getQueryBuilder(actualClass);
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/QueryParserUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/QueryParserUtil.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/QueryParserUtil.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/QueryParserUtil.java Sun Aug 11 12:19:13 2013
@@ -195,7 +195,7 @@ final public class QueryParserUtil {
if (c == '\\' || c == '+' || c == '-' || c == '!' || c == '(' || c == ')'
|| c == ':' || c == '^' || c == '[' || c == ']' || c == '\"'
|| c == '{' || c == '}' || c == '~' || c == '*' || c == '?'
- || c == '|' || c == '&') {
+ || c == '|' || c == '&' || c == '/') {
sb.append('\\');
}
sb.append(c);
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/CharStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/CharStream.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/CharStream.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/CharStream.java Sun Aug 11 12:19:13 2013
@@ -112,4 +112,4 @@ interface CharStream {
void Done();
}
-/* JavaCC - OriginalChecksum=c95f1720d9b38046dc5d294b741c44cb (do not edit this line) */
+/* JavaCC - OriginalChecksum=53b2ec7502d50e2290e86187a6c01270 (do not edit this line) */
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/ParseException.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/ParseException.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/ParseException.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/ParseException.java Sun Aug 11 12:19:13 2013
@@ -187,4 +187,4 @@ public class ParseException extends Quer
}
}
-/* JavaCC - OriginalChecksum=81401c29cf6f9909761c636b4778ccc0 (do not edit this line) */
+/* JavaCC - OriginalChecksum=4263a02db9988d7a863aa97ad2f6dc67 (do not edit this line) */
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/Token.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/Token.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/Token.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/Token.java Sun Aug 11 12:19:13 2013
@@ -128,4 +128,4 @@ public class Token implements java.io.Se
}
}
-/* JavaCC - OriginalChecksum=30bbd23e0dec26f141130dc62a4f6e9d (do not edit this line) */
+/* JavaCC - OriginalChecksum=ea8b1e55950603be28e2f63dcd544ab4 (do not edit this line) */
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/TokenMgrError.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/TokenMgrError.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/TokenMgrError.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/parser/TokenMgrError.java Sun Aug 11 12:19:13 2013
@@ -144,4 +144,4 @@ public class TokenMgrError extends Error
this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
}
}
-/* JavaCC - OriginalChecksum=3ca7fbf7de9f2424b131a5499b0a78d0 (do not edit this line) */
+/* JavaCC - OriginalChecksum=be88283d82a985d82a34dda46bcf42d5 (do not edit this line) */
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/flexible/standard/processors/AnalyzerQueryNodeProcessor.java Sun Aug 11 12:19:13 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.queryparser.fl
*/
import java.io.IOException;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.LinkedList;
import java.util.List;
@@ -116,7 +115,7 @@ public class AnalyzerQueryNodeProcessor
TokenStream source;
try {
- source = this.analyzer.tokenStream(field, new StringReader(text));
+ source = this.analyzer.tokenStream(field, text);
source.reset();
} catch (IOException e1) {
throw new RuntimeException(e1);
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/CharStream.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/CharStream.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/CharStream.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/CharStream.java Sun Aug 11 12:19:13 2013
@@ -112,4 +112,4 @@ interface CharStream {
void Done();
}
-/* JavaCC - OriginalChecksum=5ca20c9145f29a0f8909470a7f949fe4 (do not edit this line) */
+/* JavaCC - OriginalChecksum=242ae59b965491e225a44534cbc73b42 (do not edit this line) */
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/ParseException.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/ParseException.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/ParseException.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/ParseException.java Sun Aug 11 12:19:13 2013
@@ -184,4 +184,4 @@ public class ParseException extends Exce
}
}
-/* JavaCC - OriginalChecksum=be6f55e3bf157e8c96b4c06cca5ec81b (do not edit this line) */
+/* JavaCC - OriginalChecksum=bd8163f41bf2fd1bb00f025fce3dcaaf (do not edit this line) */
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/Token.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/Token.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/Token.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/Token.java Sun Aug 11 12:19:13 2013
@@ -128,4 +128,4 @@ public class Token implements java.io.Se
}
}
-/* JavaCC - OriginalChecksum=db38f23b3674db52ff034369707a0ac3 (do not edit this line) */
+/* JavaCC - OriginalChecksum=f2df701e24da1cf2d025118ce6efdd2f (do not edit this line) */
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/TokenMgrError.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/TokenMgrError.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/TokenMgrError.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/surround/parser/TokenMgrError.java Sun Aug 11 12:19:13 2013
@@ -144,4 +144,4 @@ public class TokenMgrError extends Error
this(LexicalError(EOFSeen, lexState, errorLine, errorColumn, errorAfter, curChar), reason);
}
}
-/* JavaCC - OriginalChecksum=dcdd5ccde13b91bcd8f76a86ca618852 (do not edit this line) */
+/* JavaCC - OriginalChecksum=8c69a370d9a9893140562c8bb911678c (do not edit this line) */
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LikeThisQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LikeThisQueryBuilder.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LikeThisQueryBuilder.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/LikeThisQueryBuilder.java Sun Aug 11 12:19:13 2013
@@ -4,7 +4,6 @@
package org.apache.lucene.queryparser.xml.builders;
import java.io.IOException;
-import java.io.StringReader;
import java.util.HashSet;
import java.util.Set;
@@ -75,7 +74,7 @@ public class LikeThisQueryBuilder implem
stopWordsSet = new HashSet<String>();
for (String field : fields) {
try {
- TokenStream ts = analyzer.tokenStream(field, new StringReader(stopWords));
+ TokenStream ts = analyzer.tokenStream(field, stopWords);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
ts.reset();
while (ts.incrementToken()) {
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/SpanOrTermsBuilder.java Sun Aug 11 12:19:13 2013
@@ -13,9 +13,9 @@ import org.apache.lucene.queryparser.xml
import org.w3c.dom.Element;
import java.io.IOException;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
+
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
@@ -51,7 +51,7 @@ public class SpanOrTermsBuilder extends
try {
List<SpanQuery> clausesList = new ArrayList<SpanQuery>();
- TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(value));
+ TokenStream ts = analyzer.tokenStream(fieldName, value);
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
BytesRef bytes = termAtt.getBytesRef();
ts.reset();
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsFilterBuilder.java Sun Aug 11 12:19:13 2013
@@ -3,7 +3,6 @@ package org.apache.lucene.queryparser.xm
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermToBytesRefAttribute;
-import org.apache.lucene.index.Term;
import org.apache.lucene.search.Filter;
import org.apache.lucene.queries.TermsFilter;
import org.apache.lucene.util.BytesRef;
@@ -13,7 +12,6 @@ import org.apache.lucene.queryparser.xml
import org.w3c.dom.Element;
import java.io.IOException;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;
@@ -57,9 +55,8 @@ public class TermsFilterBuilder implemen
String fieldName = DOMUtils.getAttributeWithInheritanceOrFail(e, "fieldName");
try {
- TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text));
+ TokenStream ts = analyzer.tokenStream(fieldName, text);
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
- Term term = null;
BytesRef bytes = termAtt.getBytesRef();
ts.reset();
while (ts.incrementToken()) {
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/TermsQueryBuilder.java Sun Aug 11 12:19:13 2013
@@ -15,7 +15,6 @@ import org.apache.lucene.queryparser.xml
import org.w3c.dom.Element;
import java.io.IOException;
-import java.io.StringReader;
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
@@ -53,7 +52,7 @@ public class TermsQueryBuilder implement
BooleanQuery bq = new BooleanQuery(DOMUtils.getAttribute(e, "disableCoord", false));
bq.setMinimumNumberShouldMatch(DOMUtils.getAttribute(e, "minimumNumberShouldMatch", 0));
try {
- TokenStream ts = analyzer.tokenStream(fieldName, new StringReader(text));
+ TokenStream ts = analyzer.tokenStream(fieldName, text);
TermToBytesRefAttribute termAtt = ts.addAttribute(TermToBytesRefAttribute.class);
Term term = null;
BytesRef bytes = termAtt.getBytesRef();
Modified: lucene/dev/branches/lucene4956/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/builders/TestQueryTreeBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/builders/TestQueryTreeBuilder.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/builders/TestQueryTreeBuilder.java (original)
+++ lucene/dev/branches/lucene4956/lucene/queryparser/src/test/org/apache/lucene/queryparser/flexible/core/builders/TestQueryTreeBuilder.java Sun Aug 11 12:19:13 2013
@@ -22,6 +22,8 @@ import junit.framework.Assert;
import org.apache.lucene.queryparser.flexible.core.QueryNodeException;
import org.apache.lucene.queryparser.flexible.core.nodes.FieldQueryNode;
import org.apache.lucene.queryparser.flexible.core.nodes.QueryNode;
+import org.apache.lucene.queryparser.flexible.core.nodes.QueryNodeImpl;
+import org.apache.lucene.queryparser.flexible.core.parser.EscapeQuerySyntax;
import org.apache.lucene.queryparser.flexible.core.util.UnescapedCharSequence;
import org.apache.lucene.util.LuceneTestCase;
import org.junit.Test;
@@ -35,6 +37,28 @@ public class TestQueryTreeBuilder extend
Object result = qtb.build(new FieldQueryNode(new UnescapedCharSequence("field"), "foo", 0, 0));
Assert.assertEquals("OK", result);
+ // LUCENE-4890
+ qtb = new QueryTreeBuilder();
+ qtb.setBuilder(DummyQueryNodeInterface.class, new DummyBuilder());
+ result = qtb.build(new DummyQueryNode());
+ Assert.assertEquals("OK", result);
+ }
+
+ private static interface DummyQueryNodeInterface extends QueryNode {
+
+ }
+
+ private static abstract class AbstractDummyQueryNode extends QueryNodeImpl implements DummyQueryNodeInterface {
+
+ }
+
+ private static class DummyQueryNode extends AbstractDummyQueryNode {
+
+ @Override
+ public CharSequence toQueryString(EscapeQuerySyntax escapeSyntaxParser) {
+ return "DummyQueryNode";
+ }
+
}
private static class DummyBuilder implements QueryBuilder {
Modified: lucene/dev/branches/lucene4956/lucene/replicator/ivy.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/replicator/ivy.xml?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/replicator/ivy.xml (original)
+++ lucene/dev/branches/lucene4956/lucene/replicator/ivy.xml Sun Aug 11 12:19:13 2013
@@ -39,8 +39,7 @@
<dependency org="org.eclipse.jetty" name="jetty-io" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
<dependency org="org.eclipse.jetty" name="jetty-continuation" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
<dependency org="org.eclipse.jetty" name="jetty-http" rev="&jetty.version;" transitive="false" conf="jetty->default"/>
- <dependency org="org.slf4j" name="slf4j-api" rev="1.6.6" transitive="false" conf="logging->default"/>
- <dependency org="org.slf4j" name="jcl-over-slf4j" rev="1.6.6" transitive="false" conf="logging->default"/>
+ <dependency org="commons-logging" name="commons-logging" rev="1.1.1" transitive="false" conf="logging->default"/>
<dependency org="org.eclipse.jetty.orbit" name="javax.servlet" rev="3.0.0.v201112011016" transitive="false" conf="servlet->default">
<artifact name="javax.servlet" type="orbit" ext="jar"/>
</dependency>
Modified: lucene/dev/branches/lucene4956/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java (original)
+++ lucene/dev/branches/lucene4956/lucene/replicator/src/java/org/apache/lucene/replicator/IndexAndTaxonomyRevision.java Sun Aug 11 12:19:13 2013
@@ -78,15 +78,14 @@ public class IndexAndTaxonomyRevision im
@Override
protected IndexWriterConfig createIndexWriterConfig(OpenMode openMode) {
IndexWriterConfig conf = super.createIndexWriterConfig(openMode);
- conf.setIndexDeletionPolicy(new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy()));
+ sdp = new SnapshotDeletionPolicy(conf.getIndexDeletionPolicy());
+ conf.setIndexDeletionPolicy(sdp);
return conf;
}
@Override
protected IndexWriter openIndexWriter(Directory directory, IndexWriterConfig config) throws IOException {
writer = super.openIndexWriter(directory, config);
- // must set it here because IndexWriter clones the config
- sdp = (SnapshotDeletionPolicy) writer.getConfig().getIndexDeletionPolicy();
return writer;
}
Modified: lucene/dev/branches/lucene4956/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/replicator/src/test/org/apache/lucene/replicator/LocalReplicatorTest.java Sun Aug 11 12:19:13 2013
@@ -19,6 +19,7 @@ package org.apache.lucene.replicator;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.nio.file.NoSuchFileException;
import java.util.HashMap;
import java.util.List;
import java.util.Map.Entry;
@@ -154,7 +155,7 @@ public class LocalReplicatorTest extends
try {
replicator.obtainFile(res.id, res.sourceFiles.keySet().iterator().next(), "madeUpFile");
fail("should have failed obtaining an unrecognized file");
- } catch (FileNotFoundException e) {
+ } catch (FileNotFoundException | NoSuchFileException e) {
// expected
}
}
Modified: lucene/dev/branches/lucene4956/lucene/replicator/src/test/org/apache/lucene/replicator/ReplicatorTestCase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/replicator/src/test/org/apache/lucene/replicator/ReplicatorTestCase.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/replicator/src/test/org/apache/lucene/replicator/ReplicatorTestCase.java (original)
+++ lucene/dev/branches/lucene4956/lucene/replicator/src/test/org/apache/lucene/replicator/ReplicatorTestCase.java Sun Aug 11 12:19:13 2013
@@ -17,7 +17,6 @@ package org.apache.lucene.replicator;
* limitations under the License.
*/
-import java.net.SocketException;
import java.util.Random;
import org.apache.http.conn.ClientConnectionManager;
@@ -125,12 +124,14 @@ public abstract class ReplicatorTestCase
return server;
}
- /**
- * Returns a {@link Server}'s port. This method assumes that no
- * {@link Connector}s were added to the Server besides the default one.
- */
- public static int serverPort(Server httpServer) {
- return httpServer.getConnectors()[0].getLocalPort();
+ /** Returns a {@link Server}'s port. */
+ public static int serverPort(Server server) {
+ return server.getConnectors()[0].getLocalPort();
+ }
+
+ /** Returns a {@link Server}'s host. */
+ public static String serverHost(Server server) {
+ return server.getConnectors()[0].getHost();
}
/**
Modified: lucene/dev/branches/lucene4956/lucene/replicator/src/test/org/apache/lucene/replicator/http/HttpReplicatorTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/replicator/src/test/org/apache/lucene/replicator/http/HttpReplicatorTest.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/replicator/src/test/org/apache/lucene/replicator/http/HttpReplicatorTest.java (original)
+++ lucene/dev/branches/lucene4956/lucene/replicator/src/test/org/apache/lucene/replicator/http/HttpReplicatorTest.java Sun Aug 11 12:19:13 2013
@@ -50,6 +50,7 @@ public class HttpReplicatorTest extends
private DirectoryReader reader;
private Server server;
private int port;
+ private String host;
private Directory serverIndexDir, handlerIndexDir;
private void startServer() throws Exception {
@@ -59,12 +60,14 @@ public class HttpReplicatorTest extends
replicationHandler.addServletWithMapping(servlet, ReplicationService.REPLICATION_CONTEXT + "/*");
server = newHttpServer(replicationHandler);
port = serverPort(server);
+ host = serverHost(server);
}
@Before
@Override
public void setUp() throws Exception {
super.setUp();
+ System.setProperty("org.eclipse.jetty.LEVEL", "DEBUG"); // sets stderr logging to DEBUG level
clientWorkDir = _TestUtil.getTempDir("httpReplicatorTest");
handlerIndexDir = newDirectory();
serverIndexDir = newDirectory();
@@ -81,6 +84,7 @@ public class HttpReplicatorTest extends
public void tearDown() throws Exception {
stopHttpServer(server);
IOUtils.close(reader, writer, handlerIndexDir, serverIndexDir);
+ System.clearProperty("org.eclipse.jetty.LEVEL");
super.tearDown();
}
@@ -101,7 +105,7 @@ public class HttpReplicatorTest extends
@Test
public void testBasic() throws Exception {
- Replicator replicator = new HttpReplicator("localhost", port, ReplicationService.REPLICATION_CONTEXT + "/s1",
+ Replicator replicator = new HttpReplicator(host, port, ReplicationService.REPLICATION_CONTEXT + "/s1",
getClientConnectionManager());
ReplicationClient client = new ReplicationClient(replicator, new IndexReplicationHandler(handlerIndexDir, null),
new PerSessionDirectoryFactory(clientWorkDir));
Modified: lucene/dev/branches/lucene4956/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java (original)
+++ lucene/dev/branches/lucene4956/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/FuzzyLikeThisQuery.java Sun Aug 11 12:19:13 2013
@@ -18,7 +18,6 @@ package org.apache.lucene.sandbox.querie
*/
import java.io.IOException;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
@@ -194,7 +193,7 @@ public class FuzzyLikeThisQuery extends
private void addTerms(IndexReader reader, FieldVals f) throws IOException {
if (f.queryString == null) return;
- TokenStream ts = analyzer.tokenStream(f.fieldName, new StringReader(f.queryString));
+ TokenStream ts = analyzer.tokenStream(f.fieldName, f.queryString);
CharTermAttribute termAtt = ts.addAttribute(CharTermAttribute.class);
int corpusNumDocs = reader.numDocs();
Modified: lucene/dev/branches/lucene4956/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java (original)
+++ lucene/dev/branches/lucene4956/lucene/sandbox/src/java/org/apache/lucene/sandbox/queries/SlowFuzzyTermsEnum.java Sun Aug 11 12:19:13 2013
@@ -31,9 +31,12 @@ import org.apache.lucene.util.IntsRef;
import org.apache.lucene.util.StringHelper;
import org.apache.lucene.util.UnicodeUtil;
-/** Classic fuzzy TermsEnum for enumerating all terms that are similar
+/** Potentially slow fuzzy TermsEnum for enumerating all terms that are similar
* to the specified filter term.
- *
+ * <p> If the minSimilarity or maxEdits is greater than the Automaton's
+ * allowable range, this backs off to the classic (brute force)
+ * fuzzy terms enum method by calling FuzzyTermsEnum's getAutomatonEnum.
+ * </p>
* <p>Term enumerations are always ordered by
* {@link #getComparator}. Each term in the enumeration is
* greater than all that precede it.</p>
@@ -103,18 +106,43 @@ public final class SlowFuzzyTermsEnum ex
private final IntsRef utf32 = new IntsRef(20);
/**
- * The termCompare method in FuzzyTermEnum uses Levenshtein distance to
+ * <p>The termCompare method in FuzzyTermEnum uses Levenshtein distance to
* calculate the distance between the given term and the comparing term.
+ * </p>
+ * <p>If the minSimilarity is >= 1.0, this uses the maxEdits as the comparison.
+ * Otherwise, this method uses the following logic to calculate similarity.
+ * <pre>
+ * similarity = 1 - ((float)distance / (float) (prefixLength + Math.min(textlen, targetlen)));
+ * </pre>
+ * where distance is the Levenshtein distance for the two words.
+ * </p>
+ *
*/
@Override
protected final AcceptStatus accept(BytesRef term) {
if (StringHelper.startsWith(term, prefixBytesRef)) {
UnicodeUtil.UTF8toUTF32(term, utf32);
- final float similarity = similarity(utf32.ints, realPrefixLength, utf32.length - realPrefixLength);
- if (similarity > minSimilarity) {
+ final int distance = calcDistance(utf32.ints, realPrefixLength, utf32.length - realPrefixLength);
+
+ //Integer.MIN_VALUE is the sentinel that Levenshtein stopped early
+ if (distance == Integer.MIN_VALUE){
+ return AcceptStatus.NO;
+ }
+ //no need to calc similarity, if raw is true and distance > maxEdits
+ if (raw == true && distance > maxEdits){
+ return AcceptStatus.NO;
+ }
+ final float similarity = calcSimilarity(distance, (utf32.length - realPrefixLength), text.length);
+
+ //if raw is true, then distance must also be <= maxEdits by now
+ //given the previous if statement
+ if (raw == true ||
+ (raw == false && similarity > minSimilarity)) {
boostAtt.setBoost((similarity - minSimilarity) * scale_factor);
return AcceptStatus.YES;
- } else return AcceptStatus.NO;
+ } else {
+ return AcceptStatus.NO;
+ }
} else {
return AcceptStatus.END;
}
@@ -125,52 +153,34 @@ public final class SlowFuzzyTermsEnum ex
******************************/
/**
- * <p>Similarity returns a number that is 1.0f or less (including negative numbers)
- * based on how similar the Term is compared to a target term. It returns
- * exactly 0.0f when
- * <pre>
- * editDistance > maximumEditDistance</pre>
- * Otherwise it returns:
- * <pre>
- * 1 - (editDistance / length)</pre>
- * where length is the length of the shortest term (text or target) including a
- * prefix that are identical and editDistance is the Levenshtein distance for
- * the two words.</p>
- *
+ * <p>calcDistance returns the Levenshtein distance between the query term
+ * and the target term.</p>
+ *
* <p>Embedded within this algorithm is a fail-fast Levenshtein distance
* algorithm. The fail-fast algorithm differs from the standard Levenshtein
* distance algorithm in that it is aborted if it is discovered that the
* minimum distance between the words is greater than some threshold.
- *
- * <p>To calculate the maximum distance threshold we use the following formula:
- * <pre>
- * (1 - minimumSimilarity) * length</pre>
- * where length is the shortest term including any prefix that is not part of the
- * similarity comparison. This formula was derived by solving for what maximum value
- * of distance returns false for the following statements:
- * <pre>
- * similarity = 1 - ((float)distance / (float) (prefixLength + Math.min(textlen, targetlen)));
- * return (similarity > minimumSimilarity);</pre>
- * where distance is the Levenshtein distance for the two words.
- * </p>
+
* <p>Levenshtein distance (also known as edit distance) is a measure of similarity
* between two strings where the distance is measured as the number of character
* deletions, insertions or substitutions required to transform one string to
* the other string.
* @param target the target word or phrase
- * @return the similarity, 0.0 or less indicates that it matches less than the required
- * threshold and 1.0 indicates that the text and target are identical
+ * @param offset the offset at which to start the comparison
+ * @param length the length of what's left of the string to compare
+ * @return the number of edits or Integer.MIN_VALUE if the edit distance is
+ * greater than maxDistance.
*/
- private final float similarity(final int[] target, int offset, int length) {
+ private final int calcDistance(final int[] target, int offset, int length) {
final int m = length;
final int n = text.length;
if (n == 0) {
//we don't have anything to compare. That means if we just add
//the letters for m we get the new word
- return realPrefixLength == 0 ? 0.0f : 1.0f - ((float) m / realPrefixLength);
+ return m;
}
if (m == 0) {
- return realPrefixLength == 0 ? 0.0f : 1.0f - ((float) n / realPrefixLength);
+ return n;
}
final int maxDistance = calculateMaxDistance(m);
@@ -183,7 +193,7 @@ public final class SlowFuzzyTermsEnum ex
//which is 8-3 or more precisely Math.abs(3-8).
//if our maximum edit distance is 4, then we can discard this word
//without looking at it.
- return Float.NEGATIVE_INFINITY;
+ return Integer.MIN_VALUE;
}
// init matrix d
@@ -214,7 +224,7 @@ public final class SlowFuzzyTermsEnum ex
if (j > maxDistance && bestPossibleEditDistance > maxDistance) { //equal is okay, but not greater
//the closest the target can be to the text is just too far away.
//this target is leaving the party early.
- return Float.NEGATIVE_INFINITY;
+ return Integer.MIN_VALUE;
}
// copy current distance counts to 'previous row' distance counts: swap p and d
@@ -226,12 +236,17 @@ public final class SlowFuzzyTermsEnum ex
// our last action in the above loop was to switch d and p, so p now
// actually has the most recent cost counts
+ return p[n];
+ }
+
+ private float calcSimilarity(int edits, int m, int n){
// this will return less than 0.0 when the edit distance is
// greater than the number of characters in the shorter word.
// but this was the formula that was previously used in FuzzyTermEnum,
// so it has not been changed (even though minimumSimilarity must be
// greater than 0.0)
- return 1.0f - ((float)p[n] / (float) (realPrefixLength + Math.min(n, m)));
+
+ return 1.0f - ((float)edits / (float) (realPrefixLength + Math.min(n, m)));
}
/**
Modified: lucene/dev/branches/lucene4956/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java (original)
+++ lucene/dev/branches/lucene4956/lucene/sandbox/src/test/org/apache/lucene/sandbox/queries/TestSlowFuzzyQuery.java Sun Aug 11 12:19:13 2013
@@ -43,6 +43,9 @@ import org.apache.lucene.util.LuceneTest
public class TestSlowFuzzyQuery extends LuceneTestCase {
public void testFuzziness() throws Exception {
+ //every test with SlowFuzzyQuery.defaultMinSimilarity
+ //is exercising the Automaton, not the brute force linear method
+
Directory directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
addDoc("aaaaa", writer);
@@ -194,6 +197,30 @@ public class TestSlowFuzzyQuery extends
directory.close();
}
+ public void testFuzzinessLong2() throws Exception {
+ //Lucene-5033
+ Directory directory = newDirectory();
+ RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
+ addDoc("abcdef", writer);
+ addDoc("segment", writer);
+
+ IndexReader reader = writer.getReader();
+ IndexSearcher searcher = newSearcher(reader);
+ writer.close();
+
+ SlowFuzzyQuery query;
+
+ query = new SlowFuzzyQuery(new Term("field", "abcxxxx"), 3f, 0);
+ ScoreDoc[] hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(0, hits.length);
+
+ query = new SlowFuzzyQuery(new Term("field", "abcxxxx"), 4f, 0);
+ hits = searcher.search(query, null, 1000).scoreDocs;
+ assertEquals(1, hits.length);
+ reader.close();
+ directory.close();
+ }
+
public void testFuzzinessLong() throws Exception {
Directory directory = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), directory);
@@ -385,7 +412,6 @@ public class TestSlowFuzzyQuery extends
public void testGiga() throws Exception {
- MockAnalyzer analyzer = new MockAnalyzer(random());
Directory index = newDirectory();
RandomIndexWriter w = new RandomIndexWriter(random(), index);
@@ -440,25 +466,21 @@ public class TestSlowFuzzyQuery extends
assertEquals(1, hits.length);
assertEquals("foobar", searcher.doc(hits[0].doc).get("field"));
- // TODO: cannot really be supported given the legacy scoring
- // system which scores negative, if the distance > min term len,
- // so such matches were always impossible with lucene 3.x, etc
- //
- //q = new SlowFuzzyQuery(new Term("field", "t"), 3);
- //hits = searcher.search(q, 10).scoreDocs;
- //assertEquals(1, hits.length);
- //assertEquals("test", searcher.doc(hits[0].doc).get("field"));
-
- // q = new SlowFuzzyQuery(new Term("field", "a"), 4f, 0, 50);
- // hits = searcher.search(q, 10).scoreDocs;
- // assertEquals(1, hits.length);
- // assertEquals("test", searcher.doc(hits[0].doc).get("field"));
-
- // q = new SlowFuzzyQuery(new Term("field", "a"), 6f, 0, 50);
- // hits = searcher.search(q, 10).scoreDocs;
- // assertEquals(2, hits.length);
- // assertEquals("test", searcher.doc(hits[0].doc).get("field"));
- // assertEquals("foobar", searcher.doc(hits[1].doc).get("field"));
+ q = new SlowFuzzyQuery(new Term("field", "t"), 3);
+ hits = searcher.search(q, 10).scoreDocs;
+ assertEquals(1, hits.length);
+ assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+
+ q = new SlowFuzzyQuery(new Term("field", "a"), 4f, 0, 50);
+ hits = searcher.search(q, 10).scoreDocs;
+ assertEquals(1, hits.length);
+ assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+
+ q = new SlowFuzzyQuery(new Term("field", "a"), 6f, 0, 50);
+ hits = searcher.search(q, 10).scoreDocs;
+ assertEquals(2, hits.length);
+ assertEquals("test", searcher.doc(hits[0].doc).get("field"));
+ assertEquals("foobar", searcher.doc(hits[1].doc).get("field"));
reader.close();
index.close();
Modified: lucene/dev/branches/lucene4956/lucene/site/xsl/index.xsl
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/site/xsl/index.xsl?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/site/xsl/index.xsl (original)
+++ lucene/dev/branches/lucene4956/lucene/site/xsl/index.xsl Sun Aug 11 12:19:13 2013
@@ -36,7 +36,15 @@
</head>
<body>
<div><a href="http://lucene.apache.org/core/"><img src="lucene_green_300.gif" title="Apache Lucene Logo" alt="Lucene" border="0"/></a></div>
- <h1><xsl:text>Apache Lucene </xsl:text><xsl:value-of select="$version"/><xsl:text> Documentation</xsl:text></h1>
+ <h1>
+ <xsl:text>Apache Lucene</xsl:text>
+ <span style="vertical-align: top; font-size: x-small">
+ <xsl:text>TM</xsl:text>
+ </span>
+ <xsl:text> </xsl:text>
+ <xsl:value-of select="$version"/>
+ <xsl:text> Documentation</xsl:text>
+ </h1>
<p>Lucene is a Java full-text search engine. Lucene is not a complete application,
but rather a code library and API that can easily be used to add search capabilities
to applications.</p>
@@ -71,7 +79,6 @@
<li><a href="core/org/apache/lucene/search/package-summary.html#package_description">Search and Scoring in Lucene</a>: Introduction to how Lucene scores documents.</li>
<li><a href="core/org/apache/lucene/search/similarities/TFIDFSimilarity.html">Classic Scoring Formula</a>: Formula of Lucene's classic <a href="http://en.wikipedia.org/wiki/Vector_Space_Model">Vector Space</a> implementation. (look <a href="core/org/apache/lucene/search/similarities/package-summary.html#package_description">here</a> for other models)</li>
<li><a href="queryparser/org/apache/lucene/queryparser/classic/package-summary.html#package_description">Classic QueryParser Syntax</a>: Overview of the Classic QueryParser's syntax and features.</li>
- <li><a href="facet/org/apache/lucene/facet/doc-files/userguide.html">Facet User Guide</a>: User's Guide to implementing <a href="http://en.wikipedia.org/wiki/Faceted_search">Faceted search</a>.</li>
</ul>
<h2>API Javadocs</h2>
<xsl:call-template name="modules"/>
Modified: lucene/dev/branches/lucene4956/lucene/spatial/build.xml
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/spatial/build.xml?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/spatial/build.xml (original)
+++ lucene/dev/branches/lucene4956/lucene/spatial/build.xml Sun Aug 11 12:19:13 2013
@@ -1,4 +1,22 @@
<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
<project name="spatial" default="default">
<description>
Geospatial search
Modified: lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java (original)
+++ lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/SpatialStrategy.java Sun Aug 11 12:19:13 2013
@@ -102,11 +102,20 @@ public abstract class SpatialStrategy {
public abstract Field[] createIndexableFields(Shape shape);
/**
+ * See {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point, double)} called with
+ * a multiplier of 1.0 (i.e. units of degrees).
+ */
+ public ValueSource makeDistanceValueSource(Point queryPoint) {
+ return makeDistanceValueSource(queryPoint, 1.0);
+ }
+
+ /**
* Make a ValueSource returning the distance between the center of the
* indexed shape and {@code queryPoint}. If there are multiple indexed shapes
- * then the closest one is chosen.
+ * then the closest one is chosen. The result is multiplied by {@code multiplier}, which
+ * conveniently is used to get the desired units.
*/
- public abstract ValueSource makeDistanceValueSource(Point queryPoint);
+ public abstract ValueSource makeDistanceValueSource(Point queryPoint, double multiplier);
/**
* Make a Query based principally on {@link org.apache.lucene.spatial.query.SpatialOperation}
@@ -139,7 +148,7 @@ public abstract class SpatialStrategy {
/**
* Returns a ValueSource with values ranging from 1 to 0, depending inversely
- * on the distance from {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point)}.
+ * on the distance from {@link #makeDistanceValueSource(com.spatial4j.core.shape.Point,double)}.
* The formula is {@code c/(d + c)} where 'd' is the distance and 'c' is
* one tenth the distance to the farthest edge from the center. Thus the
* scores will be 1 for indexed points at the center of the query shape and as
@@ -151,7 +160,7 @@ public abstract class SpatialStrategy {
ctx.makePoint(bbox.getMinX(), bbox.getMinY()), bbox.getMaxX(), bbox.getMaxY());
double distToEdge = diagonalDist * 0.5;
float c = (float)distToEdge * 0.1f;//one tenth
- return new ReciprocalFloatFunction(makeDistanceValueSource(queryShape.getCenter()), 1f, c, c);
+ return new ReciprocalFloatFunction(makeDistanceValueSource(queryShape.getCenter(), 1.0), 1f, c, c);
}
@Override
Modified: lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java (original)
+++ lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/BBoxStrategy.java Sun Aug 11 12:19:13 2013
@@ -135,9 +135,9 @@ public class BBoxStrategy extends Spatia
//---------------------------------
@Override
- public ValueSource makeDistanceValueSource(Point queryPoint) {
+ public ValueSource makeDistanceValueSource(Point queryPoint, double multiplier) {
return new BBoxSimilarityValueSource(
- this, new DistanceSimilarity(this.getSpatialContext(), queryPoint));
+ this, new DistanceSimilarity(this.getSpatialContext(), queryPoint, multiplier));
}
public ValueSource makeBBoxAreaSimilarityValueSource(Rectangle queryBox) {
Modified: lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/DistanceSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/DistanceSimilarity.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/DistanceSimilarity.java (original)
+++ lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/bbox/DistanceSimilarity.java Sun Aug 11 12:19:13 2013
@@ -30,13 +30,15 @@ import org.apache.lucene.search.Explanat
*/
public class DistanceSimilarity implements BBoxSimilarity {
private final Point queryPoint;
+ private final double multiplier;
private final DistanceCalculator distCalc;
private final double nullValue;
- public DistanceSimilarity(SpatialContext ctx, Point queryPoint) {
+ public DistanceSimilarity(SpatialContext ctx, Point queryPoint, double multiplier) {
this.queryPoint = queryPoint;
+ this.multiplier = multiplier;
this.distCalc = ctx.getDistCalc();
- this.nullValue = (ctx.isGeo() ? 180 : Double.MAX_VALUE);
+ this.nullValue = (ctx.isGeo() ? 180 * multiplier : Double.MAX_VALUE);
}
@Override
@@ -45,14 +47,43 @@ public class DistanceSimilarity implemen
if (indexRect == null) {
score = nullValue;
} else {
- score = distCalc.distance(queryPoint, indexRect.getCenter());
+ score = distCalc.distance(queryPoint, indexRect.getCenter()) * multiplier;
}
if (exp != null) {
exp.setValue((float)score);
exp.setDescription(this.getClass().getSimpleName());
- exp.addDetail(new Explanation(-1f,""+queryPoint));
+ exp.addDetail(new Explanation(-1f, "" + queryPoint));
exp.addDetail(new Explanation(-1f,""+indexRect));
+ exp.addDetail(new Explanation((float)multiplier,"multiplier"));
}
return score;
}
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ DistanceSimilarity that = (DistanceSimilarity) o;
+
+ if (Double.compare(that.multiplier, multiplier) != 0) return false;
+ if (Double.compare(that.nullValue, nullValue) != 0) return false;
+ if (!distCalc.equals(that.distCalc)) return false;
+ if (!queryPoint.equals(that.queryPoint)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result;
+ long temp;
+ result = queryPoint.hashCode();
+ temp = Double.doubleToLongBits(multiplier);
+ result = 31 * result + (int) (temp ^ (temp >>> 32));
+ result = 31 * result + distCalc.hashCode();
+ temp = Double.doubleToLongBits(nullValue);
+ result = 31 * result + (int) (temp ^ (temp >>> 32));
+ return result;
+ }
}
Modified: lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java?rev=1512909&r1=1512908&r2=1512909&view=diff
==============================================================================
--- lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java (original)
+++ lucene/dev/branches/lucene4956/lucene/spatial/src/java/org/apache/lucene/spatial/prefix/AbstractVisitingPrefixTreeFilter.java Sun Aug 11 12:19:13 2013
@@ -107,6 +107,14 @@ public abstract class AbstractVisitingPr
this depth. It would be nice if termsEnum knew how many terms
start with the current term without having to repeatedly next() & test to find out.
+ * Perhaps don't do intermediate seek()'s to cells above detailLevel that have Intersects
+ relation because we won't be collecting those docs any way. However seeking
+ does act as a short-circuit. So maybe do some percent of the time or when the level
+ is above some threshold.
+
+ * Each shape.relate(otherShape) result could be cached since much of the same relations
+ will be invoked when multiple segments are involved.
+
*/
protected final boolean hasIndexedLeaves;//if false then we can skip looking for them
@@ -171,11 +179,11 @@ public abstract class AbstractVisitingPr
int compare = termsEnum.getComparator().compare(thisTerm, curVNodeTerm);
if (compare > 0) {
// leap frog (termsEnum is beyond where we would otherwise seek)
- assert ! context.reader().terms(fieldName).iterator(null).seekExact(curVNodeTerm, false) : "should be absent";
+ assert ! context.reader().terms(fieldName).iterator(null).seekExact(curVNodeTerm) : "should be absent";
} else {
if (compare < 0) {
// Seek !
- TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(curVNodeTerm, true);
+ TermsEnum.SeekStatus seekStatus = termsEnum.seekCeil(curVNodeTerm);
if (seekStatus == TermsEnum.SeekStatus.END)
break; // all done
thisTerm = termsEnum.term();
@@ -339,7 +347,7 @@ public abstract class AbstractVisitingPr
}//class VisitorTemplate
/**
- * A Visitor Cell/Cell found via the query shape for {@link VisitorTemplate}.
+ * A visitor node/cell found via the query shape for {@link VisitorTemplate}.
* Sometimes these are reset(cell). It's like a LinkedList node but forms a
* tree.
*