You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mi...@apache.org on 2015/09/18 17:13:52 UTC
svn commit: r1703881 [2/5] - in /lucene/dev/branches/lucene6780: ./ lucene/
lucene/analysis/ lucene/analysis/common/
lucene/analysis/common/src/java/org/apache/lucene/analysis/hunspell/
lucene/benchmark/ lucene/benchmark/src/test/org/apache/lucene/benc...
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestConstantScoreQuery.java Fri Sep 18 15:13:49 2015
@@ -31,7 +31,7 @@ import org.apache.lucene.index.MultiRead
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
@@ -103,7 +103,7 @@ public class TestConstantScoreQuery exte
searcher.setQueryCache(null); // to assert on scorer impl
// set a similarity that does not normalize our boost away
- searcher.setSimilarity(new DefaultSimilarity() {
+ searcher.setSimilarity(new ClassicSimilarity() {
@Override
public float queryNorm(float sumOfSquaredWeights) {
return 1.0f;
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestDisjunctionMaxQuery.java Fri Sep 18 15:13:49 2015
@@ -38,7 +38,7 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
@@ -63,7 +63,7 @@ public class TestDisjunctionMaxQuery ext
* http://issues.apache.org/jira/browse/LUCENE-323
* </p>
*/
- private static class TestSimilarity extends DefaultSimilarity {
+ private static class TestSimilarity extends ClassicSimilarity {
public TestSimilarity() {}
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestElevationComparator.java Fri Sep 18 15:13:49 2015
@@ -23,7 +23,7 @@ import org.apache.lucene.document.Field;
import org.apache.lucene.document.SortedDocValuesField;
import org.apache.lucene.index.*;
import org.apache.lucene.search.FieldValueHitQueue.Entry;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.store.*;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.BytesRef;
@@ -44,7 +44,7 @@ public class TestElevationComparator ext
newIndexWriterConfig(new MockAnalyzer(random())).
setMaxBufferedDocs(2).
setMergePolicy(newLogMergePolicy(1000)).
- setSimilarity(new DefaultSimilarity())
+ setSimilarity(new ClassicSimilarity())
);
writer.addDocument(adoc(new String[] {"id", "a", "title", "ipod", "str_s", "a"}));
writer.addDocument(adoc(new String[] {"id", "b", "title", "ipod ipod", "str_s", "b"}));
@@ -57,7 +57,7 @@ public class TestElevationComparator ext
writer.close();
IndexSearcher searcher = newSearcher(r);
- searcher.setSimilarity(new DefaultSimilarity());
+ searcher.setSimilarity(new ClassicSimilarity());
runTest(searcher, true);
runTest(searcher, false);
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestFuzzyQuery.java Fri Sep 18 15:13:49 2015
@@ -31,7 +31,7 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.automaton.LevenshteinAutomata;
@@ -259,7 +259,7 @@ public class TestFuzzyQuery extends Luce
IndexReader reader = writer.getReader();
IndexSearcher searcher = newSearcher(reader);
- searcher.setSimilarity(new DefaultSimilarity()); //avoid randomisation of similarity algo by test framework
+ searcher.setSimilarity(new ClassicSimilarity()); //avoid randomisation of similarity algo by test framework
writer.close();
String searchTerms[] = { "smith", "smythe", "smdssasd" };
for (String searchTerm : searchTerms) {
@@ -296,7 +296,7 @@ public class TestFuzzyQuery extends Luce
IndexReader reader = writer.getReader();
IndexSearcher searcher = newSearcher(reader);
- searcher.setSimilarity(new DefaultSimilarity()); //avoid randomisation of similarity algo by test framework
+ searcher.setSimilarity(new ClassicSimilarity()); //avoid randomisation of similarity algo by test framework
writer.close();
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestMinShouldMatch2.java Fri Sep 18 15:13:49 2015
@@ -35,7 +35,7 @@ import org.apache.lucene.index.RandomInd
import org.apache.lucene.index.SortedSetDocValues;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.Similarity.SimScorer;
import org.apache.lucene.search.similarities.Similarity.SimWeight;
import org.apache.lucene.store.Directory;
@@ -89,7 +89,7 @@ public class TestMinShouldMatch2 extends
r = DirectoryReader.open(dir);
reader = getOnlySegmentReader(r);
searcher = new IndexSearcher(reader);
- searcher.setSimilarity(new DefaultSimilarity() {
+ searcher.setSimilarity(new ClassicSimilarity() {
@Override
public float queryNorm(float sumOfSquaredWeights) {
return 1; // we disable queryNorm, both for debugging and ease of impl
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestMultiPhraseQuery.java Fri Sep 18 15:13:49 2015
@@ -33,7 +33,7 @@ import org.apache.lucene.index.MultiFiel
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermsEnum;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.BytesRef;
@@ -334,7 +334,7 @@ public class TestMultiPhraseQuery extend
IndexReader reader = writer.getReader();
IndexSearcher searcher = newSearcher(reader);
- searcher.setSimilarity(new DefaultSimilarity() {
+ searcher.setSimilarity(new ClassicSimilarity() {
@Override
public Explanation idfExplain(CollectionStatistics collectionStats, TermStatistics termStats[]) {
return Explanation.match(10f, "just a test");
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestMultiTermConstantScore.java Fri Sep 18 15:13:49 2015
@@ -27,7 +27,7 @@ import org.apache.lucene.index.LeafReade
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.store.Directory;
import org.junit.AfterClass;
import org.junit.BeforeClass;
@@ -223,7 +223,7 @@ public class TestMultiTermConstantScore
// test for correct application of query normalization
// must use a non score normalizing method for this.
- search.setSimilarity(new DefaultSimilarity());
+ search.setSimilarity(new ClassicSimilarity());
Query q = csrq("data", "1", "6", T, T);
search.search(new BoostQuery(q, 100), new SimpleCollector() {
private int base = 0;
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java Fri Sep 18 15:13:49 2015
@@ -34,7 +34,7 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -306,7 +306,7 @@ public class TestPhraseQuery extends Luc
RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
newIndexWriterConfig(new MockAnalyzer(random()))
.setMergePolicy(newLogMergePolicy())
- .setSimilarity(new DefaultSimilarity()));
+ .setSimilarity(new ClassicSimilarity()));
Document doc = new Document();
doc.add(newTextField("field", "foo firstname lastname foo", Field.Store.YES));
@@ -324,7 +324,7 @@ public class TestPhraseQuery extends Luc
writer.close();
IndexSearcher searcher = newSearcher(reader);
- searcher.setSimilarity(new DefaultSimilarity());
+ searcher.setSimilarity(new ClassicSimilarity());
PhraseQuery query = new PhraseQuery(Integer.MAX_VALUE, "field", "firstname", "lastname");
ScoreDoc[] hits = searcher.search(query, 1000).scoreDocs;
assertEquals(3, hits.length);
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestPositionIncrement.java Fri Sep 18 15:13:49 2015
@@ -19,8 +19,8 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.io.StringReader;
-import java.nio.charset.StandardCharsets;
-import java.util.Collection;
+import java.util.ArrayList;
+import java.util.List;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.MockPayloadAnalyzer;
@@ -38,9 +38,8 @@ import org.apache.lucene.index.PostingsE
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.payloads.PayloadSpanCollector;
-import org.apache.lucene.search.payloads.PayloadSpanUtil;
import org.apache.lucene.search.spans.MultiSpansWrapper;
+import org.apache.lucene.search.spans.SpanCollector;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
@@ -201,6 +200,22 @@ public class TestPositionIncrement exten
store.close();
}
+ static class PayloadSpanCollector implements SpanCollector {
+
+ List<BytesRef> payloads = new ArrayList<>();
+
+ @Override
+ public void collectLeaf(PostingsEnum postings, int position, Term term) throws IOException {
+ if (postings.getPayload() != null)
+ payloads.add(BytesRef.deepCopyOf(postings.getPayload()));
+ }
+
+ @Override
+ public void reset() {
+ payloads.clear();
+ }
+ }
+
public void testPayloadsPos0() throws Exception {
Directory dir = newDirectory();
RandomIndexWriter writer = new RandomIndexWriter(random(), dir, new MockPayloadAnalyzer());
@@ -248,12 +263,11 @@ public class TestPositionIncrement exten
}
collector.reset();
pspans.collect(collector);
- Collection<byte[]> payloads = collector.getPayloads();
sawZero |= pspans.startPosition() == 0;
- for (byte[] bytes : payloads) {
+ for (BytesRef payload : collector.payloads) {
count++;
if (VERBOSE) {
- System.out.println(" payload: " + new String(bytes, StandardCharsets.UTF_8));
+ System.out.println(" payload: " + Term.toString(payload));
}
}
}
@@ -276,17 +290,6 @@ public class TestPositionIncrement exten
assertEquals(4, count);
assertTrue(sawZero);
- sawZero = false;
- PayloadSpanUtil psu = new PayloadSpanUtil(is.getTopReaderContext());
- Collection<byte[]> pls = psu.getPayloadsForQuery(snq);
- count = pls.size();
- for (byte[] bytes : pls) {
- String s = new String(bytes, StandardCharsets.UTF_8);
- //System.out.println(s);
- sawZero |= s.equals("pos: 0");
- }
- assertEquals(8, count);
- assertTrue(sawZero);
writer.close();
is.getIndexReader().close();
dir.close();
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestQueryRescorer.java Fri Sep 18 15:13:49 2015
@@ -30,7 +30,7 @@ import org.apache.lucene.index.LeafReade
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
import org.apache.lucene.search.BooleanClause.Occur;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
@@ -44,7 +44,7 @@ public class TestQueryRescorer extends L
IndexSearcher searcher = newSearcher(r);
// We rely on more tokens = lower score:
- searcher.setSimilarity(new DefaultSimilarity());
+ searcher.setSimilarity(new ClassicSimilarity());
return searcher;
}
@@ -70,7 +70,7 @@ public class TestQueryRescorer extends L
bq.add(new TermQuery(new Term("field", "wizard")), Occur.SHOULD);
bq.add(new TermQuery(new Term("field", "oz")), Occur.SHOULD);
IndexSearcher searcher = getSearcher(r);
- searcher.setSimilarity(new DefaultSimilarity());
+ searcher.setSimilarity(new ClassicSimilarity());
TopDocs hits = searcher.search(bq.build(), 10);
assertEquals(2, hits.totalHits);
@@ -125,7 +125,7 @@ public class TestQueryRescorer extends L
bq.add(new TermQuery(new Term("field", "wizard")), Occur.SHOULD);
bq.add(new TermQuery(new Term("field", "oz")), Occur.SHOULD);
IndexSearcher searcher = getSearcher(r);
- searcher.setSimilarity(new DefaultSimilarity());
+ searcher.setSimilarity(new ClassicSimilarity());
TopDocs hits = searcher.search(bq.build(), 10);
assertEquals(2, hits.totalHits);
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSimilarity.java Fri Sep 18 15:13:49 2015
@@ -27,7 +27,7 @@ import org.apache.lucene.index.FieldInve
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.analysis.MockAnalyzer;
import org.apache.lucene.document.Document;
@@ -38,7 +38,7 @@ import org.apache.lucene.document.Docume
*/
public class TestSimilarity extends LuceneTestCase {
- public static class SimpleSimilarity extends DefaultSimilarity {
+ public static class SimpleSimilarity extends ClassicSimilarity {
@Override
public float queryNorm(float sumOfSquaredWeights) { return 1.0f; }
@Override
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanations.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanations.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanations.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSimpleExplanations.java Fri Sep 18 15:13:49 2015
@@ -678,5 +678,10 @@ public class TestSimpleExplanations exte
qtest(query.build(), new int[] { 0,1,2,3 });
}
+
+ public void testSynonymQuery() throws Exception {
+ SynonymQuery query = new SynonymQuery(new Term(FIELD, "w1"), new Term(FIELD, "w2"));
+ qtest(query, new int[] { 0,1,2,3 });
+ }
}
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSimpleSearchEquivalence.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSimpleSearchEquivalence.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSimpleSearchEquivalence.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSimpleSearchEquivalence.java Fri Sep 18 15:13:49 2015
@@ -230,4 +230,15 @@ public class TestSimpleSearchEquivalence
assertSameScores(q1, q2);
}
+
+ public void testBooleanOrVsSynonym() throws Exception {
+ Term t1 = randomTerm();
+ Term t2 = randomTerm();
+ assertEquals(t1.field(), t2.field());
+ SynonymQuery q1 = new SynonymQuery(t1, t2);
+ BooleanQuery.Builder q2 = new BooleanQuery.Builder();
+ q2.add(new TermQuery(t1), Occur.SHOULD);
+ q2.add(new TermQuery(t2), Occur.SHOULD);
+ assertSameSet(q1, q2.build());
+ }
}
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSortRescorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSortRescorer.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSortRescorer.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestSortRescorer.java Fri Sep 18 15:13:49 2015
@@ -28,6 +28,7 @@ import org.apache.lucene.index.Directory
import org.apache.lucene.index.IndexReader;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
@@ -63,6 +64,8 @@ public class TestSortRescorer extends Lu
reader = iw.getReader();
searcher = new IndexSearcher(reader);
+ // TODO: fix this test to not be so flaky and use newSearcher
+ searcher.setSimilarity(new ClassicSimilarity());
iw.close();
}
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/TestTermScorer.java Fri Sep 18 15:13:49 2015
@@ -31,7 +31,7 @@ import org.apache.lucene.index.NumericDo
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.SlowCompositeReaderWrapper;
import org.apache.lucene.index.Term;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -52,7 +52,7 @@ public class TestTermScorer extends Luce
RandomIndexWriter writer = new RandomIndexWriter(random(), directory,
newIndexWriterConfig(new MockAnalyzer(random()))
.setMergePolicy(newLogMergePolicy())
- .setSimilarity(new DefaultSimilarity()));
+ .setSimilarity(new ClassicSimilarity()));
for (int i = 0; i < values.length; i++) {
Document doc = new Document();
doc
@@ -62,7 +62,7 @@ public class TestTermScorer extends Luce
indexReader = SlowCompositeReaderWrapper.wrap(writer.getReader());
writer.close();
indexSearcher = newSearcher(indexReader);
- indexSearcher.setSimilarity(new DefaultSimilarity());
+ indexSearcher.setSimilarity(new ClassicSimilarity());
}
@Override
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java Fri Sep 18 15:13:49 2015
@@ -52,7 +52,7 @@ public class TestSimilarity2 extends Luc
public void setUp() throws Exception {
super.setUp();
sims = new ArrayList<>();
- sims.add(new DefaultSimilarity());
+ sims.add(new ClassicSimilarity());
sims.add(new BM25Similarity());
// TODO: not great that we dup this all with TestSimilarityBase
for (BasicModel basicModel : TestSimilarityBase.BASIC_MODELS) {
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarityBase.java Fri Sep 18 15:13:49 2015
@@ -579,7 +579,7 @@ public class TestSimilarityBase extends
// LUCENE-5221
public void testDiscountOverlapsBoost() throws IOException {
- DefaultSimilarity expected = new DefaultSimilarity();
+ ClassicSimilarity expected = new ClassicSimilarity();
SimilarityBase actual = new DFRSimilarity(new BasicModelIne(), new AfterEffectB(), new NormalizationH2());
expected.setDiscountOverlaps(false);
actual.setDiscountOverlaps(false);
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/search/spans/TestSpans.java Fri Sep 18 15:13:49 2015
@@ -35,7 +35,7 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.Query;
import org.apache.lucene.search.Scorer;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -286,7 +286,7 @@ public class TestSpans extends LuceneTes
for (int i = 0, c = leaves.size(); i < c; i++) {
final LeafReaderContext ctx = leaves.get(i);
- final Similarity sim = new DefaultSimilarity() {
+ final Similarity sim = new ClassicSimilarity() {
@Override
public float sloppyFreq(int distance) {
return 0.0f;
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/util/TestQueryBuilder.java Fri Sep 18 15:13:49 2015
@@ -33,6 +33,7 @@ import org.apache.lucene.search.BooleanQ
import org.apache.lucene.search.MultiPhraseQuery;
import org.apache.lucene.search.PhraseQuery;
import org.apache.lucene.search.Query;
+import org.apache.lucene.search.SynonymQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.util.automaton.CharacterRunAutomaton;
import org.apache.lucene.util.automaton.RegExp;
@@ -162,11 +163,7 @@ public class TestQueryBuilder extends Lu
/** simple synonyms test */
public void testSynonyms() throws Exception {
- BooleanQuery.Builder expectedB = new BooleanQuery.Builder();
- expectedB.setDisableCoord(true);
- expectedB.add(new TermQuery(new Term("field", "dogs")), BooleanClause.Occur.SHOULD);
- expectedB.add(new TermQuery(new Term("field", "dog")), BooleanClause.Occur.SHOULD);
- Query expected = expectedB.build();
+ SynonymQuery expected = new SynonymQuery(new Term("field", "dogs"), new Term("field", "dog"));
QueryBuilder builder = new QueryBuilder(new MockSynonymAnalyzer());
assertEquals(expected, builder.createBooleanQuery("field", "dogs"));
assertEquals(expected, builder.createPhraseQuery("field", "dogs"));
@@ -281,25 +278,19 @@ public class TestQueryBuilder extends Lu
/** simple CJK synonym test */
public void testCJKSynonym() throws Exception {
- BooleanQuery.Builder expected = new BooleanQuery.Builder();
- expected.setDisableCoord(true);
- expected.add(new TermQuery(new Term("field", "å½")), BooleanClause.Occur.SHOULD);
- expected.add(new TermQuery(new Term("field", "å")), BooleanClause.Occur.SHOULD);
+ SynonymQuery expected = new SynonymQuery(new Term("field", "å½"), new Term("field", "å"));
QueryBuilder builder = new QueryBuilder(new MockCJKSynonymAnalyzer());
- assertEquals(expected.build(), builder.createBooleanQuery("field", "å½"));
- assertEquals(expected.build(), builder.createPhraseQuery("field", "å½"));
- assertEquals(expected.build(), builder.createBooleanQuery("field", "å½", BooleanClause.Occur.MUST));
+ assertEquals(expected, builder.createBooleanQuery("field", "å½"));
+ assertEquals(expected, builder.createPhraseQuery("field", "å½"));
+ assertEquals(expected, builder.createBooleanQuery("field", "å½", BooleanClause.Occur.MUST));
}
/** synonyms with default OR operator */
public void testCJKSynonymsOR() throws Exception {
BooleanQuery.Builder expected = new BooleanQuery.Builder();
expected.add(new TermQuery(new Term("field", "ä¸")), BooleanClause.Occur.SHOULD);
- BooleanQuery.Builder inner = new BooleanQuery.Builder();
- inner.setDisableCoord(true);
- inner.add(new TermQuery(new Term("field", "å½")), BooleanClause.Occur.SHOULD);
- inner.add(new TermQuery(new Term("field", "å")), BooleanClause.Occur.SHOULD);
- expected.add(inner.build(), BooleanClause.Occur.SHOULD);
+ SynonymQuery inner = new SynonymQuery(new Term("field", "å½"), new Term("field", "å"));
+ expected.add(inner, BooleanClause.Occur.SHOULD);
QueryBuilder builder = new QueryBuilder(new MockCJKSynonymAnalyzer());
assertEquals(expected.build(), builder.createBooleanQuery("field", "ä¸å½"));
}
@@ -308,16 +299,10 @@ public class TestQueryBuilder extends Lu
public void testCJKSynonymsOR2() throws Exception {
BooleanQuery.Builder expected = new BooleanQuery.Builder();
expected.add(new TermQuery(new Term("field", "ä¸")), BooleanClause.Occur.SHOULD);
- BooleanQuery.Builder inner = new BooleanQuery.Builder();
- inner.setDisableCoord(true);
- inner.add(new TermQuery(new Term("field", "å½")), BooleanClause.Occur.SHOULD);
- inner.add(new TermQuery(new Term("field", "å")), BooleanClause.Occur.SHOULD);
- expected.add(inner.build(), BooleanClause.Occur.SHOULD);
- BooleanQuery.Builder inner2 = new BooleanQuery.Builder();
- inner2.setDisableCoord(true);
- inner2.add(new TermQuery(new Term("field", "å½")), BooleanClause.Occur.SHOULD);
- inner2.add(new TermQuery(new Term("field", "å")), BooleanClause.Occur.SHOULD);
- expected.add(inner2.build(), BooleanClause.Occur.SHOULD);
+ SynonymQuery inner = new SynonymQuery(new Term("field", "å½"), new Term("field", "å"));
+ expected.add(inner, BooleanClause.Occur.SHOULD);
+ SynonymQuery inner2 = new SynonymQuery(new Term("field", "å½"), new Term("field", "å"));
+ expected.add(inner2, BooleanClause.Occur.SHOULD);
QueryBuilder builder = new QueryBuilder(new MockCJKSynonymAnalyzer());
assertEquals(expected.build(), builder.createBooleanQuery("field", "ä¸å½å½"));
}
@@ -326,11 +311,8 @@ public class TestQueryBuilder extends Lu
public void testCJKSynonymsAND() throws Exception {
BooleanQuery.Builder expected = new BooleanQuery.Builder();
expected.add(new TermQuery(new Term("field", "ä¸")), BooleanClause.Occur.MUST);
- BooleanQuery.Builder inner = new BooleanQuery.Builder();
- inner.setDisableCoord(true);
- inner.add(new TermQuery(new Term("field", "å½")), BooleanClause.Occur.SHOULD);
- inner.add(new TermQuery(new Term("field", "å")), BooleanClause.Occur.SHOULD);
- expected.add(inner.build(), BooleanClause.Occur.MUST);
+ SynonymQuery inner = new SynonymQuery(new Term("field", "å½"), new Term("field", "å"));
+ expected.add(inner, BooleanClause.Occur.MUST);
QueryBuilder builder = new QueryBuilder(new MockCJKSynonymAnalyzer());
assertEquals(expected.build(), builder.createBooleanQuery("field", "ä¸å½", BooleanClause.Occur.MUST));
}
@@ -339,16 +321,10 @@ public class TestQueryBuilder extends Lu
public void testCJKSynonymsAND2() throws Exception {
BooleanQuery.Builder expected = new BooleanQuery.Builder();
expected.add(new TermQuery(new Term("field", "ä¸")), BooleanClause.Occur.MUST);
- BooleanQuery.Builder inner = new BooleanQuery.Builder();
- inner.setDisableCoord(true);
- inner.add(new TermQuery(new Term("field", "å½")), BooleanClause.Occur.SHOULD);
- inner.add(new TermQuery(new Term("field", "å")), BooleanClause.Occur.SHOULD);
- expected.add(inner.build(), BooleanClause.Occur.MUST);
- BooleanQuery.Builder inner2 = new BooleanQuery.Builder();
- inner2.setDisableCoord(true);
- inner2.add(new TermQuery(new Term("field", "å½")), BooleanClause.Occur.SHOULD);
- inner2.add(new TermQuery(new Term("field", "å")), BooleanClause.Occur.SHOULD);
- expected.add(inner2.build(), BooleanClause.Occur.MUST);
+ SynonymQuery inner = new SynonymQuery(new Term("field", "å½"), new Term("field", "å"));
+ expected.add(inner, BooleanClause.Occur.MUST);
+ SynonymQuery inner2 = new SynonymQuery(new Term("field", "å½"), new Term("field", "å"));
+ expected.add(inner2, BooleanClause.Occur.MUST);
QueryBuilder builder = new QueryBuilder(new MockCJKSynonymAnalyzer());
assertEquals(expected.build(), builder.createBooleanQuery("field", "ä¸å½å½", BooleanClause.Occur.MUST));
}
Modified: lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/util/TestUnicodeUtil.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/util/TestUnicodeUtil.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/util/TestUnicodeUtil.java (original)
+++ lucene/dev/branches/lucene6780/lucene/core/src/test/org/apache/lucene/util/TestUnicodeUtil.java Fri Sep 18 15:13:49 2015
@@ -214,4 +214,14 @@ public class TestUnicodeUtil extends Luc
assertEquals(cRef.toString(), unicode);
}
}
+
+ public void testCalcUTF16toUTF8Length() {
+ int num = atLeast(5000);
+ for (int i = 0; i < num; i++) {
+ String unicode = TestUtil.randomUnicodeString(random());
+ byte[] utf8 = new byte[unicode.length() * UnicodeUtil.MAX_UTF8_BYTES_PER_CHAR];
+ int len = UnicodeUtil.UTF16toUTF8(unicode, 0, unicode.length(), utf8);
+ assertEquals(len, UnicodeUtil.calcUTF16toUTF8Length(unicode, 0, unicode.length()));
+ }
+ }
}
Modified: lucene/dev/branches/lucene6780/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionRescorer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionRescorer.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionRescorer.java (original)
+++ lucene/dev/branches/lucene6780/lucene/expressions/src/test/org/apache/lucene/expressions/TestExpressionRescorer.java Fri Sep 18 15:13:49 2015
@@ -31,6 +31,7 @@ import org.apache.lucene.search.Rescorer
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.TopDocs;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.LuceneTestCase;
@@ -65,6 +66,8 @@ public class TestExpressionRescorer exte
reader = iw.getReader();
searcher = new IndexSearcher(reader);
+ // TODO: fix this test to not be so flaky and use newSearcher
+ searcher.setSimilarity(new ClassicSimilarity());
iw.close();
}
Modified: lucene/dev/branches/lucene6780/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java (original)
+++ lucene/dev/branches/lucene6780/lucene/facet/src/test/org/apache/lucene/facet/taxonomy/TestTaxonomyFacetCounts.java Fri Sep 18 15:13:49 2015
@@ -50,7 +50,7 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
@@ -267,7 +267,7 @@ public class TestTaxonomyFacetCounts ext
Directory taxoDir = newDirectory();
IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
iwc.setSimilarity(new PerFieldSimilarityWrapper() {
- final Similarity sim = new DefaultSimilarity();
+ final Similarity sim = new ClassicSimilarity();
@Override
public Similarity get(String name) {
Modified: lucene/dev/branches/lucene6780/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java (original)
+++ lucene/dev/branches/lucene6780/lucene/highlighter/src/java/org/apache/lucene/search/highlight/QueryTermExtractor.java Fri Sep 18 15:13:49 2015
@@ -80,7 +80,7 @@ public final class QueryTermExtractor
try
{
int docFreq=reader.docFreq(new Term(fieldName,terms[i].term));
- //IDF algorithm taken from DefaultSimilarity class
+ //IDF algorithm taken from ClassicSimilarity class
float idf=(float)(Math.log(totalNumDocs/(double)(docFreq+1)) + 1.0);
terms[i].weight*=idf;
}
Modified: lucene/dev/branches/lucene6780/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java (original)
+++ lucene/dev/branches/lucene6780/lucene/highlighter/src/java/org/apache/lucene/search/highlight/WeightedSpanTermExtractor.java Fri Sep 18 15:13:49 2015
@@ -17,6 +17,16 @@ package org.apache.lucene.search.highlig
* limitations under the License.
*/
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
import org.apache.lucene.analysis.CachingTokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.index.BinaryDocValues;
@@ -59,17 +69,6 @@ import org.apache.lucene.search.spans.Sp
import org.apache.lucene.util.Bits;
import org.apache.lucene.util.IOUtils;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-
/**
* Class used to extract {@link WeightedSpanTerm}s from a {@link Query} based on whether
* {@link Term}s from the {@link Query} are contained in a supplied {@link TokenStream}.
@@ -121,33 +120,19 @@ public class WeightedSpanTermExtractor {
for (int i = 0; i < phraseQueryTerms.length; i++) {
clauses[i] = new SpanTermQuery(phraseQueryTerms[i]);
}
- int slop = phraseQuery.getSlop();
+
+ // sum position increments beyond 1
+ int positionGaps = 0;
int[] positions = phraseQuery.getPositions();
- // add largest position increment to slop
- if (positions.length > 0) {
- int lastPos = positions[0];
- int largestInc = 0;
- int sz = positions.length;
- for (int i = 1; i < sz; i++) {
- int pos = positions[i];
- int inc = pos - lastPos;
- if (inc > largestInc) {
- largestInc = inc;
- }
- lastPos = pos;
- }
- if(largestInc > 1) {
- slop += largestInc;
- }
+ if (positions.length >= 2) {
+ // positions are in increasing order. max(0,...) is just a safeguard.
+ positionGaps = Math.max(0, positions[positions.length-1] - positions[0] - positions.length + 1);
}
- boolean inorder = false;
-
- if (slop == 0) {
- inorder = true;
- }
+ //if original slop is 0 then require inOrder
+ boolean inorder = (phraseQuery.getSlop() == 0);
- SpanNearQuery sp = new SpanNearQuery(clauses, slop, inorder);
+ SpanNearQuery sp = new SpanNearQuery(clauses, phraseQuery.getSlop() + positionGaps, inorder);
extractWeightedSpanTerms(terms, sp, boost);
} else if (query instanceof TermQuery) {
extractWeightedTerms(terms, query, boost);
@@ -560,7 +545,7 @@ public class WeightedSpanTermExtractor {
while (it.hasNext()) {
WeightedSpanTerm weightedSpanTerm = terms.get(it.next());
int docFreq = reader.docFreq(new Term(fieldName, weightedSpanTerm.term));
- // IDF algorithm taken from DefaultSimilarity class
+ // IDF algorithm taken from ClassicSimilarity class
float idf = (float) (Math.log(totalNumDocs / (double) (docFreq + 1)) + 1.0);
weightedSpanTerm.weight *= idf;
}
Modified: lucene/dev/branches/lucene6780/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java (original)
+++ lucene/dev/branches/lucene6780/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterPhraseTest.java Fri Sep 18 15:13:49 2015
@@ -20,10 +20,12 @@ package org.apache.lucene.search.highlig
import java.io.IOException;
import org.apache.lucene.analysis.MockAnalyzer;
+import org.apache.lucene.analysis.MockTokenFilter;
import org.apache.lucene.analysis.MockTokenizer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.document.Field.Store;
import org.apache.lucene.analysis.tokenattributes.OffsetAttribute;
import org.apache.lucene.analysis.tokenattributes.PositionIncrementAttribute;
import org.apache.lucene.document.Document;
@@ -267,6 +269,76 @@ public class HighlighterPhraseTest exten
directory.close();
}
}
+
+ //shows the need to sum the increments in WeightedSpanTermExtractor
+ public void testStopWords() throws IOException, InvalidTokenOffsetsException {
+ MockAnalyzer stopAnalyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true,
+ MockTokenFilter.ENGLISH_STOPSET);
+ final String TEXT = "the ab the the cd the the the ef the";
+ final Directory directory = newDirectory();
+ try (IndexWriter indexWriter = new IndexWriter(directory,
+ newIndexWriterConfig(stopAnalyzer))) {
+ final Document document = new Document();
+ document.add(newTextField(FIELD, TEXT, Store.YES));
+ indexWriter.addDocument(document);
+ }
+ try (IndexReader indexReader = DirectoryReader.open(directory)) {
+ assertEquals(1, indexReader.numDocs());
+ final IndexSearcher indexSearcher = newSearcher(indexReader);
+ //equivalent of "ab the the cd the the the ef"
+ final PhraseQuery phraseQuery = new PhraseQuery.Builder()
+ .add(new Term(FIELD, "ab"), 0)
+ .add(new Term(FIELD, "cd"), 3)
+ .add(new Term(FIELD, "ef"), 7).build();
+
+ TopDocs hits = indexSearcher.search(phraseQuery, 100);
+ assertEquals(1, hits.totalHits);
+ final Highlighter highlighter = new Highlighter(
+ new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),
+ new QueryScorer(phraseQuery));
+ assertEquals(1, highlighter.getBestFragments(stopAnalyzer, FIELD, TEXT, 10).length);
+ } finally {
+ directory.close();
+ }
+ }
+
+ //shows the need to require inOrder if getSlop() == 0, not if final slop == 0
+ //in WeightedSpanTermExtractor
+ public void testInOrderWithStopWords() throws IOException, InvalidTokenOffsetsException {
+ MockAnalyzer stopAnalyzer = new MockAnalyzer(random(), MockTokenizer.WHITESPACE, true,
+ MockTokenFilter.ENGLISH_STOPSET);
+ final String TEXT = "the cd the ab the the the the the the the ab the cd the";
+ final Directory directory = newDirectory();
+ try (IndexWriter indexWriter = new IndexWriter(directory,
+ newIndexWriterConfig(stopAnalyzer))) {
+ final Document document = new Document();
+ document.add(newTextField(FIELD, TEXT, Store.YES));
+ indexWriter.addDocument(document);
+ }
+ try (IndexReader indexReader = DirectoryReader.open(directory)) {
+ assertEquals(1, indexReader.numDocs());
+ final IndexSearcher indexSearcher = newSearcher(indexReader);
+ //equivalent of "ab the cd"
+ final PhraseQuery phraseQuery = new PhraseQuery.Builder()
+ .add(new Term(FIELD, "ab"), 0)
+ .add(new Term(FIELD, "cd"), 2).build();
+
+ TopDocs hits = indexSearcher.search(phraseQuery, 100);
+ assertEquals(1, hits.totalHits);
+
+ final Highlighter highlighter = new Highlighter(
+ new SimpleHTMLFormatter(), new SimpleHTMLEncoder(),
+ new QueryScorer(phraseQuery));
+ String[] frags = highlighter.getBestFragments(stopAnalyzer, FIELD, TEXT, 10);
+ assertEquals(1, frags.length);
+ assertTrue("contains <B>ab</B> the <B>cd</B>",
+ (frags[0].contains("<B>ab</B> the <B>cd</B>")));
+ assertTrue("does not contain <B>cd</B> the <B>ab</B>",
+ (!frags[0].contains("<B>cd</B> the <B>ab</B>")));
+ } finally {
+ directory.close();
+ }
+ }
private static final class TokenStreamSparse extends TokenStream {
private Token[] tokens;
Modified: lucene/dev/branches/lucene6780/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java (original)
+++ lucene/dev/branches/lucene6780/lucene/highlighter/src/test/org/apache/lucene/search/highlight/HighlighterTest.java Fri Sep 18 15:13:49 2015
@@ -17,6 +17,8 @@ package org.apache.lucene.search.highlig
* limitations under the License.
*/
+import javax.xml.parsers.DocumentBuilder;
+import javax.xml.parsers.DocumentBuilderFactory;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
@@ -29,9 +31,6 @@ import java.util.List;
import java.util.Map;
import java.util.StringTokenizer;
-import javax.xml.parsers.DocumentBuilder;
-import javax.xml.parsers.DocumentBuilderFactory;
-
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.analysis.CachingTokenFilter;
@@ -56,11 +55,9 @@ import org.apache.lucene.index.IndexRead
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
-import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.StoredDocument;
import org.apache.lucene.index.Term;
import org.apache.lucene.queries.CommonTermsQuery;
-import org.apache.lucene.queries.CustomScoreProvider;
import org.apache.lucene.queries.CustomScoreQuery;
import org.apache.lucene.search.BooleanClause.Occur;
import org.apache.lucene.search.BooleanQuery;
@@ -81,12 +78,12 @@ import org.apache.lucene.search.TermRang
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.WildcardQuery;
import org.apache.lucene.search.highlight.SynonymTokenizer.TestHighlightRunner;
-import org.apache.lucene.search.join.QueryBitSetProducer;
import org.apache.lucene.search.join.BitSetProducer;
+import org.apache.lucene.search.join.QueryBitSetProducer;
import org.apache.lucene.search.join.ScoreMode;
import org.apache.lucene.search.join.ToChildBlockJoinQuery;
import org.apache.lucene.search.join.ToParentBlockJoinQuery;
-import org.apache.lucene.search.payloads.SpanPayloadCheckQuery;
+import org.apache.lucene.queries.payloads.SpanPayloadCheckQuery;
import org.apache.lucene.search.spans.SpanMultiTermQueryWrapper;
import org.apache.lucene.search.spans.SpanNearQuery;
import org.apache.lucene.search.spans.SpanNotQuery;
@@ -1957,7 +1954,7 @@ public class HighlighterTest extends Bas
}
try (IndexReader reader = DirectoryReader.open(dir)) {
Query query = new SpanPayloadCheckQuery(new SpanTermQuery(new Term(FIELD_NAME, "words")),
- Collections.singleton("pos: 1".getBytes("UTF-8")));//just match the first "word" occurrence
+ Collections.singletonList(new BytesRef("pos: 1")));//just match the first "word" occurrence
IndexSearcher searcher = newSearcher(reader);
QueryScorer scorer = new QueryScorer(query, searcher.getIndexReader(), FIELD_NAME);
scorer.setUsePayloads(true);
Modified: lucene/dev/branches/lucene6780/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java (original)
+++ lucene/dev/branches/lucene6780/lucene/memory/src/test/org/apache/lucene/index/memory/TestMemoryIndex.java Fri Sep 18 15:13:49 2015
@@ -30,7 +30,7 @@ import org.apache.lucene.search.IndexSea
import org.apache.lucene.search.MatchAllDocsQuery;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.similarities.BM25Similarity;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.util.LuceneTestCase;
import org.apache.lucene.util.TestUtil;
import org.junit.Before;
@@ -90,7 +90,7 @@ public class TestMemoryIndex extends Luc
assertThat(mi.search(new TermQuery(new Term("f1", "wibble"))), not(is(0.0f)));
// check we can set the Similarity again
- mi.setSimilarity(new DefaultSimilarity());
+ mi.setSimilarity(new ClassicSimilarity());
}
@@ -144,7 +144,7 @@ public class TestMemoryIndex extends Luc
float n1 = reader.getNormValues("f1").get(0);
// Norms are re-computed when we change the Similarity
- mi.setSimilarity(new DefaultSimilarity() {
+ mi.setSimilarity(new ClassicSimilarity() {
@Override
public float lengthNorm(FieldInvertState state) {
return 74;
Modified: lucene/dev/branches/lucene6780/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java (original)
+++ lucene/dev/branches/lucene6780/lucene/misc/src/java/org/apache/lucene/misc/SweetSpotSimilarity.java Fri Sep 18 15:13:49 2015
@@ -17,7 +17,7 @@
package org.apache.lucene.misc;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.index.FieldInvertState;
/**
@@ -38,7 +38,7 @@ import org.apache.lucene.index.FieldInve
*
* @see <a href="doc-files/ss.gnuplot">A Gnuplot file used to generate some of the visualizations refrenced from each function.</a>
*/
-public class SweetSpotSimilarity extends DefaultSimilarity {
+public class SweetSpotSimilarity extends ClassicSimilarity {
private int ln_min = 1;
private int ln_max = 1;
Modified: lucene/dev/branches/lucene6780/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java (original)
+++ lucene/dev/branches/lucene6780/lucene/misc/src/test/org/apache/lucene/misc/SweetSpotSimilarityTest.java Fri Sep 18 15:13:49 2015
@@ -18,7 +18,7 @@
package org.apache.lucene.misc;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.PerFieldSimilarityWrapper;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
@@ -44,7 +44,7 @@ public class SweetSpotSimilarityTest ext
final SweetSpotSimilarity ss = new SweetSpotSimilarity();
ss.setLengthNormFactors(1,1,0.5f,true);
- Similarity d = new DefaultSimilarity();
+ Similarity d = new ClassicSimilarity();
Similarity s = ss;
@@ -200,7 +200,7 @@ public class SweetSpotSimilarityTest ext
SweetSpotSimilarity ss = new SweetSpotSimilarity();
- TFIDFSimilarity d = new DefaultSimilarity();
+ TFIDFSimilarity d = new ClassicSimilarity();
TFIDFSimilarity s = ss;
// tf equal
Modified: lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/TermsQuery.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/TermsQuery.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/TermsQuery.java (original)
+++ lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/TermsQuery.java Fri Sep 18 15:13:49 2015
@@ -182,8 +182,7 @@ public class TermsQuery extends Query im
builder.append(' ');
}
first = false;
- builder.append(iterator.field()).append(':');
- builder.append(term.utf8ToString());
+ builder.append(new Term(iterator.field(), term).toString());
}
return builder.toString();
Modified: lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java (original)
+++ lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/IDFValueSource.java Fri Sep 18 15:13:49 2015
@@ -50,7 +50,7 @@ public class IDFValueSource extends DocF
IndexSearcher searcher = (IndexSearcher)context.get("searcher");
TFIDFSimilarity sim = asTFIDF(searcher.getSimilarity(true), field);
if (sim == null) {
- throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
+ throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as ClassicSimilarity)");
}
int docfreq = searcher.getIndexReader().docFreq(new Term(indexedField, indexedBytes));
float idf = sim.idf(docfreq, searcher.getIndexReader().maxDoc());
Modified: lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java (original)
+++ lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/NormValueSource.java Fri Sep 18 15:13:49 2015
@@ -60,7 +60,7 @@ public class NormValueSource extends Val
IndexSearcher searcher = (IndexSearcher)context.get("searcher");
final TFIDFSimilarity similarity = IDFValueSource.asTFIDF(searcher.getSimilarity(true), field);
if (similarity == null) {
- throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
+ throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as ClassicSimilarity)");
}
final NumericDocValues norms = readerContext.reader().getNormValues(field);
Modified: lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java (original)
+++ lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/function/valuesource/TFValueSource.java Fri Sep 18 15:13:49 2015
@@ -56,7 +56,7 @@ public class TFValueSource extends TermF
IndexSearcher searcher = (IndexSearcher)context.get("searcher");
final TFIDFSimilarity similarity = IDFValueSource.asTFIDF(searcher.getSimilarity(true), indexedField);
if (similarity == null) {
- throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as DefaultSimilarity)");
+ throw new UnsupportedOperationException("requires a TFIDFSimilarity (such as ClassicSimilarity)");
}
return new FloatDocValues(this) {
Modified: lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java (original)
+++ lucene/dev/branches/lucene6780/lucene/queries/src/java/org/apache/lucene/queries/mlt/MoreLikeThis.java Fri Sep 18 15:13:49 2015
@@ -32,7 +32,7 @@ import org.apache.lucene.search.BooleanQ
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.TFIDFSimilarity;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.CharsRefBuilder;
@@ -313,7 +313,7 @@ public final class MoreLikeThis {
* Constructor requiring an IndexReader.
*/
public MoreLikeThis(IndexReader ir) {
- this(ir, new DefaultSimilarity());
+ this(ir, new ClassicSimilarity());
}
public MoreLikeThis(IndexReader ir, TFIDFSimilarity sim) {
Modified: lucene/dev/branches/lucene6780/lucene/queries/src/test/org/apache/lucene/queries/TermsQueryTest.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/queries/src/test/org/apache/lucene/queries/TermsQueryTest.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/queries/src/test/org/apache/lucene/queries/TermsQueryTest.java (original)
+++ lucene/dev/branches/lucene6780/lucene/queries/src/test/org/apache/lucene/queries/TermsQueryTest.java Fri Sep 18 15:13:49 2015
@@ -321,4 +321,9 @@ public class TermsQueryTest extends Luce
wrapped.close();
dir.close();
}
+
+ public void testBinaryToString() {
+ TermsQuery query = new TermsQuery(new Term("field", new BytesRef(new byte[] { (byte) 0xff, (byte) 0xfe })));
+ assertEquals("field:[ff fe]", query.toString());
+ }
}
Modified: lucene/dev/branches/lucene6780/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java (original)
+++ lucene/dev/branches/lucene6780/lucene/queries/src/test/org/apache/lucene/queries/function/TestLongNormValueSource.java Fri Sep 18 15:13:49 2015
@@ -49,7 +49,7 @@ public class TestLongNormValueSource ext
static IndexSearcher searcher;
static Analyzer analyzer;
- private static Similarity sim = new PreciseDefaultSimilarity();
+ private static Similarity sim = new PreciseClassicSimilarity();
@BeforeClass
public static void beforeClass() throws Exception {
@@ -120,10 +120,10 @@ public class TestLongNormValueSource ext
/** Encodes norm as 4-byte float. */
-class PreciseDefaultSimilarity extends TFIDFSimilarity {
+class PreciseClassicSimilarity extends TFIDFSimilarity {
/** Sole constructor: parameter-free */
- public PreciseDefaultSimilarity() {}
+ public PreciseClassicSimilarity() {}
/** Implemented as <code>overlap / maxOverlap</code>. */
@Override
Modified: lucene/dev/branches/lucene6780/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java (original)
+++ lucene/dev/branches/lucene6780/lucene/queries/src/test/org/apache/lucene/queries/function/TestValueSources.java Fri Sep 18 15:13:49 2015
@@ -80,7 +80,7 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.TermQuery;
-import org.apache.lucene.search.similarities.DefaultSimilarity;
+import org.apache.lucene.search.similarities.ClassicSimilarity;
import org.apache.lucene.search.similarities.Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
@@ -224,7 +224,7 @@ public class TestValueSources extends Lu
public void testIDF() throws Exception {
Similarity saved = searcher.getSimilarity(true);
try {
- searcher.setSimilarity(new DefaultSimilarity());
+ searcher.setSimilarity(new ClassicSimilarity());
ValueSource vs = new IDFValueSource("bogus", "bogus", "text", new BytesRef("test"));
assertHits(new FunctionQuery(vs), new float[] { 1.0f, 1.0f });
assertAllExist(vs);
@@ -343,7 +343,7 @@ public class TestValueSources extends Lu
Similarity saved = searcher.getSimilarity(true);
try {
// no norm field (so agnostic to indexed similarity)
- searcher.setSimilarity(new DefaultSimilarity());
+ searcher.setSimilarity(new ClassicSimilarity());
ValueSource vs = new NormValueSource("byte");
assertHits(new FunctionQuery(vs), new float[] { 0f, 0f });
@@ -395,7 +395,7 @@ public class TestValueSources extends Lu
Similarity saved = searcher.getSimilarity(true);
try {
- searcher.setSimilarity(new DefaultSimilarity());
+ searcher.setSimilarity(new ClassicSimilarity());
ValueSource vs = new QueryValueSource(new TermQuery(new Term("string","bar")), 42F);
assertHits(new FunctionQuery(vs), new float[] { 42F, 1.4054651F });
@@ -502,7 +502,7 @@ public class TestValueSources extends Lu
Similarity saved = searcher.getSimilarity(true);
try {
// no norm field (so agnostic to indexed similarity)
- searcher.setSimilarity(new DefaultSimilarity());
+ searcher.setSimilarity(new ClassicSimilarity());
ValueSource vs = new TFValueSource("bogus", "bogus", "text", new BytesRef("test"));
assertHits(new FunctionQuery(vs),
Modified: lucene/dev/branches/lucene6780/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java (original)
+++ lucene/dev/branches/lucene6780/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/MultiFieldQueryParser.java Fri Sep 18 15:13:49 2015
@@ -98,7 +98,7 @@ public class MultiFieldQueryParser exten
@Override
protected Query getFieldQuery(String field, String queryText, int slop) throws ParseException {
if (field == null) {
- List<BooleanClause> clauses = new ArrayList<>();
+ List<Query> clauses = new ArrayList<>();
for (int i = 0; i < fields.length; i++) {
Query q = super.getFieldQuery(fields[i], queryText, true);
if (q != null) {
@@ -111,12 +111,12 @@ public class MultiFieldQueryParser exten
}
}
q = applySlop(q,slop);
- clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD));
+ clauses.add(q);
}
}
if (clauses.size() == 0) // happens for stopwords
return null;
- return getBooleanQuery(clauses, true);
+ return getMultiFieldQuery(clauses);
}
Query q = super.getFieldQuery(field, queryText, true);
q = applySlop(q,slop);
@@ -144,7 +144,7 @@ public class MultiFieldQueryParser exten
@Override
protected Query getFieldQuery(String field, String queryText, boolean quoted) throws ParseException {
if (field == null) {
- List<BooleanClause> clauses = new ArrayList<>();
+ List<Query> clauses = new ArrayList<>();
for (int i = 0; i < fields.length; i++) {
Query q = super.getFieldQuery(fields[i], queryText, quoted);
if (q != null) {
@@ -156,12 +156,12 @@ public class MultiFieldQueryParser exten
q = new BoostQuery(q, boost.floatValue());
}
}
- clauses.add(new BooleanClause(q, BooleanClause.Occur.SHOULD));
+ clauses.add(q);
}
}
if (clauses.size() == 0) // happens for stopwords
return null;
- return getBooleanQuery(clauses, true);
+ return getMultiFieldQuery(clauses);
}
Query q = super.getFieldQuery(field, queryText, quoted);
return q;
@@ -172,12 +172,11 @@ public class MultiFieldQueryParser exten
protected Query getFuzzyQuery(String field, String termStr, float minSimilarity) throws ParseException
{
if (field == null) {
- List<BooleanClause> clauses = new ArrayList<>();
+ List<Query> clauses = new ArrayList<>();
for (int i = 0; i < fields.length; i++) {
- clauses.add(new BooleanClause(getFuzzyQuery(fields[i], termStr, minSimilarity),
- BooleanClause.Occur.SHOULD));
+ clauses.add(getFuzzyQuery(fields[i], termStr, minSimilarity));
}
- return getBooleanQuery(clauses, true);
+ return getMultiFieldQuery(clauses);
}
return super.getFuzzyQuery(field, termStr, minSimilarity);
}
@@ -186,12 +185,11 @@ public class MultiFieldQueryParser exten
protected Query getPrefixQuery(String field, String termStr) throws ParseException
{
if (field == null) {
- List<BooleanClause> clauses = new ArrayList<>();
+ List<Query> clauses = new ArrayList<>();
for (int i = 0; i < fields.length; i++) {
- clauses.add(new BooleanClause(getPrefixQuery(fields[i], termStr),
- BooleanClause.Occur.SHOULD));
+ clauses.add(getPrefixQuery(fields[i], termStr));
}
- return getBooleanQuery(clauses, true);
+ return getMultiFieldQuery(clauses);
}
return super.getPrefixQuery(field, termStr);
}
@@ -199,12 +197,11 @@ public class MultiFieldQueryParser exten
@Override
protected Query getWildcardQuery(String field, String termStr) throws ParseException {
if (field == null) {
- List<BooleanClause> clauses = new ArrayList<>();
+ List<Query> clauses = new ArrayList<>();
for (int i = 0; i < fields.length; i++) {
- clauses.add(new BooleanClause(getWildcardQuery(fields[i], termStr),
- BooleanClause.Occur.SHOULD));
+ clauses.add(getWildcardQuery(fields[i], termStr));
}
- return getBooleanQuery(clauses, true);
+ return getMultiFieldQuery(clauses);
}
return super.getWildcardQuery(field, termStr);
}
@@ -213,12 +210,11 @@ public class MultiFieldQueryParser exten
@Override
protected Query getRangeQuery(String field, String part1, String part2, boolean startInclusive, boolean endInclusive) throws ParseException {
if (field == null) {
- List<BooleanClause> clauses = new ArrayList<>();
+ List<Query> clauses = new ArrayList<>();
for (int i = 0; i < fields.length; i++) {
- clauses.add(new BooleanClause(getRangeQuery(fields[i], part1, part2, startInclusive, endInclusive),
- BooleanClause.Occur.SHOULD));
+ clauses.add(getRangeQuery(fields[i], part1, part2, startInclusive, endInclusive));
}
- return getBooleanQuery(clauses, true);
+ return getMultiFieldQuery(clauses);
}
return super.getRangeQuery(field, part1, part2, startInclusive, endInclusive);
}
@@ -229,15 +225,28 @@ public class MultiFieldQueryParser exten
protected Query getRegexpQuery(String field, String termStr)
throws ParseException {
if (field == null) {
- List<BooleanClause> clauses = new ArrayList<>();
+ List<Query> clauses = new ArrayList<>();
for (int i = 0; i < fields.length; i++) {
- clauses.add(new BooleanClause(getRegexpQuery(fields[i], termStr),
- BooleanClause.Occur.SHOULD));
+ clauses.add(getRegexpQuery(fields[i], termStr));
}
- return getBooleanQuery(clauses, true);
+ return getMultiFieldQuery(clauses);
}
return super.getRegexpQuery(field, termStr);
}
+
+ /** Creates a multifield query */
+ // TODO: investigate more general approach by default, e.g. DisjunctionMaxQuery?
+ protected Query getMultiFieldQuery(List<Query> queries) throws ParseException {
+ if (queries.isEmpty()) {
+ return null; // all clause words were filtered away by the analyzer.
+ }
+ BooleanQuery.Builder query = newBooleanQuery();
+ query.setDisableCoord(true);
+ for (Query sub : queries) {
+ query.add(sub, BooleanClause.Occur.SHOULD);
+ }
+ return query.build();
+ }
/**
* Parses a query which searches on the fields specified.
Modified: lucene/dev/branches/lucene6780/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java (original)
+++ lucene/dev/branches/lucene6780/lucene/queryparser/src/java/org/apache/lucene/queryparser/classic/QueryParserBase.java Fri Sep 18 15:13:49 2015
@@ -116,7 +116,7 @@ public abstract class QueryParserBase ex
try {
// TopLevelQuery is a Query followed by the end-of-input (EOF)
Query res = TopLevelQuery(field);
- return res!=null ? res : newBooleanQuery(false).build();
+ return res!=null ? res : newBooleanQuery().build();
}
catch (ParseException | TokenMgrError tme) {
// rethrow to include the original query:
@@ -688,30 +688,10 @@ public abstract class QueryParserBase ex
* @exception org.apache.lucene.queryparser.classic.ParseException throw in overridden method to disallow
*/
protected Query getBooleanQuery(List<BooleanClause> clauses) throws ParseException {
- return getBooleanQuery(clauses, false);
- }
-
- /**
- * Factory method for generating query, given a set of clauses.
- * By default creates a boolean query composed of clauses passed in.
- *
- * Can be overridden by extending classes, to modify query being
- * returned.
- *
- * @param clauses List that contains {@link org.apache.lucene.search.BooleanClause} instances
- * to join.
- * @param disableCoord true if coord scoring should be disabled.
- *
- * @return Resulting {@link org.apache.lucene.search.Query} object.
- * @exception org.apache.lucene.queryparser.classic.ParseException throw in overridden method to disallow
- */
- protected Query getBooleanQuery(List<BooleanClause> clauses, boolean disableCoord)
- throws ParseException
- {
if (clauses.size()==0) {
return null; // all clause words were filtered away by the analyzer.
}
- BooleanQuery.Builder query = newBooleanQuery(disableCoord);
+ BooleanQuery.Builder query = newBooleanQuery();
for(final BooleanClause clause: clauses) {
query.add(clause);
}
Modified: lucene/dev/branches/lucene6780/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BoostingTermBuilder.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BoostingTermBuilder.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BoostingTermBuilder.java (original)
+++ lucene/dev/branches/lucene6780/lucene/queryparser/src/java/org/apache/lucene/queryparser/xml/builders/BoostingTermBuilder.java Fri Sep 18 15:13:49 2015
@@ -3,8 +3,8 @@ package org.apache.lucene.queryparser.xm
import org.apache.lucene.index.Term;
import org.apache.lucene.queryparser.xml.DOMUtils;
import org.apache.lucene.queryparser.xml.ParserException;
-import org.apache.lucene.search.payloads.AveragePayloadFunction;
-import org.apache.lucene.search.payloads.PayloadScoreQuery;
+import org.apache.lucene.queries.payloads.AveragePayloadFunction;
+import org.apache.lucene.queries.payloads.PayloadScoreQuery;
import org.apache.lucene.search.spans.SpanBoostQuery;
import org.apache.lucene.search.spans.SpanQuery;
import org.apache.lucene.search.spans.SpanTermQuery;
Modified: lucene/dev/branches/lucene6780/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiAnalyzer.java
URL: http://svn.apache.org/viewvc/lucene/dev/branches/lucene6780/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiAnalyzer.java?rev=1703881&r1=1703880&r2=1703881&view=diff
==============================================================================
--- lucene/dev/branches/lucene6780/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiAnalyzer.java (original)
+++ lucene/dev/branches/lucene6780/lucene/queryparser/src/test/org/apache/lucene/queryparser/classic/TestMultiAnalyzer.java Fri Sep 18 15:13:49 2015
@@ -50,12 +50,12 @@ public class TestMultiAnalyzer extends B
assertEquals("\"foo foobar blah\"", qp.parse("\"foo foobar blah\"").toString());
// two tokens at the same position:
- assertEquals("(multi multi2) foo", qp.parse("multi foo").toString());
- assertEquals("foo (multi multi2)", qp.parse("foo multi").toString());
- assertEquals("(multi multi2) (multi multi2)", qp.parse("multi multi").toString());
- assertEquals("+(foo (multi multi2)) +(bar (multi multi2))",
+ assertEquals("Synonym(multi multi2) foo", qp.parse("multi foo").toString());
+ assertEquals("foo Synonym(multi multi2)", qp.parse("foo multi").toString());
+ assertEquals("Synonym(multi multi2) Synonym(multi multi2)", qp.parse("multi multi").toString());
+ assertEquals("+(foo Synonym(multi multi2)) +(bar Synonym(multi multi2))",
qp.parse("+(foo multi) +(bar multi)").toString());
- assertEquals("+(foo (multi multi2)) field:\"bar (multi multi2)\"",
+ assertEquals("+(foo Synonym(multi multi2)) field:\"bar (multi multi2)\"",
qp.parse("+(foo multi) field:\"bar multi\"").toString());
// phrases:
@@ -65,12 +65,12 @@ public class TestMultiAnalyzer extends B
qp.parse("\"foo multi foobar multi\"").toString());
// fields:
- assertEquals("(field:multi field:multi2) field:foo", qp.parse("field:multi field:foo").toString());
+ assertEquals("Synonym(field:multi field:multi2) field:foo", qp.parse("field:multi field:foo").toString());
assertEquals("field:\"(multi multi2) foo\"", qp.parse("field:\"multi foo\"").toString());
// three tokens at one position:
- assertEquals("triplemulti multi3 multi2", qp.parse("triplemulti").toString());
- assertEquals("foo (triplemulti multi3 multi2) foobar",
+ assertEquals("Synonym(multi2 multi3 triplemulti)", qp.parse("triplemulti").toString());
+ assertEquals("foo Synonym(multi2 multi3 triplemulti) foobar",
qp.parse("foo triplemulti foobar").toString());
// phrase with non-default slop:
@@ -89,7 +89,7 @@ public class TestMultiAnalyzer extends B
// non-default operator:
qp.setDefaultOperator(QueryParserBase.AND_OPERATOR);
- assertEquals("+(multi multi2) +foo", qp.parse("multi foo").toString());
+ assertEquals("+Synonym(multi multi2) +foo", qp.parse("multi foo").toString());
}