You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/01/03 10:51:49 UTC
[01/18] lucene-solr:jira/solr-11702: SOLR-11426: removed infostream
debug logging from master
Repository: lucene-solr
Updated Branches:
refs/heads/jira/solr-11702 19d2af7b5 -> 3e6afede4
SOLR-11426: removed infostream debug logging from master
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/6b368c04
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/6b368c04
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/6b368c04
Branch: refs/heads/jira/solr-11702
Commit: 6b368c04846fbb7ae769da6bee4506dcbf26c278
Parents: 137675a
Author: Erick Erickson <er...@apache.org>
Authored: Wed Dec 27 09:07:53 2017 -0800
Committer: Erick Erickson <er...@apache.org>
Committed: Wed Dec 27 09:07:53 2017 -0800
----------------------------------------------------------------------
solr/core/src/test/org/apache/solr/core/TestLazyCores.java | 7 -------
1 file changed, 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/6b368c04/solr/core/src/test/org/apache/solr/core/TestLazyCores.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestLazyCores.java b/solr/core/src/test/org/apache/solr/core/TestLazyCores.java
index afc6b5a..9d03ca4 100644
--- a/solr/core/src/test/org/apache/solr/core/TestLazyCores.java
+++ b/solr/core/src/test/org/apache/solr/core/TestLazyCores.java
@@ -787,8 +787,6 @@ public class TestLazyCores extends SolrTestCaseJ4 {
// Cores 2, 3, 6, 7, 8, 9 are transient
@Test
public void testNoCommit() throws Exception {
- String infoStream = System.getProperty("solr.tests.infostream");
- System.setProperty("solr.tests.infostream","true");
DirectUpdateHandler2.commitOnClose = true;
CoreContainer cc = init();
@@ -837,11 +835,6 @@ public class TestLazyCores extends SolrTestCaseJ4 {
} finally {
cc.shutdown();
}
- if (infoStream != null) {
- System.setProperty("solr.tests.infostream", infoStream);
- } else {
- System.clearProperty("solr.tests.infostream");
- }
}
private void add10(SolrCore core) throws IOException {
[05/18] lucene-solr:jira/solr-11702: Revert "LUCENE-8110: Fix
potential IndexOutOfBoundsException in *Classifier.getClasses(?, int)."
Posted by da...@apache.org.
Revert "LUCENE-8110: Fix potential IndexOutOfBoundsException in *Classifier.getClasses(?,int)."
This reverts commit af41d02eae6a58fd450553f9a09c9325ddf6e0ab.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/152d223b
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/152d223b
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/152d223b
Branch: refs/heads/jira/solr-11702
Commit: 152d223b3235459a30f6a8b1cb5331bec46dfb3d
Parents: 287062d
Author: Christine Poerschke <cp...@apache.org>
Authored: Thu Dec 28 15:11:39 2017 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Thu Dec 28 15:11:39 2017 +0000
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 ---
.../java/org/apache/lucene/classification/BM25NBClassifier.java | 1 -
.../org/apache/lucene/classification/KNearestFuzzyClassifier.java | 1 -
.../apache/lucene/classification/KNearestNeighborClassifier.java | 1 -
.../apache/lucene/classification/SimpleNaiveBayesClassifier.java | 1 -
.../document/SimpleNaiveBayesDocumentClassifier.java | 1 -
6 files changed, 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152d223b/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 84ba235..0da2cfe 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -104,9 +104,6 @@ Bug Fixes
* LUCENE-8077: Fixed bug in how CheckIndex verifies doc-value iterators.
(Xiaoshan Sun via Adrien Grand)
-* LUCENE-8110: Fix potential IndexOutOfBoundsException in *Classifier.getClasses(?,int).
- (Christine Poerschke)
-
======================= Lucene 7.2.0 =======================
API Changes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152d223b/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java
index 39a5b2f..1a74416 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java
@@ -124,7 +124,6 @@ public class BM25NBClassifier implements Classifier<BytesRef> {
public List<ClassificationResult<BytesRef>> getClasses(String text, int max) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = assignClassNormalizedList(text);
Collections.sort(assignedClasses);
- max = Math.min(max, assignedClasses.size());
return assignedClasses.subList(0, max);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152d223b/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
index 65effe4..cbd241b 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
@@ -140,7 +140,6 @@ public class KNearestFuzzyClassifier implements Classifier<BytesRef> {
TopDocs knnResults = knnSearch(text);
List<ClassificationResult<BytesRef>> assignedClasses = buildListFromTopDocs(knnResults);
Collections.sort(assignedClasses);
- max = Math.min(max, assignedClasses.size());
return assignedClasses.subList(0, max);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152d223b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
index f145912..f0391f4 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
@@ -162,7 +162,6 @@ public class KNearestNeighborClassifier implements Classifier<BytesRef> {
TopDocs knnResults = knnSearch(text);
List<ClassificationResult<BytesRef>> assignedClasses = buildListFromTopDocs(knnResults);
Collections.sort(assignedClasses);
- max = Math.min(max, assignedClasses.size());
return assignedClasses.subList(0, max);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152d223b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
index 3bb701f..3509df5 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
@@ -132,7 +132,6 @@ public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
public List<ClassificationResult<BytesRef>> getClasses(String text, int max) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = assignClassNormalizedList(text);
Collections.sort(assignedClasses);
- max = Math.min(max, assignedClasses.size());
return assignedClasses.subList(0, max);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/152d223b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
index 097fe0c..6bc8573 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
@@ -105,7 +105,6 @@ public class SimpleNaiveBayesDocumentClassifier extends SimpleNaiveBayesClassifi
public List<ClassificationResult<BytesRef>> getClasses(Document document, int max) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = assignNormClasses(document);
Collections.sort(assignedClasses);
- max = Math.min(max, assignedClasses.size());
return assignedClasses.subList(0, max);
}
[09/18] lucene-solr:jira/solr-11702: LUCENE-7993: Faster phrases if
total hit counts are not required.
Posted by da...@apache.org.
LUCENE-7993: Faster phrases if total hit counts are not required.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c95dc6d9
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c95dc6d9
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c95dc6d9
Branch: refs/heads/jira/solr-11702
Commit: c95dc6d95743f4a9a1ffe9baa04c3a9d1e3acdf9
Parents: b2f2481
Author: Adrien Grand <jp...@gmail.com>
Authored: Fri Dec 29 09:14:32 2017 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Fri Dec 29 10:06:00 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 ++
.../apache/lucene/search/ExactPhraseScorer.java | 23 ++++++++--
.../apache/lucene/search/MultiPhraseQuery.java | 14 +++---
.../org/apache/lucene/search/PhraseQuery.java | 14 +++---
.../apache/lucene/search/TestPhraseQuery.java | 47 ++++++++++++++++++++
5 files changed, 84 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c95dc6d9/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0da2cfe..ef7e005 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -61,6 +61,9 @@ Optimizations
* LUCENE-4100: Disjunctions now support faster collection of top hits when the
total hit count is not required. (Stefan Pohl, Adrien Grand, Robert Muir)
+* LUCENE-7993: Phrase queries are now faster if total hit counts are not
+ required. (Adrien Grand)
+
======================= Lucene 7.3.0 =======================
API Changes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c95dc6d9/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java b/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java
index 85a242e..f4a7ca7 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ExactPhraseScorer.java
@@ -43,15 +43,17 @@ final class ExactPhraseScorer extends Scorer {
private int freq;
private final Similarity.SimScorer docScorer;
- private final boolean needsScores;
+ private final boolean needsScores, needsTotalHitCount;
private float matchCost;
+ private float minCompetitiveScore;
ExactPhraseScorer(Weight weight, PhraseQuery.PostingsAndFreq[] postings,
- Similarity.SimScorer docScorer, boolean needsScores,
+ Similarity.SimScorer docScorer, ScoreMode scoreMode,
float matchCost) throws IOException {
super(weight);
this.docScorer = docScorer;
- this.needsScores = needsScores;
+ this.needsScores = scoreMode.needsScores();
+ this.needsTotalHitCount = scoreMode != ScoreMode.TOP_SCORES;
List<DocIdSetIterator> iterators = new ArrayList<>();
List<PostingsAndPosition> postingsAndPositions = new ArrayList<>();
@@ -66,10 +68,25 @@ final class ExactPhraseScorer extends Scorer {
}
@Override
+ public void setMinCompetitiveScore(float minScore) {
+ minCompetitiveScore = minScore;
+ }
+
+ @Override
public TwoPhaseIterator twoPhaseIterator() {
return new TwoPhaseIterator(conjunction) {
@Override
public boolean matches() throws IOException {
+ if (needsTotalHitCount == false && minCompetitiveScore > 0) {
+ int minFreq = postings[0].postings.freq();
+ for (int i = 1; i < postings.length; ++i) {
+ minFreq = Math.min(postings[i].postings.freq(), minFreq);
+ }
+ if (docScorer.score(docID(), minFreq) < minCompetitiveScore) {
+ // The maximum score we could get is less than the min competitive score
+ return false;
+ }
+ }
return phraseFreq() > 0;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c95dc6d9/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
index 44a5ad0..34361a7 100644
--- a/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/MultiPhraseQuery.java
@@ -185,13 +185,13 @@ public class MultiPhraseQuery extends Query {
private final Similarity similarity;
private final Similarity.SimWeight stats;
private final Map<Term,TermContext> termContexts = new HashMap<>();
- private final boolean needsScores;
+ private final ScoreMode scoreMode;
- public MultiPhraseWeight(IndexSearcher searcher, boolean needsScores, float boost)
+ public MultiPhraseWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost)
throws IOException {
super(MultiPhraseQuery.this);
- this.needsScores = needsScores;
- this.similarity = searcher.getSimilarity(needsScores);
+ this.scoreMode = scoreMode;
+ this.similarity = searcher.getSimilarity(scoreMode.needsScores());
final IndexReaderContext context = searcher.getTopReaderContext();
// compute idf
@@ -283,11 +283,11 @@ public class MultiPhraseQuery extends Query {
if (slop == 0) {
return new ExactPhraseScorer(this, postingsFreqs,
similarity.simScorer(stats, context),
- needsScores, totalMatchCost);
+ scoreMode, totalMatchCost);
} else {
return new SloppyPhraseScorer(this, postingsFreqs, slop,
similarity.simScorer(stats, context),
- needsScores, totalMatchCost);
+ scoreMode.needsScores(), totalMatchCost);
}
}
@@ -335,7 +335,7 @@ public class MultiPhraseQuery extends Query {
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
- return new MultiPhraseWeight(searcher, scoreMode.needsScores(), boost);
+ return new MultiPhraseWeight(searcher, scoreMode, boost);
}
/** Prints a user-readable version of this query. */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c95dc6d9/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
index e0b60be..3d359b4 100644
--- a/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/PhraseQuery.java
@@ -353,10 +353,10 @@ public class PhraseQuery extends Query {
private class PhraseWeight extends Weight {
private final Similarity similarity;
private final Similarity.SimWeight stats;
- private final boolean needsScores;
+ private final ScoreMode scoreMode;
private transient TermContext states[];
- public PhraseWeight(IndexSearcher searcher, boolean needsScores, float boost)
+ public PhraseWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost)
throws IOException {
super(PhraseQuery.this);
final int[] positions = PhraseQuery.this.getPositions();
@@ -365,8 +365,8 @@ public class PhraseQuery extends Query {
} else if (positions[0] != 0) {
throw new IllegalStateException("PhraseWeight requires that the first position is 0, call rewrite first");
}
- this.needsScores = needsScores;
- this.similarity = searcher.getSimilarity(needsScores);
+ this.scoreMode = scoreMode;
+ this.similarity = searcher.getSimilarity(scoreMode.needsScores());
final IndexReaderContext context = searcher.getTopReaderContext();
states = new TermContext[terms.length];
TermStatistics termStats[] = new TermStatistics[terms.length];
@@ -434,11 +434,11 @@ public class PhraseQuery extends Query {
if (slop == 0) { // optimize exact case
return new ExactPhraseScorer(this, postingsFreqs,
similarity.simScorer(stats, context),
- needsScores, totalMatchCost);
+ scoreMode, totalMatchCost);
} else {
return new SloppyPhraseScorer(this, postingsFreqs, slop,
similarity.simScorer(stats, context),
- needsScores, totalMatchCost);
+ scoreMode.needsScores(), totalMatchCost);
}
}
@@ -510,7 +510,7 @@ public class PhraseQuery extends Query {
@Override
public Weight createWeight(IndexSearcher searcher, ScoreMode scoreMode, float boost) throws IOException {
- return new PhraseWeight(searcher, scoreMode.needsScores(), boost);
+ return new PhraseWeight(searcher, scoreMode, boost);
}
/** Prints a user-readable version of this query. */
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c95dc6d9/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java b/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
index 2bbd0dd..eb31128 100644
--- a/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
+++ b/lucene/core/src/test/org/apache/lucene/search/TestPhraseQuery.java
@@ -19,6 +19,8 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Random;
@@ -33,7 +35,10 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.TextField;
+import org.apache.lucene.document.Field.Store;
+import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
+import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig.OpenMode;
import org.apache.lucene.index.RandomIndexWriter;
import org.apache.lucene.index.Term;
@@ -713,4 +718,46 @@ public class TestPhraseQuery extends LuceneTestCase {
builder.add(new Term("field", "three"), 4);
});
}
+
+ static String[] DOCS = new String[] {
+ "a b c d e f g h",
+ "b c b",
+ "c d d d e f g b",
+ "c b a b c",
+ "a a b b c c d d",
+ "a b c d a b c d a b c d"
+ };
+
+ public void testTopPhrases() throws IOException {
+ Directory dir = newDirectory();
+ IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
+ String[] docs = Arrays.copyOf(DOCS, DOCS.length);
+ Collections.shuffle(Arrays.asList(docs), random());
+ for (String value : DOCS) {
+ Document doc = new Document();
+ doc.add(new TextField("f", value, Store.NO));
+ w.addDocument(doc);
+ }
+ IndexReader r = DirectoryReader.open(w);
+ w.close();
+ IndexSearcher searcher = newSearcher(r);
+ for (Query query : Arrays.asList(
+ new PhraseQuery("f", "b", "c"), // common phrase
+ new PhraseQuery("f", "e", "f"), // always appear next to each other
+ new PhraseQuery("f", "d", "d") // repeated term
+ )) {
+ for (int topN = 1; topN <= 2; ++topN) {
+ TopScoreDocCollector collector1 = TopScoreDocCollector.create(topN, null, true);
+ searcher.search(query, collector1);
+ ScoreDoc[] hits1 = collector1.topDocs().scoreDocs;
+ TopScoreDocCollector collector2 = TopScoreDocCollector.create(topN, null, false);
+ searcher.search(query, collector2);
+ ScoreDoc[] hits2 = collector2.topDocs().scoreDocs;
+ assertTrue("" + query, hits1.length > 0);
+ CheckHits.checkEqual(query, hits1, hits2);
+ }
+ }
+ r.close();
+ dir.close();
+ }
}
[06/18] lucene-solr:jira/solr-11702: SOLR-11799: Fix NPE and class
cast exceptions in the TimeSeriesStream
Posted by da...@apache.org.
SOLR-11799: Fix NPE and class cast exceptions in the TimeSeriesStream
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/0c4fb312
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/0c4fb312
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/0c4fb312
Branch: refs/heads/jira/solr-11702
Commit: 0c4fb31205aa68e4d432ce3a67f5cac6bb5a9681
Parents: 152d223
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu Dec 28 11:14:31 2017 -0500
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu Dec 28 11:14:31 2017 -0500
----------------------------------------------------------------------
.../solrj/io/stream/TimeSeriesStream.java | 12 +++---
.../solrj/io/stream/StreamExpressionTest.java | 45 ++++++++++++++++++++
2 files changed, 52 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c4fb312/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
index 12b04de..3c2c2b5 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/TimeSeriesStream.java
@@ -389,11 +389,13 @@ public class TimeSeriesStream extends TupleStream implements Expressible {
for(Metric metric : _metrics) {
String identifier = metric.getIdentifier();
if(!identifier.startsWith("count(")) {
- double d = (double)bucket.get("facet_"+m);
- if(metric.outputLong) {
- t.put(identifier, Math.round(d));
- } else {
- t.put(identifier, d);
+ if(bucket.get("facet_"+m) != null) {
+ Number d = (Number) bucket.get("facet_" + m);
+ if (metric.outputLong) {
+ t.put(identifier, Math.round(d.doubleValue()));
+ } else {
+ t.put(identifier, d.doubleValue());
+ }
}
++m;
} else {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/0c4fb312/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index c564700..bc3dee5 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -5493,6 +5493,51 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertTrue(tuples.get(3).getDouble("max(price_f)").equals(400D));
assertTrue(tuples.get(3).getDouble("min(price_f)").equals(400D));
+
+ expr = "timeseries("+COLLECTIONORALIAS+", q=\"*:*\", start=\"2012-01-01T01:00:00.000Z\", " +
+ "end=\"2016-12-01T01:00:00.000Z\", " +
+ "gap=\"+1YEAR\", " +
+ "field=\"test_dt\", " +
+ "format=\"yyyy-MM\", " +
+ "count(*), sum(price_f), max(price_f), min(price_f))";
+ paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", expr);
+ paramsLoc.set("qt", "/stream");
+
+ solrStream = new SolrStream(url, paramsLoc);
+
+ solrStream.setStreamContext(context);
+ tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 5);
+ assertTrue(tuples.get(0).get("test_dt").equals("2012-01"));
+ assertTrue(tuples.get(0).getLong("count(*)").equals(0L));
+ assertTrue(tuples.get(0).getDouble("sum(price_f)") == null);
+ assertTrue(tuples.get(0).getDouble("max(price_f)") == null);
+ assertTrue(tuples.get(0).getDouble("min(price_f)") == null);
+
+ assertTrue(tuples.get(1).get("test_dt").equals("2013-01"));
+ assertTrue(tuples.get(1).getLong("count(*)").equals(100L));
+ assertTrue(tuples.get(1).getDouble("sum(price_f)").equals(10000D));
+ assertTrue(tuples.get(1).getDouble("max(price_f)").equals(100D));
+ assertTrue(tuples.get(1).getDouble("min(price_f)").equals(100D));
+
+ assertTrue(tuples.get(2).get("test_dt").equals("2014-01"));
+ assertTrue(tuples.get(2).getLong("count(*)").equals(50L));
+ assertTrue(tuples.get(2).getDouble("sum(price_f)").equals(25000D));
+ assertTrue(tuples.get(2).getDouble("max(price_f)").equals(500D));
+ assertTrue(tuples.get(2).getDouble("min(price_f)").equals(500D));
+
+ assertTrue(tuples.get(3).get("test_dt").equals("2015-01"));
+ assertTrue(tuples.get(3).getLong("count(*)").equals(50L));
+ assertTrue(tuples.get(3).getDouble("sum(price_f)").equals(15000D));
+ assertTrue(tuples.get(3).getDouble("max(price_f)").equals(300D));
+ assertTrue(tuples.get(3).getDouble("min(price_f)").equals(300D));
+
+ assertTrue(tuples.get(4).get("test_dt").equals("2016-01"));
+ assertTrue(tuples.get(4).getLong("count(*)").equals(50L));
+ assertTrue(tuples.get(4).getDouble("sum(price_f)").equals(20000D));
+ assertTrue(tuples.get(4).getDouble("max(price_f)").equals(400D));
+ assertTrue(tuples.get(4).getDouble("min(price_f)").equals(400D));
}
@Test
[10/18] lucene-solr:jira/solr-11702: LUCENE-8010: Fix similarities so
that they pass tests.
Posted by da...@apache.org.
LUCENE-8010: Fix similarities so that they pass tests.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/b2f24816
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/b2f24816
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/b2f24816
Branch: refs/heads/jira/solr-11702
Commit: b2f248164c1a3ddf213a56778d55c9252a022f18
Parents: 8e439a0
Author: Adrien Grand <jp...@gmail.com>
Authored: Thu Dec 28 17:44:01 2017 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Fri Dec 29 10:06:00 2017 +0100
----------------------------------------------------------------------
.../lucene/search/similarities/Axiomatic.java | 28 +++++++++++++++++++-
.../search/similarities/AxiomaticF1EXP.java | 2 +-
.../search/similarities/AxiomaticF1LOG.java | 2 +-
.../search/similarities/AxiomaticF3EXP.java | 2 +-
.../search/similarities/AxiomaticF3LOG.java | 2 +-
.../search/similarities/DistributionSPL.java | 26 +++++++++++++++---
.../lucene/search/similarities/LambdaDF.java | 7 ++++-
.../lucene/search/similarities/LambdaTTF.java | 7 ++++-
.../search/similarities/TestAxiomaticF1EXP.java | 4 ---
.../search/similarities/TestAxiomaticF1LOG.java | 4 ---
.../search/similarities/TestAxiomaticF3EXP.java | 7 +----
.../search/similarities/TestAxiomaticF3LOG.java | 7 +----
.../similarities/TestDistributionSPL.java | 4 ---
.../search/similarities/TestSimilarity2.java | 7 +++++
.../search/similarities/RandomSimilarity.java | 11 +++++---
15 files changed, 81 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java b/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
index 403773e..553fd42 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
@@ -17,6 +17,7 @@
package org.apache.lucene.search.similarities;
+import java.util.ArrayList;
import java.util.List;
import org.apache.lucene.search.Explanation;
@@ -101,11 +102,13 @@ public abstract class Axiomatic extends SimilarityBase {
@Override
public double score(BasicStats stats, double freq, double docLen) {
- return tf(stats, freq, docLen)
+ double score = tf(stats, freq, docLen)
* ln(stats, freq, docLen)
* tfln(stats, freq, docLen)
* idf(stats, freq, docLen)
- gamma(stats, freq, docLen);
+ // AxiomaticF3 similarities might produce negative scores due to their gamma component
+ return Math.max(0, score);
}
@Override
@@ -115,6 +118,29 @@ public abstract class Axiomatic extends SimilarityBase {
}
@Override
+ protected Explanation explain(
+ BasicStats stats, int doc, Explanation freq, double docLen) {
+ List<Explanation> subs = new ArrayList<>();
+ explain(subs, stats, doc, freq.getValue(), docLen);
+
+ double score = tf(stats, freq.getValue(), docLen)
+ * ln(stats, freq.getValue(), docLen)
+ * tfln(stats, freq.getValue(), docLen)
+ * idf(stats, freq.getValue(), docLen)
+ - gamma(stats, freq.getValue(), docLen);
+
+ Explanation explanation = Explanation.match((float) score,
+ "score(" + getClass().getSimpleName() + ", doc=" + doc + ", freq=" + freq.getValue() +"), computed from:",
+ subs);
+ if (score < 0) {
+ explanation = Explanation.match(0, "max of:",
+ Explanation.match(0, "Minimum legal score"),
+ explanation);
+ }
+ return explanation;
+ }
+
+ @Override
protected void explain(List<Explanation> subs, BasicStats stats, int doc,
double freq, double docLen) {
if (stats.getBoost() != 1.0d) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF1EXP.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF1EXP.java b/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF1EXP.java
index c026feb..ca5c42b 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF1EXP.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF1EXP.java
@@ -60,7 +60,7 @@ public class AxiomaticF1EXP extends Axiomatic {
*/
@Override
protected double tf(BasicStats stats, double freq, double docLen) {
- if (freq <= 0.0) return 0.0;
+ freq += 1; // otherwise gives negative scores for freqs < 1
return 1 + Math.log(1 + Math.log(freq));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF1LOG.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF1LOG.java b/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF1LOG.java
index 2e19255..6ef3587 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF1LOG.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF1LOG.java
@@ -52,7 +52,7 @@ public class AxiomaticF1LOG extends Axiomatic {
*/
@Override
protected double tf(BasicStats stats, double freq, double docLen) {
- if (freq <= 0.0) return 0.0;
+ freq += 1; // otherwise gives negative scores for freqs < 1
return 1 + Math.log(1 + Math.log(freq));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF3EXP.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF3EXP.java b/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF3EXP.java
index 635dc68..a54c754 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF3EXP.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF3EXP.java
@@ -58,7 +58,7 @@ public class AxiomaticF3EXP extends Axiomatic {
*/
@Override
protected double tf(BasicStats stats, double freq, double docLen) {
- if (freq <= 0.0) return 0.0;
+ freq += 1; // otherwise gives negative scores for freqs < 1
return 1 + Math.log(1 + Math.log(freq));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF3LOG.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF3LOG.java b/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF3LOG.java
index 4753e4e..194b70a 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF3LOG.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/AxiomaticF3LOG.java
@@ -47,7 +47,7 @@ public class AxiomaticF3LOG extends Axiomatic {
*/
@Override
protected double tf(BasicStats stats, double freq, double docLen) {
- if (freq <= 0.0) return 0.0;
+ freq += 1; // otherwise gives negative scores for freqs < 1
return 1 + Math.log(1 + Math.log(freq));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java b/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java
index fc05d72..2ab44df 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/DistributionSPL.java
@@ -34,11 +34,29 @@ public class DistributionSPL extends Distribution {
@Override
public final double score(BasicStats stats, double tfn, double lambda) {
- if (lambda == 1d) {
- lambda = 0.99d;
+ assert lambda != 1;
+
+ // tfn/(tfn+1) -> 1 - 1/(tfn+1), guaranteed to be non decreasing when tfn increases
+ double q = 1 - 1 / (tfn + 1);
+ if (q == 1) {
+ q = Math.nextDown(1.0);
+ }
+
+ double pow = Math.pow(lambda, q);
+ if (pow == lambda) {
+ // this can happen because of floating-point rounding
+ // but then we return infinity when taking the log, so we enforce
+ // that pow is different from lambda
+ if (lambda < 1) {
+ // x^y > x when x < 1 and y < 1
+ pow = Math.nextUp(lambda);
+ } else {
+ // x^y < x when x > 1 and y < 1
+ pow = Math.nextDown(lambda);
+ }
}
- return -Math.log(
- (Math.pow(lambda, (tfn / (tfn + 1))) - lambda) / (1 - lambda));
+
+ return -Math.log((pow - lambda) / (1 - lambda));
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaDF.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaDF.java b/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaDF.java
index 7dc320d..6b7dbb2 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaDF.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaDF.java
@@ -30,7 +30,12 @@ public class LambdaDF extends Lambda {
@Override
public final float lambda(BasicStats stats) {
- return (stats.getDocFreq()+1F) / (stats.getNumberOfDocuments()+1F);
+ float lambda = (float) ((stats.getDocFreq() + 1.0) / (stats.getNumberOfDocuments() + 1.0));
+ if (lambda == 1) {
+ // Distribution SPL cannot work with values of lambda that are equal to 1
+ lambda = Math.nextDown(lambda);
+ }
+ return lambda;
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaTTF.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaTTF.java b/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaTTF.java
index 6dc54a3..72eae4c 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaTTF.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/LambdaTTF.java
@@ -30,7 +30,12 @@ public class LambdaTTF extends Lambda {
@Override
public final float lambda(BasicStats stats) {
- return (stats.getTotalTermFreq()+1F) / (stats.getNumberOfDocuments()+1F);
+ float lambda = (float) ((stats.getTotalTermFreq() + 1.0) / (stats.getNumberOfDocuments() + 1.0));
+ if (lambda == 1) {
+ // Distribution SPL cannot work with values of lambda that are equal to 1
+ lambda = Math.nextUp(lambda);
+ }
+ return lambda;
}
public final Explanation explain(BasicStats stats) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF1EXP.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF1EXP.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF1EXP.java
index 16da903..9e2edf1 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF1EXP.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF1EXP.java
@@ -16,10 +16,6 @@
*/
package org.apache.lucene.search.similarities;
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
-
-// returns NaN scores for sloppy freqs < 1 (due to log without floor)
-@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8010")
public class TestAxiomaticF1EXP extends AxiomaticTestCase {
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF1LOG.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF1LOG.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF1LOG.java
index 88ad18e..0d6ba48 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF1LOG.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF1LOG.java
@@ -16,10 +16,6 @@
*/
package org.apache.lucene.search.similarities;
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
-
-// returns NaN scores for sloppy freqs < 1 (due to log without floor)
-@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8010")
public class TestAxiomaticF1LOG extends AxiomaticTestCase {
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF3EXP.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF3EXP.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF3EXP.java
index 69ab719..63ad87a 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF3EXP.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF3EXP.java
@@ -16,16 +16,11 @@
*/
package org.apache.lucene.search.similarities;
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
-
-// returns negative scores at least, but it (now) warns it has problems
-@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8010")
public class TestAxiomaticF3EXP extends AxiomaticTestCase {
@Override
protected final Similarity getAxiomaticModel(float s, int queryLen, float k) {
- // TODO: use the randomized parameters and not these hardcoded ones
- return new AxiomaticF3EXP(0.25f, 1);
+ return new AxiomaticF3EXP(s, queryLen);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF3LOG.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF3LOG.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF3LOG.java
index 6863277..cb92984 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF3LOG.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestAxiomaticF3LOG.java
@@ -16,16 +16,11 @@
*/
package org.apache.lucene.search.similarities;
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
-
-// returns negative scores at least, but it (now) warns it has problems
-@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8010")
public class TestAxiomaticF3LOG extends AxiomaticTestCase {
@Override
protected final Similarity getAxiomaticModel(float s, int queryLen, float k) {
- // TODO: use the randomized parameters and not these hardcoded ones
- return new AxiomaticF3LOG(0.25f, 1);
+ return new AxiomaticF3LOG(s, queryLen);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/core/src/test/org/apache/lucene/search/similarities/TestDistributionSPL.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestDistributionSPL.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestDistributionSPL.java
index 984915a..20a2b32 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestDistributionSPL.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestDistributionSPL.java
@@ -16,10 +16,6 @@
*/
package org.apache.lucene.search.similarities;
-import org.apache.lucene.util.LuceneTestCase.AwaitsFix;
-
-// scores go infinite, but it warns it has problems
-@AwaitsFix(bugUrl = "https://issues.apache.org/jira/browse/LUCENE-8010")
public class TestDistributionSPL extends DistributionTestCase {
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java
index 6fd38bd..49dc154 100644
--- a/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java
+++ b/lucene/core/src/test/org/apache/lucene/search/similarities/TestSimilarity2.java
@@ -54,6 +54,13 @@ public class TestSimilarity2 extends LuceneTestCase {
sims = new ArrayList<>();
sims.add(new ClassicSimilarity());
sims.add(new BM25Similarity());
+ sims.add(new BooleanSimilarity());
+ sims.add(new AxiomaticF1EXP());
+ sims.add(new AxiomaticF1LOG());
+ sims.add(new AxiomaticF2EXP());
+ sims.add(new AxiomaticF2LOG());
+ sims.add(new AxiomaticF3EXP(0.25f, 3));
+ sims.add(new AxiomaticF3LOG(0.25f, 3));
// TODO: not great that we dup this all with TestSimilarityBase
for (BasicModel basicModel : TestSimilarityBase.BASIC_MODELS) {
for (AfterEffect afterEffect : TestSimilarityBase.AFTER_EFFECTS) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/b2f24816/lucene/test-framework/src/java/org/apache/lucene/search/similarities/RandomSimilarity.java
----------------------------------------------------------------------
diff --git a/lucene/test-framework/src/java/org/apache/lucene/search/similarities/RandomSimilarity.java b/lucene/test-framework/src/java/org/apache/lucene/search/similarities/RandomSimilarity.java
index 444e8ef..0925aee 100644
--- a/lucene/test-framework/src/java/org/apache/lucene/search/similarities/RandomSimilarity.java
+++ b/lucene/test-framework/src/java/org/apache/lucene/search/similarities/RandomSimilarity.java
@@ -89,8 +89,12 @@ public class RandomSimilarity extends PerFieldSimilarityWrapper {
allSims = new ArrayList<>();
allSims.add(new ClassicSimilarity());
allSims.add(new BM25Similarity());
- // We cannot do this, because this similarity behaves in "non-traditional" ways:
- // allSims.add(new BooleanSimilarity());
+ allSims.add(new AxiomaticF1EXP());
+ allSims.add(new AxiomaticF1LOG());
+ allSims.add(new AxiomaticF2EXP());
+ allSims.add(new AxiomaticF2LOG());
+
+ allSims.add(new BooleanSimilarity());
for (BasicModel basicModel : BASIC_MODELS) {
for (AfterEffect afterEffect : AFTER_EFFECTS) {
for (Normalization normalization : NORMALIZATIONS) {
@@ -105,8 +109,7 @@ public class RandomSimilarity extends PerFieldSimilarityWrapper {
}
}
}
- /* TODO: enable Dirichlet
- allSims.add(new LMDirichletSimilarity()); */
+ allSims.add(new LMDirichletSimilarity());
allSims.add(new LMJelinekMercerSimilarity(0.1f));
allSims.add(new LMJelinekMercerSimilarity(0.7f));
for (Independence independence : INDEPENDENCE_MEASURES) {
[12/18] lucene-solr:jira/solr-11702: LUCENE-8109: Propagate
information about the minimum score in BooleanQuery.
Posted by da...@apache.org.
LUCENE-8109: Propagate information about the minimum score in BooleanQuery.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/33f421e7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/33f421e7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/33f421e7
Branch: refs/heads/jira/solr-11702
Commit: 33f421e798d47da6482ed5ff3c55efe7d8f350f8
Parents: bc9836f
Author: Adrien Grand <jp...@gmail.com>
Authored: Fri Dec 29 13:24:23 2017 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Fri Dec 29 14:35:05 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 5 +
.../apache/lucene/search/ConjunctionScorer.java | 17 +-
.../lucene/search/MaxScoreSumPropagator.java | 157 +++++++++++++++
.../apache/lucene/search/ReqOptSumScorer.java | 8 +-
.../org/apache/lucene/search/WANDScorer.java | 20 +-
.../search/TestMaxScoreSumPropagator.java | 198 +++++++++++++++++++
6 files changed, 380 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/33f421e7/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index a78770a..0ec795a 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -64,6 +64,11 @@ Optimizations
* LUCENE-7993: Phrase queries are now faster if total hit counts are not
required. (Adrien Grand)
+* LUCENE-8109: Boolean queries propagate information about the minimum
+ competitive score in order to make collection faster if there are disjunctions
+ or phrase queries as sub queries, which know how to leverage this information
+ to run faster. (Adrien Grand)
+
======================= Lucene 7.3.0 =======================
API Changes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/33f421e7/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java b/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java
index 2a8b6e6..75895fc 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ConjunctionScorer.java
@@ -27,6 +27,7 @@ class ConjunctionScorer extends Scorer {
final DocIdSetIterator disi;
final Scorer[] scorers;
final Collection<Scorer> required;
+ final MaxScoreSumPropagator maxScorePropagator;
/** Create a new {@link ConjunctionScorer}, note that {@code scorers} must be a subset of {@code required}. */
ConjunctionScorer(Weight weight, Collection<Scorer> required, Collection<Scorer> scorers) {
@@ -35,6 +36,7 @@ class ConjunctionScorer extends Scorer {
this.disi = ConjunctionDISI.intersectScorers(required);
this.scorers = scorers.toArray(new Scorer[scorers.size()]);
this.required = required;
+ this.maxScorePropagator = new MaxScoreSumPropagator(scorers);
}
@Override
@@ -63,22 +65,13 @@ class ConjunctionScorer extends Scorer {
@Override
public float maxScore() {
- // We iterate in the same order as #score() so no need to worry
- // about floating-point errors: we would do the same errors in
- // #score()
- double sum = 0d;
- for (Scorer scorer : scorers) {
- sum += scorer.maxScore();
- }
- return (float) sum;
+ return maxScorePropagator.maxScore();
}
@Override
public void setMinCompetitiveScore(float score) {
- if (scorers.length == 1) {
- scorers[0].setMinCompetitiveScore(score);
- }
- // TODO: handle the case when there are multiple scoring clauses too
+ // Propagate to sub clauses.
+ maxScorePropagator.setMinCompetitiveScore(score);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/33f421e7/lucene/core/src/java/org/apache/lucene/search/MaxScoreSumPropagator.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/MaxScoreSumPropagator.java b/lucene/core/src/java/org/apache/lucene/search/MaxScoreSumPropagator.java
new file mode 100644
index 0000000..27c8933
--- /dev/null
+++ b/lucene/core/src/java/org/apache/lucene/search/MaxScoreSumPropagator.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.util.Collection;
+
+import org.apache.lucene.util.InPlaceMergeSorter;
+import org.apache.lucene.util.MathUtil;
+
+/**
+ * Utility class to propagate scoring information in {@link BooleanQuery}, which
+ * compute the score as the sum of the scores of its matching clauses.
+ * This helps propagate information about the maximum produced score
+ */
+final class MaxScoreSumPropagator {
+
+ /**
+ * Return an array which, at index i, stores the sum of all entries of
+ * {@code v} except the one at index i.
+ */
+ private static double[] computeSumOfComplement(float[] v) {
+ // We do not use subtraction on purpose because it would defeat the
+ // upperbound formula that we use for sums.
+ // Naive approach would be O(n^2), but we can do O(n) by computing the
+ // sum for i<j and i>j and then sum them.
+ double[] sum1 = new double[v.length];
+ for (int i = 1; i < sum1.length; ++i) {
+ sum1[i] = sum1[i-1] + v[i-1];
+ }
+
+ double[] sum2 = new double[v.length];
+ for (int i = sum2.length - 2; i >= 0; --i) {
+ sum2[i] = sum2[i+1] + v[i+1];
+ }
+
+ double[] result = new double[v.length];
+ for (int i = 0; i < result.length; ++i) {
+ result[i] = sum1[i] + sum2[i];
+ }
+ return result;
+ }
+
+ private final int numClauses;
+ private final float maxScore;
+ private final Scorer[] scorers;
+ private final double[] sumOfOtherMaxScores;
+
+ MaxScoreSumPropagator(Collection<? extends Scorer> scorerList) {
+ numClauses = scorerList.size();
+ scorers = scorerList.toArray(new Scorer[numClauses]);
+ // We'll need max scores multiple times so we cache them
+ float[] maxScores = new float[numClauses];
+ for (int i = 0; i < numClauses; ++i) {
+ maxScores[i] = scorers[i].maxScore();
+ }
+ // Sort by decreasing max score
+ new InPlaceMergeSorter() {
+ @Override
+ protected void swap(int i, int j) {
+ Scorer tmp = scorers[i];
+ scorers[i] = scorers[j];
+ scorers[j] = tmp;
+ float tmpF = maxScores[i];
+ maxScores[i] = maxScores[j];
+ maxScores[j] = tmpF;
+ }
+ @Override
+ protected int compare(int i, int j) {
+ return Float.compare(maxScores[j], maxScores[i]);
+ }
+ }.sort(0, scorers.length);
+
+ sumOfOtherMaxScores = computeSumOfComplement(maxScores);
+ if (numClauses == 0) {
+ maxScore = 0;
+ } else {
+ maxScore = sumUpperBound(maxScores[0] + sumOfOtherMaxScores[0]);
+ }
+ }
+
+ public float maxScore() {
+ return maxScore;
+ }
+
+ public void setMinCompetitiveScore(float minScoreSum) {
+ for (int i = 0; i < numClauses; ++i) {
+ double sumOfOtherMaxScores = this.sumOfOtherMaxScores[i];
+ float minCompetitiveScore = getMinCompetitiveScore(minScoreSum, sumOfOtherMaxScores);
+ if (minCompetitiveScore <= 0) {
+ // given that scorers are sorted by decreasing max score, next scorers will
+ // have 0 as a minimum competitive score too
+ break;
+ }
+ scorers[i].setMinCompetitiveScore(minCompetitiveScore);
+ }
+ }
+
+ /**
+ * Return the minimum score that a Scorer must produce in order for a hit to
+ * be competitive.
+ */
+ private float getMinCompetitiveScore(float minScoreSum, double sumOfOtherMaxScores) {
+ assert numClauses > 0;
+ if (minScoreSum <= sumOfOtherMaxScores) {
+ return 0f;
+ }
+
+ // We need to find a value 'minScore' so that 'minScore + sumOfOtherMaxScores <= minScoreSum'
+ // TODO: is there an efficient way to find the greatest value that meets this requirement?
+ float minScore = (float) (minScoreSum - sumOfOtherMaxScores);
+ int iters = 0;
+ while (sumUpperBound(minScore + sumOfOtherMaxScores) > minScoreSum) {
+ // Important: use ulp of minScoreSum and not minScore to make sure that we
+ // converge quickly.
+ minScore -= Math.ulp(minScoreSum);
+ // this should converge in at most two iterations:
+ // - one because of the subtraction rounding error
+ // - one because of the error introduced by sumUpperBound
+ assert ++iters <= 2: iters;
+ }
+ return Math.max(minScore, 0f);
+ }
+
+ private float sumUpperBound(double sum) {
+ if (numClauses <= 2) {
+ // When there are only two clauses, the sum is always the same regardless
+ // of the order.
+ return (float) sum;
+ }
+
+ // The error of sums depends on the order in which values are summed up. In
+ // order to avoid this issue, we compute an upper bound of the value that
+ // the sum may take. If the max relative error is b, then it means that two
+ // sums are always within 2*b of each other.
+ // For conjunctions, we could skip this error factor since the order in which
+ // scores are summed up is predictable, but in practice, this wouldn't help
+ // much since the delta that is introduced by this error factor is usually
+ // cancelled by the float cast.
+ double b = MathUtil.sumRelativeErrorBound(numClauses);
+ return (float) ((1.0 + 2 * b) * sum);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/33f421e7/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java b/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
index 6b2ace8..b91d253 100644
--- a/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/ReqOptSumScorer.java
@@ -18,6 +18,7 @@ package org.apache.lucene.search;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
/** A Scorer for queries with a required part and an optional part.
@@ -35,6 +36,7 @@ class ReqOptSumScorer extends Scorer {
private boolean optIsRequired;
private final DocIdSetIterator approximation;
private final TwoPhaseIterator twoPhase;
+ final MaxScoreSumPropagator maxScorePropagator;
/** Construct a <code>ReqOptScorer</code>.
* @param reqScorer The required scorer. This must match.
@@ -51,6 +53,7 @@ class ReqOptSumScorer extends Scorer {
this.optScorer = optScorer;
this.reqMaxScore = reqScorer.maxScore();
+ this.maxScorePropagator = new MaxScoreSumPropagator(Arrays.asList(reqScorer, optScorer));
final TwoPhaseIterator reqTwoPhase = reqScorer.twoPhaseIterator();
this.optTwoPhase = optScorer.twoPhaseIterator();
@@ -208,14 +211,17 @@ class ReqOptSumScorer extends Scorer {
@Override
public float maxScore() {
- return reqScorer.maxScore() + optScorer.maxScore();
+ return maxScorePropagator.maxScore();
}
@Override
public void setMinCompetitiveScore(float minScore) {
+ // Potentially move to a conjunction
if (optIsRequired == false && minScore > reqMaxScore) {
optIsRequired = true;
}
+ // And also propagate to sub clauses.
+ maxScorePropagator.setMinCompetitiveScore(minScore);
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/33f421e7/lucene/core/src/java/org/apache/lucene/search/WANDScorer.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/WANDScorer.java b/lucene/core/src/java/org/apache/lucene/search/WANDScorer.java
index f5f647e..314ccf1 100644
--- a/lucene/core/src/java/org/apache/lucene/search/WANDScorer.java
+++ b/lucene/core/src/java/org/apache/lucene/search/WANDScorer.java
@@ -26,8 +26,6 @@ import java.util.Collection;
import java.util.List;
import java.util.OptionalInt;
-import org.apache.lucene.util.MathUtil;
-
/**
* This implements the WAND (Weak AND) algorithm for dynamic pruning
* described in "Efficient Query Evaluation using a Two-Level Retrieval
@@ -122,7 +120,7 @@ final class WANDScorer extends Scorer {
int tailSize;
final long cost;
- final float maxScore;
+ final MaxScoreSumPropagator maxScorePropagator;
WANDScorer(Weight weight, Collection<Scorer> scorers) {
super(weight);
@@ -145,12 +143,10 @@ final class WANDScorer extends Scorer {
// Use a scaling factor of 0 if all max scores are either 0 or +Infty
this.scalingFactor = scalingFactor.orElse(0);
- double maxScoreSum = 0;
for (Scorer scorer : scorers) {
DisiWrapper w = new DisiWrapper(scorer);
float maxScore = scorer.maxScore();
w.maxScore = scaleMaxScore(maxScore, this.scalingFactor);
- maxScoreSum += maxScore;
addLead(w);
}
@@ -159,12 +155,7 @@ final class WANDScorer extends Scorer {
cost += w.cost;
}
this.cost = cost;
- // The error of sums depends on the order in which values are summed up. In
- // order to avoid this issue, we compute an upper bound of the value that
- // the sum may take. If the max relative error is b, then it means that two
- // sums are always within 2*b of each other.
- double maxScoreRelativeErrorBound = MathUtil.sumRelativeErrorBound(scorers.size());
- this.maxScore = (float) ((1.0 + 2 * maxScoreRelativeErrorBound) * maxScoreSum);
+ this.maxScorePropagator = new MaxScoreSumPropagator(scorers);
}
// returns a boolean so that it can be called from assert
@@ -195,10 +186,15 @@ final class WANDScorer extends Scorer {
@Override
public void setMinCompetitiveScore(float minScore) {
+ // Let this disjunction know about the new min score so that it can skip
+ // over clauses that produce low scores.
assert minScore >= 0;
long scaledMinScore = scaleMinScore(minScore, scalingFactor);
assert scaledMinScore >= minCompetitiveScore;
minCompetitiveScore = scaledMinScore;
+
+ // And also propagate to sub clauses.
+ maxScorePropagator.setMinCompetitiveScore(minScore);
}
@Override
@@ -386,7 +382,7 @@ final class WANDScorer extends Scorer {
@Override
public float maxScore() {
- return maxScore;
+ return maxScorePropagator.maxScore();
}
@Override
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/33f421e7/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreSumPropagator.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreSumPropagator.java b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreSumPropagator.java
new file mode 100644
index 0000000..e52a490
--- /dev/null
+++ b/lucene/core/src/test/org/apache/lucene/search/TestMaxScoreSumPropagator.java
@@ -0,0 +1,198 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.search;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.lucene.util.TestUtil;
+
+public class TestMaxScoreSumPropagator extends LuceneTestCase {
+
+ private static class FakeScorer extends Scorer {
+
+ final float maxScore;
+ float minCompetitiveScore;
+
+ FakeScorer(float maxScore) {
+ super(null);
+ this.maxScore = maxScore;
+ }
+
+ @Override
+ public int docID() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public float score() throws IOException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public DocIdSetIterator iterator() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public float maxScore() {
+ return maxScore;
+ }
+
+ @Override
+ public void setMinCompetitiveScore(float minCompetitiveScore) {
+ this.minCompetitiveScore = minCompetitiveScore;
+ }
+ }
+
+ public void test0Clause() {
+ MaxScoreSumPropagator p = new MaxScoreSumPropagator(Collections.emptyList());
+ assertEquals(0f, p.maxScore(), 0f);
+ p.setMinCompetitiveScore(0f); // no exception
+ p.setMinCompetitiveScore(0.5f); // no exception
+ }
+
+ public void test1Clause() {
+ FakeScorer a = new FakeScorer(1);
+
+ MaxScoreSumPropagator p = new MaxScoreSumPropagator(Collections.singletonList(a));
+ assertEquals(1f, p.maxScore(), 0f);
+ p.setMinCompetitiveScore(0f);
+ assertEquals(0f, a.minCompetitiveScore, 0f);
+ p.setMinCompetitiveScore(0.5f);
+ assertEquals(0.5f, a.minCompetitiveScore, 0f);
+ p.setMinCompetitiveScore(1f);
+ assertEquals(1f, a.minCompetitiveScore, 0f);
+ }
+
+ public void test2Clauses() {
+ FakeScorer a = new FakeScorer(1);
+ FakeScorer b = new FakeScorer(2);
+
+ MaxScoreSumPropagator p = new MaxScoreSumPropagator(Arrays.asList(a, b));
+ assertEquals(3f, p.maxScore(), 0f);
+
+ p.setMinCompetitiveScore(1f);
+ assertEquals(0f, a.minCompetitiveScore, 0f);
+ assertEquals(0f, b.minCompetitiveScore, 0f);
+
+ p.setMinCompetitiveScore(2f);
+ assertEquals(0f, a.minCompetitiveScore, 0f);
+ assertEquals(1f, b.minCompetitiveScore, 0f);
+
+ p.setMinCompetitiveScore(2.5f);
+ assertEquals(0.5f, a.minCompetitiveScore, 0f);
+ assertEquals(1.5f, b.minCompetitiveScore, 0f);
+
+ p.setMinCompetitiveScore(3f);
+ assertEquals(1f, a.minCompetitiveScore, 0f);
+ assertEquals(2f, b.minCompetitiveScore, 0f);
+ }
+
+ public void test3Clauses() {
+ FakeScorer a = new FakeScorer(1);
+ FakeScorer b = new FakeScorer(2);
+ FakeScorer c = new FakeScorer(1.5f);
+
+ MaxScoreSumPropagator p = new MaxScoreSumPropagator(Arrays.asList(a, b, c));
+ assertEquals(4.5f, p.maxScore(), 0f);
+
+ p.setMinCompetitiveScore(1f);
+ assertEquals(0f, a.minCompetitiveScore, 0f);
+ assertEquals(0f, b.minCompetitiveScore, 0f);
+ assertEquals(0f, c.minCompetitiveScore, 0f);
+
+ p.setMinCompetitiveScore(2f);
+ assertEquals(0f, a.minCompetitiveScore, 0f);
+ assertEquals(0f, b.minCompetitiveScore, 0f);
+ assertEquals(0f, c.minCompetitiveScore, 0f);
+
+ p.setMinCompetitiveScore(3f);
+ assertEquals(0f, a.minCompetitiveScore, 0f);
+ assertEquals(0.5f, b.minCompetitiveScore, 0f);
+ assertEquals(0f, c.minCompetitiveScore, 0f);
+
+ p.setMinCompetitiveScore(4f);
+ assertEquals(0.5f, a.minCompetitiveScore, 0f);
+ assertEquals(1.5f, b.minCompetitiveScore, 0f);
+ assertEquals(1f, c.minCompetitiveScore, 0f);
+ }
+
+ public void test2ClausesRandomScore() {
+ for (int iter = 0; iter < 10; ++iter) {
+ FakeScorer a = new FakeScorer(random().nextFloat());
+ FakeScorer b = new FakeScorer(Math.nextUp(a.maxScore()) + random().nextFloat());
+
+ MaxScoreSumPropagator p = new MaxScoreSumPropagator(Arrays.asList(a, b));
+ assertEquals(a.maxScore() + b.maxScore(), p.maxScore(), 0f);
+ assertMinCompetitiveScore(Arrays.asList(a, b), p, Math.nextUp(a.maxScore()));
+ assertMinCompetitiveScore(Arrays.asList(a, b), p, (a.maxScore() + b.maxScore()) / 2);
+ assertMinCompetitiveScore(Arrays.asList(a, b), p, Math.nextDown(a.maxScore() + b.maxScore()));
+ assertMinCompetitiveScore(Arrays.asList(a, b), p, a.maxScore() + b.maxScore());
+ }
+ }
+
+ public void testNClausesRandomScore() {
+ for (int iter = 0; iter < 100; ++iter) {
+ List<FakeScorer> scorers = new ArrayList<>();
+ int numScorers = TestUtil.nextInt(random(), 3, 4 << random().nextInt(8));
+ double sumOfMaxScore = 0;
+ for (int i = 0; i < numScorers; ++i) {
+ float maxScore = random().nextFloat();
+ scorers.add(new FakeScorer(maxScore));
+ sumOfMaxScore += maxScore;
+ }
+
+ MaxScoreSumPropagator p = new MaxScoreSumPropagator(scorers);
+ assertTrue(p.maxScore() >= (float) sumOfMaxScore);
+ for (int i = 0; i < 10; ++i) {
+ final float minCompetitiveScore = random().nextFloat() * numScorers;
+ assertMinCompetitiveScore(scorers, p, minCompetitiveScore);
+ // reset
+ for (FakeScorer scorer : scorers) {
+ scorer.minCompetitiveScore = 0;
+ }
+ }
+ }
+ }
+
+ private void assertMinCompetitiveScore(Collection<FakeScorer> scorers, MaxScoreSumPropagator p, float minCompetitiveScore) {
+ p.setMinCompetitiveScore(minCompetitiveScore);
+
+ for (FakeScorer scorer : scorers) {
+ if (scorer.minCompetitiveScore == 0f) {
+ // no propagation is performed, still visiting all hits
+ break;
+ }
+ double scoreSum = scorer.minCompetitiveScore;
+ for (FakeScorer scorer2 : scorers) {
+ if (scorer2 != scorer) {
+ scoreSum += scorer2.maxScore();
+ }
+ }
+ assertTrue(
+ "scoreSum=" + scoreSum + ", minCompetitiveScore=" + minCompetitiveScore,
+ (float) scoreSum <= minCompetitiveScore);
+ }
+ }
+
+}
[04/18] lucene-solr:jira/solr-11702: SOLR-11793: Reduce code
duplication w.r.t. RestTestHarness(es).
Posted by da...@apache.org.
SOLR-11793: Reduce code duplication w.r.t. RestTestHarness(es).
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/287062df
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/287062df
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/287062df
Branch: refs/heads/jira/solr-11702
Commit: 287062df37b6e4a1f158fc53418baa9ae40eeeda
Parents: af41d02
Author: Christine Poerschke <cp...@apache.org>
Authored: Thu Dec 28 12:31:05 2017 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Thu Dec 28 12:31:05 2017 +0000
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../org/apache/solr/cloud/TestCryptoKeys.java | 33 +++----------
.../org/apache/solr/core/TestCustomStream.java | 23 +--------
.../apache/solr/core/TestDynamicLoading.java | 29 ++---------
.../apache/solr/handler/TestConfigReload.java | 25 +---------
.../handler/TestSolrConfigHandlerCloud.java | 28 ++---------
.../TestSolrConfigHandlerConcurrent.java | 22 +--------
.../solr/schema/TestBulkSchemaConcurrent.java | 32 +++---------
.../apache/solr/schema/TestCloudSchemaless.java | 43 ++++++++---------
.../processor/TestNamedUpdateProcessors.java | 51 ++++++++------------
.../cloud/AbstractFullDistribZkTestBase.java | 33 +++++++++++++
11 files changed, 104 insertions(+), 217 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/287062df/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index b951d0d..e14d771 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -91,6 +91,8 @@ Other Changes
* SOLR-11703: Solr Should Send Log Notifications if Ulimits are too low (Kevin Cowan via Erick Eickson)
+* SOLR-11793: Reduce code duplication w.r.t. RestTestHarness(es). (Christine Poerschke)
+
================== 7.2.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/287062df/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java b/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java
index 84bec90..b2044b1 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestCryptoKeys.java
@@ -19,12 +19,9 @@ package org.apache.solr.cloud;
import java.io.FileInputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.List;
import java.util.Map;
-import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.util.Utils;
@@ -41,24 +38,6 @@ import static java.util.Arrays.asList;
import static org.apache.solr.handler.TestSolrConfigHandlerCloud.compareValues;
public class TestCryptoKeys extends AbstractFullDistribZkTestBase {
- private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
-
- private void setupHarnesses() {
- for (final SolrClient client : clients) {
- RestTestHarness harness = new RestTestHarness(() -> ((HttpSolrClient) client).getBaseURL());
- restTestHarnesses.add(harness);
- }
- }
-
-
- @Override
- public void distribTearDown() throws Exception {
- super.distribTearDown();
- for (RestTestHarness r : restTestHarnesses) {
- r.close();
- }
- }
-
public TestCryptoKeys() {
super();
@@ -68,7 +47,7 @@ public class TestCryptoKeys extends AbstractFullDistribZkTestBase {
@Test
public void test() throws Exception {
System.setProperty("enable.runtime.lib", "true");
- setupHarnesses();
+ setupRestTestHarnesses();
String pk1sig = "G8LEW7uJ1is81Aqqfl3Sld3qDtOxPuVFeTLJHFJWecgDvUkmJNFXmf7nkHOVlXnDWahp1vqZf0W02VHXg37lBw==";
String pk2sig = "pCyBQycB/0YvLVZfKLDIIqG1tFwM/awqzkp2QNpO7R3ThTqmmrj11wEJFDRLkY79efuFuQPHt40EE7jrOKoj9jLNELsfEqvU3jw9sZKiDONY+rV9Bj9QPeW8Pgt+F9Y1";
String wrongKeySig = "xTk2hTipfpb+J5s4x3YZGOXkmHWtnJz05Vvd8RTm/Q1fbQVszR7vMk6dQ1URxX08fcg4HvxOo8g9bG2TSMOGjg==";
@@ -123,7 +102,7 @@ public class TestCryptoKeys extends AbstractFullDistribZkTestBase {
String payload = "{\n" +
"'create-requesthandler' : { 'name' : '/runtime', 'class': 'org.apache.solr.core.RuntimeLibReqHandler' , 'runtimeLib':true }" +
"}";
- RestTestHarness client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ RestTestHarness client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
@@ -137,7 +116,7 @@ public class TestCryptoKeys extends AbstractFullDistribZkTestBase {
payload = "{\n" +
"'add-runtimelib' : { 'name' : 'signedjar' ,'version':1}\n" +
"}";
- client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
@@ -156,7 +135,7 @@ public class TestCryptoKeys extends AbstractFullDistribZkTestBase {
payload = "{\n" +
"'update-runtimelib' : { 'name' : 'signedjar' ,'version':1, 'sig': 'QKqHtd37QN02iMW9UEgvAO9g9qOOuG5vEBNkbUsN7noc2hhXKic/ABFIOYJA9PKw61mNX2EmNFXOcO3WClYdSw=='}\n" +
"}";
- client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
@@ -175,7 +154,7 @@ public class TestCryptoKeys extends AbstractFullDistribZkTestBase {
payload = "{\n" +
"'update-runtimelib' : { 'name' : 'signedjar' ,'version':1, 'sig': 'YkTQgOtvcM/H/5EQdABGl3wjjrPhonAGlouIx59vppBy2cZEofX3qX1yZu5sPNRmJisNXEuhHN2149dxeUmk2Q=='}\n" +
"}";
- client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
@@ -196,7 +175,7 @@ public class TestCryptoKeys extends AbstractFullDistribZkTestBase {
payload = "{\n" +
"'update-runtimelib' : { 'name' : 'signedjar' ,'version':1, 'sig': 'VJPMTxDf8Km3IBj2B5HWkIOqeM/o+HHNobOYCNA3WjrEVfOMZbMMqS1Lo7uLUUp//RZwOGkOhrUhuPNY1z2CGEIKX2/m8VGH64L14d52oSvFiwhoTDDuuyjW1TFGu35D'}\n" +
"}";
- client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/287062df/solr/core/src/test/org/apache/solr/core/TestCustomStream.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestCustomStream.java b/solr/core/src/test/org/apache/solr/core/TestCustomStream.java
index 0a70374..856c865 100644
--- a/solr/core/src/test/org/apache/solr/core/TestCustomStream.java
+++ b/solr/core/src/test/org/apache/solr/core/TestCustomStream.java
@@ -17,11 +17,8 @@
package org.apache.solr.core;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.List;
-import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.handler.TestBlobHandler;
@@ -33,32 +30,16 @@ import org.junit.Test;
* Created by caomanhdat on 6/3/16.
*/
public class TestCustomStream extends AbstractFullDistribZkTestBase {
- private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
-
- private void setupHarnesses() {
- for (final SolrClient client : clients) {
- RestTestHarness harness = new RestTestHarness(() -> ((HttpSolrClient)client).getBaseURL());
- restTestHarnesses.add(harness);
- }
- }
@BeforeClass
public static void enableRuntimeLib() throws Exception {
System.setProperty("enable.runtime.lib", "true");
}
- @Override
- public void distribTearDown() throws Exception {
- super.distribTearDown();
- for (RestTestHarness r : restTestHarnesses) {
- r.close();
- }
- }
-
@Test
public void testDynamicLoadingCustomStream() throws Exception {
System.setProperty("enable.runtime.lib", "true");
- setupHarnesses();
+ setupRestTestHarnesses();
String blobName = "colltest";
@@ -73,7 +54,7 @@ public class TestCustomStream extends AbstractFullDistribZkTestBase {
"'create-expressible' : { 'name' : 'hello', 'class': 'org.apache.solr.core.HelloStream' }\n" +
"}";
- RestTestHarness client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ RestTestHarness client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client,"/config",payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/287062df/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java b/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java
index 4dca763..0a27269 100644
--- a/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java
+++ b/solr/core/src/test/org/apache/solr/core/TestDynamicLoading.java
@@ -16,7 +16,6 @@
*/
package org.apache.solr.core;
-import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.handler.TestBlobHandler;
@@ -30,9 +29,7 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Arrays;
-import java.util.List;
import java.util.Map;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
@@ -41,32 +38,16 @@ import static java.util.Arrays.asList;
import static org.apache.solr.handler.TestSolrConfigHandlerCloud.compareValues;
public class TestDynamicLoading extends AbstractFullDistribZkTestBase {
- private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
-
- private void setupHarnesses() {
- for (final SolrClient client : clients) {
- RestTestHarness harness = new RestTestHarness(() -> ((HttpSolrClient)client).getBaseURL());
- restTestHarnesses.add(harness);
- }
- }
@BeforeClass
public static void enableRuntimeLib() throws Exception {
System.setProperty("enable.runtime.lib", "true");
}
- @Override
- public void distribTearDown() throws Exception {
- super.distribTearDown();
- for (RestTestHarness r : restTestHarnesses) {
- r.close();
- }
- }
-
@Test
public void testDynamicLoading() throws Exception {
System.setProperty("enable.runtime.lib", "true");
- setupHarnesses();
+ setupRestTestHarnesses();
String blobName = "colltest";
boolean success = false;
@@ -78,7 +59,7 @@ public class TestDynamicLoading extends AbstractFullDistribZkTestBase {
String payload = "{\n" +
"'add-runtimelib' : { 'name' : 'colltest' ,'version':1}\n" +
"}";
- RestTestHarness client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ RestTestHarness client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
@@ -92,7 +73,7 @@ public class TestDynamicLoading extends AbstractFullDistribZkTestBase {
"'create-requesthandler' : { 'name' : '/test1', 'class': 'org.apache.solr.core.BlobStoreTestRequestHandler' ,registerPath: '/solr,/v2', 'runtimeLib' : true }\n" +
"}";
- client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client,"/config",payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
@@ -156,7 +137,7 @@ public class TestDynamicLoading extends AbstractFullDistribZkTestBase {
"'create-searchcomponent' : { 'name' : 'get', 'class': 'org.apache.solr.core.RuntimeLibSearchComponent' , 'runtimeLib':true }," +
"'create-queryResponseWriter' : { 'name' : 'json1', 'class': 'org.apache.solr.core.RuntimeLibResponseWriter' , 'runtimeLib':true }" +
"}";
- client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
Map result = TestSolrConfigHandler.testForResponseElement(client,
@@ -197,7 +178,7 @@ public class TestDynamicLoading extends AbstractFullDistribZkTestBase {
payload = "{\n" +
"'update-runtimelib' : { 'name' : 'colltest' ,'version':2}\n" +
"}";
- client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/287062df/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java b/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java
index ab7a74d..bac990c 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestConfigReload.java
@@ -31,8 +31,6 @@ import java.util.concurrent.TimeUnit;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
@@ -42,7 +40,6 @@ import org.apache.solr.common.cloud.ZkConfigManager;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.core.SolrConfig;
-import org.apache.solr.util.RestTestHarness;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
import org.apache.zookeeper.data.Stat;
@@ -55,32 +52,14 @@ import org.slf4j.LoggerFactory;
public class TestConfigReload extends AbstractFullDistribZkTestBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
-
- private void setupHarnesses() {
- for (final SolrClient client : clients) {
- RestTestHarness harness = new RestTestHarness(() -> ((HttpSolrClient)client).getBaseURL());
- restTestHarnesses.add(harness);
- }
- }
-
- @Override
- public void distribTearDown() throws Exception {
- super.distribTearDown();
- for (RestTestHarness h : restTestHarnesses) {
- h.close();
- }
- }
@Test
public void test() throws Exception {
- setupHarnesses();
+ setupRestTestHarnesses();
try {
reloadTest();
} finally {
- for (RestTestHarness h : restTestHarnesses) {
- h.close();
- }
+ closeRestTestHarnesses();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/287062df/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerCloud.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerCloud.java b/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerCloud.java
index f143c02..944fa65 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerCloud.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerCloud.java
@@ -23,9 +23,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.function.Predicate;
-import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.LukeRequest;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.common.cloud.DocCollection;
@@ -44,26 +42,10 @@ import static java.util.Arrays.asList;
import static org.apache.solr.handler.TestBlobHandler.getAsString;
public class TestSolrConfigHandlerCloud extends AbstractFullDistribZkTestBase {
- private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
-
- private void setupHarnesses() {
- for (final SolrClient client : clients) {
- RestTestHarness harness = new RestTestHarness(((HttpSolrClient) client)::getBaseURL);
- restTestHarnesses.add(harness);
- }
- }
-
- @Override
- public void distribTearDown() throws Exception {
- super.distribTearDown();
- for (RestTestHarness r : restTestHarnesses) {
- r.close();
- }
- }
@Test
public void test() throws Exception {
- setupHarnesses();
+ setupRestTestHarnesses();
testReqHandlerAPIs();
testReqParams();
testAdminPath();
@@ -71,7 +53,7 @@ public class TestSolrConfigHandlerCloud extends AbstractFullDistribZkTestBase {
private void testAdminPath() throws Exception{
String testServerBaseUrl = getRandomServer(cloudClient,"collection1");
- RestTestHarness writeHarness = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ RestTestHarness writeHarness = randomRestTestHarness();
String payload = "{\n" +
"'create-requesthandler' : { 'name' : '/admin/luke', " +
"'class': 'org.apache.solr.handler.DumpRequestHandler'}}";
@@ -93,7 +75,7 @@ public class TestSolrConfigHandlerCloud extends AbstractFullDistribZkTestBase {
private void testReqHandlerAPIs() throws Exception {
String testServerBaseUrl = getRandomServer(cloudClient,"collection1");
- RestTestHarness writeHarness = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ RestTestHarness writeHarness = randomRestTestHarness();
TestSolrConfigHandler.reqhandlertests(writeHarness, testServerBaseUrl , cloudClient);
}
@@ -115,7 +97,7 @@ public class TestSolrConfigHandlerCloud extends AbstractFullDistribZkTestBase {
urls.add(""+replica.get(ZkStateReader.BASE_URL_PROP) + "/"+replica.get(ZkStateReader.CORE_NAME_PROP));
}
- RestTestHarness writeHarness = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ RestTestHarness writeHarness = randomRestTestHarness();
String payload = " {\n" +
" 'set' : {'x': {" +
" 'a':'A val',\n" +
@@ -190,7 +172,7 @@ public class TestSolrConfigHandlerCloud extends AbstractFullDistribZkTestBase {
- writeHarness = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ writeHarness = randomRestTestHarness();
payload = " {\n" +
" 'set' : {'y':{\n" +
" 'c':'CY val',\n" +
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/287062df/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java b/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
index d077c84..bf982ad 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestSolrConfigHandlerConcurrent.java
@@ -34,9 +34,7 @@ import org.apache.http.HttpEntity;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.util.EntityUtils;
import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
@@ -55,29 +53,13 @@ import org.slf4j.LoggerFactory;
public class TestSolrConfigHandlerConcurrent extends AbstractFullDistribZkTestBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
-
- private void setupHarnesses() {
- for (final SolrClient client : clients) {
- RestTestHarness harness = new RestTestHarness(() -> ((HttpSolrClient)client).getBaseURL());
- restTestHarnesses.add(harness);
- }
- }
-
- @Override
- public void distribTearDown() throws Exception {
- super.distribTearDown();
- for (RestTestHarness h : restTestHarnesses) {
- h.close();
- }
- }
@Test
public void test() throws Exception {
Map editable_prop_map = (Map) Utils.fromJSONResource("EditableSolrConfigAttributes.json");
Map caches = (Map) editable_prop_map.get("query");
- setupHarnesses();
+ setupRestTestHarnesses();
List<Thread> threads = new ArrayList<>(caches.size());
final List<List> collectErrors = new ArrayList<>();
@@ -129,7 +111,7 @@ public class TestSolrConfigHandlerConcurrent extends AbstractFullDistribZkTestBa
Set<String> errmessages = new HashSet<>();
for(int i =1;i<2;i++){//make it ahigher number
- RestTestHarness publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
+ RestTestHarness publisher = randomRestTestHarness(r);
String response;
String val1;
String val2;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/287062df/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java b/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java
index b3958f8..2a079f9 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestBulkSchemaConcurrent.java
@@ -30,8 +30,6 @@ import java.util.Set;
import java.util.concurrent.TimeUnit;
import org.apache.solr.SolrTestCaseJ4;
-import org.apache.solr.client.solrj.SolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
@@ -45,7 +43,6 @@ import org.slf4j.LoggerFactory;
public class TestBulkSchemaConcurrent extends AbstractFullDistribZkTestBase {
private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
- private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
@BeforeClass
public static void initSysProperties() {
@@ -57,26 +54,11 @@ public class TestBulkSchemaConcurrent extends AbstractFullDistribZkTestBase {
return "solrconfig-managed-schema.xml";
}
- private void setupHarnesses() {
- for (final SolrClient client : clients) {
- RestTestHarness harness = new RestTestHarness(() -> ((HttpSolrClient)client).getBaseURL());
- restTestHarnesses.add(harness);
- }
- }
-
- @Override
- public void distribTearDown() throws Exception {
- super.distribTearDown();
- for (RestTestHarness r : restTestHarnesses) {
- r.close();
- }
- }
-
@Test
public void test() throws Exception {
final int threadCount = 5;
- setupHarnesses();
+ setupRestTestHarnesses();
Thread[] threads = new Thread[threadCount];
final List<List> collectErrors = new ArrayList<>();
@@ -148,7 +130,7 @@ public class TestBulkSchemaConcurrent extends AbstractFullDistribZkTestBase {
payload = payload.replace("replaceDynamicCopyFieldDest", dynamicCopyFldDest);
payload = payload.replace("myNewFieldTypeName", newFieldTypeName);
- RestTestHarness publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
+ RestTestHarness publisher = randomRestTestHarness(r);
String response = publisher.post("/schema", SolrTestCaseJ4.json(payload));
Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
Object errors = map.get("errors");
@@ -159,7 +141,7 @@ public class TestBulkSchemaConcurrent extends AbstractFullDistribZkTestBase {
//get another node
Set<String> errmessages = new HashSet<>();
- RestTestHarness harness = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
+ RestTestHarness harness = randomRestTestHarness(r);
try {
long startTime = System.nanoTime();
long maxTimeoutMillis = 100000;
@@ -218,7 +200,7 @@ public class TestBulkSchemaConcurrent extends AbstractFullDistribZkTestBase {
payload = payload.replace("replaceDynamicField", dynamicFldName);
payload = payload.replace("myNewFieldTypeName", newFieldTypeName);
- RestTestHarness publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
+ RestTestHarness publisher = randomRestTestHarness(r);
String response = publisher.post("/schema", SolrTestCaseJ4.json(payload));
Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
Object errors = map.get("errors");
@@ -229,7 +211,7 @@ public class TestBulkSchemaConcurrent extends AbstractFullDistribZkTestBase {
//get another node
Set<String> errmessages = new HashSet<>();
- RestTestHarness harness = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
+ RestTestHarness harness = randomRestTestHarness(r);
try {
long startTime = System.nanoTime();
long maxTimeoutMillis = 100000;
@@ -280,7 +262,7 @@ public class TestBulkSchemaConcurrent extends AbstractFullDistribZkTestBase {
payload = payload.replace("replaceDynamicCopyFieldDest",dynamicCopyFldDest);
payload = payload.replace("myNewFieldTypeName", newFieldTypeName);
- RestTestHarness publisher = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
+ RestTestHarness publisher = randomRestTestHarness(r);
String response = publisher.post("/schema", SolrTestCaseJ4.json(payload));
Map map = (Map) ObjectBuilder.getVal(new JSONParser(new StringReader(response)));
Object errors = map.get("errors");
@@ -291,7 +273,7 @@ public class TestBulkSchemaConcurrent extends AbstractFullDistribZkTestBase {
//get another node
Set<String> errmessages = new HashSet<>();
- RestTestHarness harness = restTestHarnesses.get(r.nextInt(restTestHarnesses.size()));
+ RestTestHarness harness = randomRestTestHarness(r);
try {
long startTime = System.nanoTime();
long maxTimeoutMillis = 100000;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/287062df/solr/core/src/test/org/apache/solr/schema/TestCloudSchemaless.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/schema/TestCloudSchemaless.java b/solr/core/src/test/org/apache/solr/schema/TestCloudSchemaless.java
index b479024..c8f65b7 100644
--- a/solr/core/src/test/org/apache/solr/schema/TestCloudSchemaless.java
+++ b/solr/core/src/test/org/apache/solr/schema/TestCloudSchemaless.java
@@ -18,7 +18,6 @@ package org.apache.solr.schema;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
-import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.cloud.AbstractFullDistribZkTestBase;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
@@ -39,6 +38,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;
+import java.util.function.UnaryOperator;
/**
* Tests a schemaless collection configuration with SolrCloud
@@ -51,9 +51,7 @@ public class TestCloudSchemaless extends AbstractFullDistribZkTestBase {
@After
public void teardDown() throws Exception {
super.tearDown();
- for (RestTestHarness h : restTestHarnesses) {
- h.close();
- }
+ closeRestTestHarnesses();
}
public TestCloudSchemaless() {
@@ -75,15 +73,6 @@ public class TestCloudSchemaless extends AbstractFullDistribZkTestBase {
return extraServlets;
}
- private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
-
- private void setupHarnesses() {
- for (final SolrClient client : clients) {
- RestTestHarness harness = new RestTestHarness(() -> ((HttpSolrClient)client).getBaseURL());
- restTestHarnesses.add(harness);
- }
- }
-
private String[] getExpectedFieldResponses(int numberOfDocs) {
String[] expectedAddFields = new String[1 + numberOfDocs];
expectedAddFields[0] = SUCCESS_XPATH;
@@ -99,7 +88,7 @@ public class TestCloudSchemaless extends AbstractFullDistribZkTestBase {
@Test
@ShardsFixed(num = 8)
public void test() throws Exception {
- setupHarnesses();
+ setupRestTestHarnesses();
// First, add a bunch of documents in a single update with the same new field.
// This tests that the replicas properly handle schema additions.
@@ -127,16 +116,24 @@ public class TestCloudSchemaless extends AbstractFullDistribZkTestBase {
String [] expectedFields = getExpectedFieldResponses(docNumber);
// Check that all the fields were added
- for (RestTestHarness client : restTestHarnesses) {
- String request = "/schema/fields?wt=xml";
- String response = client.query(request);
- String result = BaseTestHarness.validateXPath(response, expectedFields);
- if (result != null) {
- String msg = "QUERY FAILED: xpath=" + result + " request=" + request + " response=" + response;
- log.error(msg);
- fail(msg);
+ forAllRestTestHarnesses( new UnaryOperator<RestTestHarness>() {
+ @Override
+ public RestTestHarness apply(RestTestHarness client) {
+ try {
+ String request = "/schema/fields?wt=xml";
+ String response = client.query(request);
+ String result = BaseTestHarness.validateXPath(response, expectedFields);
+ if (result != null) {
+ String msg = "QUERY FAILED: xpath=" + result + " request=" + request + " response=" + response;
+ log.error(msg);
+ fail(msg);
+ }
+ } catch (Exception ex) {
+ fail("Caught exception: "+ex);
+ }
+ return client;
}
- }
+ });
// Now, let's ensure that writing the same field with two different types fails
int failTrials = 50;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/287062df/solr/core/src/test/org/apache/solr/update/processor/TestNamedUpdateProcessors.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/update/processor/TestNamedUpdateProcessors.java b/solr/core/src/test/org/apache/solr/update/processor/TestNamedUpdateProcessors.java
index 8db4f96..41ed9c7 100644
--- a/solr/core/src/test/org/apache/solr/update/processor/TestNamedUpdateProcessors.java
+++ b/solr/core/src/test/org/apache/solr/update/processor/TestNamedUpdateProcessors.java
@@ -20,14 +20,12 @@ import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
-import java.util.List;
+import java.util.function.UnaryOperator;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
-import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrQuery;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
import org.apache.solr.client.solrj.request.UpdateRequest;
@@ -44,28 +42,11 @@ import org.apache.solr.util.SimplePostTool;
import org.junit.Test;
public class TestNamedUpdateProcessors extends AbstractFullDistribZkTestBase {
- private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
-
- private void setupHarnesses() {
- for (final SolrClient client : clients) {
- RestTestHarness harness = new RestTestHarness(() -> ((HttpSolrClient) client).getBaseURL());
- restTestHarnesses.add(harness);
- }
- }
-
-
- @Override
- public void distribTearDown() throws Exception {
- super.distribTearDown();
- for (RestTestHarness r : restTestHarnesses) {
- r.close();
- }
- }
@Test
public void test() throws Exception {
System.setProperty("enable.runtime.lib", "true");
- setupHarnesses();
+ setupRestTestHarnesses();
String blobName = "colltest";
@@ -81,7 +62,7 @@ public class TestNamedUpdateProcessors extends AbstractFullDistribZkTestBase {
String payload = "{\n" +
"'add-runtimelib' : { 'name' : 'colltest' ,'version':1}\n" +
"}";
- RestTestHarness client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ RestTestHarness client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
TestSolrConfigHandler.testForResponseElement(client,
null,
@@ -96,16 +77,24 @@ public class TestNamedUpdateProcessors extends AbstractFullDistribZkTestBase {
"'create-updateprocessor' : { 'name' : 'maxFld', 'class': 'solr.MaxFieldValueUpdateProcessorFactory', 'fieldName':'mul_s'} \n" +
"}";
- client = restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ client = randomRestTestHarness();
TestSolrConfigHandler.runConfigCommand(client, "/config", payload);
- for (RestTestHarness restTestHarness : restTestHarnesses) {
- TestSolrConfigHandler.testForResponseElement(restTestHarness,
- null,
- "/config/overlay",
- null,
- Arrays.asList("overlay", "updateProcessor", "firstFld", "fieldName"),
- "test_s", 10);
- }
+ forAllRestTestHarnesses( new UnaryOperator<RestTestHarness>() {
+ @Override
+ public RestTestHarness apply(RestTestHarness restTestHarness) {
+ try {
+ TestSolrConfigHandler.testForResponseElement(restTestHarness,
+ null,
+ "/config/overlay",
+ null,
+ Arrays.asList("overlay", "updateProcessor", "firstFld", "fieldName"),
+ "test_s", 10);
+ } catch (Exception ex) {
+ fail("Caught exception: "+ex);
+ }
+ return restTestHarness;
+ }
+ });
SolrInputDocument doc = new SolrInputDocument();
doc.addField("id", "123");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/287062df/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
----------------------------------------------------------------------
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
index a597f3d..9f0ff20 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
@@ -33,9 +33,11 @@ import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Properties;
+import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.UnaryOperator;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.client.solrj.SolrClient;
@@ -83,6 +85,7 @@ import org.apache.solr.update.SolrCmdDistributor;
import org.apache.solr.update.SolrIndexWriter;
import org.apache.solr.util.RTimer;
import org.apache.solr.util.RefCounted;
+import org.apache.solr.util.RestTestHarness;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.KeeperException;
@@ -136,6 +139,8 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
private boolean cloudInit;
protected boolean useJettyDataDir = true;
+ private List<RestTestHarness> restTestHarnesses = new ArrayList<>();
+
protected Map<URI,SocketProxy> proxies = new HashMap<>();
public static class CloudJettyRunner {
@@ -1552,6 +1557,7 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
if (VERBOSE || printLayoutOnTearDown) {
super.printLayout();
}
+ closeRestTestHarnesses(); // TODO: close here or later?
if (commonCloudSolrClient != null) {
commonCloudSolrClient.close();
}
@@ -2239,4 +2245,31 @@ public abstract class AbstractFullDistribZkTestBase extends AbstractDistribZkTes
return CollectionAdminRequest.requestStatus(requestId).process(client);
}
+ protected void setupRestTestHarnesses() {
+ for (final SolrClient client : clients) {
+ RestTestHarness harness = new RestTestHarness(() -> ((HttpSolrClient) client).getBaseURL());
+ restTestHarnesses.add(harness);
+ }
+ }
+
+ protected void closeRestTestHarnesses() throws IOException {
+ for (RestTestHarness h : restTestHarnesses) {
+ h.close();
+ }
+ }
+
+ protected RestTestHarness randomRestTestHarness() {
+ return restTestHarnesses.get(random().nextInt(restTestHarnesses.size()));
+ }
+
+ protected RestTestHarness randomRestTestHarness(Random random) {
+ return restTestHarnesses.get(random.nextInt(restTestHarnesses.size()));
+ }
+
+ protected void forAllRestTestHarnesses(UnaryOperator<RestTestHarness> op) {
+ for (RestTestHarness h : restTestHarnesses) {
+ op.apply(h);
+ }
+ }
+
}
[11/18] lucene-solr:jira/solr-11702: LUCENE-8111:
IndexOrDocValuesQuery Javadoc references outdated method name.
Posted by da...@apache.org.
LUCENE-8111: IndexOrDocValuesQuery Javadoc references outdated method name.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/bc9836fd
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/bc9836fd
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/bc9836fd
Branch: refs/heads/jira/solr-11702
Commit: bc9836fd50319e5b0bdcf09e8b53954cee5d6cad
Parents: c95dc6d
Author: Adrien Grand <jp...@gmail.com>
Authored: Fri Dec 29 10:04:32 2017 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Fri Dec 29 10:06:00 2017 +0100
----------------------------------------------------------------------
lucene/CHANGES.txt | 5 +++++
.../java/org/apache/lucene/search/IndexOrDocValuesQuery.java | 2 +-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bc9836fd/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index ef7e005..a78770a 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -107,6 +107,11 @@ Bug Fixes
* LUCENE-8077: Fixed bug in how CheckIndex verifies doc-value iterators.
(Xiaoshan Sun via Adrien Grand)
+Other
+
+* LUCENE-8111: IndexOrDocValuesQuery Javadoc references outdated method name.
+ (Kai Chan via Adrien Grand)
+
======================= Lucene 7.2.0 =======================
API Changes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/bc9836fd/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java b/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java
index e679e81..f89924d 100644
--- a/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java
+++ b/lucene/core/src/java/org/apache/lucene/search/IndexOrDocValuesQuery.java
@@ -37,7 +37,7 @@ import org.apache.lucene.index.Term;
* String field;
* long minValue, maxValue;
* Query pointQuery = LongPoint.newRangeQuery(field, minValue, maxValue);
- * Query dvQuery = SortedNumericDocValuesField.newRangeQuery(field, minValue, maxValue);
+ * Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(field, minValue, maxValue);
* Query query = new IndexOrDocValuesQuery(pointQuery, dvQuery);
* </pre>
* The above query will be efficient as it will use points in the case that they
[18/18] lucene-solr:jira/solr-11702: SOLR-11702: Replica should check
the connection between its and leader before doing recovery
Posted by da...@apache.org.
SOLR-11702: Replica should check the connection between its and leader before doing recovery
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/3e6afede
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/3e6afede
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/3e6afede
Branch: refs/heads/jira/solr-11702
Commit: 3e6afede46d9824bea07a6df87214fa991c7c209
Parents: 764c1f2
Author: Cao Manh Dat <da...@apache.org>
Authored: Wed Jan 3 17:51:31 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Wed Jan 3 17:51:31 2018 +0700
----------------------------------------------------------------------
.../org/apache/solr/cloud/RecoveryStrategy.java | 52 ++++++++-
.../apache/solr/cloud/HttpPartitionTest.java | 107 ++++++++++++-------
2 files changed, 120 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3e6afede/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
index 95ba3f1..63dfe19 100644
--- a/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
+++ b/solr/core/src/java/org/apache/solr/cloud/RecoveryStrategy.java
@@ -35,8 +35,10 @@ import org.apache.solr.client.solrj.impl.HttpSolrClient.HttpUriRequestResponse;
import org.apache.solr.client.solrj.request.AbstractUpdateRequest;
import org.apache.solr.client.solrj.request.CoreAdminRequest.WaitForState;
import org.apache.solr.client.solrj.request.UpdateRequest;
+import org.apache.solr.client.solrj.response.SolrPingResponse;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.SolrException.ErrorCode;
+import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
@@ -521,8 +523,12 @@ public class RecoveryStrategy implements Runnable, Closeable {
while (!successfulRecovery && !Thread.currentThread().isInterrupted() && !isClosed()) { // don't use interruption or it will close channels though
try {
CloudDescriptor cloudDesc = core.getCoreDescriptor().getCloudDescriptor();
- final Replica leader = zkStateReader.getLeaderRetry(cloudDesc.getCollectionName(), cloudDesc.getShardId());
- //TODO we should try to ping to the leader to ensure the network is healthy
+ final Replica leader = pingLeader(ourUrl, core.getCoreDescriptor(), true);
+ if (isClosed()) {
+ LOG.info("RecoveryStrategy has been closed");
+ break;
+ }
+
boolean isLeader = leader.getCoreUrl().equals(ourUrl);
if (isLeader && !cloudDesc.isLeader()) {
throw new SolrException(ErrorCode.SERVER_ERROR, "Cloud state still says we are leader.");
@@ -738,6 +744,48 @@ public class RecoveryStrategy implements Runnable, Closeable {
LOG.info("Finished recovery process, successful=[{}]", Boolean.toString(successfulRecovery));
}
+ private final Replica pingLeader(String ourUrl, CoreDescriptor coreDesc, boolean mayPutReplicaAsDown) throws Exception {
+ int numTried = 0;
+ while (true) {
+ CloudDescriptor cloudDesc = coreDesc.getCloudDescriptor();
+ DocCollection docCollection = zkStateReader.getClusterState().getCollection(cloudDesc.getCollectionName());
+ if (mayPutReplicaAsDown && numTried == 1 &&
+ docCollection.getReplica(coreDesc.getCloudDescriptor().getCoreNodeName()).getState() == Replica.State.ACTIVE) {
+ // this operation may take a long time, by putting replica into DOWN state, client won't query this replica
+ zkController.publish(coreDesc, Replica.State.DOWN);
+ }
+ numTried++;
+ final Replica leaderReplica = zkStateReader.getLeaderRetry(
+ cloudDesc.getCollectionName(), cloudDesc.getShardId());
+
+ if (isClosed()) {
+ return leaderReplica;
+ }
+
+ if (leaderReplica.getCoreUrl().equals(ourUrl)) {
+ return leaderReplica;
+ }
+
+ try (HttpSolrClient httpSolrClient = new HttpSolrClient.Builder(leaderReplica.getCoreUrl())
+ .withSocketTimeout(1000)
+ .withConnectionTimeout(1000)
+ .build()) {
+ SolrPingResponse resp = httpSolrClient.ping();
+ return leaderReplica;
+ } catch (IOException e) {
+ LOG.info("Failed to connect leader {} on recovery, try again", leaderReplica.getBaseUrl(), e);
+ Thread.sleep(500);
+ } catch (Exception e) {
+ if (e.getCause() instanceof IOException) {
+ LOG.info("Failed to connect leader {} on recovery, try again", leaderReplica.getBaseUrl(), e);
+ Thread.sleep(500);
+ } else {
+ return leaderReplica;
+ }
+ }
+ }
+ }
+
public static Runnable testing_beforeReplayBufferingUpdates;
final private Future<RecoveryInfo> replay(SolrCore core)
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/3e6afede/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
index ff083cd..24f4ff6 100644
--- a/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/HttpPartitionTest.java
@@ -31,6 +31,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.lucene.util.LuceneTestCase.Slow;
import org.apache.solr.JSONTestUtil;
import org.apache.solr.SolrTestCaseJ4.SuppressSSL;
+import org.apache.solr.client.solrj.SolrClient;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
@@ -46,10 +47,13 @@ import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.ZkCoreNodeProps;
import org.apache.solr.common.cloud.ZkStateReader;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.SolrCore;
import org.apache.solr.update.UpdateLog;
import org.apache.solr.util.RTimer;
+import org.apache.solr.util.TimeOut;
+import org.apache.zookeeper.KeeperException;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -148,6 +152,7 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase {
sendDoc(1, 2);
+ JettySolrRunner leaderJetty = getJettyOnPort(getReplicaPort(getShardLeader(testCollectionName, "shard1", 1000)));
List<Replica> notLeaders =
ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 3, maxWaitSecsToSeeAllActive);
assertTrue("Expected 2 non-leader replicas for collection " + testCollectionName
@@ -160,27 +165,21 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase {
// Now introduce a network partition between the leader and 1 replica, so a minRf of 2 is still achieved
log.info("partitioning replica : " + notLeaders.get(0));
SocketProxy proxy0 = getProxyForReplica(notLeaders.get(0));
+ SocketProxy leaderProxy = getProxyForReplica(getShardLeader(testCollectionName, "shard1", 1000));
proxy0.close();
+ // leader still can connect to replica 2, by closing leaderProxy, replica 1 can not do recovery
+ leaderProxy.close();
// indexing during a partition
- int achievedRf = sendDoc(2, 2);
+ int achievedRf = sendDoc(2, 2, leaderJetty);
assertEquals("Unexpected achieved replication factor", 2, achievedRf);
-
+ try (ZkShardTerms zkShardTerms = new ZkShardTerms(testCollectionName, "shard1", cloudClient.getZkStateReader().getZkClient())) {
+ assertFalse(zkShardTerms.canBecomeLeader(notLeaders.get(0).getName()));
+ }
Thread.sleep(sleepMsBeforeHealPartition);
-
- // Verify that the partitioned replica is DOWN
- ZkStateReader zkr = cloudClient.getZkStateReader();
- zkr.forceUpdateCollection(testCollectionName);; // force the state to be fresh
- ClusterState cs = zkr.getClusterState();
- Collection<Slice> slices = cs.getCollection(testCollectionName).getActiveSlices();
- Slice slice = slices.iterator().next();
- Replica partitionedReplica = slice.getReplica(notLeaders.get(0).getName());
- assertEquals("The partitioned replica did not go into recovering",
- Replica.State.RECOVERING.toString(), partitionedReplica.getStr(ZkStateReader.STATE_PROP));
- log.info("un-partitioning replica : " + notLeaders.get(0));
-
proxy0.reopen();
+ leaderProxy.reopen();
notLeaders =
ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 3, maxWaitSecsToSeeAllActive);
@@ -193,8 +192,10 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase {
proxy0.close();
SocketProxy proxy1 = getProxyForReplica(notLeaders.get(1));
proxy1.close();
+ leaderProxy = getProxyForReplica(getShardLeader(testCollectionName, "shard1", 1000));
+ leaderProxy.close();
- achievedRf = sendDoc(3, 2);
+ achievedRf = sendDoc(3, 2, leaderJetty);
assertEquals("Unexpected achieved replication factor", 1, achievedRf);
Thread.sleep(sleepMsBeforeHealPartition);
@@ -204,6 +205,7 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase {
proxy0.reopen();
proxy1.reopen();
+ leaderProxy.reopen();
notLeaders =
ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 3, maxWaitSecsToSeeAllActive);
@@ -238,30 +240,27 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase {
Replica notLeader =
ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 2, maxWaitSecsToSeeAllActive).get(0);
-
+ JettySolrRunner leaderJetty = getJettyOnPort(getReplicaPort(getShardLeader(testCollectionName, "shard1", 1000)));
+
// ok, now introduce a network partition between the leader and the replica
SocketProxy proxy = getProxyForReplica(notLeader);
+ SocketProxy leaderProxy = getProxyForReplica(getShardLeader(testCollectionName, "shard1", 1000));
proxy.close();
-
+ leaderProxy.close();
+
// indexing during a partition
- sendDoc(2);
-
- // Have the partition last at least 1 sec
- // While this gives the impression that recovery is timing related, this is
- // really only
- // to give time for the state to be written to ZK before the test completes.
- // In other words,
- // without a brief pause, the test finishes so quickly that it doesn't give
- // time for the recovery process to kick-in
- Thread.sleep(sleepMsBeforeHealPartition);
+ sendDoc(2, null, leaderJetty);
+ // replica should publish itself as DOWN if the network is not healed after some amount time
+ waitForDownState(testCollectionName, notLeader.getName(), 10000);
proxy.reopen();
+ leaderProxy.reopen();
List<Replica> notLeaders =
ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 2, maxWaitSecsToSeeAllActive);
- int achievedRf = sendDoc(3, 2);
+ int achievedRf = sendDoc(3);
if (achievedRf == 1) {
// this case can happen when leader reuse an connection get established before network partition
// TODO: Remove when SOLR-11776 get committed
@@ -293,21 +292,25 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase {
if (d % 10 == 0) {
if (hasPartition) {
proxy.reopen();
+ leaderProxy.reopen();
hasPartition = false;
} else {
if (d >= 10) {
proxy.close();
+ leaderProxy.close();
hasPartition = true;
Thread.sleep(sleepMsBeforeHealPartition);
}
}
}
- sendDoc(d + 4); // 4 is offset as we've already indexed 1-3
+ // always send doc directly to leader without going through proxy
+ sendDoc(d + 4, null, leaderJetty); // 4 is offset as we've already indexed 1-3
}
// restore connectivity if lost
if (hasPartition) {
proxy.reopen();
+ leaderProxy.reopen();
}
notLeaders = ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 2, maxWaitSecsToSeeAllActive);
@@ -328,7 +331,24 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase {
// try to clean up
attemptCollectionDelete(cloudClient, testCollectionName);
}
-
+
+ private void waitForDownState(String collection, String replicaName, long ms) throws KeeperException, InterruptedException {
+ TimeOut timeOut = new TimeOut(ms, TimeUnit.MILLISECONDS, TimeSource.CURRENT_TIME);
+ Replica.State replicaState = Replica.State.ACTIVE;
+ while (!timeOut.hasTimedOut()) {
+ ZkStateReader zkr = cloudClient.getZkStateReader();
+ zkr.forceUpdateCollection(collection);; // force the state to be fresh
+ ClusterState cs = zkr.getClusterState();
+ Collection<Slice> slices = cs.getCollection(collection).getActiveSlices();
+ Slice slice = slices.iterator().next();
+ Replica partitionedReplica = slice.getReplica(replicaName);
+ replicaState = partitionedReplica.getState();
+ if (replicaState == Replica.State.DOWN) return;
+ }
+ assertEquals("The partitioned replica did not published it self as down",
+ Replica.State.DOWN, replicaState);
+ }
+
protected void testRf3() throws Exception {
// create a collection that has 1 shard but 2 replicas
String testCollectionName = "c8n_1x3";
@@ -344,27 +364,30 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase {
+ " but found " + notLeaders.size() + "; clusterState: "
+ printClusterStateInfo(testCollectionName),
notLeaders.size() == 2);
+ JettySolrRunner leaderJetty = getJettyOnPort(getReplicaPort(getShardLeader(testCollectionName, "shard1", 1000)));
// ok, now introduce a network partition between the leader and the replica
SocketProxy proxy0 = getProxyForReplica(notLeaders.get(0));
+ SocketProxy leaderProxy = getProxyForReplica(getShardLeader(testCollectionName, "shard1", 1000));
proxy0.close();
+ leaderProxy.close();
// indexing during a partition
- sendDoc(2);
+ sendDoc(2, null, leaderJetty);
Thread.sleep(sleepMsBeforeHealPartition);
-
proxy0.reopen();
SocketProxy proxy1 = getProxyForReplica(notLeaders.get(1));
-
proxy1.close();
- sendDoc(3);
+ sendDoc(3, null, leaderJetty);
Thread.sleep(sleepMsBeforeHealPartition);
proxy1.reopen();
+
+ leaderProxy.reopen();
// sent 4 docs in so far, verify they are on the leader and replica
notLeaders = ensureAllReplicasAreActive(testCollectionName, "shard1", 1, 3, maxWaitSecsToSeeAllActive);
@@ -522,8 +545,19 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase {
protected int sendDoc(int docId) throws Exception {
return sendDoc(docId, null);
}
-
+
+ // Send doc directly to a server (without going through proxy)
+ protected int sendDoc(int docId, Integer minRf, JettySolrRunner leaderJetty) throws IOException, SolrServerException {
+ try (HttpSolrClient solrClient = new HttpSolrClient.Builder(leaderJetty.getBaseUrl().toString()).build()) {
+ return sendDoc(docId, minRf, solrClient, cloudClient.getDefaultCollection());
+ }
+ }
+
protected int sendDoc(int docId, Integer minRf) throws Exception {
+ return sendDoc(docId, minRf, cloudClient, cloudClient.getDefaultCollection());
+ }
+
+ protected int sendDoc(int docId, Integer minRf, SolrClient solrClient, String collection) throws IOException, SolrServerException {
SolrInputDocument doc = new SolrInputDocument();
doc.addField(id, String.valueOf(docId));
doc.addField("a_t", "hello" + docId);
@@ -533,8 +567,7 @@ public class HttpPartitionTest extends AbstractFullDistribZkTestBase {
up.setParam(UpdateRequest.MIN_REPFACT, String.valueOf(minRf));
}
up.add(doc);
-
- return cloudClient.getMinAchievedReplicationFactor(cloudClient.getDefaultCollection(), cloudClient.request(up));
+ return cloudClient.getMinAchievedReplicationFactor(collection, solrClient.request(up, collection));
}
/**
[13/18] lucene-solr:jira/solr-11702: LUCENE-8010: Axiomatic
similarities must incorporate the boost as a multiplicative factor.
Posted by da...@apache.org.
LUCENE-8010: Axiomatic similarities must incorporate the boost as a multiplicative factor.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8dfb0190
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8dfb0190
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8dfb0190
Branch: refs/heads/jira/solr-11702
Commit: 8dfb0190762922ea2e61dc565616e17aa02ebb5a
Parents: 33f421e
Author: Adrien Grand <jp...@gmail.com>
Authored: Fri Dec 29 15:59:21 2017 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Fri Dec 29 15:59:21 2017 +0100
----------------------------------------------------------------------
.../java/org/apache/lucene/search/similarities/Axiomatic.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8dfb0190/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
----------------------------------------------------------------------
diff --git a/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java b/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
index 553fd42..df683c1 100644
--- a/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
+++ b/lucene/core/src/java/org/apache/lucene/search/similarities/Axiomatic.java
@@ -107,6 +107,7 @@ public abstract class Axiomatic extends SimilarityBase {
* tfln(stats, freq, docLen)
* idf(stats, freq, docLen)
- gamma(stats, freq, docLen);
+ score *= stats.boost;
// AxiomaticF3 similarities might produce negative scores due to their gamma component
return Math.max(0, score);
}
@@ -132,6 +133,11 @@ public abstract class Axiomatic extends SimilarityBase {
Explanation explanation = Explanation.match((float) score,
"score(" + getClass().getSimpleName() + ", doc=" + doc + ", freq=" + freq.getValue() +"), computed from:",
subs);
+ if (stats.boost != 1f) {
+ explanation = Explanation.match((float) (score * stats.boost), "Boosted score, computed as (score * boost) from:",
+ explanation,
+ Explanation.match((float) stats.boost, "Query boost"));
+ }
if (score < 0) {
explanation = Explanation.match(0, "max of:",
Explanation.match(0, "Minimum legal score"),
[02/18] lucene-solr:jira/solr-11702: SOLR-11555: If the query terms
reduce to nothing, filter(clause) produces an NPE whereas fq=clause does not
Posted by da...@apache.org.
SOLR-11555: If the query terms reduce to nothing, filter(clause) produces an NPE whereas fq=clause does not
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/05ab5e10
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/05ab5e10
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/05ab5e10
Branch: refs/heads/jira/solr-11702
Commit: 05ab5e10ae2309a307951d753ec0c141b6c12f7d
Parents: 6b368c0
Author: Erick Erickson <er...@apache.org>
Authored: Wed Dec 27 12:05:31 2017 -0800
Committer: Erick Erickson <er...@apache.org>
Committed: Wed Dec 27 12:05:31 2017 -0800
----------------------------------------------------------------------
solr/CHANGES.txt | 3 +++
.../java/org/apache/solr/query/FilterQuery.java | 5 +++++
.../apache/solr/search/TestSolrQueryParser.java | 20 ++++++++++++++++++++
3 files changed, 28 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/05ab5e10/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 68003cb..b951d0d 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -68,6 +68,9 @@ Bug Fixes
* SOLR-11783: Rename core in solr standalone mode is not persisted (Erick Erickson)
+* SOLR-11555: If the query terms reduce to nothing, filter(clause) produces an NPE whereas
+ fq=clause does not (Erick Erickson)
+
Optimizations
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/05ab5e10/solr/core/src/java/org/apache/solr/query/FilterQuery.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/query/FilterQuery.java b/solr/core/src/java/org/apache/solr/query/FilterQuery.java
index 7a74798..02cb88b 100644
--- a/solr/core/src/java/org/apache/solr/query/FilterQuery.java
+++ b/solr/core/src/java/org/apache/solr/query/FilterQuery.java
@@ -22,6 +22,7 @@ import org.apache.lucene.index.IndexReader;
import org.apache.lucene.search.BoostQuery;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.IndexSearcher;
+import org.apache.lucene.search.MatchNoDocsQuery;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreMode;
import org.apache.lucene.search.Weight;
@@ -34,6 +35,10 @@ public class FilterQuery extends ExtendedQueryBase {
protected final Query q;
public FilterQuery(Query q) {
+ if (q == null) {
+ this.q = new MatchNoDocsQuery();
+ return;
+ }
this.q = q;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/05ab5e10/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
index 1cf0a0c..b69fe9e 100644
--- a/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
+++ b/solr/core/src/test/org/apache/solr/search/TestSolrQueryParser.java
@@ -491,6 +491,26 @@ public class TestSolrQueryParser extends SolrTestCaseJ4 {
, "/response/docs/[0]/score==10.0"
);
+ assertU(adoc("id", "40", "wdf_nocase", "just some text, don't want NPE"));
+ assertU(commit());
+
+ // See SOLR-11555. If wdff removes all the characters, an NPE occurs.
+ // try q and fq
+ assertJQ(req("q", "filter(wdf_nocase:&)", "fl", "id", "debug", "query")
+ , "/response/numFound==0"
+ );
+ assertJQ(req("fq", "filter(wdf_nocase:.,)", "fl", "id", "debug", "query")
+ , "/response/numFound==0"
+ );
+
+ // Insure the same behavior as with bare clause, just not filter
+ assertJQ(req("q", "wdf_nocase:&", "fl", "id", "debug", "query")
+ , "/response/numFound==0"
+ );
+ assertJQ(req("fq", "wdf_nocase:.,", "fl", "id", "debug", "query")
+ , "/response/numFound==0"
+ );
+
}
[16/18] lucene-solr:jira/solr-11702: SOLR-11808: Add sumSq Stream
Evaluator
Posted by da...@apache.org.
SOLR-11808: Add sumSq Stream Evaluator
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/692e5243
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/692e5243
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/692e5243
Branch: refs/heads/jira/solr-11702
Commit: 692e5243e9ae550de74fe75d9c2db5af5efda580
Parents: fbc8508
Author: Joel Bernstein <jb...@apache.org>
Authored: Sun Dec 31 13:48:02 2017 -0500
Committer: Joel Bernstein <jb...@apache.org>
Committed: Sun Dec 31 13:48:23 2017 -0500
----------------------------------------------------------------------
.../org/apache/solr/handler/StreamHandler.java | 1 +
.../client/solrj/io/eval/SumSqEvaluator.java | 56 ++++++++++++++++++++
.../solrj/io/stream/StreamExpressionTest.java | 17 ++++++
3 files changed, 74 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/692e5243/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index ae15260..73dea3c 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -287,6 +287,7 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("integrate", IntegrateEvaluator.class)
.withFunctionName("density", DensityEvaluator.class)
.withFunctionName("mannWhitney", MannWhitneyUEvaluator.class)
+ .withFunctionName("sumSq", SumSqEvaluator.class)
// Boolean Stream Evaluators
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/692e5243/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SumSqEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SumSqEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SumSqEvaluator.java
new file mode 100644
index 0000000..a2097ec
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/SumSqEvaluator.java
@@ -0,0 +1,56 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Locale;
+
+import org.apache.commons.math3.stat.StatUtils;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+public class SumSqEvaluator extends RecursiveObjectEvaluator implements OneValueWorker {
+ private static final long serialVersionUID = 1;
+
+ public SumSqEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+ }
+
+ @Override
+ public Object doWork(Object value) throws IOException {
+ if(null == value){
+ return value;
+ }
+ else if(!(value instanceof List<?>)){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - found type %s for value, expecting a List",toExpression(constructingFactory), value.getClass().getSimpleName()));
+ }
+
+ List<Number> list = (List<Number>)value;
+
+ if(0 == list.size()){
+ return list;
+ }
+
+ double[] vec = new double[list.size()];
+ for(int i=0; i< vec.length; i++) {
+ vec[i] = list.get(i).doubleValue();
+ }
+
+ return StatUtils.sumSq(vec);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/692e5243/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index 4af8e43..2662fd2 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -7550,6 +7550,23 @@ public class StreamExpressionTest extends SolrCloudTestCase {
}
@Test
+ public void testSumSq() throws Exception {
+ String cexpr = "sumSq(array(-3,-2.5, 10))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Number sumSq = (Number)tuples.get(0).get("return-value");
+ assertEquals(sumSq.doubleValue(), 115.25D, 0.0D);
+ }
+
+
+ @Test
public void testMonteCarlo() throws Exception {
String cexpr = "let(a=constantDistribution(10), b=constantDistribution(20), c=monteCarlo(add(sample(a), sample(b)), 10))";
ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
[17/18] lucene-solr:jira/solr-11702: Merge branch 'master' into
jira/solr-11702
Posted by da...@apache.org.
Merge branch 'master' into jira/solr-11702
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/764c1f27
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/764c1f27
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/764c1f27
Branch: refs/heads/jira/solr-11702
Commit: 764c1f27a6c0decf921c3f6844991740fae1f0d2
Parents: 19d2af7 692e524
Author: Cao Manh Dat <da...@apache.org>
Authored: Tue Jan 2 10:20:37 2018 +0700
Committer: Cao Manh Dat <da...@apache.org>
Committed: Tue Jan 2 10:20:37 2018 +0700
----------------------------------------------------------------------
lucene/CHANGES.txt | 13 ++
.../apache/lucene/search/ConjunctionScorer.java | 17 +-
.../apache/lucene/search/ExactPhraseScorer.java | 23 ++-
.../lucene/search/IndexOrDocValuesQuery.java | 2 +-
.../lucene/search/MaxScoreSumPropagator.java | 157 +++++++++++++++
.../apache/lucene/search/MultiPhraseQuery.java | 14 +-
.../org/apache/lucene/search/PhraseQuery.java | 14 +-
.../apache/lucene/search/ReqOptSumScorer.java | 8 +-
.../org/apache/lucene/search/WANDScorer.java | 20 +-
.../lucene/search/similarities/Axiomatic.java | 34 +++-
.../search/similarities/AxiomaticF1EXP.java | 2 +-
.../search/similarities/AxiomaticF1LOG.java | 2 +-
.../search/similarities/AxiomaticF3EXP.java | 2 +-
.../search/similarities/AxiomaticF3LOG.java | 2 +-
.../search/similarities/DistributionSPL.java | 26 ++-
.../lucene/search/similarities/LambdaDF.java | 7 +-
.../lucene/search/similarities/LambdaTTF.java | 7 +-
.../search/TestMaxScoreSumPropagator.java | 198 +++++++++++++++++++
.../apache/lucene/search/TestPhraseQuery.java | 47 +++++
.../search/similarities/TestAxiomaticF1EXP.java | 4 -
.../search/similarities/TestAxiomaticF1LOG.java | 4 -
.../search/similarities/TestAxiomaticF3EXP.java | 7 +-
.../search/similarities/TestAxiomaticF3LOG.java | 7 +-
.../similarities/TestDistributionSPL.java | 4 -
.../search/similarities/TestSimilarity2.java | 7 +
.../lucene/search/grouping/TestGrouping.java | 3 +
.../FastVectorHighlighterTest.java | 8 +-
.../search/similarities/RandomSimilarity.java | 11 +-
solr/CHANGES.txt | 7 +
.../org/apache/solr/handler/StreamHandler.java | 2 +
.../java/org/apache/solr/query/FilterQuery.java | 5 +
.../org/apache/solr/cloud/TestCryptoKeys.java | 33 +---
.../org/apache/solr/core/TestCustomStream.java | 23 +--
.../apache/solr/core/TestDynamicLoading.java | 29 +--
.../org/apache/solr/core/TestLazyCores.java | 7 -
.../apache/solr/handler/TestConfigReload.java | 25 +--
.../handler/TestSolrConfigHandlerCloud.java | 28 +--
.../TestSolrConfigHandlerConcurrent.java | 22 +--
.../solr/schema/TestBulkSchemaConcurrent.java | 32 +--
.../apache/solr/schema/TestCloudSchemaless.java | 43 ++--
.../apache/solr/search/TestSolrQueryParser.java | 20 ++
.../processor/TestNamedUpdateProcessors.java | 51 ++---
solr/solr-ref-guide/src/documents-screen.adoc | 2 +-
solr/solr-ref-guide/src/index-replication.adoc | 6 +-
solr/solr-ref-guide/src/indexupgrader-tool.adoc | 2 +-
.../src/uploading-data-with-index-handlers.adoc | 6 +-
.../solrj/io/eval/MannWhitneyUEvaluator.java | 62 ++++++
.../client/solrj/io/eval/SumSqEvaluator.java | 56 ++++++
.../solrj/io/stream/TimeSeriesStream.java | 12 +-
.../solrj/io/stream/StreamExpressionTest.java | 80 +++++++-
.../cloud/AbstractFullDistribZkTestBase.java | 33 ++++
.../web/js/angular/controllers/collections.js | 1 -
.../web/js/angular/controllers/core-overview.js | 12 --
solr/webapp/web/js/angular/controllers/cores.js | 15 --
.../web/js/angular/controllers/dataimport.js | 1 -
solr/webapp/web/js/angular/services.js | 7 +-
solr/webapp/web/partials/core_overview.html | 4 -
solr/webapp/web/partials/cores.html | 5 -
58 files changed, 919 insertions(+), 362 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/764c1f27/solr/test-framework/src/java/org/apache/solr/cloud/AbstractFullDistribZkTestBase.java
----------------------------------------------------------------------
[03/18] lucene-solr:jira/solr-11702: LUCENE-8110: Fix potential
IndexOutOfBoundsException in *Classifier.getClasses(?, int).
Posted by da...@apache.org.
LUCENE-8110: Fix potential IndexOutOfBoundsException in *Classifier.getClasses(?,int).
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/af41d02e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/af41d02e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/af41d02e
Branch: refs/heads/jira/solr-11702
Commit: af41d02eae6a58fd450553f9a09c9325ddf6e0ab
Parents: 05ab5e1
Author: Christine Poerschke <cp...@apache.org>
Authored: Thu Dec 28 12:28:41 2017 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Thu Dec 28 12:28:41 2017 +0000
----------------------------------------------------------------------
lucene/CHANGES.txt | 3 +++
.../java/org/apache/lucene/classification/BM25NBClassifier.java | 1 +
.../org/apache/lucene/classification/KNearestFuzzyClassifier.java | 1 +
.../apache/lucene/classification/KNearestNeighborClassifier.java | 1 +
.../apache/lucene/classification/SimpleNaiveBayesClassifier.java | 1 +
.../document/SimpleNaiveBayesDocumentClassifier.java | 1 +
6 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af41d02e/lucene/CHANGES.txt
----------------------------------------------------------------------
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 0da2cfe..84ba235 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -104,6 +104,9 @@ Bug Fixes
* LUCENE-8077: Fixed bug in how CheckIndex verifies doc-value iterators.
(Xiaoshan Sun via Adrien Grand)
+* LUCENE-8110: Fix potential IndexOutOfBoundsException in *Classifier.getClasses(?,int).
+ (Christine Poerschke)
+
======================= Lucene 7.2.0 =======================
API Changes
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af41d02e/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java
index 1a74416..39a5b2f 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/BM25NBClassifier.java
@@ -124,6 +124,7 @@ public class BM25NBClassifier implements Classifier<BytesRef> {
public List<ClassificationResult<BytesRef>> getClasses(String text, int max) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = assignClassNormalizedList(text);
Collections.sort(assignedClasses);
+ max = Math.min(max, assignedClasses.size());
return assignedClasses.subList(0, max);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af41d02e/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
index cbd241b..65effe4 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/KNearestFuzzyClassifier.java
@@ -140,6 +140,7 @@ public class KNearestFuzzyClassifier implements Classifier<BytesRef> {
TopDocs knnResults = knnSearch(text);
List<ClassificationResult<BytesRef>> assignedClasses = buildListFromTopDocs(knnResults);
Collections.sort(assignedClasses);
+ max = Math.min(max, assignedClasses.size());
return assignedClasses.subList(0, max);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af41d02e/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
index f0391f4..f145912 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/KNearestNeighborClassifier.java
@@ -162,6 +162,7 @@ public class KNearestNeighborClassifier implements Classifier<BytesRef> {
TopDocs knnResults = knnSearch(text);
List<ClassificationResult<BytesRef>> assignedClasses = buildListFromTopDocs(knnResults);
Collections.sort(assignedClasses);
+ max = Math.min(max, assignedClasses.size());
return assignedClasses.subList(0, max);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af41d02e/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
index 3509df5..3bb701f 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/SimpleNaiveBayesClassifier.java
@@ -132,6 +132,7 @@ public class SimpleNaiveBayesClassifier implements Classifier<BytesRef> {
public List<ClassificationResult<BytesRef>> getClasses(String text, int max) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = assignClassNormalizedList(text);
Collections.sort(assignedClasses);
+ max = Math.min(max, assignedClasses.size());
return assignedClasses.subList(0, max);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/af41d02e/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
----------------------------------------------------------------------
diff --git a/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
index 6bc8573..097fe0c 100644
--- a/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
+++ b/lucene/classification/src/java/org/apache/lucene/classification/document/SimpleNaiveBayesDocumentClassifier.java
@@ -105,6 +105,7 @@ public class SimpleNaiveBayesDocumentClassifier extends SimpleNaiveBayesClassifi
public List<ClassificationResult<BytesRef>> getClasses(Document document, int max) throws IOException {
List<ClassificationResult<BytesRef>> assignedClasses = assignNormClasses(document);
Collections.sort(assignedClasses);
+ max = Math.min(max, assignedClasses.size());
return assignedClasses.subList(0, max);
}
[08/18] lucene-solr:jira/solr-11702: SOLR-7733: remove optimize from
the UI.
Posted by da...@apache.org.
SOLR-7733: remove optimize from the UI.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/8e439a0a
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/8e439a0a
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/8e439a0a
Branch: refs/heads/jira/solr-11702
Commit: 8e439a0a5c37a95ae632719b8901b225462b80bf
Parents: fbea59b
Author: Erick Erickson <er...@apache.org>
Authored: Thu Dec 28 20:41:48 2017 -0800
Committer: Erick Erickson <er...@apache.org>
Committed: Thu Dec 28 20:41:48 2017 -0800
----------------------------------------------------------------------
solr/CHANGES.txt | 2 ++
solr/solr-ref-guide/src/documents-screen.adoc | 2 +-
solr/solr-ref-guide/src/index-replication.adoc | 6 +++++-
solr/solr-ref-guide/src/indexupgrader-tool.adoc | 2 +-
.../src/uploading-data-with-index-handlers.adoc | 6 +++++-
.../webapp/web/js/angular/controllers/collections.js | 1 -
.../web/js/angular/controllers/core-overview.js | 12 ------------
solr/webapp/web/js/angular/controllers/cores.js | 15 ---------------
solr/webapp/web/js/angular/controllers/dataimport.js | 1 -
solr/webapp/web/js/angular/services.js | 7 ++-----
solr/webapp/web/partials/core_overview.html | 4 ----
solr/webapp/web/partials/cores.html | 5 -----
12 files changed, 16 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e439a0a/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index e14d771..c8e36f2 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -93,6 +93,8 @@ Other Changes
* SOLR-11793: Reduce code duplication w.r.t. RestTestHarness(es). (Christine Poerschke)
+* SOLR-7733: remove "optimize" from the UI. (Erick Erickson)
+
================== 7.2.0 ==================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e439a0a/solr/solr-ref-guide/src/documents-screen.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/documents-screen.adoc b/solr/solr-ref-guide/src/documents-screen.adoc
index abeeb9e..83da713 100644
--- a/solr/solr-ref-guide/src/documents-screen.adoc
+++ b/solr/solr-ref-guide/src/documents-screen.adoc
@@ -68,7 +68,7 @@ Then you can choose when documents should be added to the index (Commit Within),
== Solr Command
-The Solr Command option allows you use XML or JSON to perform specific actions on documents, such as defining documents to be added or deleted, updating only certain fields of documents, or commit and optimize commands on the index.
+The Solr Command option allows you use XML or JSON to perform specific actions on documents, such as defining documents to be added or deleted, updating only certain fields of documents, or commit commands on the index.
The documents should be structured as they would be if using `/update` on the command line.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e439a0a/solr/solr-ref-guide/src/index-replication.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/index-replication.adoc b/solr/solr-ref-guide/src/index-replication.adoc
index de1e8d6..7e07e49 100644
--- a/solr/solr-ref-guide/src/index-replication.adoc
+++ b/solr/solr-ref-guide/src/index-replication.adoc
@@ -54,7 +54,7 @@ Distribution::
The copying of an index from the master server to all slaves. The distribution process takes advantage of Lucene's index file structure.
Inserts and Deletes::
-As inserts and deletes occur in the index, the directory remains unchanged. Documents are always inserted into newly created files. Documents that are deleted are not removed from the files. They are flagged in the file, deletable, and are not removed from the files until the index is optimized.
+As inserts and deletes occur in the index, the directory remains unchanged. Documents are always inserted into newly created segment files. Documents that are deleted are not removed from the segment files. They are flagged in the file, deletable, and are not removed from the segments until the segment is merged as part of normal index updates.
Master and Slave::
A Solr replication master is a single node which receives all updates initially and keeps everything organized. Solr replication slave nodes receive no updates directly, instead all changes (such as inserts, updates, deletes, etc.) are made against the single master node. Changes made on the master are distributed to all the slave nodes which service all query requests from the clients.
@@ -65,6 +65,8 @@ An update is a single change request against a single Solr instance. It may be a
Optimization::
A process that compacts the index and merges segments in order to improve query performance. Optimization should only be run on the master nodes. An optimized index may give query performance gains compared to an index that has become fragmented over a period of time with many updates. Distributing an optimized index requires a much longer time than the distribution of new segments to an un-optimized index.
+WARNING: optimizing is not recommended unless it can be performed regularly as it may lead to a significantly larger portion of the index consisting of deleted documents than would normally be the case.
+
Segments::
A self contained subset of an index consisting of some documents and data structures related to the inverted index of terms in those documents.
@@ -356,3 +358,5 @@ Consider this example: on a three-slave one-master configuration, distributing a
Optimizing on the master allows for a straight-forward optimization operation. No query slaves need to be taken out of service. The optimized index can be distributed in the background as queries are being normally serviced. The optimization can occur at any time convenient to the application providing index updates.
While optimizing may have some benefits in some situations, a rapidly changing index will not retain those benefits for long, and since optimization is an intensive process, it may be better to consider other options, such as lowering the merge factor (discussed in the section on <<indexconfig-in-solrconfig.adoc#merge-factors,Index Configuration>>).
+
+TIP: Do not elect to optimize your index unless you have tangible evidence that it will significantly improve your search performance. Recent changes in Solr/Lucene have dramatically lessened the need to optimize as discussed at the above link.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e439a0a/solr/solr-ref-guide/src/indexupgrader-tool.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/indexupgrader-tool.adoc b/solr/solr-ref-guide/src/indexupgrader-tool.adoc
index ff60c42..bb01eea 100644
--- a/solr/solr-ref-guide/src/indexupgrader-tool.adoc
+++ b/solr/solr-ref-guide/src/indexupgrader-tool.adoc
@@ -42,5 +42,5 @@ Upgrading large indexes may take a long time. As a rule of thumb, the upgrade pr
[WARNING]
====
-This tool may reorder documents if the index was partially upgraded before execution (e.g., documents were added). If your application relies on monotonicity of document IDs (i.e., the order in which the documents were added to the index is preserved), do a full forceMerge instead.
+This tool may reorder documents if the index was partially upgraded before execution (e.g., documents were added). If your application relies on monotonicity of document IDs (i.e., the order in which the documents were added to the index is preserved), do a full optimize instead.
====
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e439a0a/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc b/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc
index c872485..a240dcf 100644
--- a/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc
+++ b/solr/solr-ref-guide/src/uploading-data-with-index-handlers.adoc
@@ -87,7 +87,9 @@ The `<commit>` operation writes all documents loaded since the last commit to on
Commits may be issued explicitly with a `<commit/>` message, and can also be triggered from `<autocommit>` parameters in `solrconfig.xml`.
-The `<optimize>` operation requests Solr to merge internal data structures in order to improve search performance. For a large index, optimization will take some time to complete, but by merging many small segment files into a larger one, search performance will improve. If you are using Solr's replication mechanism to distribute searches across many systems, be aware that after an optimize, a complete index will need to be transferred. In contrast, post-commit transfers are usually much smaller.
+The `<optimize>` operation requests Solr to merge internal data structures. For a large index, optimization will take some time to complete, but by merging many small segment files into a larger one, search performance may improve. If you are using Solr's replication mechanism to distribute searches across many systems, be aware that after an optimize, a complete index will need to be transferred.
+
+WARNING: You should only consider using optimize on static indexes, i.e. indexes that can be optimized as part of the regular update process (say once-a-day updates). Applications requiring NRT functionalty are discouraged from using optimize.
The `<commit>` and `<optimize>` elements accept these optional attributes:
@@ -96,6 +98,8 @@ Default is `true`. Blocks until a new searcher is opened and registered as the m
`expungeDeletes`:: (commit only) Default is `false`. Merges segments that have more than 10% deleted docs, expunging them in the process.
+WARNING: expungeDeletes is "less expensive" than optimize, but the same warnings apply.
+
`maxSegments`:: (optimize only) Default is `1`. Merges the segments down to no more than this number of segments.
Here are examples of <commit> and <optimize> using optional attributes:
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e439a0a/solr/webapp/web/js/angular/controllers/collections.js
----------------------------------------------------------------------
diff --git a/solr/webapp/web/js/angular/controllers/collections.js b/solr/webapp/web/js/angular/controllers/collections.js
index 22c5e7f..251298a 100644
--- a/solr/webapp/web/js/angular/controllers/collections.js
+++ b/solr/webapp/web/js/angular/controllers/collections.js
@@ -15,7 +15,6 @@
limitations under the License.
*/
-// @todo test optimize (delete stuff, watch button appear, test button/form)
solrAdminApp.controller('CollectionsController',
function($scope, $routeParams, $location, $timeout, Collections, Zookeeper, Constants){
$scope.resetMenu("collections", Constants.IS_ROOT_PAGE);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e439a0a/solr/webapp/web/js/angular/controllers/core-overview.js
----------------------------------------------------------------------
diff --git a/solr/webapp/web/js/angular/controllers/core-overview.js b/solr/webapp/web/js/angular/controllers/core-overview.js
index 7aa9685..b11329c 100644
--- a/solr/webapp/web/js/angular/controllers/core-overview.js
+++ b/solr/webapp/web/js/angular/controllers/core-overview.js
@@ -30,18 +30,6 @@ function($scope, $rootScope, $routeParams, Luke, CoreSystem, Update, Replication
);
};
- $scope.optimizeIndex = function(core) {
- Update.optimize({core: $routeParams.core},
- function(response) {
- $scope.refresh();
- delete $scope.indexMessage;
- },
- function(error) {
- $scope.statisticsDisabled = true;
- $scope.indexMessage = "Optimize broken.";
- });
- };
-
$scope.refreshReplication = function() {
Replication.details({core: $routeParams.core},
function(data) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e439a0a/solr/webapp/web/js/angular/controllers/cores.js
----------------------------------------------------------------------
diff --git a/solr/webapp/web/js/angular/controllers/cores.js b/solr/webapp/web/js/angular/controllers/cores.js
index 0188617..202b622 100644
--- a/solr/webapp/web/js/angular/controllers/cores.js
+++ b/solr/webapp/web/js/angular/controllers/cores.js
@@ -15,7 +15,6 @@
limitations under the License.
*/
-// @todo test optimize (delete stuff, watch button appear, test button/form)
solrAdminApp.controller('CoreAdminController',
function($scope, $routeParams, $location, $timeout, $route, Cores, Update, Constants){
$scope.resetMenu("cores", Constants.IS_ROOT_PAGE);
@@ -176,20 +175,6 @@ solrAdminApp.controller('CoreAdminController',
$scope.showSwap = false;
};
- $scope.optimizeCore = function() {
- Update.optimize({core: $scope.selectedCore},
- function(successData) {
- $scope.optimizeSuccess = true;
- $timeout(function() {$scope.optimizeSuccess=false}, 1000);
- $scope.refresh();
- },
- function(failureData) {
- $scope.optimizeFailure = true;
- $timeout(function () {$scope.optimizeFailure=false}, 1000);
- $scope.refresh();
- });
- };
-
$scope.refresh();
}
);
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e439a0a/solr/webapp/web/js/angular/controllers/dataimport.js
----------------------------------------------------------------------
diff --git a/solr/webapp/web/js/angular/controllers/dataimport.js b/solr/webapp/web/js/angular/controllers/dataimport.js
index 3a26324..cebf27a 100644
--- a/solr/webapp/web/js/angular/controllers/dataimport.js
+++ b/solr/webapp/web/js/angular/controllers/dataimport.js
@@ -98,7 +98,6 @@ solrAdminApp.controller('DataImportController',
verbose: false,
clean: true,
commit: true,
- optimize: false,
showDebug: false,
custom: "",
core: $routeParams.core
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e439a0a/solr/webapp/web/js/angular/services.js
----------------------------------------------------------------------
diff --git a/solr/webapp/web/js/angular/services.js b/solr/webapp/web/js/angular/services.js
index a1df37e..4d89347 100644
--- a/solr/webapp/web/js/angular/services.js
+++ b/solr/webapp/web/js/angular/services.js
@@ -35,8 +35,7 @@ solrAdminServices.factory('System',
"deleteReplica": {params:{action: "DELETEREPLICA"}},
"addReplica": {params:{action: "ADDREPLICA"}},
"deleteShard": {params:{action: "DELETESHARD"}},
- "reload": {method: "GET", params:{action:"RELOAD", core: "@core"}},
- "optimize": {params:{}}
+ "reload": {method: "GET", params:{action:"RELOAD", core: "@core"}}
});
}])
.factory('Cores',
@@ -49,8 +48,7 @@ solrAdminServices.factory('System',
"unload": {params:{action: "UNLOAD", core: "@core"}},
"rename": {params:{action: "RENAME"}},
"swap": {params:{action: "SWAP"}},
- "reload": {method: "GET", params:{action:"RELOAD", core: "@core"}, headers:{doNotIntercept: "true"}},
- "optimize": {params:{}}
+ "reload": {method: "GET", params:{action:"RELOAD", core: "@core"}, headers:{doNotIntercept: "true"}}
});
}])
.factory('Logging',
@@ -106,7 +104,6 @@ solrAdminServices.factory('System',
.factory('Update',
['$resource', function($resource) {
return $resource(':core/:handler', {core: '@core', wt:'json', _:Date.now(), handler:'update'}, {
- "optimize": {params: { optimize: "true"}},
"commit": {params: {commit: "true"}},
"post": {headers: {'Content-type': 'application/json'}, method: "POST", params: {handler: '@handler'}},
"postJson": {headers: {'Content-type': 'application/json'}, method: "POST", params: {handler: '@handler'}},
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e439a0a/solr/webapp/web/partials/core_overview.html
----------------------------------------------------------------------
diff --git a/solr/webapp/web/partials/core_overview.html b/solr/webapp/web/partials/core_overview.html
index 3b92fc3..f1826f6 100644
--- a/solr/webapp/web/partials/core_overview.html
+++ b/solr/webapp/web/partials/core_overview.html
@@ -51,10 +51,6 @@ limitations under the License.
<dt class="index_segmentCount">Segment Count:</dt>
<dd class="index_segmentCount value">{{index.segmentCount}}</dd>
- <dt class="index_optimized">Optimized:</dt>
- <dd class="index_optimized ico value" ng-class="index.hasDeletions ? 'ico-0' : 'ico-1'"><span></span>
- <a ng-click="optimizeIndex()" ng-show="index.hasDeletions">optimize now</a></dd>
-
<dt class="index_current">Current:</dt>
<dd class="ico" ng-class="index.current ?'ico-1' : 'ico-0'"><span></span></dd>
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/8e439a0a/solr/webapp/web/partials/cores.html
----------------------------------------------------------------------
diff --git a/solr/webapp/web/partials/cores.html b/solr/webapp/web/partials/cores.html
index 7a989ba..e426e4c 100644
--- a/solr/webapp/web/partials/cores.html
+++ b/solr/webapp/web/partials/cores.html
@@ -198,11 +198,6 @@ limitations under the License.
<dd>{{core.index.deletedDocs}}</dd>
</dl></li>
- <li class="optimized"><dl class="clearfix">
- <dt><span>optimized:</span></dt>
- <dd class="ico" ng-class="core.index.hasDeletions ? 'ico-0' : 'ico-1'"><span></span></dd>
- </dl></li>
-
<li class="current"><dl class="clearfix">
<dt><span>current:</span></dt>
<dd class="ico" ng-class="core.index.current ?'ico-1' : 'ico-0'"><span></span></dd>
[07/18] lucene-solr:jira/solr-11702: SOLR-11172: Add Mann-Whitney U
test Stream Evaluator
Posted by da...@apache.org.
SOLR-11172: Add Mann-Whitney U test Stream Evaluator
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fbea59b0
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fbea59b0
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fbea59b0
Branch: refs/heads/jira/solr-11702
Commit: fbea59b0864768356f1057a0a099d8a54887d272
Parents: 0c4fb31
Author: Joel Bernstein <jb...@apache.org>
Authored: Thu Dec 28 14:14:26 2017 -0500
Committer: Joel Bernstein <jb...@apache.org>
Committed: Thu Dec 28 14:14:26 2017 -0500
----------------------------------------------------------------------
.../org/apache/solr/handler/StreamHandler.java | 1 +
.../solrj/io/eval/MannWhitneyUEvaluator.java | 62 ++++++++++++++++++++
.../solrj/io/stream/StreamExpressionTest.java | 18 +++++-
3 files changed, 80 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fbea59b0/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
index fa1aaaa..ae15260 100644
--- a/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/StreamHandler.java
@@ -286,6 +286,7 @@ public class StreamHandler extends RequestHandlerBase implements SolrCoreAware,
.withFunctionName("multiVariateNormalDistribution", MultiVariateNormalDistributionEvaluator.class)
.withFunctionName("integrate", IntegrateEvaluator.class)
.withFunctionName("density", DensityEvaluator.class)
+ .withFunctionName("mannWhitney", MannWhitneyUEvaluator.class)
// Boolean Stream Evaluators
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fbea59b0/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MannWhitneyUEvaluator.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MannWhitneyUEvaluator.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MannWhitneyUEvaluator.java
new file mode 100644
index 0000000..defa919
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/eval/MannWhitneyUEvaluator.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.client.solrj.io.eval;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.commons.math3.stat.inference.MannWhitneyUTest;
+import org.apache.solr.client.solrj.io.Tuple;
+import org.apache.solr.client.solrj.io.stream.expr.StreamExpression;
+import org.apache.solr.client.solrj.io.stream.expr.StreamFactory;
+
+
+public class MannWhitneyUEvaluator extends RecursiveNumericListEvaluator implements ManyValueWorker {
+ protected static final long serialVersionUID = 1L;
+
+ public MannWhitneyUEvaluator(StreamExpression expression, StreamFactory factory) throws IOException {
+ super(expression, factory);
+
+ if(containedEvaluators.size() < 1){
+ throw new IOException(String.format(Locale.ROOT,"Invalid expression %s - expecting at least one value but found %d",expression,containedEvaluators.size()));
+ }
+ }
+
+ @Override
+ public Object doWork(Object... values) throws IOException {
+ List<double[]> mannWhitneyUInput = Arrays.stream(values)
+ .map(value -> ((List<Number>) value).stream().mapToDouble(Number::doubleValue).toArray())
+ .collect(Collectors.toList());
+ if(mannWhitneyUInput.size() == 2) {
+ MannWhitneyUTest mannwhitneyutest = new MannWhitneyUTest();
+ double u = mannwhitneyutest.mannWhitneyU(mannWhitneyUInput.get(0), mannWhitneyUInput.get(1));
+ double p = mannwhitneyutest.mannWhitneyUTest(mannWhitneyUInput.get(0), mannWhitneyUInput.get(1));
+ Map<String,Number> m = new HashMap<>();
+ m.put("u-statistic", u);
+ m.put("p-value", p);
+ return new Tuple(m);
+ }else{
+ throw new IOException(String.format(Locale.ROOT,"%s(...) only works with a list of 2 arrays but a list of %d array(s) was provided.", constructingFactory.getFunctionName(getClass()), mannWhitneyUInput.size()));
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fbea59b0/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
index bc3dee5..4af8e43 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/io/stream/StreamExpressionTest.java
@@ -7403,7 +7403,6 @@ public class StreamExpressionTest extends SolrCloudTestCase {
}
}
-
@Test
public void testAnova() throws Exception {
String cexpr = "anova(array(1,2,3,5,4,6), array(5,2,3,5,4,6), array(1,2,7,5,4,6))";
@@ -7513,6 +7512,23 @@ public class StreamExpressionTest extends SolrCloudTestCase {
assertEquals((double) out.get(3), 5.5, .0);
}
+ @Test
+ public void testMannWhitney() throws Exception {
+ String cexpr = "mannWhitney(array(0.15,0.10,0.11,0.24,0.08,0.08,0.10,0.10,0.10,0.12,0.04,0.07), " +
+ "array(0.10,0.20,0.30,0.10,0.10,0.02,0.05,0.07))";
+ ModifiableSolrParams paramsLoc = new ModifiableSolrParams();
+ paramsLoc.set("expr", cexpr);
+ paramsLoc.set("qt", "/stream");
+ String url = cluster.getJettySolrRunners().get(0).getBaseUrl().toString()+"/"+COLLECTIONORALIAS;
+ TupleStream solrStream = new SolrStream(url, paramsLoc);
+ StreamContext context = new StreamContext();
+ solrStream.setStreamContext(context);
+ List<Tuple> tuples = getTuples(solrStream);
+ assertTrue(tuples.size() == 1);
+ Map out = (Map)tuples.get(0).get("return-value");
+ assertEquals((double) out.get("u-statistic"), 52.5, .1);
+ assertEquals((double) out.get("p-value"), 0.7284, .001);
+ }
@Test
public void testMovingMedian() throws Exception {
[15/18] lucene-solr:jira/solr-11702: LUCENE-8010: Remove assumptions
about how the similarity orders hits from FastVectorHighlighterTest.
Posted by da...@apache.org.
LUCENE-8010: Remove assumptions about how the similarity orders hits from FastVectorHighlighterTest.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/fbc8508e
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/fbc8508e
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/fbc8508e
Branch: refs/heads/jira/solr-11702
Commit: fbc8508e11ce80343d1052e5b05ee0e922c5bad7
Parents: deb7644
Author: Adrien Grand <jp...@gmail.com>
Authored: Sat Dec 30 15:46:15 2017 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Sat Dec 30 15:46:15 2017 +0100
----------------------------------------------------------------------
.../search/vectorhighlight/FastVectorHighlighterTest.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/fbc8508e/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java
----------------------------------------------------------------------
diff --git a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java
index 0683e8d..ddcce5a 100644
--- a/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java
+++ b/lucene/highlighter/src/test/org/apache/lucene/search/vectorhighlight/FastVectorHighlighterTest.java
@@ -324,7 +324,9 @@ public class FastVectorHighlighterTest extends LuceneTestCase {
public void testCommonTermsQueryHighlight() throws IOException {
Directory dir = newDirectory();
- IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET)));
+ IndexWriter writer = new IndexWriter(dir,
+ newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.SIMPLE, true, MockTokenFilter.ENGLISH_STOPSET))
+ .setMergePolicy(newLogMergePolicy())); // don't reorder doc ids
FieldType type = new FieldType(TextField.TYPE_STORED);
type.setStoreTermVectorOffsets(true);
type.setStoreTermVectorPositions(true);
@@ -353,11 +355,11 @@ public class FastVectorHighlighterTest extends LuceneTestCase {
TopDocs hits = searcher.search(query, 10);
assertEquals(2, hits.totalHits);
FieldQuery fieldQuery = highlighter.getFieldQuery(query, reader);
- String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[0].doc, "field", 1000, 1);
+ String[] bestFragments = highlighter.getBestFragments(fieldQuery, reader, 1, "field", 1000, 1);
assertEquals("This piece of <b>text</b> refers to Kennedy at the beginning then has a longer piece of <b>text</b> that is <b>very</b> <b>long</b> in the middle and finally ends with another reference to Kennedy", bestFragments[0]);
fieldQuery = highlighter.getFieldQuery(query, reader);
- bestFragments = highlighter.getBestFragments(fieldQuery, reader, hits.scoreDocs[1].doc, "field", 1000, 1);
+ bestFragments = highlighter.getBestFragments(fieldQuery, reader, 0, "field", 1000, 1);
assertEquals("Hello this is a piece of <b>text</b> that is <b>very</b> <b>long</b> and contains too much preamble and the meat is really here which says kennedy has been shot", bestFragments[0]);
reader.close();
[14/18] lucene-solr:jira/solr-11702: LUCENE-8010: TestGrouping
expects that longer fields produce lower scores.
Posted by da...@apache.org.
LUCENE-8010: TestGrouping expects that longer fields produce lower scores.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/deb76449
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/deb76449
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/deb76449
Branch: refs/heads/jira/solr-11702
Commit: deb7644912f6339e0647795c732049af2e60fc75
Parents: 8dfb019
Author: Adrien Grand <jp...@gmail.com>
Authored: Fri Dec 29 15:59:50 2017 +0100
Committer: Adrien Grand <jp...@gmail.com>
Committed: Fri Dec 29 15:59:50 2017 +0100
----------------------------------------------------------------------
.../src/test/org/apache/lucene/search/grouping/TestGrouping.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/deb76449/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
----------------------------------------------------------------------
diff --git a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
index 9779c43..f59df53 100644
--- a/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
+++ b/lucene/grouping/src/test/org/apache/lucene/search/grouping/TestGrouping.java
@@ -59,6 +59,7 @@ import org.apache.lucene.search.Sort;
import org.apache.lucene.search.SortField;
import org.apache.lucene.search.TermQuery;
import org.apache.lucene.search.Weight;
+import org.apache.lucene.search.similarities.BM25Similarity;
import org.apache.lucene.store.Directory;
import org.apache.lucene.util.BytesRef;
import org.apache.lucene.util.LuceneTestCase;
@@ -135,6 +136,8 @@ public class TestGrouping extends LuceneTestCase {
w.addDocument(doc);
IndexSearcher indexSearcher = newSearcher(w.getReader());
+ // This test relies on the fact that longer fields produce lower scores
+ indexSearcher.setSimilarity(new BM25Similarity());
w.close();
final Sort groupSort = Sort.RELEVANCE;