You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by va...@apache.org on 2015/08/05 15:30:53 UTC
svn commit: r1694210 - in /lucene/dev/trunk/solr: CHANGES.txt
core/src/java/org/apache/solr/search/stats/ExactStatsCache.java
core/src/test/org/apache/solr/search/stats/TestDistribIDF.java
Author: varun
Date: Wed Aug 5 13:30:53 2015
New Revision: 1694210
URL: http://svn.apache.org/r1694210
Log:
SOLR-7818: Distributed stats is only calculated with the terms that are present in the last shard of a distributed request
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1694210&r1=1694209&r2=1694210&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Wed Aug 5 13:30:53 2015
@@ -275,6 +275,9 @@ Bug Fixes
* SOLR-7756: ExactStatsCache and LRUStatsCache will throw an NPE when a term is not present on a shard.
(Varun Thacker, Anshum Gupta)
+* SOLR-7818: Distributed stats is only calculated with the terms that are present in the last shard
+ of a distributed request. (Varun Thacker, Anshum Gupta)
+
Optimizations
----------------------
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java?rev=1694210&r1=1694209&r2=1694210&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/stats/ExactStatsCache.java Wed Aug 5 13:30:53 2015
@@ -17,6 +17,7 @@ package org.apache.solr.search.stats;
* limitations under the License.
*/
+import com.google.common.collect.Lists;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.Term;
import org.apache.lucene.index.TermContext;
@@ -91,6 +92,7 @@ public class ExactStatsCache extends Sta
@Override
public void mergeToGlobalStats(SolrQueryRequest req, List<ShardResponse> responses) {
+ Set<Object> allTerms = new HashSet<>();
for (ShardResponse r : responses) {
LOG.debug("Merging to global stats, shard={}, response={}", r.getShard(), r.getSolrResponse().getResponse());
String shard = r.getShard();
@@ -103,16 +105,16 @@ public class ExactStatsCache extends Sta
addToPerShardTermStats(req, shard, termStatsString);
}
List<Object> terms = nl.getAll(TERMS_KEY);
- if (terms != null) {
- req.getContext().put(TERMS_KEY, terms);
- }
-
+ allTerms.addAll(terms);
String colStatsString = (String) nl.get(COL_STATS_KEY);
Map<String,CollectionStats> colStats = StatsUtil.colStatsMapFromString(colStatsString);
if (colStats != null) {
addToPerShardColStats(req, shard, colStats);
}
}
+ if (allTerms.size() > 0) {
+ req.getContext().put(TERMS_KEY, Lists.newArrayList(allTerms));
+ }
if (LOG.isDebugEnabled()) printStats(req);
}
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java?rev=1694210&r1=1694209&r2=1694210&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/search/stats/TestDistribIDF.java Wed Aug 5 13:30:53 2015
@@ -138,6 +138,50 @@ public class TestDistribIDF extends Solr
}
}
+ @Test
+ public void testMultiCollectionQuery() throws Exception {
+ // collection1 and collection2 are collections which have distributed idf enabled
+ // collection1_local and collection2_local don't have distributed idf available
+ // Only one doc has cat:football in each collection
+ // When doing queries across collections we want to test that the query takes into account
+ // distributed idf for the collection=collection1,collection2 query.
+ // The way we verify is that score should be the same when querying across collection1 and collection2
+ // But should be different when querying across collection1_local and collection2_local
+ // since the idf is calculated per shard
+
+ createCollection("collection1", "conf1");
+ createCollection("collection1_local", "conf2");
+ createCollection("collection2", "conf1");
+ createCollection("collection2_local", "conf2");
+
+ addDocsRandomly();
+
+ //Test against all nodes
+ for (JettySolrRunner jettySolrRunner : solrCluster.getJettySolrRunners()) {
+ SolrClient solrClient = new HttpSolrClient(jettySolrRunner.getBaseUrl().toString());
+ SolrClient solrClient_local = new HttpSolrClient(jettySolrRunner.getBaseUrl().toString());
+
+ SolrQuery query = new SolrQuery("cat:football");
+ query.setFields("*,score").add("collection", "collection1,collection2");
+ QueryResponse queryResponse = solrClient.query("collection1", query);
+ assertEquals(2, queryResponse.getResults().getNumFound());
+ float score1 = (float) queryResponse.getResults().get(0).get("score");
+ float score2 = (float) queryResponse.getResults().get(1).get("score");
+ assertEquals("Doc1 score=" + score1 + " Doc2 score=" + score2, 0, Float.compare(score1, score2));
+
+
+ query = new SolrQuery("cat:football");
+ query.setFields("*,score").add("collection", "collection1_local,collection2_local");
+ queryResponse = solrClient_local.query("collection1_local", query);
+ assertEquals(2, queryResponse.getResults().getNumFound());
+ assertEquals(2, queryResponse.getResults().get(0).get("id"));
+ assertEquals(1, queryResponse.getResults().get(1).get("id"));
+ float score1_local = (float) queryResponse.getResults().get(0).get("score");
+ float score2_local = (float) queryResponse.getResults().get(1).get("score");
+ assertEquals("Doc1 score=" + score1_local + " Doc2 score=" + score2_local, 1, Float.compare(score1_local, score2_local));
+ }
+ }
+
private void createCollection(String name, String config) throws Exception {
createCollection(name, config, CompositeIdRouter.NAME);
}