You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by sh...@apache.org on 2013/07/02 18:10:16 UTC
svn commit: r1498992 - in /lucene/dev/trunk/solr: ./
core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/
core/src/test/org/apache/solr/
Author: shalin
Date: Tue Jul 2 16:10:16 2013
New Revision: 1498992
URL: http://svn.apache.org/r1498992
Log:
SOLR-3369: shards.tolerant=true is broken for group queries
Modified:
lucene/dev/trunk/solr/CHANGES.txt
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
Modified: lucene/dev/trunk/solr/CHANGES.txt
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/CHANGES.txt?rev=1498992&r1=1498991&r2=1498992&view=diff
==============================================================================
--- lucene/dev/trunk/solr/CHANGES.txt (original)
+++ lucene/dev/trunk/solr/CHANGES.txt Tue Jul 2 16:10:16 2013
@@ -237,6 +237,9 @@ Bug Fixes
* SOLR-4974: Outgrowth of SOLR-4960 that includes transient cores and pending cores
(Erick Erickson)
+* SOLR-3369: shards.tolerant=true is broken for group queries
+ (Russell Black, Martijn van Groningen, Jabouille jean Charles, Ryan McKinley via shalin)
+
Optimizations
----------------------
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java?rev=1498992&r1=1498991&r2=1498992&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java Tue Jul 2 16:10:16 2013
@@ -20,8 +20,11 @@ package org.apache.solr.search.grouping.
import org.apache.lucene.search.Sort;
import org.apache.lucene.search.grouping.SearchGroup;
import org.apache.lucene.util.BytesRef;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.handler.component.ShardRequest;
import org.apache.solr.handler.component.ShardResponse;
@@ -31,6 +34,8 @@ import org.apache.solr.search.grouping.d
import org.apache.solr.search.grouping.distributed.shardresultserializer.SearchGroupsResultTransformer;
import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.util.*;
/**
@@ -61,7 +66,38 @@ public class SearchGroupShardResponsePro
try {
int maxElapsedTime = 0;
int hitCountDuringFirstPhase = 0;
+
+ NamedList<Object> shardInfo = null;
+ if (rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) {
+ shardInfo = new SimpleOrderedMap<Object>();
+ rb.rsp.getValues().add(ShardParams.SHARDS_INFO + ".firstPhase", shardInfo);
+ }
+
for (ShardResponse srsp : shardRequest.responses) {
+ if (shardInfo != null) {
+ SimpleOrderedMap<Object> nl = new SimpleOrderedMap<Object>();
+
+ if (srsp.getException() != null) {
+ Throwable t = srsp.getException();
+ if (t instanceof SolrServerException) {
+ t = ((SolrServerException) t).getCause();
+ }
+ nl.add("error", t.toString());
+ StringWriter trace = new StringWriter();
+ t.printStackTrace(new PrintWriter(trace));
+ nl.add("trace", trace.toString());
+ } else {
+ nl.add("numFound", (Integer) srsp.getSolrResponse().getResponse().get("totalHitCount"));
+ }
+ if (srsp.getSolrResponse() != null) {
+ nl.add("time", srsp.getSolrResponse().getElapsedTime());
+ }
+
+ shardInfo.add(srsp.getShard(), nl);
+ }
+ if (rb.req.getParams().getBool(ShardParams.SHARDS_TOLERANT, false) && srsp.getException() != null) {
+ continue; // continue if there was an error and we're tolerant.
+ }
maxElapsedTime = (int) Math.max(maxElapsedTime, srsp.getSolrResponse().getElapsedTime());
@SuppressWarnings("unchecked")
NamedList<NamedList> firstPhaseResult = (NamedList<NamedList>) srsp.getSolrResponse().getResponse().get("firstPhase");
Modified: lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java?rev=1498992&r1=1498991&r2=1498992&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java (original)
+++ lucene/dev/trunk/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java Tue Jul 2 16:10:16 2013
@@ -23,8 +23,11 @@ import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.grouping.GroupDocs;
import org.apache.lucene.search.grouping.TopGroups;
import org.apache.lucene.util.BytesRef;
+import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.params.ShardParams;
import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.handler.component.ShardDoc;
import org.apache.solr.handler.component.ShardRequest;
@@ -35,6 +38,8 @@ import org.apache.solr.search.grouping.d
import org.apache.solr.search.grouping.distributed.shardresultserializer.TopGroupsResultTransformer;
import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringWriter;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -76,18 +81,66 @@ public class TopGroupsShardResponseProce
}
TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb);
+
+ NamedList<Object> shardInfo = null;
+ if (rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) {
+ shardInfo = new SimpleOrderedMap<Object>();
+ rb.rsp.getValues().add(ShardParams.SHARDS_INFO, shardInfo);
+ }
+
for (ShardResponse srsp : shardRequest.responses) {
+ SimpleOrderedMap<Object> individualShardInfo = null;
+ if (shardInfo != null) {
+ individualShardInfo = new SimpleOrderedMap<Object>();
+
+ if (srsp.getException() != null) {
+ Throwable t = srsp.getException();
+ if (t instanceof SolrServerException) {
+ t = ((SolrServerException) t).getCause();
+ }
+ individualShardInfo.add("error", t.toString());
+ StringWriter trace = new StringWriter();
+ t.printStackTrace(new PrintWriter(trace));
+ individualShardInfo.add("trace", trace.toString());
+ } else {
+ // summary for successful shard response is added down below
+ }
+ if (srsp.getSolrResponse() != null) {
+ individualShardInfo.add("time", srsp.getSolrResponse().getElapsedTime());
+ }
+
+ shardInfo.add(srsp.getShard(), individualShardInfo);
+ }
+ if (rb.req.getParams().getBool(ShardParams.SHARDS_TOLERANT, false) && srsp.getException() != null) {
+ continue; // continue if there was an error and we're tolerant.
+ }
NamedList<NamedList> secondPhaseResult = (NamedList<NamedList>) srsp.getSolrResponse().getResponse().get("secondPhase");
Map<String, ?> result = serializer.transformToNative(secondPhaseResult, groupSort, sortWithinGroup, srsp.getShard());
+ int numFound = 0;
+ float maxScore = Float.NaN;
for (String field : commandTopGroups.keySet()) {
TopGroups<BytesRef> topGroups = (TopGroups<BytesRef>) result.get(field);
if (topGroups == null) {
continue;
}
+ if (individualShardInfo != null) { // keep track of this when shards.info=true
+ numFound += topGroups.totalHitCount;
+ if (Float.isNaN(maxScore) || topGroups.maxScore > maxScore) maxScore = topGroups.maxScore;
+ }
commandTopGroups.get(field).add(topGroups);
}
for (String query : queries) {
- commandTopDocs.get(query).add((QueryCommandResult) result.get(query));
+ QueryCommandResult queryCommandResult = (QueryCommandResult) result.get(query);
+ if (individualShardInfo != null) { // keep track of this when shards.info=true
+ numFound += queryCommandResult.getMatches();
+ float thisMax = queryCommandResult.getTopDocs().getMaxScore();
+ if (Float.isNaN(maxScore) || thisMax > maxScore) maxScore = thisMax;
+ }
+ commandTopDocs.get(query).add(queryCommandResult);
+ }
+ if (individualShardInfo != null) { // when shards.info=true
+ individualShardInfo.add("numFound", numFound);
+ individualShardInfo.add("maxScore", maxScore);
}
}
try {
Modified: lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDistributedSearch.java
URL: http://svn.apache.org/viewvc/lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDistributedSearch.java?rev=1498992&r1=1498991&r2=1498992&view=diff
==============================================================================
--- lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDistributedSearch.java (original)
+++ lucene/dev/trunk/solr/core/src/test/org/apache/solr/TestDistributedSearch.java Tue Jul 2 16:10:16 2013
@@ -351,7 +351,7 @@ public class TestDistributedSearch exten
ChaosMonkey.stop(downJetty);
downJettys.add(downJetty);
}
-
+
queryPartialResults(upShards, upClients,
"q","*:*",
"facet","true",
@@ -359,7 +359,27 @@ public class TestDistributedSearch exten
"facet.limit",5,
ShardParams.SHARDS_INFO,"true",
ShardParams.SHARDS_TOLERANT,"true");
-
+
+ queryPartialResults(upShards, upClients,
+ "q", "*:*",
+ "facet", "true",
+ "facet.query", i1 + ":[1 TO 50]",
+ ShardParams.SHARDS_INFO, "true",
+ ShardParams.SHARDS_TOLERANT, "true");
+
+ // test group query
+ queryPartialResults(upShards, upClients,
+ "q", "*:*",
+ "rows", 100,
+ "fl", "id," + i1,
+ "group", "true",
+ "group.query", t1 + ":kings OR " + t1 + ":eggs",
+ "group.limit", 10,
+ "sort", i1 + " asc, id asc",
+ CommonParams.TIME_ALLOWED, 1,
+ ShardParams.SHARDS_INFO, "true",
+ ShardParams.SHARDS_TOLERANT, "true");
+
// restart the jettys
for (JettySolrRunner downJetty : downJettys) {
downJetty.start();