You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by mk...@apache.org on 2019/03/04 14:42:44 UTC
[lucene-solr] branch master updated: SOLR-9882: reporting
timeAllowed breach as partialResults instead of 500 error
This is an automated email from the ASF dual-hosted git repository.
mkhl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/lucene-solr.git
The following commit(s) were added to refs/heads/master by this push:
new b8d569a SOLR-9882: reporting timeAllowed breach as partialResults instead of 500 error
b8d569a is described below
commit b8d569aff0c4417b0f9cd52d54455ab9b66236a1
Author: Mikhail Khludnev <mk...@apache.org>
AuthorDate: Wed Feb 20 16:24:52 2019 +0300
SOLR-9882: reporting timeAllowed breach as partialResults instead of 500 error
---
solr/CHANGES.txt | 3 +
.../client/solrj/embedded/JettySolrRunner.java | 20 +-
.../apache/solr/handler/RequestHandlerBase.java | 5 +-
.../solr/handler/component/FacetComponent.java | 12 ++
.../solr/handler/component/QueryComponent.java | 33 ++-
.../solr/handler/component/ResponseBuilder.java | 7 +-
.../solr/handler/component/SearchHandler.java | 21 +-
.../java/org/apache/solr/request/SimpleFacets.java | 7 +-
.../apache/solr/response/BasicResultContext.java | 1 +
.../org/apache/solr/search/SolrIndexSearcher.java | 2 +-
.../org/apache/solr/search/facet/FacetModule.java | 14 +-
.../org/apache/solr/search/facet/FacetRequest.java | 9 +-
.../SearchGroupShardResponseProcessor.java | 4 +-
.../TopGroupsShardResponseProcessor.java | 4 +-
.../configsets/exitable-directory/conf/schema.xml | 1 +
.../exitable-directory/conf/solrconfig.xml | 79 +++----
.../cloud/CloudExitableDirectoryReaderTest.java | 185 +++++++++++++++--
.../solr/cloud/TrollingIndexReaderFactory.java | 229 +++++++++++++++++++++
.../apache/solr/cloud/MiniSolrCloudCluster.java | 61 +++++-
.../org/apache/solr/cloud/SolrCloudTestCase.java | 8 +-
20 files changed, 588 insertions(+), 117 deletions(-)
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index bd04156..0d3b41e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -87,6 +87,9 @@ Bug Fixes
all the cores.
(Danyal Prout via shalin)
+* SOLR-9882: 500 error code on breaching timeAllowed by core and distributed (fsv) search,
+ old and json facets (Mikhail Khludnev)
+
Improvements
----------------------
* SOLR-12999: Index replication could delete segments before downloading segments from master if there is not enough
diff --git a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
index 986c286..ba94104 100644
--- a/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
+++ b/solr/core/src/java/org/apache/solr/client/solrj/embedded/JettySolrRunner.java
@@ -54,11 +54,11 @@ import org.apache.solr.common.util.SolrjNamedThreadFactory;
import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.servlet.SolrDispatchFilter;
+import org.apache.solr.util.TimeOut;
import org.eclipse.jetty.alpn.server.ALPNServerConnectionFactory;
import org.eclipse.jetty.http2.HTTP2Cipher;
import org.eclipse.jetty.http2.server.HTTP2CServerConnectionFactory;
import org.eclipse.jetty.http2.server.HTTP2ServerConnectionFactory;
-import org.apache.solr.util.TimeOut;
import org.eclipse.jetty.server.Connector;
import org.eclipse.jetty.server.HttpConfiguration;
import org.eclipse.jetty.server.HttpConnectionFactory;
@@ -66,6 +66,7 @@ import org.eclipse.jetty.server.SecureRequestCustomizer;
import org.eclipse.jetty.server.Server;
import org.eclipse.jetty.server.ServerConnector;
import org.eclipse.jetty.server.SslConnectionFactory;
+import org.eclipse.jetty.server.handler.HandlerWrapper;
import org.eclipse.jetty.server.handler.gzip.GzipHandler;
import org.eclipse.jetty.server.session.DefaultSessionIdManager;
import org.eclipse.jetty.servlet.FilterHolder;
@@ -333,6 +334,8 @@ public class JettySolrRunner {
server.setConnectors(new Connector[] {connector});
}
+ HandlerWrapper chain;
+ {
// Initialize the servlets
final ServletContextHandler root = new ServletContextHandler(server, config.context, ServletContextHandler.SESSIONS);
@@ -391,11 +394,15 @@ public class JettySolrRunner {
System.clearProperty("hostPort");
}
});
-
// for some reason, there must be a servlet for this to get applied
root.addServlet(Servlet404.class, "/*");
+ chain = root;
+ }
+
+ chain = injectJettyHandlers(chain);
+
GzipHandler gzipHandler = new GzipHandler();
- gzipHandler.setHandler(root);
+ gzipHandler.setHandler(chain);
gzipHandler.setMinGzipSize(0);
gzipHandler.setCheckGzExists(false);
@@ -406,6 +413,13 @@ public class JettySolrRunner {
server.setHandler(gzipHandler);
}
+ /** descendants may inject own handler chaining it to the given root
+ * and then returning that own one*/
+ protected HandlerWrapper injectJettyHandlers(HandlerWrapper chain) {
+ return chain;
+ }
+
+
/**
* @return the {@link SolrDispatchFilter} for this node
*/
diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
index a398eb7..ee7923f 100644
--- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
+++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java
@@ -200,9 +200,8 @@ public abstract class RequestHandlerBase implements SolrRequestHandler, SolrInfo
// count timeouts
NamedList header = rsp.getResponseHeader();
if(header != null) {
- Object partialResults = header.get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY);
- boolean timedOut = partialResults == null ? false : (Boolean)partialResults;
- if( timedOut ) {
+ if( Boolean.TRUE.equals(header.getBooleanArg(
+ SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY)) ) {
numTimeouts.mark();
rsp.setHttpCaching(false);
}
diff --git a/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java b/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
index e01c958..e339a75 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/FacetComponent.java
@@ -47,6 +47,7 @@ import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.request.SimpleFacets;
import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.schema.FieldType;
import org.apache.solr.schema.PointField;
import org.apache.solr.search.QueryParsing;
@@ -711,6 +712,17 @@ public class FacetComponent extends SearchComponent {
NamedList facet_counts = null;
try {
facet_counts = (NamedList) srsp.getSolrResponse().getResponse().get("facet_counts");
+ if (facet_counts==null) {
+ NamedList<?> responseHeader = (NamedList<?>)srsp.getSolrResponse().getResponse().get("responseHeader");
+ if (Boolean.TRUE.equals(responseHeader.getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
+ continue;
+ } else {
+ log.warn("corrupted response on "+srsp.getShardRequest()+": "+srsp.getSolrResponse());
+ throw new SolrException(ErrorCode.SERVER_ERROR,
+ "facet_counts is absent in response from " + srsp.getNodeName() +
+ ", but "+SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY+" hasn't been responded");
+ }
+ }
} catch (Exception ex) {
if (ShardParams.getShardsTolerantAsBool(rb.req.getParams())) {
continue; // looks like a shard did not return anything
diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
index e937370..4dab304 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
@@ -31,6 +31,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
+import org.apache.lucene.index.ExitableDirectoryReader;
import org.apache.lucene.index.IndexReaderContext;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.ReaderUtil;
@@ -384,6 +385,7 @@ public class QueryComponent extends SearchComponent
// TODO: See SOLR-5595
boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES,false);
if(fsv){
+ try {
NamedList<Object[]> sortVals = new NamedList<>(); // order is important for the sort fields
IndexReaderContext topReaderContext = searcher.getTopReaderContext();
List<LeafReaderContext> leaves = topReaderContext.leaves();
@@ -394,13 +396,12 @@ public class QueryComponent extends SearchComponent
leaves=null;
}
- DocList docList = rb.getResults().docList;
+ final DocList docs = rb.getResults().docList;
// sort ids from lowest to highest so we can access them in order
- int nDocs = docList.size();
+ int nDocs = docs.size();
final long[] sortedIds = new long[nDocs];
final float[] scores = new float[nDocs]; // doc scores, parallel to sortedIds
- DocList docs = rb.getResults().docList;
DocIterator it = docs.iterator();
for (int i=0; i<nDocs; i++) {
sortedIds[i] = (((long)it.nextDoc()) << 32) | i;
@@ -476,8 +477,13 @@ public class QueryComponent extends SearchComponent
sortVals.add(sortField.getField(), vals);
}
-
rsp.add("sort_values", sortVals);
+ }catch(ExitableDirectoryReader.ExitingReaderException x) {
+ // it's hard to understand where we stopped, so yield nothing
+ // search handler will flag partial results
+ rsp.add("sort_values",new NamedList<>() );
+ throw x;
+ }
}
}
@@ -860,7 +866,7 @@ public class QueryComponent extends SearchComponent
}
if (responseHeader != null) {
- if (Boolean.TRUE.equals(responseHeader.get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
+ if (Boolean.TRUE.equals(responseHeader.getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
partialResults = true;
}
if (!Boolean.TRUE.equals(segmentTerminatedEarly)) {
@@ -880,6 +886,9 @@ public class QueryComponent extends SearchComponent
numFound += docs.getNumFound();
NamedList sortFieldValues = (NamedList)(srsp.getSolrResponse().getResponse().get("sort_values"));
+ if (sortFieldValues.size()==0 && partialResults) {
+ continue; //fsv timeout yields empty sort_vlaues
+ }
NamedList unmarshalledSortFieldValues = unmarshalSortValues(ss, sortFieldValues, schema);
// go through every doc in this response, construct a ShardDoc, and
@@ -958,9 +967,8 @@ public class QueryComponent extends SearchComponent
populateNextCursorMarkFromMergedShards(rb);
if (partialResults) {
- if(rb.rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY) == null) {
- rb.rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
- }
+ rb.rsp.getResponseHeader().asShallowMap()
+ .put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
}
if (segmentTerminatedEarly != null) {
final Object existingSegmentTerminatedEarly = rb.rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY);
@@ -1158,6 +1166,13 @@ public class QueryComponent extends SearchComponent
continue;
}
+ {
+ NamedList<?> responseHeader = (NamedList<?>)srsp.getSolrResponse().getResponse().get("responseHeader");
+ if (responseHeader!=null && Boolean.TRUE.equals(responseHeader.getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
+ rb.rsp.getResponseHeader().asShallowMap()
+ .put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
+ }
+ }
SolrDocumentList docs = (SolrDocumentList) srsp.getSolrResponse().getResponse().get("response");
for (SolrDocument doc : docs) {
Object id = doc.getFieldValue(keyFieldName);
@@ -1436,7 +1451,7 @@ public class QueryComponent extends SearchComponent
ResultContext ctx = new BasicResultContext(rb);
rsp.addResponse(ctx);
- rsp.getToLog().add("hits", rb.getResults().docList.matches());
+ rsp.getToLog().add("hits", rb.getResults()==null || rb.getResults().docList==null ? 0 : rb.getResults().docList.matches());
if ( ! rb.req.getParams().getBool(ShardParams.IS_SHARD,false) ) {
if (null != rb.getNextCursorMark()) {
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
index 1f9e2d5..f757ad7 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
@@ -30,6 +30,7 @@ import org.apache.solr.request.SolrRequestInfo;
import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.CursorMark;
import org.apache.solr.search.DocListAndSet;
+import org.apache.solr.search.DocSlice;
import org.apache.solr.search.QParser;
import org.apache.solr.search.QueryCommand;
import org.apache.solr.search.QueryResult;
@@ -460,7 +461,11 @@ public class ResponseBuilder
public void setResult(QueryResult result) {
setResults(result.getDocListAndSet());
if (result.isPartialResults()) {
- rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
+ rsp.getResponseHeader().asShallowMap()
+ .put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
+ if(getResults().docList==null) {
+ getResults().docList = new DocSlice(0, 0, new int[] {}, new float[] {}, 0, 0);
+ }
}
final Boolean segmentTerminatedEarly = result.getSegmentTerminatedEarly();
if (segmentTerminatedEarly != null) {
diff --git a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
index d4c680c..89cfa2e 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/SearchHandler.java
@@ -16,6 +16,9 @@
*/
package org.apache.solr.handler.component;
+import static org.apache.solr.common.params.CommonParams.DISTRIB;
+import static org.apache.solr.common.params.CommonParams.PATH;
+
import java.io.PrintWriter;
import java.io.StringWriter;
import java.lang.invoke.MethodHandles;
@@ -53,9 +56,6 @@ import org.apache.solr.util.plugin.SolrCoreAware;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import static org.apache.solr.common.params.CommonParams.DISTRIB;
-import static org.apache.solr.common.params.CommonParams.PATH;
-
/**
*
@@ -315,17 +315,16 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware ,
}
} catch (ExitableDirectoryReader.ExitingReaderException ex) {
log.warn( "Query: " + req.getParamString() + "; " + ex.getMessage());
- SolrDocumentList r = (SolrDocumentList) rb.rsp.getResponse();
- if(r == null)
- r = new SolrDocumentList();
- r.setNumFound(0);
- rb.rsp.addResponse(r);
+ if( rb.rsp.getResponse() == null) {
+ rb.rsp.addResponse(new SolrDocumentList());
+ }
if(rb.isDebug()) {
NamedList debug = new NamedList();
debug.add("explain", new NamedList());
rb.rsp.add("debug", debug);
}
- rb.rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
+ rb.rsp.getResponseHeader().asShallowMap()
+ .put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
} finally {
SolrQueryTimeoutImpl.reset();
}
@@ -413,9 +412,7 @@ public class SearchHandler extends RequestHandlerBase implements SolrCoreAware ,
throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, srsp.getException());
}
} else {
- if(rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY) == null) {
- rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
- }
+ rsp.getResponseHeader().asShallowMap().put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
}
}
diff --git a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
index 1bea80a..35ef0fe 100644
--- a/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
+++ b/solr/core/src/java/org/apache/solr/request/SimpleFacets.java
@@ -38,6 +38,7 @@ import java.util.concurrent.Semaphore;
import java.util.function.Predicate;
import java.util.stream.Stream;
+import org.apache.lucene.index.ExitableDirectoryReader;
import org.apache.lucene.index.LeafReader;
import org.apache.lucene.index.LeafReaderContext;
import org.apache.lucene.index.MultiPostingsEnum;
@@ -828,7 +829,11 @@ public class SimpleFacets {
return result;
} catch (SolrException se) {
throw se;
- } catch (Exception e) {
+ }
+ catch(ExitableDirectoryReader.ExitingReaderException timeout) {
+ throw timeout;
+ }
+ catch (Exception e) {
throw new SolrException(ErrorCode.SERVER_ERROR,
"Exception during facet.field: " + facetValue, e);
} finally {
diff --git a/solr/core/src/java/org/apache/solr/response/BasicResultContext.java b/solr/core/src/java/org/apache/solr/response/BasicResultContext.java
index 792e483..579c0ad 100644
--- a/solr/core/src/java/org/apache/solr/response/BasicResultContext.java
+++ b/solr/core/src/java/org/apache/solr/response/BasicResultContext.java
@@ -31,6 +31,7 @@ public class BasicResultContext extends ResultContext {
private SolrQueryRequest req;
public BasicResultContext(DocList docList, ReturnFields returnFields, SolrIndexSearcher searcher, Query query, SolrQueryRequest req) {
+ assert docList!=null;
this.docList = docList;
this.returnFields = returnFields;
this.searcher = searcher;
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index a659d66..ab0996d 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -1713,7 +1713,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
set = DocSetUtil.getDocSet(setCollector, this);
totalHits = topCollector.getTotalHits();
- assert (totalHits == set.size());
+ assert (totalHits == set.size()) || qr.isPartialResults();
TopDocs topDocs = topCollector.topDocs(0, len);
if (cmd.getSort() != null && query instanceof RankQuery == false && (cmd.getFlags() & GET_SCORES) != 0) {
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java b/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java
index a5d55fb..5da2cef 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetModule.java
@@ -36,6 +36,7 @@ import org.apache.solr.handler.component.ResponseBuilder;
import org.apache.solr.handler.component.SearchComponent;
import org.apache.solr.handler.component.ShardRequest;
import org.apache.solr.handler.component.ShardResponse;
+import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.search.QueryContext;
import org.noggit.CharArr;
import org.noggit.JSONWriter;
@@ -142,7 +143,8 @@ public class FacetModule extends SearchComponent {
rb.req.getContext().put("FacetDebugInfo", fdebug);
}
- final Object results = facetState.facetRequest.process(fcontext);
+ Object results = facetState.facetRequest.process(fcontext);
+ // ExitableDirectory timeout causes absent "facets"
rb.rsp.add("facets", results);
}
@@ -167,7 +169,7 @@ public class FacetModule extends SearchComponent {
}
// Check if there are any refinements possible
- if (facetState.mcontext.getSubsWithRefinement(facetState.facetRequest).isEmpty()) {
+ if ((facetState.mcontext==null) ||facetState.mcontext.getSubsWithRefinement(facetState.facetRequest).isEmpty()) {
clearFaceting(rb.outgoing);
return ResponseBuilder.STAGE_DONE;
}
@@ -277,7 +279,13 @@ public class FacetModule extends SearchComponent {
NamedList<Object> top = rsp.getResponse();
if (top == null) continue; // shards.tolerant=true will cause this to happen on exceptions/errors
Object facet = top.get("facets");
- if (facet == null) continue;
+ if (facet == null) {
+ SimpleOrderedMap shardResponseHeader = (SimpleOrderedMap)rsp.getResponse().get("responseHeader");
+ if(Boolean.TRUE.equals(shardResponseHeader.getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
+ rb.rsp.getResponseHeader().asShallowMap().put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
+ }
+ continue;
+ }
if (facetState.merger == null) {
facetState.merger = facetState.facetRequest.createFacetMerger(facet);
facetState.mcontext = new FacetMerger.Context( sreq.responses.size() );
diff --git a/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java b/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java
index 752a71e..fd8ce79 100644
--- a/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java
+++ b/solr/core/src/java/org/apache/solr/search/facet/FacetRequest.java
@@ -407,11 +407,14 @@ public abstract class FacetRequest {
debugInfo.setProcessor(facetProcessor.getClass().getSimpleName());
debugInfo.putInfoItem("domainSize", (long) fcontext.base.size());
RTimer timer = new RTimer();
- facetProcessor.process();
- debugInfo.setElapse((long) timer.getTime());
+ try {
+ facetProcessor.process();
+ }finally {
+ debugInfo.setElapse((long) timer.getTime());
+ }
}
- return facetProcessor.getResponse(); // note: not captured in elapsed time above; good/bad?
+ return facetProcessor.getResponse();
}
public abstract FacetProcessor createFacetProcessor(FacetContext fcontext);
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
index edad958..163c38d 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/SearchGroupShardResponseProcessor.java
@@ -100,9 +100,7 @@ public class SearchGroupShardResponseProcessor implements ShardResponseProcessor
shardInfo.add(srsp.getShard(), nl);
}
if (ShardParams.getShardsTolerantAsBool(rb.req.getParams()) && srsp.getException() != null) {
- if(rb.rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY) == null) {
- rb.rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
- }
+ rb.rsp.getResponseHeader().asShallowMap().put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
continue; // continue if there was an error and we're tolerant.
}
maxElapsedTime = (int) Math.max(maxElapsedTime, srsp.getSolrResponse().getElapsedTime());
diff --git a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
index e2dd299..2db6b22 100644
--- a/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
+++ b/solr/core/src/java/org/apache/solr/search/grouping/distributed/responseprocessor/TopGroupsShardResponseProcessor.java
@@ -111,9 +111,7 @@ public class TopGroupsShardResponseProcessor implements ShardResponseProcessor {
shardInfo.add(srsp.getShard(), individualShardInfo);
}
if (ShardParams.getShardsTolerantAsBool(rb.req.getParams()) && srsp.getException() != null) {
- if(rb.rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY) == null) {
- rb.rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
- }
+ rb.rsp.getResponseHeader().asShallowMap().put(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
continue; // continue if there was an error and we're tolerant.
}
NamedList<NamedList> secondPhaseResult = (NamedList<NamedList>) srsp.getSolrResponse().getResponse().get("secondPhase");
diff --git a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml
index 52a63f4..72f148c 100644
--- a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml
+++ b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/schema.xml
@@ -24,5 +24,6 @@
<field name="_version_" type="long" indexed="true" stored="true"/>
<field name="_root_" type="string" indexed="true" stored="true" multiValued="false" required="false"/>
<field name="id" type="string" indexed="true" stored="true"/>
+ <field name="num" type="int" indexed="true" stored="true"/>
<uniqueKey>id</uniqueKey>
</schema>
diff --git a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml
index af1036f..5d466e8 100644
--- a/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml
+++ b/solr/core/src/test-files/solr/configsets/exitable-directory/conf/solrconfig.xml
@@ -19,6 +19,7 @@
<config>
<jmx />
+ <metrics/>
<luceneMatchVersion>${tests.luceneMatchVersion:LATEST}</luceneMatchVersion>
@@ -32,6 +33,9 @@
</directoryFactory>
<schemaFactory class="ClassicIndexSchemaFactory"/>
+ <indexReaderFactory name="IndexReaderFactory"
+ class="org.apache.solr.cloud.TrollingIndexReaderFactory"></indexReaderFactory >
+
<dataDir>${solr.data.dir:}</dataDir>
<!-- an update processor the explicitly excludes distrib to test
@@ -51,67 +55,32 @@
</updateLog>
</updateHandler>
- <updateRequestProcessorChain name="dedupe">
- <processor class="org.apache.solr.update.processor.SignatureUpdateProcessorFactory">
- <bool name="enabled">true</bool>
- <bool name="overwriteDupes">true</bool>
- <str name="fields">v_t,t_field</str>
- <str name="signatureClass">org.apache.solr.update.processor.TextProfileSignature</str>
- </processor>
- <processor class="solr.RunUpdateProcessorFactory" />
- </updateRequestProcessorChain>
- <updateRequestProcessorChain name="stored_sig">
- <!-- this chain is valid even though the signature field is not
- indexed, because we are not asking for dups to be overwritten
- -->
- <processor class="org.apache.solr.update.processor.SignatureUpdateProcessorFactory">
- <bool name="enabled">true</bool>
- <str name="signatureField">non_indexed_signature_sS</str>
- <bool name="overwriteDupes">false</bool>
- <str name="fields">v_t,t_field</str>
- <str name="signatureClass">org.apache.solr.update.processor.TextProfileSignature</str>
- </processor>
- <processor class="solr.RunUpdateProcessorFactory" />
- </updateRequestProcessorChain>
-
- <updateRequestProcessorChain name="distrib-dup-test-chain-explicit">
- <!-- explicit test using processors before and after distrib -->
- <processor class="solr.RegexReplaceProcessorFactory">
- <str name="fieldName">regex_dup_A_s</str>
- <str name="pattern">x</str>
- <str name="replacement">x_x</str>
- </processor>
- <processor class="solr.DistributedUpdateProcessorFactory" />
- <processor class="solr.RegexReplaceProcessorFactory">
- <str name="fieldName">regex_dup_B_s</str>
- <str name="pattern">x</str>
- <str name="replacement">x_x</str>
- </processor>
- <processor class="solr.RunUpdateProcessorFactory" />
- </updateRequestProcessorChain>
-
- <updateRequestProcessorChain name="distrib-dup-test-chain-implicit">
- <!-- implicit test w/o distrib declared-->
- <processor class="solr.RegexReplaceProcessorFactory">
- <str name="fieldName">regex_dup_A_s</str>
- <str name="pattern">x</str>
- <str name="replacement">x_x</str>
- </processor>
- <processor class="solr.RegexReplaceProcessorFactory">
- <str name="fieldName">regex_dup_B_s</str>
- <str name="pattern">x</str>
- <str name="replacement">x_x</str>
- </processor>
- <processor class="solr.RunUpdateProcessorFactory" />
- </updateRequestProcessorChain>
-
+ <query>
+ <filterCache class="solr.FastLRUCache"
+ size="0"
+ initialSize="0"
+ autowarmCount="0"/>
+ <queryResultCache class="solr.LRUCache"
+ size="0"
+ initialSize="0"
+ autowarmCount="0"/>
+ <documentCache class="solr.LRUCache"
+ size="0"
+ initialSize="0"
+ autowarmCount="0"/>
+ <fieldValueCache class="solr.FastLRUCache"
+ size="0"
+ autowarmCount="0"
+ showItems="0" />
+ </query>
+
<searchComponent name="delayingSearchComponent"
class="org.apache.solr.search.DelayingSearchComponent"/>
<requestHandler name="/select" class="solr.SearchHandler">
<arr name="first-components">
<str>delayingSearchComponent</str>
- </arr>
+ </arr>
</requestHandler>
</config>
diff --git a/solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java b/solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java
index a4dc86e..3a45500 100644
--- a/solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/CloudExitableDirectoryReaderTest.java
@@ -16,65 +16,100 @@
*/
package org.apache.solr.cloud;
+import java.lang.invoke.MethodHandles;
+import java.util.LinkedHashMap;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import org.apache.lucene.util.TestUtil;
+import org.apache.solr.client.solrj.embedded.JettySolrRunner;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.UpdateRequest;
import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.MiniSolrCloudCluster.JettySolrRunnerWithMetrics;
+import static org.apache.solr.cloud.TrollingIndexReaderFactory.*;
import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.handler.component.FacetComponent;
+import org.apache.solr.handler.component.QueryComponent;
import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.search.facet.FacetModule;
import org.junit.BeforeClass;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.carrotsearch.randomizedtesting.annotations.Repeat;
+import com.codahale.metrics.Metered;
+import com.codahale.metrics.MetricRegistry;
/**
* Distributed test for {@link org.apache.lucene.index.ExitableDirectoryReader}
*/
public class CloudExitableDirectoryReaderTest extends SolrCloudTestCase {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
private static final int NUM_DOCS_PER_TYPE = 20;
private static final String sleep = "2";
private static final String COLLECTION = "exitable";
+ private static Map<String, Metered> fiveHundredsByNode;
@BeforeClass
public static void setupCluster() throws Exception {
- configureCluster(2)
- .addConfig("conf", TEST_PATH().resolve("configsets").resolve("exitable-directory").resolve("conf"))
+ Builder clusterBuilder = configureCluster(2)
+ .addConfig("conf", TEST_PATH().resolve("configsets").resolve("exitable-directory").resolve("conf"));
+ clusterBuilder.withMetrics(true);
+ clusterBuilder
.configure();
CollectionAdminRequest.createCollection(COLLECTION, "conf", 2, 1)
.processAndWait(cluster.getSolrClient(), DEFAULT_TIMEOUT);
cluster.getSolrClient().waitForState(COLLECTION, DEFAULT_TIMEOUT, TimeUnit.SECONDS,
(n, c) -> DocCollection.isFullyActive(n, c, 2, 1));
- }
- @Test
- public void test() throws Exception {
+ fiveHundredsByNode = new LinkedHashMap<>();
+ for (JettySolrRunner jetty: cluster.getJettySolrRunners()) {
+ MetricRegistry metricRegistry = ((JettySolrRunnerWithMetrics)jetty).getMetricRegistry();
+ Metered httpOk = (Metered) metricRegistry.getMetrics()
+ .get("org.eclipse.jetty.servlet.ServletContextHandler.2xx-responses");
+ assertTrue("expeting some http activity during collection creation",httpOk.getCount()>0);
+
+ Metered old = fiveHundredsByNode.put(jetty.getNodeName(),
+ (Metered) metricRegistry.getMetrics()
+ .get("org.eclipse.jetty.servlet.ServletContextHandler.5xx-responses"));
+ assertNull("expecting uniq nodenames",old);
+ }
+
indexDocs();
- doTimeoutTests();
}
- public void indexDocs() throws Exception {
- int counter = 1;
+ public static void indexDocs() throws Exception {
+ int counter;
+ counter = 1;
UpdateRequest req = new UpdateRequest();
for(; (counter % NUM_DOCS_PER_TYPE) != 0; counter++ )
- req.add(sdoc("id", Integer.toString(counter), "name", "a" + counter));
+ req.add(sdoc("id", Integer.toString(counter), "name", "a" + counter,
+ "num",""+counter));
counter++;
for(; (counter % NUM_DOCS_PER_TYPE) != 0; counter++ )
- req.add(sdoc("id", Integer.toString(counter), "name", "b" + counter));
+ req.add(sdoc("id", Integer.toString(counter), "name", "b" + counter,
+ "num",""+counter));
counter++;
for(; counter % NUM_DOCS_PER_TYPE != 0; counter++ )
- req.add(sdoc("id", Integer.toString(counter), "name", "dummy term doc" + counter));
+ req.add(sdoc("id", Integer.toString(counter), "name", "dummy term doc" + counter,
+ "num",""+counter));
req.commit(cluster.getSolrClient(), COLLECTION);
}
- public void doTimeoutTests() throws Exception {
+ @Test
+ public void test() throws Exception {
assertPartialResults(params("q", "name:a*", "timeAllowed", "1", "sleep", sleep));
/*
@@ -99,18 +134,136 @@ public class CloudExitableDirectoryReaderTest extends SolrCloudTestCase {
assertSuccess(params("q","name:b*")); // no time limitation
}
+ @Test
+ public void testWhitebox() throws Exception {
+
+ try (Trap catchIds = catchTrace(
+ new CheckMethodName("doProcessSearchByIds"), () -> {})) {
+ assertPartialResults(params("q", "{!cache=false}name:a*", "sort", "query($q,1) asc"),
+ () -> assertTrue(catchIds.hasCaught()));
+ } catch (AssertionError ae) {
+ Trap.dumpLastStackTraces(log);
+ throw ae;
+ }
+
+ // the point is to catch sort_values (fsv) timeout, between search and facet
+ // I haven't find a way to encourage fsv to read index
+ try (Trap catchFSV = catchTrace(
+ new CheckMethodName("doFieldSortValues"), () -> {})) {
+ assertPartialResults(params("q", "{!cache=false}name:a*", "sort", "query($q,1) asc"),
+ () -> assertTrue(catchFSV.hasCaught()));
+ } catch (AssertionError ae) {
+ Trap.dumpLastStackTraces(log);
+ throw ae;
+ }
+
+ try (Trap catchClass = catchClass(
+ QueryComponent.class.getSimpleName(), () -> { })) {
+ assertPartialResults(params("q", "{!cache=false}name:a*"),
+ ()->assertTrue(catchClass.hasCaught()));
+ }catch(AssertionError ae) {
+ Trap.dumpLastStackTraces(log);
+ throw ae;
+ }
+ try(Trap catchClass = catchClass(FacetComponent.class.getSimpleName())){
+ assertPartialResults(params("q", "{!cache=false}name:a*", "facet","true", "facet.method", "enum",
+ "facet.field", "id"),
+ ()->assertTrue(catchClass.hasCaught()));
+ }catch(AssertionError ae) {
+ Trap.dumpLastStackTraces(log);
+ throw ae;
+ }
+
+ try (Trap catchClass = catchClass(FacetModule.class.getSimpleName())) {
+ assertPartialResults(params("q", "{!cache=false}name:a*", "json.facet", "{ ids: {"
+ + " type: range, field : num, start : 0, end : 100, gap : 10 }}"),
+ () -> assertTrue(catchClass.hasCaught()));
+ } catch (AssertionError ae) {
+ Trap.dumpLastStackTraces(log);
+ throw ae;
+ }
+ }
+
+ @Test
+ @Repeat(iterations=5)
+ public void testCreepThenBite() throws Exception {
+ int creep=100;
+ ModifiableSolrParams params = params("q", "{!cache=false}name:a*");
+ SolrParams cases[] = new SolrParams[] {
+ params( "sort","query($q,1) asc"),
+ params("rows","0", "facet","true", "facet.method", "enum", "facet.field", "name"),
+ params("rows","0", "json.facet","{ ids: { type: range, field : num, start : 1, end : 99, gap : 9 }}")
+ }; //add more cases here
+
+ params.add(cases[random().nextInt(cases.length)]);
+ for (; ; creep*=1.5) {
+ final int boundary = creep;
+ try(Trap catchClass = catchCount(boundary)){
+
+ params.set("boundary", boundary);
+ QueryResponse rsp = cluster.getSolrClient().query(COLLECTION,
+ params);
+ assertEquals(""+rsp, rsp.getStatus(), 0);
+ assertNo500s(""+rsp);
+ if (!isPartial(rsp)) {
+ assertFalse(catchClass.hasCaught());
+ break;
+ }
+ assertTrue(catchClass.hasCaught());
+ }catch(AssertionError ae) {
+ Trap.dumpLastStackTraces(log);
+ throw ae;
+ }
+ }
+ int numBites = atLeast(100);
+ for(int bite=0; bite<numBites; bite++) {
+ int boundary = random().nextInt(creep);
+ try(Trap catchCount = catchCount(boundary)){
+ params.set("boundary", boundary);
+ QueryResponse rsp = cluster.getSolrClient().query(COLLECTION,
+ params);
+ assertEquals(""+rsp, rsp.getStatus(), 0);
+ assertNo500s(""+rsp);
+ assertEquals(""+creep+" ticks were sucessful; trying "+boundary+" yields "+rsp,
+ catchCount.hasCaught(), isPartial(rsp));
+ }catch(AssertionError ae) {
+ Trap.dumpLastStackTraces(log);
+ throw ae;
+ }
+ }
+ }
+
+ public boolean isPartial(QueryResponse rsp) {
+ return Boolean.TRUE.equals(rsp.getHeader().getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY));
+ }
+
+ public void assertNo500s(String msg) {
+ assertTrue(msg,fiveHundredsByNode.values().stream().allMatch((m)->m.getCount()==0));
+ }
+
/**
* execute a request, verify that we get an expected error
*/
public void assertPartialResults(ModifiableSolrParams p) throws Exception {
+ assertPartialResults(p, ()->{});
+ }
+
+ public void assertPartialResults(ModifiableSolrParams p, Runnable postRequestCheck) throws Exception {
QueryResponse rsp = cluster.getSolrClient().query(COLLECTION, p);
- assertEquals(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY+" were expected",
- true, rsp.getHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY));
+ postRequestCheck.run();
+ assertEquals(rsp.getStatus(), 0);
+ assertEquals(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY+" were expected at "+rsp,
+ true, rsp.getHeader().getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY));
+ assertNo500s(""+rsp);
}
public void assertSuccess(ModifiableSolrParams p) throws Exception {
- QueryResponse response = cluster.getSolrClient().query(COLLECTION, p);
- assertEquals("Wrong #docs in response", NUM_DOCS_PER_TYPE - 1, response.getResults().getNumFound());
+ QueryResponse rsp = cluster.getSolrClient().query(COLLECTION, p);
+ assertEquals(rsp.getStatus(), 0);
+ assertEquals("Wrong #docs in response", NUM_DOCS_PER_TYPE - 1, rsp.getResults().getNumFound());
+ assertNotEquals(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY+" weren't expected "+rsp,
+ true, rsp.getHeader().getBooleanArg(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY));
+ assertNo500s(""+rsp);
}
}
diff --git a/solr/core/src/test/org/apache/solr/cloud/TrollingIndexReaderFactory.java b/solr/core/src/test/org/apache/solr/cloud/TrollingIndexReaderFactory.java
new file mode 100644
index 0000000..553ed6f
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/cloud/TrollingIndexReaderFactory.java
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.solr.cloud;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+
+import org.apache.lucene.index.DirectoryReader;
+import org.apache.lucene.index.ExitableDirectoryReader;
+import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.QueryTimeout;
+import org.apache.lucene.store.Directory;
+import org.apache.lucene.util.LuceneTestCase;
+import org.apache.solr.core.SolrCore;
+import org.apache.solr.core.StandardIndexReaderFactory;
+
+public class TrollingIndexReaderFactory extends StandardIndexReaderFactory {
+
+ private static volatile Trap trap;
+ private final static BlockingQueue<List<Object>> lastStacktraces = new LinkedBlockingQueue<List<Object>>();
+ private final static long startTime = ManagementFactory.getRuntimeMXBean().getStartTime();
+ private static final int keepStackTraceLines = 20;
+ protected static final int maxTraces = 4;
+
+
+ private static Trap setTrap(Trap troll) {
+ trap = troll;
+ return troll;
+ }
+
+ public static abstract class Trap implements Closeable{
+ protected abstract boolean shouldExit();
+ public abstract boolean hasCaught();
+ @Override
+ public final void close() throws IOException {
+ setTrap(null);
+ }
+ @Override
+ public abstract String toString();
+
+ public static void dumpLastStackTraces(org.slf4j.Logger log) {
+ ArrayList<List<Object>> stacks = new ArrayList<>();
+ lastStacktraces.drainTo(stacks);
+ StringBuilder out = new StringBuilder("the last caught stacktraces: \n");
+ for(List<Object> stack : stacks) {
+ int l=0;
+ for (Object line : stack) {
+ if (l++>0) {
+ out.append('\t');
+ }
+ out.append(line);
+ out.append('\n');
+ }
+ out.append('\n');
+ }
+ log.error("the last caught traces {}", out);
+ }
+ }
+
+ static final class CheckMethodName implements Predicate<StackTraceElement> {
+ private final String methodName;
+
+ CheckMethodName(String methodName) {
+ this.methodName = methodName;
+ }
+
+ @Override
+ public boolean test(StackTraceElement trace) {
+ return trace.getMethodName().equals(methodName);
+ }
+
+ @Override
+ public String toString() {
+ return "hunting for "+methodName+"()";
+ }
+ }
+
+ public static Trap catchClass(String className) {
+ return catchClass(className, ()->{});
+ }
+
+ public static Trap catchClass(String className, Runnable onCaught) {
+ Predicate<StackTraceElement> judge = new Predicate<StackTraceElement>() {
+ @Override
+ public boolean test(StackTraceElement trace) {
+ return trace.getClassName().indexOf(className)>=0;
+ }
+ @Override
+ public String toString() {
+ return "className contains "+className;
+ }
+ };
+ return catchTrace(judge, onCaught) ;
+ }
+
+ public static Trap catchTrace(Predicate<StackTraceElement> judge, Runnable onCaught) {
+ return setTrap(new Trap() {
+
+ private boolean trigered;
+
+ @Override
+ protected boolean shouldExit() {
+ Exception e = new Exception("stack sniffer");
+ e.fillInStackTrace();
+ StackTraceElement[] stackTrace = e.getStackTrace();
+ for(StackTraceElement trace : stackTrace) {
+ if (judge.test(trace)) {
+ trigered = true;
+ recordStackTrace(stackTrace);
+ onCaught.run();
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public boolean hasCaught() {
+ return trigered;
+ }
+
+ @Override
+ public String toString() {
+ return ""+judge;
+ }
+ });
+ }
+
+ public static Trap catchCount(int boundary) {
+ return setTrap(new Trap() {
+
+ private AtomicInteger count = new AtomicInteger();
+
+ @Override
+ public String toString() {
+ return ""+count.get()+"th tick of "+boundary+" allowed";
+ }
+
+ private boolean trigered;
+
+ @Override
+ protected boolean shouldExit() {
+ int now = count.incrementAndGet();
+ boolean trigger = now==boundary
+ || (now>boundary && LuceneTestCase.rarely(LuceneTestCase.random()));
+ if (trigger) {
+ Exception e = new Exception("stack sniffer");
+ e.fillInStackTrace();
+ recordStackTrace(e.getStackTrace());
+ trigered = true;
+ }
+ return trigger;
+ }
+
+ @Override
+ public boolean hasCaught() {
+ return trigered;
+ }
+ });
+ }
+
+ private static void recordStackTrace(StackTraceElement[] stackTrace) {
+ //keep the last n limited traces.
+ //e.printStackTrace();
+ ArrayList<Object> stack = new ArrayList<Object>();
+ stack.add(""+ (new Date().getTime()-startTime)+" ("+Thread.currentThread().getName()+")");
+ for (int l=2; l<stackTrace.length && l<keepStackTraceLines; l++) {
+ stack.add(stackTrace[l]);
+ }
+ lastStacktraces.add(stack);
+ // triming queue
+ while(lastStacktraces.size()>maxTraces) {
+ try {
+ lastStacktraces.poll(100, TimeUnit.MILLISECONDS);
+ } catch (InterruptedException e1) {
+ e1.printStackTrace();
+ }
+ }
+ }
+
+ @Override
+ public DirectoryReader newReader(Directory indexDir, SolrCore core) throws IOException {
+ DirectoryReader newReader = super.newReader(indexDir, core);
+ return wrap(newReader);
+ }
+
+ private ExitableDirectoryReader wrap(DirectoryReader newReader) throws IOException {
+ return new ExitableDirectoryReader(newReader, new QueryTimeout() {
+ @Override
+ public boolean shouldExit() {
+ return trap!=null && trap.shouldExit();
+ }
+
+ @Override
+ public String toString() {
+ return ""+trap;
+ }
+ });
+ }
+
+ @Override
+ public DirectoryReader newReader(IndexWriter writer, SolrCore core) throws IOException {
+ DirectoryReader newReader = super.newReader(writer, core);
+ return wrap(newReader);
+ }
+}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
index 2fbeba8..deaa4a8 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/MiniSolrCloudCluster.java
@@ -16,7 +16,6 @@
*/
package org.apache.solr.cloud;
-import javax.servlet.Filter;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.nio.charset.Charset;
@@ -44,6 +43,8 @@ import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
+import javax.servlet.Filter;
+
import org.apache.lucene.util.LuceneTestCase;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.embedded.JettyConfig;
@@ -71,10 +72,13 @@ import org.apache.solr.common.util.TimeSource;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.util.TimeOut;
import org.apache.zookeeper.KeeperException;
+import org.eclipse.jetty.server.handler.HandlerWrapper;
import org.eclipse.jetty.servlet.ServletHolder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.codahale.metrics.MetricRegistry;
+
/**
* "Mini" SolrCloud cluster to be used for testing
*/
@@ -124,9 +128,11 @@ public class MiniSolrCloudCluster {
private final Path baseDir;
private final CloudSolrClient solrClient;
private final JettyConfig jettyConfig;
+ private final boolean trackJettyMetrics;
private final AtomicInteger nodeIds = new AtomicInteger();
+
/**
* Create a MiniSolrCloudCluster with default solr.xml
*
@@ -230,10 +236,32 @@ public class MiniSolrCloudCluster {
*/
MiniSolrCloudCluster(int numServers, Path baseDir, String solrXml, JettyConfig jettyConfig,
ZkTestServer zkTestServer, Optional<String> securityJson) throws Exception {
+ this(numServers, baseDir, solrXml, jettyConfig,
+ zkTestServer,securityJson, false);
+ }
+ /**
+ * Create a MiniSolrCloudCluster.
+ * Note - this constructor visibility is changed to package protected so as to
+ * discourage its usage. Ideally *new* functionality should use {@linkplain SolrCloudTestCase}
+ * to configure any additional parameters.
+ *
+ * @param numServers number of Solr servers to start
+ * @param baseDir base directory that the mini cluster should be run from
+ * @param solrXml solr.xml file to be uploaded to ZooKeeper
+ * @param jettyConfig Jetty configuration
+ * @param zkTestServer ZkTestServer to use. If null, one will be created
+ * @param securityJson A string representation of security.json file (optional).
+ * @param trackJettyMetrics supply jetties with metrics registry
+ *
+ * @throws Exception if there was an error starting the cluster
+ */
+ MiniSolrCloudCluster(int numServers, Path baseDir, String solrXml, JettyConfig jettyConfig,
+ ZkTestServer zkTestServer, Optional<String> securityJson, boolean trackJettyMetrics) throws Exception {
Objects.requireNonNull(securityJson);
this.baseDir = Objects.requireNonNull(baseDir);
this.jettyConfig = Objects.requireNonNull(jettyConfig);
+ this.trackJettyMetrics = trackJettyMetrics;
log.info("Starting cluster of {} servers in {}", numServers, baseDir);
@@ -433,12 +461,14 @@ public class MiniSolrCloudCluster {
Path runnerPath = createInstancePath(name);
String context = getHostContextSuitableForServletContext(hostContext);
JettyConfig newConfig = JettyConfig.builder(config).setContext(context).build();
- JettySolrRunner jetty = new JettySolrRunner(runnerPath.toString(), newConfig);
+ JettySolrRunner jetty = !trackJettyMetrics
+ ? new JettySolrRunner(runnerPath.toString(), newConfig)
+ :new JettySolrRunnerWithMetrics(runnerPath.toString(), newConfig);
jetty.start();
jettys.add(jetty);
return jetty;
}
-
+
/**
* Start a new Solr instance, using the default config
*
@@ -774,4 +804,29 @@ public class MiniSolrCloudCluster {
throw new TimeoutException("Waiting for Jetty to stop timed out");
}
}
+
+ /** @lucene.experimental */
+ public static final class JettySolrRunnerWithMetrics extends JettySolrRunner {
+ public JettySolrRunnerWithMetrics(String solrHome, JettyConfig config) {
+ super(solrHome, config);
+ }
+
+ private volatile MetricRegistry metricRegistry;
+
+ @Override
+ protected HandlerWrapper injectJettyHandlers(HandlerWrapper chain) {
+ metricRegistry = new MetricRegistry();
+ com.codahale.metrics.jetty9.InstrumentedHandler metrics
+ = new com.codahale.metrics.jetty9.InstrumentedHandler(
+ metricRegistry);
+ metrics.setHandler(chain);
+ return metrics;
+ }
+
+ /** @return optional subj. It may be null, if it's not yet created. */
+ public MetricRegistry getMetricRegistry() {
+ return metricRegistry;
+ }
+ }
+
}
diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
index acb09be..647d7b7 100644
--- a/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
+++ b/solr/test-framework/src/java/org/apache/solr/cloud/SolrCloudTestCase.java
@@ -107,6 +107,7 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
private List<Config> configs = new ArrayList<>();
private Map<String, Object> clusterProperties = new HashMap<>();
+ private boolean trackJettyMetrics;
/**
* Create a builder
* @param nodeCount the number of nodes in the cluster
@@ -191,6 +192,10 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
return this;
}
+ public Builder withMetrics(boolean trackJettyMetrics) {
+ this.trackJettyMetrics = trackJettyMetrics;
+ return this;
+ }
/**
* Configure and run the {@link MiniSolrCloudCluster}
* @throws Exception if an error occurs on startup
@@ -204,7 +209,8 @@ public class SolrCloudTestCase extends SolrTestCaseJ4 {
* @throws Exception if an error occurs on startup
*/
public MiniSolrCloudCluster build() throws Exception {
- MiniSolrCloudCluster cluster = new MiniSolrCloudCluster(nodeCount, baseDir, solrxml, jettyConfig, null, securityJson);
+ MiniSolrCloudCluster cluster = new MiniSolrCloudCluster(nodeCount, baseDir, solrxml, jettyConfig,
+ null, securityJson, trackJettyMetrics);
CloudSolrClient client = cluster.getSolrClient();
for (Config config : configs) {
((ZkClientClusterStateProvider)client.getClusterStateProvider()).uploadConfig(config.path, config.name);