You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by cp...@apache.org on 2016/02/09 22:28:42 UTC
[3/3] lucene-solr git commit: make Lucene's
EarlyTerminatingSortingCollector available in Solr via optional
segmentTerminateEarly=(false|true) parameter
make Lucene's EarlyTerminatingSortingCollector available in Solr via optional segmentTerminateEarly=(false|true) parameter
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/58ba778c
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/58ba778c
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/58ba778c
Branch: refs/heads/jira/solr-5730-master
Commit: 58ba778c71ecdcff20444e305464803643bc33b2
Parents: 3a9887f
Author: Christine Poerschke <cp...@apache.org>
Authored: Tue Feb 9 21:17:33 2016 +0000
Committer: Christine Poerschke <cp...@apache.org>
Committed: Tue Feb 9 21:17:33 2016 +0000
----------------------------------------------------------------------
.../solr/handler/component/QueryComponent.java | 41 ++-
.../solr/handler/component/ResponseBuilder.java | 4 +
.../apache/solr/response/SolrQueryResponse.java | 1 +
.../org/apache/solr/search/QueryCommand.java | 12 +
.../org/apache/solr/search/QueryResult.java | 9 +
.../apache/solr/search/SolrIndexSearcher.java | 20 ++
.../solr/update/DefaultSolrCoreState.java | 19 ++
.../org/apache/solr/update/SolrCoreState.java | 8 +
.../solr/cloud/TestMiniSolrCloudCluster.java | 311 +++++++++++++++++++
.../solr/response/TestSolrQueryResponse.java | 6 +
.../apache/solr/common/params/CommonParams.java | 6 +
11 files changed, 434 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/58ba778c/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
index c855936..e4aaf25 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/QueryComponent.java
@@ -373,11 +373,17 @@ public class QueryComponent extends SearchComponent
QueryResult result = new QueryResult();
+ cmd.setSegmentTerminateEarly(params.getBool(CommonParams.SEGMENT_TERMINATE_EARLY, CommonParams.SEGMENT_TERMINATE_EARLY_DEFAULT));
+ if (cmd.getSegmentTerminateEarly()) {
+ result.setSegmentTerminatedEarly(Boolean.FALSE);
+ }
+
//
// grouping / field collapsing
//
GroupingSpecification groupingSpec = rb.getGroupingSpec();
if (groupingSpec != null) {
+ cmd.setSegmentTerminateEarly(false); // not supported, silently ignore any segmentTerminateEarly flag
try {
boolean needScores = (cmd.getFlags() & SolrIndexSearcher.GET_SCORES) != 0;
if (params.getBool(GroupParams.GROUP_DISTRIBUTED_FIRST, false)) {
@@ -983,8 +989,10 @@ public class QueryComponent extends SearchComponent
long numFound = 0;
Float maxScore=null;
boolean partialResults = false;
+ Boolean segmentTerminatedEarly = null;
for (ShardResponse srsp : sreq.responses) {
SolrDocumentList docs = null;
+ NamedList<?> responseHeader = null;
if(shardInfo!=null) {
SimpleOrderedMap<Object> nl = new SimpleOrderedMap<>();
@@ -1003,6 +1011,11 @@ public class QueryComponent extends SearchComponent
}
}
else {
+ responseHeader = (NamedList<?>)srsp.getSolrResponse().getResponse().get("responseHeader");
+ final Object rhste = (responseHeader == null ? null : responseHeader.get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY));
+ if (rhste != null) {
+ nl.add(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY, rhste);
+ }
docs = (SolrDocumentList)srsp.getSolrResponse().getResponse().get("response");
nl.add("numFound", docs.getNumFound());
nl.add("maxScore", docs.getMaxScore());
@@ -1024,9 +1037,22 @@ public class QueryComponent extends SearchComponent
docs = (SolrDocumentList)srsp.getSolrResponse().getResponse().get("response");
}
- NamedList<?> responseHeader = (NamedList<?>)srsp.getSolrResponse().getResponse().get("responseHeader");
- if (responseHeader != null && Boolean.TRUE.equals(responseHeader.get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
- partialResults = true;
+ if (responseHeader == null) { // could have been initialized in the shards info block above
+ responseHeader = (NamedList<?>)srsp.getSolrResponse().getResponse().get("responseHeader");
+ }
+
+ if (responseHeader != null) {
+ if (Boolean.TRUE.equals(responseHeader.get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY))) {
+ partialResults = true;
+ }
+ if (!Boolean.TRUE.equals(segmentTerminatedEarly)) {
+ final Object ste = responseHeader.get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY);
+ if (Boolean.TRUE.equals(ste)) {
+ segmentTerminatedEarly = Boolean.TRUE;
+ } else if (Boolean.FALSE.equals(ste)) {
+ segmentTerminatedEarly = Boolean.FALSE;
+ }
+ }
}
// calculate global maxScore and numDocsFound
@@ -1118,6 +1144,15 @@ public class QueryComponent extends SearchComponent
rb.rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
}
}
+ if (segmentTerminatedEarly != null) {
+ final Object existingSegmentTerminatedEarly = rb.rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY);
+ if(existingSegmentTerminatedEarly == null) {
+ rb.rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY, segmentTerminatedEarly);
+ } else if (!Boolean.TRUE.equals(existingSegmentTerminatedEarly) && Boolean.TRUE.equals(segmentTerminatedEarly)) {
+ rb.rsp.getResponseHeader().remove(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY);
+ rb.rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY, segmentTerminatedEarly);
+ }
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/58ba778c/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
index 8f05e26..5dd4b5c 100644
--- a/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
+++ b/solr/core/src/java/org/apache/solr/handler/component/ResponseBuilder.java
@@ -453,6 +453,10 @@ public class ResponseBuilder
if (result.isPartialResults()) {
rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
}
+ final Boolean segmentTerminatedEarly = result.getSegmentTerminatedEarly();
+ if (segmentTerminatedEarly != null) {
+ rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY, segmentTerminatedEarly);
+ }
if (null != cursorMark) {
assert null != result.getNextCursorMark() : "using cursor but no next cursor set";
this.setNextCursorMark(result.getNextCursorMark());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/58ba778c/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java b/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java
index 277848b..f1ccd08 100644
--- a/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java
+++ b/solr/core/src/java/org/apache/solr/response/SolrQueryResponse.java
@@ -67,6 +67,7 @@ import org.apache.solr.search.SolrReturnFields;
public class SolrQueryResponse {
public static final String NAME = "response";
public static final String RESPONSE_HEADER_PARTIAL_RESULTS_KEY = "partialResults";
+ public static final String RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY = "segmentTerminatedEarly";
private static final String RESPONSE_HEADER_KEY = "responseHeader";
private static final String RESPONSE_KEY = "response";
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/58ba778c/solr/core/src/java/org/apache/solr/search/QueryCommand.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/QueryCommand.java b/solr/core/src/java/org/apache/solr/search/QueryCommand.java
index 129a5c2..553e022 100755
--- a/solr/core/src/java/org/apache/solr/search/QueryCommand.java
+++ b/solr/core/src/java/org/apache/solr/search/QueryCommand.java
@@ -207,4 +207,16 @@ public class QueryCommand {
}
}
+ public boolean getSegmentTerminateEarly() {
+ return (flags & SolrIndexSearcher.SEGMENT_TERMINATE_EARLY) != 0;
+ }
+
+ public QueryCommand setSegmentTerminateEarly(boolean segmentSegmentTerminateEarly) {
+ if (segmentSegmentTerminateEarly) {
+ return setFlags(SolrIndexSearcher.SEGMENT_TERMINATE_EARLY);
+ } else {
+ return clearFlags(SolrIndexSearcher.SEGMENT_TERMINATE_EARLY);
+ }
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/58ba778c/solr/core/src/java/org/apache/solr/search/QueryResult.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/QueryResult.java b/solr/core/src/java/org/apache/solr/search/QueryResult.java
index c1d6528..419a8f4 100755
--- a/solr/core/src/java/org/apache/solr/search/QueryResult.java
+++ b/solr/core/src/java/org/apache/solr/search/QueryResult.java
@@ -22,6 +22,7 @@ package org.apache.solr.search;
public class QueryResult {
private boolean partialResults;
+ private Boolean segmentTerminatedEarly;
private DocListAndSet docListAndSet;
private CursorMark nextCursorMark;
@@ -57,6 +58,14 @@ public class QueryResult {
this.partialResults = partialResults;
}
+ public Boolean getSegmentTerminatedEarly() {
+ return segmentTerminatedEarly;
+ }
+
+ public void setSegmentTerminatedEarly(Boolean segmentTerminatedEarly) {
+ this.segmentTerminatedEarly = segmentTerminatedEarly;
+ }
+
public void setDocListAndSet(DocListAndSet listSet) {
docListAndSet = listSet;
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/58ba778c/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
index 66c4795..8057a97 100644
--- a/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
+++ b/solr/core/src/java/org/apache/solr/search/SolrIndexSearcher.java
@@ -70,6 +70,7 @@ import org.apache.lucene.search.Collector;
import org.apache.lucene.search.ConstantScoreQuery;
import org.apache.lucene.search.DocIdSet;
import org.apache.lucene.search.DocIdSetIterator;
+import org.apache.lucene.search.EarlyTerminatingSortingCollector;
import org.apache.lucene.search.Explanation;
import org.apache.lucene.search.FieldDoc;
import org.apache.lucene.search.IndexSearcher;
@@ -219,6 +220,20 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
private void buildAndRunCollectorChain(QueryResult qr, Query query, Collector collector, QueryCommand cmd,
DelegatingCollector postFilter) throws IOException {
+ EarlyTerminatingSortingCollector earlyTerminatingSortingCollector = null;
+ if (cmd.getSegmentTerminateEarly()) {
+ final Sort cmdSort = cmd.getSort();
+ final int cmdLen = cmd.getLen();
+ final Sort mergeSort = core.getSolrCoreState().getMergePolicySort();
+
+ if (cmdSort == null || cmdLen <= 0 || mergeSort == null ||
+ !EarlyTerminatingSortingCollector.canEarlyTerminate(cmdSort, mergeSort)) {
+ log.warn("unsupported combination: segmentTerminateEarly=true cmdSort={} cmdLen={} mergeSort={}", cmdSort, cmdLen, mergeSort);
+ } else {
+ collector = earlyTerminatingSortingCollector = new EarlyTerminatingSortingCollector(collector, cmdSort, cmd.getLen(), mergeSort);
+ }
+ }
+
final boolean terminateEarly = cmd.getTerminateEarly();
if (terminateEarly) {
collector = new EarlyTerminatingCollector(collector, cmd.getLen());
@@ -244,6 +259,10 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
((DelegatingCollector) collector).finish();
}
throw etce;
+ } finally {
+ if (earlyTerminatingSortingCollector != null) {
+ qr.setSegmentTerminatedEarly(earlyTerminatingSortingCollector.terminatedEarly());
+ }
}
if (collector instanceof DelegatingCollector) {
((DelegatingCollector) collector).finish();
@@ -1465,6 +1484,7 @@ public class SolrIndexSearcher extends IndexSearcher implements Closeable, SolrI
public static final int GET_DOCSET = 0x40000000;
static final int NO_CHECK_FILTERCACHE = 0x20000000;
static final int NO_SET_QCACHE = 0x10000000;
+ static final int SEGMENT_TERMINATE_EARLY = 0x08;
public static final int TERMINATE_EARLY = 0x04;
public static final int GET_DOCLIST = 0x02; // get the documents actually returned in a response
public static final int GET_SCORES = 0x01;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/58ba778c/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
index 8e58c6c..a83183d 100644
--- a/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
+++ b/solr/core/src/java/org/apache/solr/update/DefaultSolrCoreState.java
@@ -28,6 +28,9 @@ import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.index.MergePolicy;
+import org.apache.lucene.index.SortingMergePolicy;
+import org.apache.lucene.search.Sort;
import org.apache.solr.cloud.ActionThrottle;
import org.apache.solr.cloud.RecoveryStrategy;
import org.apache.solr.common.SolrException;
@@ -239,6 +242,22 @@ public final class DefaultSolrCoreState extends SolrCoreState implements Recover
core.getSolrConfig().indexConfig, core.getDeletionPolicy(), core.getCodec());
}
+ public Sort getMergePolicySort() throws IOException {
+ Sort mergePolicySort = null;
+ lock(iwLock.readLock());
+ try {
+ if (indexWriter != null) {
+ final MergePolicy mergePolicy = indexWriter.getConfig().getMergePolicy();
+ if (mergePolicy instanceof SortingMergePolicy) {
+ mergePolicySort = ((SortingMergePolicy)mergePolicy).getSort();
+ }
+ }
+ } finally {
+ iwLock.readLock().unlock();
+ }
+ return mergePolicySort;
+ }
+
@Override
public DirectoryFactory getDirectoryFactory() {
return directoryFactory;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/58ba778c/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
index efcf7b3..42727b4 100644
--- a/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
+++ b/solr/core/src/java/org/apache/solr/update/SolrCoreState.java
@@ -21,6 +21,7 @@ import java.lang.invoke.MethodHandles;
import java.util.concurrent.locks.Lock;
import org.apache.lucene.index.IndexWriter;
+import org.apache.lucene.search.Sort;
import org.apache.solr.cloud.ActionThrottle;
import org.apache.solr.core.CoreContainer;
import org.apache.solr.core.CoreDescriptor;
@@ -126,6 +127,13 @@ public abstract class SolrCoreState {
public abstract void rollbackIndexWriter(SolrCore core) throws IOException;
/**
+ * Get the current Sort of the current IndexWriter's MergePolicy..
+ *
+ * @throws IOException If there is a low-level I/O error.
+ */
+ public abstract Sort getMergePolicySort() throws IOException;
+
+ /**
* @return the {@link DirectoryFactory} that should be used.
*/
public abstract DirectoryFactory getDirectoryFactory();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/58ba778c/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java
index 0830563..68feb7d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java
+++ b/solr/core/src/test/org/apache/solr/cloud/TestMiniSolrCloudCluster.java
@@ -21,12 +21,15 @@ import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.net.URL;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
import com.carrotsearch.randomizedtesting.rules.SystemPropertiesRestoreRule;
@@ -41,12 +44,17 @@ import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.client.solrj.response.RequestStatusState;
import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.cloud.ClusterState;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.cloud.SolrZkClient;
import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ShardParams;
+import org.apache.solr.common.util.SimpleOrderedMap;
import org.apache.solr.core.CoreDescriptor;
+import org.apache.solr.response.SolrQueryResponse;
import org.apache.solr.util.RevertDefaultThreadHandlerRule;
import org.junit.ClassRule;
import org.junit.Rule;
@@ -451,4 +459,307 @@ public class TestMiniSolrCloudCluster extends LuceneTestCase {
}
}
+ private class TestSegmentTerminateEarlyState {
+
+ final Boolean smpfEnabled = random().nextBoolean() ? null : new Boolean(random().nextBoolean());
+
+ final String keyField = "id";
+ final String timestampField = "timestamp";
+ final String oddField = "odd_l1"; // <dynamicField name="*_l1" type="long" indexed="true" stored="true" multiValued="false"/>
+ final String quadField = "quad_l1"; // <dynamicField name="*_l1" type="long" indexed="true" stored="true" multiValued="false"/>
+
+ final Set<Integer> minTimestampDocKeys = new HashSet<>();
+ final Set<Integer> maxTimestampDocKeys = new HashSet<>();
+
+ Integer minTimestampMM = null;
+ Integer maxTimestampMM = null;
+
+ int numDocs = 0;
+
+ private void addDocuments(CloudSolrClient cloudSolrClient,
+ int numCommits, int numDocsPerCommit, boolean optimize) throws Exception {
+ for (int cc = 1; cc <= numCommits; ++cc) {
+ for (int nn = 1; nn <= numDocsPerCommit; ++nn) {
+ ++numDocs;
+ final Integer docKey = new Integer(numDocs);
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.setField(keyField, ""+docKey);
+ final int MM = random().nextInt(60);
+ if (minTimestampMM == null || MM <= minTimestampMM.intValue()) {
+ if (minTimestampMM != null && MM < minTimestampMM.intValue()) {
+ minTimestampDocKeys.clear();
+ }
+ minTimestampMM = new Integer(MM);
+ minTimestampDocKeys.add(docKey);
+ }
+ if (maxTimestampMM == null || maxTimestampMM.intValue() <= MM) {
+ if (maxTimestampMM != null && maxTimestampMM.intValue() < MM) {
+ maxTimestampDocKeys.clear();
+ }
+ maxTimestampMM = new Integer(MM);
+ maxTimestampDocKeys.add(docKey);
+ }
+ doc.setField(timestampField, "2016-01-01T00:"+MM+":00Z");
+ doc.setField(oddField, ""+(numDocs % 2));
+ doc.setField(quadField, ""+(numDocs % 4)+1);
+ cloudSolrClient.add(doc);
+ }
+ cloudSolrClient.commit();
+ }
+ if (optimize) {
+ cloudSolrClient.optimize();
+ }
+ }
+
+ private void queryTimestampDescending(CloudSolrClient cloudSolrClient) throws Exception {
+ assertFalse(maxTimestampDocKeys.isEmpty());
+ assertTrue("numDocs="+numDocs+" is not even", (numDocs%2)==0);
+ final Long oddFieldValue = new Long(maxTimestampDocKeys.iterator().next().intValue()%2);
+ final SolrQuery query = new SolrQuery(oddField+":"+oddFieldValue);
+ query.setSort(timestampField, SolrQuery.ORDER.desc);
+ query.setFields(keyField, oddField, timestampField);
+ query.setRows(1);
+ // CommonParams.SEGMENT_TERMINATE_EARLY parameter intentionally absent
+ final QueryResponse rsp = cloudSolrClient.query(query);
+ // check correctness of the results count
+ assertEquals("numFound", numDocs/2, rsp.getResults().getNumFound());
+ // check correctness of the first result
+ if (rsp.getResults().getNumFound() > 0) {
+ final SolrDocument solrDocument0 = rsp.getResults().get(0);
+ assertTrue(keyField+" of ("+solrDocument0+") is not in maxTimestampDocKeys("+maxTimestampDocKeys+")",
+ maxTimestampDocKeys.contains(solrDocument0.getFieldValue(keyField)));
+ assertEquals(oddField, oddFieldValue, solrDocument0.getFieldValue(oddField));
+ }
+ // check segmentTerminatedEarly flag
+ assertNull("responseHeader.segmentTerminatedEarly present in "+rsp.getResponseHeader(),
+ rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY));
+ }
+
+ private void queryTimestampDescendingSegmentTerminateEarlyYes(CloudSolrClient cloudSolrClient) throws Exception {
+ assertFalse(maxTimestampDocKeys.isEmpty());
+ assertTrue("numDocs="+numDocs+" is not even", (numDocs%2)==0);
+ final Long oddFieldValue = new Long(maxTimestampDocKeys.iterator().next().intValue()%2);
+ final SolrQuery query = new SolrQuery(oddField+":"+oddFieldValue);
+ query.setSort(timestampField, SolrQuery.ORDER.desc);
+ query.setFields(keyField, oddField, timestampField);
+ final int rowsWanted = 1;
+ query.setRows(rowsWanted);
+ final Boolean shardsInfoWanted = (random().nextBoolean() ? null : new Boolean(random().nextBoolean()));
+ if (shardsInfoWanted != null) {
+ query.set(ShardParams.SHARDS_INFO, shardsInfoWanted.booleanValue());
+ }
+ query.set(CommonParams.SEGMENT_TERMINATE_EARLY, true);
+ final QueryResponse rsp = cloudSolrClient.query(query);
+ // check correctness of the results count
+ if (!Boolean.FALSE.equals(smpfEnabled)) {
+ assertTrue("numFound", rowsWanted <= rsp.getResults().getNumFound());
+ assertTrue("numFound", rsp.getResults().getNumFound() <= numDocs/2);
+ } else {
+ assertEquals("numFound", numDocs/2, rsp.getResults().getNumFound());
+ }
+ // check correctness of the first result
+ if (rsp.getResults().getNumFound() > 0) {
+ final SolrDocument solrDocument0 = rsp.getResults().get(0);
+ assertTrue(keyField+" of ("+solrDocument0+") is not in maxTimestampDocKeys("+maxTimestampDocKeys+")",
+ maxTimestampDocKeys.contains(solrDocument0.getFieldValue(keyField)));
+ assertEquals(oddField, oddFieldValue, rsp.getResults().get(0).getFieldValue(oddField));
+ }
+ // check segmentTerminatedEarly flag
+ assertNotNull("responseHeader.segmentTerminatedEarly missing in "+rsp.getResponseHeader(),
+ rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY));
+ if (!Boolean.FALSE.equals(smpfEnabled)) {
+ assertTrue("responseHeader.segmentTerminatedEarly missing/false in "+rsp.getResponseHeader(),
+ Boolean.TRUE.equals(rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY)));
+ } else {
+ assertTrue("responseHeader.segmentTerminatedEarly missing/true in "+rsp.getResponseHeader(),
+ Boolean.FALSE.equals(rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY)));
+ }
+ // check shards info
+ final Object shardsInfo = rsp.getResponse().get(ShardParams.SHARDS_INFO);
+ if (!Boolean.TRUE.equals(shardsInfoWanted)) {
+ assertNull(ShardParams.SHARDS_INFO, shardsInfo);
+ } else {
+ assertNotNull(ShardParams.SHARDS_INFO, shardsInfo);
+ int segmentTerminatedEarlyShardsCount = 0;
+ for (Map.Entry<String, ?> si : (SimpleOrderedMap<?>)shardsInfo) {
+ if (Boolean.TRUE.equals(((SimpleOrderedMap)si.getValue()).get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY))) {
+ segmentTerminatedEarlyShardsCount += 1;
+ }
+ }
+ // check segmentTerminatedEarly flag within shards info
+ if (!Boolean.FALSE.equals(smpfEnabled)) {
+ assertTrue(segmentTerminatedEarlyShardsCount+" shards reported "+SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY,
+ (0<segmentTerminatedEarlyShardsCount));
+ } else {
+ assertEquals("shards reporting "+SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY,
+ 0, segmentTerminatedEarlyShardsCount);
+ }
+ }
+ }
+
+ private void queryTimestampDescendingSegmentTerminateEarlyNo(CloudSolrClient cloudSolrClient) throws Exception {
+ assertFalse(maxTimestampDocKeys.isEmpty());
+ assertTrue("numDocs="+numDocs+" is not even", (numDocs%2)==0);
+ final Long oddFieldValue = new Long(maxTimestampDocKeys.iterator().next().intValue()%2);
+ final SolrQuery query = new SolrQuery(oddField+":"+oddFieldValue);
+ query.setSort(timestampField, SolrQuery.ORDER.desc);
+ query.setFields(keyField, oddField, timestampField);
+ query.setRows(1);
+ final Boolean shardsInfoWanted = (random().nextBoolean() ? null : new Boolean(random().nextBoolean()));
+ if (shardsInfoWanted != null) {
+ query.set(ShardParams.SHARDS_INFO, shardsInfoWanted.booleanValue());
+ }
+ query.set(CommonParams.SEGMENT_TERMINATE_EARLY, false);
+ final QueryResponse rsp = cloudSolrClient.query(query);
+ // check correctness of the results count
+ assertEquals("numFound", numDocs/2, rsp.getResults().getNumFound());
+ // check correctness of the first result
+ if (rsp.getResults().getNumFound() > 0) {
+ final SolrDocument solrDocument0 = rsp.getResults().get(0);
+ assertTrue(keyField+" of ("+solrDocument0+") is not in maxTimestampDocKeys("+maxTimestampDocKeys+")",
+ maxTimestampDocKeys.contains(solrDocument0.getFieldValue(keyField)));
+ assertEquals(oddField, oddFieldValue, rsp.getResults().get(0).getFieldValue(oddField));
+ }
+ // check segmentTerminatedEarly flag
+ assertNull("responseHeader.segmentTerminatedEarly present in "+rsp.getResponseHeader(),
+ rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY));
+ assertFalse("responseHeader.segmentTerminatedEarly present/true in "+rsp.getResponseHeader(),
+ Boolean.TRUE.equals(rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY)));
+ // check shards info
+ final Object shardsInfo = rsp.getResponse().get(ShardParams.SHARDS_INFO);
+ if (!Boolean.TRUE.equals(shardsInfoWanted)) {
+ assertNull(ShardParams.SHARDS_INFO, shardsInfo);
+ } else {
+ assertNotNull(ShardParams.SHARDS_INFO, shardsInfo);
+ int segmentTerminatedEarlyShardsCount = 0;
+ for (Map.Entry<String, ?> si : (SimpleOrderedMap<?>)shardsInfo) {
+ if (Boolean.TRUE.equals(((SimpleOrderedMap)si.getValue()).get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY))) {
+ segmentTerminatedEarlyShardsCount += 1;
+ }
+ }
+ assertEquals("shards reporting "+SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY,
+ 0, segmentTerminatedEarlyShardsCount);
+ }
+ }
+
+ private void queryTimestampDescendingSegmentTerminateEarlyYesGrouped(CloudSolrClient cloudSolrClient) throws Exception {
+ assertFalse(maxTimestampDocKeys.isEmpty());
+ assertTrue("numDocs="+numDocs+" is not even", (numDocs%2)==0);
+ final Long oddFieldValue = new Long(maxTimestampDocKeys.iterator().next().intValue()%2);
+ final SolrQuery query = new SolrQuery(oddField+":"+oddFieldValue);
+ query.setSort(timestampField, SolrQuery.ORDER.desc);
+ query.setFields(keyField, oddField, timestampField);
+ query.setRows(1);
+ query.set(CommonParams.SEGMENT_TERMINATE_EARLY, true);
+ assertTrue("numDocs="+numDocs+" is not quad-able", (numDocs%4)==0);
+ query.add("group.field", quadField);
+ query.set("group", true);
+ final QueryResponse rsp = cloudSolrClient.query(query);
+ // check correctness of the results count
+ assertEquals("matches", numDocs/2, rsp.getGroupResponse().getValues().get(0).getMatches());
+ // check correctness of the first result
+ if (rsp.getGroupResponse().getValues().get(0).getMatches() > 0) {
+ final SolrDocument solrDocument = rsp.getGroupResponse().getValues().get(0).getValues().get(0).getResult().get(0);
+ assertTrue(keyField+" of ("+solrDocument+") is not in maxTimestampDocKeys("+maxTimestampDocKeys+")",
+ maxTimestampDocKeys.contains(solrDocument.getFieldValue(keyField)));
+ assertEquals(oddField, oddFieldValue, solrDocument.getFieldValue(oddField));
+ }
+ // check segmentTerminatedEarly flag
+ // at present segmentTerminateEarly cannot be used with grouped queries
+ assertFalse("responseHeader.segmentTerminatedEarly present/true in "+rsp.getResponseHeader(),
+ Boolean.TRUE.equals(rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY)));
+ }
+
+ private void queryTimestampAscendingSegmentTerminateEarlyYes(CloudSolrClient cloudSolrClient) throws Exception {
+ assertFalse(minTimestampDocKeys.isEmpty());
+ assertTrue("numDocs="+numDocs+" is not even", (numDocs%2)==0);
+ final Long oddFieldValue = new Long(minTimestampDocKeys.iterator().next().intValue()%2);
+ final SolrQuery query = new SolrQuery(oddField+":"+oddFieldValue);
+ query.setSort(timestampField, SolrQuery.ORDER.asc); // a sort order that is _not_ compatible with the merge sort order
+ query.setFields(keyField, oddField, timestampField);
+ query.setRows(1);
+ query.set(CommonParams.SEGMENT_TERMINATE_EARLY, true);
+ final QueryResponse rsp = cloudSolrClient.query(query);
+ // check correctness of the results count
+ assertEquals("numFound", numDocs/2, rsp.getResults().getNumFound());
+ // check correctness of the first result
+ if (rsp.getResults().getNumFound() > 0) {
+ final SolrDocument solrDocument0 = rsp.getResults().get(0);
+ assertTrue(keyField+" of ("+solrDocument0+") is not in minTimestampDocKeys("+minTimestampDocKeys+")",
+ minTimestampDocKeys.contains(solrDocument0.getFieldValue(keyField)));
+ assertEquals(oddField, oddFieldValue, solrDocument0.getFieldValue(oddField));
+ }
+ // check segmentTerminatedEarly flag
+ assertNotNull("responseHeader.segmentTerminatedEarly missing in "+rsp.getResponseHeader(),
+ rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY));
+ // segmentTerminateEarly cannot be used with incompatible sort orders
+ assertTrue("responseHeader.segmentTerminatedEarly missing/true in "+rsp.getResponseHeader(),
+ Boolean.FALSE.equals(rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY)));
+ }
+ }
+
+ @Test
+ public void testSegmentTerminateEarly() throws Exception {
+
+ final String collectionName = "testSegmentTerminateEarlyCollection";
+ final String solr_sortingMergePolicyFactory_enable = "solr.sortingMergePolicyFactory.enable";
+
+ final TestSegmentTerminateEarlyState tstes = new TestSegmentTerminateEarlyState();
+ if (tstes.smpfEnabled != null) {
+ System.setProperty(solr_sortingMergePolicyFactory_enable, tstes.smpfEnabled.toString());
+ }
+
+ File solrXml = new File(SolrTestCaseJ4.TEST_HOME(), "solr-no-core.xml");
+ Builder jettyConfig = JettyConfig.builder();
+ jettyConfig.waitForLoadingCoresToFinish(null);
+ final MiniSolrCloudCluster miniCluster = createMiniSolrCloudCluster();
+ final CloudSolrClient cloudSolrClient = miniCluster.getSolrClient();
+ cloudSolrClient.setDefaultCollection(collectionName);
+
+ try {
+ // create collection
+ {
+ final String asyncId = (random().nextBoolean() ? null : "asyncId("+collectionName+".create)="+random().nextInt());
+ final Map<String, String> collectionProperties = new HashMap<>();
+ collectionProperties.put(CoreDescriptor.CORE_CONFIG, "solrconfig-sortingmergepolicyfactory.xml");
+ createCollection(miniCluster, collectionName, null, asyncId, Boolean.TRUE, collectionProperties);
+ if (asyncId != null) {
+ final RequestStatusState state = AbstractFullDistribZkTestBase.getRequestStateAfterCompletion(asyncId, 330, cloudSolrClient);
+ assertSame("did not see async createCollection completion", RequestStatusState.COMPLETED, state);
+ }
+ }
+
+ try (SolrZkClient zkClient = new SolrZkClient
+ (miniCluster.getZkServer().getZkAddress(), AbstractZkTestCase.TIMEOUT, 45000, null);
+ ZkStateReader zkStateReader = new ZkStateReader(zkClient)) {
+ AbstractDistribZkTestBase.waitForRecoveriesToFinish(collectionName, zkStateReader, true, true, 330);
+
+ // add some documents, then optimize to get merged-sorted segments
+ tstes.addDocuments(cloudSolrClient, 10, 10, true);
+
+ // CommonParams.SEGMENT_TERMINATE_EARLY parameter intentionally absent
+ tstes.queryTimestampDescending(cloudSolrClient);
+
+ // add a few more documents, but don't optimize to have some not-merge-sorted segments
+ tstes.addDocuments(cloudSolrClient, 2, 10, false);
+
+ // CommonParams.SEGMENT_TERMINATE_EARLY parameter now present
+ tstes.queryTimestampDescendingSegmentTerminateEarlyYes(cloudSolrClient);
+ tstes.queryTimestampDescendingSegmentTerminateEarlyNo(cloudSolrClient);
+
+ // CommonParams.SEGMENT_TERMINATE_EARLY parameter present but it won't be used
+ tstes.queryTimestampDescendingSegmentTerminateEarlyYesGrouped(cloudSolrClient);
+ tstes.queryTimestampAscendingSegmentTerminateEarlyYes(cloudSolrClient); // uses a sort order that is _not_ compatible with the merge sort order
+
+ // delete the collection we created earlier
+ miniCluster.deleteCollection(collectionName);
+ AbstractDistribZkTestBase.waitForCollectionToDisappear(collectionName, zkStateReader, true, true, 330);
+ }
+ }
+ finally {
+ miniCluster.shutdown();
+ }
+ System.clearProperty(solr_sortingMergePolicyFactory_enable);
+ }
+
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/58ba778c/solr/core/src/test/org/apache/solr/response/TestSolrQueryResponse.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/response/TestSolrQueryResponse.java b/solr/core/src/test/org/apache/solr/response/TestSolrQueryResponse.java
index e938afc..8b17dc6 100644
--- a/solr/core/src/test/org/apache/solr/response/TestSolrQueryResponse.java
+++ b/solr/core/src/test/org/apache/solr/response/TestSolrQueryResponse.java
@@ -45,6 +45,12 @@ public class TestSolrQueryResponse extends LuceneTestCase {
}
@Test
+ public void testResponseHeaderSegmentTerminatedEarly() throws Exception {
+ assertEquals("SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY value changed",
+ "segmentTerminatedEarly", SolrQueryResponse.RESPONSE_HEADER_SEGMENT_TERMINATED_EARLY_KEY);
+ }
+
+ @Test
public void testValues() throws Exception {
final SolrQueryResponse response = new SolrQueryResponse();
assertEquals("values initially not empty", 0, response.getValues().size());
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/58ba778c/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
index 5429328..5ccd70f 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonParams.java
@@ -152,6 +152,12 @@ public interface CommonParams {
public static final String STREAM_CONTENTTYPE = "stream.contentType";
/**
+ * Whether or not the search may be terminated early within a segment.
+ */
+ public static final String SEGMENT_TERMINATE_EARLY = "segmentTerminateEarly";
+ public static final boolean SEGMENT_TERMINATE_EARLY_DEFAULT = false;
+
+ /**
* Timeout value in milliseconds. If not set, or the value is >= 0, there is no timeout.
*/
public static final String TIME_ALLOWED = "timeAllowed";