You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by dw...@apache.org on 2021/03/10 09:49:31 UTC
[lucene] 01/09: Initial patch.
This is an automated email from the ASF dual-hosted git repository.
dweiss pushed a commit to branch jira/solr-12730
in repository https://gitbox.apache.org/repos/asf/lucene.git
commit 3d91b8e27736b6a7456d710a467a7c723205d47a
Author: Andrzej Bialecki <ab...@apache.org>
AuthorDate: Wed Oct 24 13:28:21 2018 +0200
Initial patch.
---
.../solr/cloud/api/collections/SplitShardCmd.java | 9 ++++-
.../solr/cloud/autoscaling/IndexSizeTrigger.java | 26 ++++++++++++++-
.../solr/cloud/autoscaling/TriggerEvent.java | 2 ++
.../solr/handler/admin/CollectionsHandler.java | 9 ++++-
.../cloud/autoscaling/IndexSizeTriggerTest.java | 30 ++++++++++-------
.../cloud/autoscaling/SplitShardSuggester.java | 14 +++++++-
.../client/solrj/cloud/autoscaling/Suggester.java | 15 ++++++++-
.../solrj/request/CollectionAdminRequest.java | 10 ++++++
.../org/apache/solr/common/cloud/DocRouter.java | 38 +++++++++++++++++++++-
.../solr/common/params/CommonAdminParams.java | 4 ++-
10 files changed, 138 insertions(+), 19 deletions(-)
diff --git a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
index aa4909d..b9370ac 100644
--- a/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
+++ b/solr/core/src/java/org/apache/solr/cloud/api/collections/SplitShardCmd.java
@@ -686,6 +686,13 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd {
boolean firstReplicaNrt) {
String splitKey = message.getStr("split.key");
String rangesStr = message.getStr(CoreAdminParams.RANGES);
+ String fuzzStr = message.getStr(CommonAdminParams.SPLIT_FUZZ, "0");
+ float fuzz = 0.0f;
+ try {
+ fuzz = Float.parseFloat(fuzzStr);
+ } catch (Exception e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "?Invalid numeric value of 'fuzz': " + fuzzStr);
+ }
DocRouter.Range range = parentSlice.getRange();
if (range == null) {
@@ -752,7 +759,7 @@ public class SplitShardCmd implements OverseerCollectionMessageHandler.Cmd {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
"A shard can only be split into "+MIN_NUM_SUB_SHARDS+" to " + MAX_NUM_SUB_SHARDS
+ " subshards in one split request. Provided "+NUM_SUB_SHARDS+"=" + numSubShards);
- subRanges.addAll(router.partitionRange(numSubShards, range));
+ subRanges.addAll(router.partitionRange(numSubShards, range, fuzz));
}
for (int i = 0; i < subRanges.size(); i++) {
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
index 25083ae..533ec53 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
@@ -41,11 +41,13 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CommonAdminParams;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.metrics.SolrCoreMetricManager;
+import org.apache.solr.update.SolrIndexSplitter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,6 +65,8 @@ public class IndexSizeTrigger extends TriggerBase {
public static final String BELOW_OP_PROP = "belowOp";
public static final String COLLECTIONS_PROP = "collections";
public static final String MAX_OPS_PROP = "maxOps";
+ public static final String SPLIT_FUZZ_PROP = CommonAdminParams.SPLIT_FUZZ;
+ public static final String SPLIT_METHOD_PROP = CommonAdminParams.SPLIT_METHOD;
public static final String BYTES_SIZE_PROP = "__bytes__";
public static final String DOCS_SIZE_PROP = "__docs__";
@@ -76,6 +80,8 @@ public class IndexSizeTrigger extends TriggerBase {
private long aboveBytes, aboveDocs, belowBytes, belowDocs;
private int maxOps;
+ private SolrIndexSplitter.SplitMethod splitMethod;
+ private float splitFuzz;
private CollectionParams.CollectionAction aboveOp, belowOp;
private final Set<String> collections = new HashSet<>();
private final Map<String, Long> lastAboveEventMap = new ConcurrentHashMap<>();
@@ -85,7 +91,7 @@ public class IndexSizeTrigger extends TriggerBase {
super(TriggerEventType.INDEXSIZE, name);
TriggerUtils.validProperties(validProperties,
ABOVE_BYTES_PROP, ABOVE_DOCS_PROP, BELOW_BYTES_PROP, BELOW_DOCS_PROP,
- COLLECTIONS_PROP, MAX_OPS_PROP);
+ COLLECTIONS_PROP, MAX_OPS_PROP, SPLIT_FUZZ_PROP);
}
@Override
@@ -165,6 +171,18 @@ public class IndexSizeTrigger extends TriggerBase {
} catch (Exception e) {
throw new TriggerValidationException(getName(), MAX_OPS_PROP, "invalid value: '" + maxOpsStr + "': " + e.getMessage());
}
+ String methodStr = (String)properties.getOrDefault(CommonAdminParams.SPLIT_METHOD, SolrIndexSplitter.SplitMethod.LINK.toLower());
+ splitMethod = SolrIndexSplitter.SplitMethod.get(methodStr);
+ if (splitMethod == null) {
+ throw new TriggerValidationException(getName(), SPLIT_METHOD_PROP, "Unknown value '" + CommonAdminParams.SPLIT_METHOD +
+ ": " + methodStr);
+ }
+ String fuzzStr = String.valueOf(properties.getOrDefault(SPLIT_FUZZ_PROP, 0.0f));
+ try {
+ splitFuzz = Float.parseFloat(fuzzStr);
+ } catch (Exception e) {
+ throw new TriggerValidationException(getName(), SPLIT_FUZZ_PROP, "invalid value: '" + fuzzStr + "': " + e.getMessage());
+ }
}
@Override
@@ -383,6 +401,12 @@ public class IndexSizeTrigger extends TriggerBase {
}
TriggerEvent.Op op = new TriggerEvent.Op(aboveOp);
op.addHint(Suggester.Hint.COLL_SHARD, new Pair<>(coll, r.getShard()));
+ Map<String, Object> params = new HashMap<>();
+ params.put(CommonAdminParams.SPLIT_METHOD, splitMethod.toLower());
+ if (splitFuzz > 0) {
+ params.put(CommonAdminParams.SPLIT_FUZZ, splitFuzz);
+ }
+ op.addHint(Suggester.Hint.PARAMS, params);
ops.add(op);
Long time = lastAboveEventMap.get(r.getCore());
if (time != null && eventTime.get() > time) {
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java
index 8e3a348..c61556c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java
@@ -64,6 +64,8 @@ public class TriggerEvent implements MapWriter {
if (hint.multiValued) {
Collection<?> values = value instanceof Collection ? (Collection) value : Collections.singletonList(value);
((Set) hints.computeIfAbsent(hint, h -> new LinkedHashSet<>())).addAll(values);
+ } else if (value instanceof Map) {
+ hints.put(hint, value);
} else {
hints.put(hint, value == null ? null : String.valueOf(value));
}
diff --git a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
index dfb3c6b..462da77 100644
--- a/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
+++ b/solr/core/src/java/org/apache/solr/handler/admin/CollectionsHandler.java
@@ -144,6 +144,7 @@ import static org.apache.solr.common.params.CollectionParams.CollectionAction.*;
import static org.apache.solr.common.params.CommonAdminParams.ASYNC;
import static org.apache.solr.common.params.CommonAdminParams.IN_PLACE_MOVE;
import static org.apache.solr.common.params.CommonAdminParams.NUM_SUB_SHARDS;
+import static org.apache.solr.common.params.CommonAdminParams.SPLIT_FUZZ;
import static org.apache.solr.common.params.CommonAdminParams.SPLIT_METHOD;
import static org.apache.solr.common.params.CommonAdminParams.WAIT_FOR_FINAL_STATE;
import static org.apache.solr.common.params.CommonParams.NAME;
@@ -640,6 +641,7 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
String rangesStr = req.getParams().get(CoreAdminParams.RANGES);
String splitKey = req.getParams().get("split.key");
String numSubShards = req.getParams().get(NUM_SUB_SHARDS);
+ String fuzz = req.getParams().get(SPLIT_FUZZ);
if (splitKey == null && shard == null) {
throw new SolrException(ErrorCode.BAD_REQUEST, "At least one of shard, or split.key should be specified.");
@@ -656,6 +658,10 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
throw new SolrException(ErrorCode.BAD_REQUEST,
"numSubShards can not be specified with split.key or ranges parameters");
}
+ if (fuzz != null && (splitKey != null || rangesStr != null)) {
+ throw new SolrException(ErrorCode.BAD_REQUEST,
+ "fuzz can not be specified with split.key or ranges parameters");
+ }
Map<String, Object> map = copy(req.getParams(), null,
COLLECTION_PROP,
@@ -665,7 +671,8 @@ public class CollectionsHandler extends RequestHandlerBase implements Permission
WAIT_FOR_FINAL_STATE,
TIMING,
SPLIT_METHOD,
- NUM_SUB_SHARDS);
+ NUM_SUB_SHARDS,
+ SPLIT_FUZZ);
return copyPropertiesWithPrefix(req.getParams(), map, COLL_PROP_PREFIX);
}),
DELETESHARD_OP(DELETESHARD, (req, rsp, h) -> {
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
index 996532a..c933c0a 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
@@ -93,7 +93,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
configureCluster(2)
.addConfig("conf", configset("cloud-minimal"))
.configure();
- if (random().nextBoolean()) {
+ if (random().nextBoolean() || true) {
cloudManager = cluster.getJettySolrRunner(0).getCoreContainer().getZkController().getSolrCloudManager();
solrClient = cluster.getSolrClient();
loader = cluster.getJettySolrRunner(0).getCoreContainer().getResourceLoader();
@@ -501,7 +501,7 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
}
assertTrue("maxSize should be non-zero", maxSize > 0);
- int aboveBytes = maxSize * 2 / 3;
+ int aboveBytes = maxSize * 9 / 10;
long waitForSeconds = 3 + random().nextInt(5);
@@ -570,6 +570,16 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
boolean await = finished.await(90000 / SPEED, TimeUnit.MILLISECONDS);
assertTrue("did not finish processing in time", await);
+ // suspend the trigger to avoid generating more events
+ String suspendTriggerCommand = "{" +
+ "'suspend-trigger' : {" +
+ "'name' : 'index_size_trigger4'" +
+ "}" +
+ "}";
+ req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
+ response = solrClient.request(req);
+ assertEquals(response.get("result").toString(), "success");
+
assertEquals(1, listenerEvents.size());
List<CapturedEvent> events = listenerEvents.get("capturing4");
assertNotNull("'capturing4' events not found", events);
@@ -613,15 +623,6 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
listenerEvents.clear();
finished = new CountDownLatch(1);
- // suspend the trigger first so that we can safely delete all docs
- String suspendTriggerCommand = "{" +
- "'suspend-trigger' : {" +
- "'name' : 'index_size_trigger4'" +
- "}" +
- "}";
- req = createAutoScalingRequest(SolrRequest.METHOD.POST, suspendTriggerCommand);
- response = solrClient.request(req);
- assertEquals(response.get("result").toString(), "success");
for (int j = 0; j < 10; j++) {
UpdateRequest ureq = new UpdateRequest();
@@ -641,7 +642,12 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
ur.setParam(UpdateParams.OPEN_SEARCHER, "true");
solrClient.request(ur, collectionName);
- // resume trigger
+ // wait for the segments to merge to reduce the index size
+ cloudManager.getTimeSource().sleep(50000);
+
+ solrClient.commit(collectionName, true, true);
+
+ // resume the trigger
req = createAutoScalingRequest(SolrRequest.METHOD.POST, resumeTriggerCommand);
response = solrClient.request(req);
assertEquals(response.get("result").toString(), "success");
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
index 2c1d7df..8aafef8 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
@@ -17,11 +17,13 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
import java.util.Collections;
+import java.util.Map;
import java.util.Set;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CommonAdminParams;
import org.apache.solr.common.util.Pair;
/**
@@ -44,6 +46,16 @@ class SplitShardSuggester extends Suggester {
throw new RuntimeException("split-shard requires exactly one pair of 'collection' and 'shard'");
}
Pair<String, String> collShard = shards.iterator().next();
- return CollectionAdminRequest.splitShard(collShard.first()).setShardName(collShard.second());
+ Map<String, Object> params = (Map<String, Object>)hints.getOrDefault(Hint.PARAMS, Collections.emptyMap());
+ Float splitFuzz = (Float)params.get(CommonAdminParams.SPLIT_FUZZ);
+ CollectionAdminRequest.SplitShard req = CollectionAdminRequest.splitShard(collShard.first()).setShardName(collShard.second());
+ if (splitFuzz != null) {
+ req.setSplitFuzz(splitFuzz);
+ }
+ String splitMethod = (String)params.get(CommonAdminParams.SPLIT_METHOD);
+ if (splitMethod != null) {
+ req.setSplitMethod(splitMethod);
+ }
+ return req;
}
}
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index bba5906..39ad8bf 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -95,7 +95,15 @@ public abstract class Suggester implements MapWriter {
Collection<?> values = value instanceof Collection ? (Collection) value : Collections.singletonList(value);
((Set) hints.computeIfAbsent(hint, h -> new HashSet<>())).addAll(values);
} else {
- hints.put(hint, value == null ? null : String.valueOf(value));
+ if (value == null) {
+ hints.put(hint, null);
+ } else {
+ if ((value instanceof Map) || (value instanceof Number)) {
+ hints.put(hint, value);
+ } else {
+ hints.put(hint, String.valueOf(value));
+ }
+ }
}
return this;
}
@@ -372,6 +380,11 @@ public abstract class Suggester implements MapWriter {
NUMBER(true, o -> {
if (!(o instanceof Number)) throw new RuntimeException("NUMBER hint must be a number");
}),
+ PARAMS(false, o -> {
+ if (!(o instanceof Map)) {
+ throw new RuntimeException("PARAMS hint must be a Map<String, Object>");
+ }
+ }),
REPLICA(true);
public final boolean multiValued;
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
index 4f26984..3769c2e 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/CollectionAdminRequest.java
@@ -1154,6 +1154,7 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
protected String shard;
protected String splitMethod;
protected Integer numSubShards;
+ protected Float splitFuzz;
private Properties properties;
@@ -1183,6 +1184,15 @@ public abstract class CollectionAdminRequest<T extends CollectionAdminResponse>
return splitMethod;
}
+ public SplitShard setSplitFuzz(float splitFuzz) {
+ this.splitFuzz = splitFuzz;
+ return this;
+ }
+
+ public Float getSplitFuzz() {
+ return splitFuzz;
+ }
+
public SplitShard setSplitKey(String splitKey) {
this.splitKey = splitKey;
return this;
diff --git a/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java b/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java
index 846c25e..c1471b5 100644
--- a/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java
+++ b/solr/solrj/src/java/org/apache/solr/common/cloud/DocRouter.java
@@ -30,6 +30,7 @@ import java.util.HashSet;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Random;
import static org.apache.solr.common.cloud.DocCollection.DOC_ROUTER;
@@ -153,24 +154,59 @@ public abstract class DocRouter {
}
/**
- * Returns the range for each partition
+ * Split the range into partitions.
+ * @param partitions number of partitions
+ * @param range range to split
*/
public List<Range> partitionRange(int partitions, Range range) {
+ return partitionRange(partitions, range, 0.0f);
+ }
+
+ /**
+ * Split the range into partitions with inexact sizes.
+ * @param partitions number of partitions
+ * @param range range to split
+ * @param fuzz value between 0 (inclusive) and 0.5 indicating inexact split, i.e. percentage
+ * of variation in resulting ranges - odd ranges will be larger and even ranges will be smaller
+ * by up to that percentage.
+ */
+ public List<Range> partitionRange(int partitions, Range range, float fuzz) {
int min = range.min;
int max = range.max;
assert max >= min;
+ if (fuzz > 0.5f) {
+ throw new IllegalArgumentException("'fuzz' parameter must be <= 0.5f but was " + fuzz);
+ }
if (partitions == 0) return Collections.EMPTY_LIST;
long rangeSize = (long)max - (long)min;
long rangeStep = Math.max(1, rangeSize / partitions);
+ long fuzzStep = Math.round(rangeStep * fuzz / 2.0f);
+ // use a predictable pseudo-random
+ Random r = null;
+ if (fuzzStep > 2) {
+ r = new Random(0);
+ // limit randomness to half the range
+ fuzzStep = fuzzStep / 2;
+ }
List<Range> ranges = new ArrayList<>(partitions);
long start = min;
long end = start;
+ boolean odd = true;
while (end < max) {
end = start + rangeStep;
+ if (fuzzStep > 0) {
+ long currentFuzz = r != null ? fuzzStep + r.nextLong() % fuzzStep : fuzzStep;
+ if (odd) {
+ end = end + currentFuzz;
+ } else {
+ end = end - currentFuzz;
+ }
+ odd = !odd;
+ }
// make last range always end exactly on MAX_VALUE
if (ranges.size() == partitions - 1) {
end = max;
diff --git a/solr/solrj/src/java/org/apache/solr/common/params/CommonAdminParams.java b/solr/solrj/src/java/org/apache/solr/common/params/CommonAdminParams.java
index c080342..13d304c 100644
--- a/solr/solrj/src/java/org/apache/solr/common/params/CommonAdminParams.java
+++ b/solr/solrj/src/java/org/apache/solr/common/params/CommonAdminParams.java
@@ -27,8 +27,10 @@ public interface CommonAdminParams
String IN_PLACE_MOVE = "inPlaceMove";
/** Method to use for shard splitting. */
String SPLIT_METHOD = "splitMethod";
- /** **/
+ /** Number of sub-shards to create. **/
String NUM_SUB_SHARDS = "numSubShards";
/** Timeout for replicas to become active. */
String TIMEOUT = "timeout";
+ /** Inexact shard splitting factor. */
+ String SPLIT_FUZZ = "splitFuzz";
}