You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by ab...@apache.org on 2018/10/31 19:29:47 UTC
lucene-solr:branch_7x: SOLR-12942: Add an option in IndexSizeTrigger
to select the split shard method.
Repository: lucene-solr
Updated Branches:
refs/heads/branch_7x 487e5befa -> a852ebc59
SOLR-12942: Add an option in IndexSizeTrigger to select the split shard method.
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/a852ebc5
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/a852ebc5
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/a852ebc5
Branch: refs/heads/branch_7x
Commit: a852ebc5988fbca4af6173fbacc1bfd34f0899b4
Parents: 487e5be
Author: Andrzej Bialecki <ab...@apache.org>
Authored: Wed Oct 31 18:47:34 2018 +0100
Committer: Andrzej Bialecki <ab...@apache.org>
Committed: Wed Oct 31 18:48:43 2018 +0100
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../cloud/autoscaling/IndexSizeTrigger.java | 25 +++++-
.../solr/cloud/autoscaling/TriggerEvent.java | 2 +
.../solr/cloud/overseer/ReplicaMutator.java | 4 +-
.../cloud/autoscaling/IndexSizeTriggerTest.java | 81 ++++++++++++++++++++
.../src/solrcloud-autoscaling-triggers.adoc | 7 ++
.../cloud/autoscaling/SplitShardSuggester.java | 10 ++-
.../solrj/cloud/autoscaling/Suggester.java | 15 +++-
8 files changed, 138 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a852ebc5/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index be31fe7..ee7f3c7 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -90,6 +90,8 @@ New Features
* SOLR-12862: Add log10 Stream Evaluator and allow the pow Stream Evaluator to accept a vector of exponents (Joel Bernstein)
+* SOLR-12942: Add an option in IndexSizeTrigger to select the split shard method. (ab)
+
Other Changes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a852ebc5/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
index 25083ae..31322c0 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/IndexSizeTrigger.java
@@ -41,11 +41,13 @@ import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.Slice;
import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CommonAdminParams;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.StrUtils;
import org.apache.solr.common.util.Utils;
import org.apache.solr.core.SolrResourceLoader;
import org.apache.solr.metrics.SolrCoreMetricManager;
+import org.apache.solr.update.SolrIndexSplitter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -63,6 +65,7 @@ public class IndexSizeTrigger extends TriggerBase {
public static final String BELOW_OP_PROP = "belowOp";
public static final String COLLECTIONS_PROP = "collections";
public static final String MAX_OPS_PROP = "maxOps";
+ public static final String SPLIT_METHOD_PROP = CommonAdminParams.SPLIT_METHOD;
public static final String BYTES_SIZE_PROP = "__bytes__";
public static final String DOCS_SIZE_PROP = "__docs__";
@@ -76,6 +79,7 @@ public class IndexSizeTrigger extends TriggerBase {
private long aboveBytes, aboveDocs, belowBytes, belowDocs;
private int maxOps;
+ private SolrIndexSplitter.SplitMethod splitMethod;
private CollectionParams.CollectionAction aboveOp, belowOp;
private final Set<String> collections = new HashSet<>();
private final Map<String, Long> lastAboveEventMap = new ConcurrentHashMap<>();
@@ -85,7 +89,7 @@ public class IndexSizeTrigger extends TriggerBase {
super(TriggerEventType.INDEXSIZE, name);
TriggerUtils.validProperties(validProperties,
ABOVE_BYTES_PROP, ABOVE_DOCS_PROP, BELOW_BYTES_PROP, BELOW_DOCS_PROP,
- COLLECTIONS_PROP, MAX_OPS_PROP);
+ COLLECTIONS_PROP, MAX_OPS_PROP, SPLIT_METHOD_PROP);
}
@Override
@@ -165,6 +169,12 @@ public class IndexSizeTrigger extends TriggerBase {
} catch (Exception e) {
throw new TriggerValidationException(getName(), MAX_OPS_PROP, "invalid value: '" + maxOpsStr + "': " + e.getMessage());
}
+ String methodStr = (String)properties.getOrDefault(CommonAdminParams.SPLIT_METHOD, SolrIndexSplitter.SplitMethod.REWRITE.toLower());
+ splitMethod = SolrIndexSplitter.SplitMethod.get(methodStr);
+ if (splitMethod == null) {
+ throw new TriggerValidationException(getName(), SPLIT_METHOD_PROP, "Unknown value '" + CommonAdminParams.SPLIT_METHOD +
+ ": " + methodStr);
+ }
}
@Override
@@ -234,7 +244,7 @@ public class IndexSizeTrigger extends TriggerBase {
DocCollection docCollection = clusterState.getCollection(coll);
shards.forEach((sh, replicas) -> {
- // check only the leader of a replica in active shard
+ // check only the leader replica in an active shard
Slice s = docCollection.getSlice(sh);
if (s.getState() != Slice.State.ACTIVE) {
return;
@@ -244,6 +254,10 @@ public class IndexSizeTrigger extends TriggerBase {
if (r == null) {
return;
}
+ // not on this node
+ if (!r.getNodeName().equals(node)) {
+ return;
+ }
// find ReplicaInfo
ReplicaInfo info = null;
for (ReplicaInfo ri : replicas) {
@@ -279,14 +293,14 @@ public class IndexSizeTrigger extends TriggerBase {
} else {
// verify that it's a Number
if (!(size instanceof Number)) {
- log.warn("invalid size value - not a number: '" + size + "' is " + size.getClass().getName());
+ log.warn("invalid size value for tag " + tag + " - not a number: '" + size + "' is " + size.getClass().getName());
return;
}
ReplicaInfo currentInfo = currentSizes.computeIfAbsent(info.getCore(), k -> (ReplicaInfo)info.clone());
if (tag.contains("INDEX")) {
currentInfo.getVariables().put(BYTES_SIZE_PROP, ((Number) size).longValue());
- } else {
+ } else if (tag.contains("SEARCHER")) {
currentInfo.getVariables().put(DOCS_SIZE_PROP, ((Number) size).longValue());
}
}
@@ -383,6 +397,9 @@ public class IndexSizeTrigger extends TriggerBase {
}
TriggerEvent.Op op = new TriggerEvent.Op(aboveOp);
op.addHint(Suggester.Hint.COLL_SHARD, new Pair<>(coll, r.getShard()));
+ Map<String, Object> params = new HashMap<>();
+ params.put(CommonAdminParams.SPLIT_METHOD, splitMethod.toLower());
+ op.addHint(Suggester.Hint.PARAMS, params);
ops.add(op);
Long time = lastAboveEventMap.get(r.getCore());
if (time != null && eventTime.get() > time) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a852ebc5/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java
index 8e3a348..c61556c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java
+++ b/solr/core/src/java/org/apache/solr/cloud/autoscaling/TriggerEvent.java
@@ -64,6 +64,8 @@ public class TriggerEvent implements MapWriter {
if (hint.multiValued) {
Collection<?> values = value instanceof Collection ? (Collection) value : Collections.singletonList(value);
((Set) hints.computeIfAbsent(hint, h -> new LinkedHashSet<>())).addAll(values);
+ } else if (value instanceof Map) {
+ hints.put(hint, value);
} else {
hints.put(hint, value == null ? null : String.valueOf(value));
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a852ebc5/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java b/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
index 68a42b9..82a0610 100644
--- a/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
+++ b/solr/core/src/java/org/apache/solr/cloud/overseer/ReplicaMutator.java
@@ -394,7 +394,7 @@ public class ReplicaMutator {
}
}
if (allActive) {
- log.info("Shard: {} - all replicas are active. Finding status of fellow sub-shards", sliceName);
+ log.info("Shard: {} - all {} replicas are active. Finding status of fellow sub-shards", sliceName, slice.getReplicasMap().size());
// find out about other sub shards
Map<String, Slice> allSlicesCopy = new HashMap<>(collection.getSlicesMap());
List<Slice> subShardSlices = new ArrayList<>();
@@ -413,7 +413,7 @@ public class ReplicaMutator {
break outer;
}
}
- log.info("Shard: {} - Fellow sub-shard: {} has all replicas active", sliceName, otherSlice.getName());
+ log.info("Shard: {} - Fellow sub-shard: {} has all {} replicas active", sliceName, otherSlice.getName(), otherSlice.getReplicasMap().size());
subShardSlices.add(otherSlice);
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a852ebc5/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
index 996532a..ec2315d 100644
--- a/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/autoscaling/IndexSizeTriggerTest.java
@@ -46,6 +46,7 @@ import org.apache.solr.cloud.autoscaling.sim.SimCloudManager;
import org.apache.solr.common.SolrInputDocument;
import org.apache.solr.common.cloud.ZkNodeProps;
import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CommonAdminParams;
import org.apache.solr.common.params.CommonParams;
import org.apache.solr.common.params.UpdateParams;
import org.apache.solr.common.util.NamedList;
@@ -207,6 +208,9 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
} else {
fail("unexpected shard name " + p.second());
}
+ Map<String, Object> params = (Map<String, Object>)op.getHints().get(Suggester.Hint.PARAMS);
+ assertNotNull("params are null: " + op, params);
+ assertEquals("splitMethod: " + op, "rewrite", params.get(CommonAdminParams.SPLIT_METHOD));
}
assertTrue("shard1 should be split", shard1);
assertTrue("shard2 should be split", shard2);
@@ -822,6 +826,83 @@ public class IndexSizeTriggerTest extends SolrCloudTestCase {
assertEquals("number of ops: " + ops, 3, ops.size());
}
+ @Test
+ public void testSplitMethodConfig() throws Exception {
+ String collectionName = "testSplitMethod_collection";
+ CollectionAdminRequest.Create create = CollectionAdminRequest.createCollection(collectionName,
+ "conf", 2, 2).setMaxShardsPerNode(2);
+ create.process(solrClient);
+ CloudTestUtils.waitForState(cloudManager, "failed to create " + collectionName, collectionName,
+ CloudTestUtils.clusterShape(2, 2, false, true));
+
+ long waitForSeconds = 3 + random().nextInt(5);
+ Map<String, Object> props = createTriggerProps(waitForSeconds);
+ props.put(CommonAdminParams.SPLIT_METHOD, "link");
+ try (IndexSizeTrigger trigger = new IndexSizeTrigger("index_size_trigger6")) {
+ trigger.configure(loader, cloudManager, props);
+ trigger.init();
+ trigger.setProcessor(noFirstRunProcessor);
+ trigger.run();
+
+ for (int i = 0; i < 25; i++) {
+ SolrInputDocument doc = new SolrInputDocument("id", "id-" + i);
+ solrClient.add(collectionName, doc);
+ }
+ solrClient.commit(collectionName);
+
+ AtomicBoolean fired = new AtomicBoolean(false);
+ AtomicReference<TriggerEvent> eventRef = new AtomicReference<>();
+ trigger.setProcessor(event -> {
+ if (fired.compareAndSet(false, true)) {
+ eventRef.set(event);
+ long currentTimeNanos = timeSource.getTimeNs();
+ long eventTimeNanos = event.getEventTime();
+ long waitForNanos = TimeUnit.NANOSECONDS.convert(waitForSeconds, TimeUnit.SECONDS) - WAIT_FOR_DELTA_NANOS;
+ if (currentTimeNanos - eventTimeNanos <= waitForNanos) {
+ fail("processor was fired before the configured waitFor period: currentTimeNanos=" + currentTimeNanos + ", eventTimeNanos=" + eventTimeNanos + ",waitForNanos=" + waitForNanos);
+ }
+ } else {
+ fail("IndexSizeTrigger was fired more than once!");
+ }
+ return true;
+ });
+ trigger.run();
+ TriggerEvent ev = eventRef.get();
+ // waitFor delay - should not produce any event yet
+ assertNull("waitFor not elapsed but produced an event", ev);
+ timeSource.sleep(TimeUnit.MILLISECONDS.convert(waitForSeconds + 1, TimeUnit.SECONDS));
+ trigger.run();
+ ev = eventRef.get();
+ assertNotNull("should have fired an event", ev);
+ List<TriggerEvent.Op> ops = (List<TriggerEvent.Op>) ev.getProperty(TriggerEvent.REQUESTED_OPS);
+ assertNotNull("should contain requestedOps", ops);
+ assertEquals("number of ops: " + ops, 2, ops.size());
+ boolean shard1 = false;
+ boolean shard2 = false;
+ for (TriggerEvent.Op op : ops) {
+ assertEquals(CollectionParams.CollectionAction.SPLITSHARD, op.getAction());
+ Set<Pair<String, String>> hints = (Set<Pair<String, String>>)op.getHints().get(Suggester.Hint.COLL_SHARD);
+ assertNotNull("hints", hints);
+ assertEquals("hints", 1, hints.size());
+ Pair<String, String> p = hints.iterator().next();
+ assertEquals(collectionName, p.first());
+ if (p.second().equals("shard1")) {
+ shard1 = true;
+ } else if (p.second().equals("shard2")) {
+ shard2 = true;
+ } else {
+ fail("unexpected shard name " + p.second());
+ }
+ Map<String, Object> params = (Map<String, Object>)op.getHints().get(Suggester.Hint.PARAMS);
+ assertNotNull("params are null: " + op, params);
+ assertEquals("splitMethod: " + op, "link", params.get(CommonAdminParams.SPLIT_METHOD));
+ }
+ assertTrue("shard1 should be split", shard1);
+ assertTrue("shard2 should be split", shard2);
+ }
+
+ }
+
private Map<String, Object> createTriggerProps(long waitForSeconds) {
Map<String, Object> props = new HashMap<>();
props.put("event", "indexSize");
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a852ebc5/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
----------------------------------------------------------------------
diff --git a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
index 97b9dd7..d091537 100644
--- a/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
+++ b/solr/solr-ref-guide/src/solrcloud-autoscaling-triggers.adoc
@@ -304,6 +304,13 @@ changes in a highly dynamic situation, which may lead to more serious threshold
but it also limits the maximum load on the cluster that the large number of requested
operations may cause. The default value is 10.
+`splitMethod`::
+One of the supported methods for index splitting to use. Default value is `rewrite`, which is
+slow and puts a high CPU load on the shard leader but results in optimized sub-shard indexes.
+The `link` method is much faster and puts very little load on the shard leader but results in
+indexes that are initially as large as the parent shard's index, which slows down replication and
+may lead to excessive initial disk space consumption on replicas.
+
Events generated by this trigger contain additional details about the shards
that exceeded thresholds and the types of violations (upper / lower bounds, bytes / docs metrics).
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a852ebc5/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
index 2c1d7df..559ff8a 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/SplitShardSuggester.java
@@ -17,11 +17,13 @@
package org.apache.solr.client.solrj.cloud.autoscaling;
import java.util.Collections;
+import java.util.Map;
import java.util.Set;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.common.params.CollectionParams;
+import org.apache.solr.common.params.CommonAdminParams;
import org.apache.solr.common.util.Pair;
/**
@@ -44,6 +46,12 @@ class SplitShardSuggester extends Suggester {
throw new RuntimeException("split-shard requires exactly one pair of 'collection' and 'shard'");
}
Pair<String, String> collShard = shards.iterator().next();
- return CollectionAdminRequest.splitShard(collShard.first()).setShardName(collShard.second());
+ Map<String, Object> params = (Map<String, Object>)hints.getOrDefault(Hint.PARAMS, Collections.emptyMap());
+ CollectionAdminRequest.SplitShard req = CollectionAdminRequest.splitShard(collShard.first()).setShardName(collShard.second());
+ String splitMethod = (String)params.get(CommonAdminParams.SPLIT_METHOD);
+ if (splitMethod != null) {
+ req.setSplitMethod(splitMethod);
+ }
+ return req;
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/a852ebc5/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
index e4d297e..f052ae9 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/Suggester.java
@@ -95,7 +95,15 @@ public abstract class Suggester implements MapWriter {
Collection<?> values = value instanceof Collection ? (Collection) value : Collections.singletonList(value);
((Set) hints.computeIfAbsent(hint, h -> new HashSet<>())).addAll(values);
} else {
- hints.put(hint, value == null ? null : String.valueOf(value));
+ if (value == null) {
+ hints.put(hint, null);
+ } else {
+ if ((value instanceof Map) || (value instanceof Number)) {
+ hints.put(hint, value);
+ } else {
+ hints.put(hint, String.valueOf(value));
+ }
+ }
}
return this;
}
@@ -374,6 +382,11 @@ public abstract class Suggester implements MapWriter {
NUMBER(true, o -> {
if (!(o instanceof Number)) throw new RuntimeException("NUMBER hint must be a number");
}),
+ PARAMS(false, o -> {
+ if (!(o instanceof Map)) {
+ throw new RuntimeException("PARAMS hint must be a Map<String, Object>");
+ }
+ }),
REPLICA(true);
public final boolean multiValued;