You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by da...@apache.org on 2018/01/23 10:30:38 UTC
[08/41] lucene-solr:jira/solr-11702: SOLR-11064: Collection APIs
should use the disk space hint when using policy framework
SOLR-11064: Collection APIs should use the disk space hint when using policy framework
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/d99799c7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/d99799c7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/d99799c7
Branch: refs/heads/jira/solr-11702
Commit: d99799c75c79b488d4db26c968d4e1a9cf415c6b
Parents: 1c4b417
Author: Noble Paul <no...@apache.org>
Authored: Wed Jan 17 03:04:34 2018 +1100
Committer: Noble Paul <no...@apache.org>
Committed: Wed Jan 17 03:04:34 2018 +1100
----------------------------------------------------------------------
solr/CHANGES.txt | 2 +
.../DelegatingClusterStateProvider.java | 7 ++
.../solrj/cloud/autoscaling/PolicyHelper.java | 33 ++++++++
.../client/solrj/impl/ClusterStateProvider.java | 7 +-
.../solrj/cloud/autoscaling/TestPolicy.java | 83 ++++++++++++++++++++
5 files changed, 131 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d99799c7/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index ecdcca1..4fd3ff1 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -87,6 +87,8 @@ New Features
* SOLR-3218: Added range faceting support for CurrencyFieldType. This includes both "facet.range" as well
as json.facet's "type:range" (Andrew Morrison, Jan Høydahl, Vitaliy Zhovtyuk, hossman)
+* SOLR-11064: Collection APIs should use the disk space hint when using policy framework (noble)
+
Bug Fixes
----------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d99799c7/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingClusterStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingClusterStateProvider.java
index e512ab3..e0b9bac 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingClusterStateProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/DelegatingClusterStateProvider.java
@@ -24,6 +24,7 @@ import java.util.Set;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
/**
* Base class for overriding some behavior of {@link ClusterStateProvider}
@@ -90,6 +91,12 @@ public class DelegatingClusterStateProvider implements ClusterStateProvider {
}
@Override
+ public DocCollection getCollection(String name) throws IOException {
+ ClusterState cs = getClusterState();
+ return cs == null ? null : cs.getCollectionOrNull(name);
+ }
+
+ @Override
public void connect() {
if (delegate != null) {
delegate.connect();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d99799c7/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
index 489a9f9..ae7c9af 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/cloud/autoscaling/PolicyHelper.java
@@ -21,7 +21,9 @@ package org.apache.solr.client.solrj.cloud.autoscaling;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.EnumMap;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
@@ -32,6 +34,7 @@ import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.DocCollection;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ReplicaPosition;
import org.apache.solr.common.util.Pair;
@@ -40,7 +43,10 @@ import org.apache.solr.common.util.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static java.util.Collections.emptyMap;
+import static java.util.Collections.singletonList;
import static java.util.concurrent.TimeUnit.MILLISECONDS;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.FREEDISK;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CoreAdminParams.NODE;
import static org.apache.solr.common.util.Utils.time;
@@ -106,6 +112,30 @@ public class PolicyHelper {
}
session = sessionWrapper.session;
+ Map<String, Double> diskSpaceReqd = new HashMap<>();
+ try {
+ DocCollection coll = cloudManager.getClusterStateProvider().getCollection(collName);
+ if (coll != null) {
+ for (String shardName : shardNames) {
+ Replica ldr = coll.getLeader(shardName);
+ if (ldr != null) {
+ Map<String, Map<String, List<ReplicaInfo>>> details = cloudManager.getNodeStateProvider().getReplicaInfo(ldr.getNodeName(),
+ Collections.singleton(FREEDISK.perReplicaValue));
+ ReplicaInfo replicaInfo = details.getOrDefault(collName, emptyMap()).getOrDefault(shardName, singletonList(null)).get(0);
+ if (replicaInfo != null) {
+ Object idxSz = replicaInfo.getVariables().get(FREEDISK.perReplicaValue);
+ if (idxSz != null) {
+ diskSpaceReqd.put(shardName, 1.5 * (Double) Suggestion.ConditionType.FREEDISK.validate(null, idxSz, false));
+ }
+ }
+ }
+
+ }
+ }
+ } catch (IOException e) {
+ /*ignore*/
+ }
+
Map<Replica.Type, Integer> typeVsCount = new EnumMap<>(Replica.Type.class);
typeVsCount.put(Replica.Type.NRT, nrtReplicas);
@@ -123,6 +153,9 @@ public class PolicyHelper {
suggester = suggester.hint(Hint.TARGET_NODE, nodeName);
}
}
+ if (diskSpaceReqd.get(shardName) != null) {
+ suggester.hint(Hint.MINFREEDISK, diskSpaceReqd.get(shardName));
+ }
SolrRequest op = suggester.getSuggestion();
if (op == null) {
throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No node can satisfy the rules " +
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d99799c7/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java
index 3041a13..c04b80d 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/ClusterStateProvider.java
@@ -17,12 +17,13 @@
package org.apache.solr.client.solrj.impl;
import java.io.IOException;
-import java.util.Map;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import org.apache.solr.common.SolrCloseable;
import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
public interface ClusterStateProvider extends SolrCloseable {
@@ -48,6 +49,10 @@ public interface ClusterStateProvider extends SolrCloseable {
*/
ClusterState getClusterState() throws IOException;
+ default DocCollection getCollection(String name) throws IOException{
+ return getClusterState().getCollectionOrNull(name);
+ }
+
/**
* Obtain cluster properties.
* @return configured cluster properties, or an empty map, never null.
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/d99799c7/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
index 3a5caf9..2c119f3 100644
--- a/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/cloud/autoscaling/TestPolicy.java
@@ -40,6 +40,8 @@ import org.apache.solr.client.solrj.cloud.DistributedQueueFactory;
import org.apache.solr.client.solrj.cloud.autoscaling.Suggester.Hint;
import org.apache.solr.client.solrj.impl.ClusterStateProvider;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.DocRouter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.ReplicaPosition;
import org.apache.solr.common.cloud.ZkStateReader;
@@ -54,6 +56,7 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import static org.apache.solr.client.solrj.cloud.autoscaling.Suggestion.ConditionType.FREEDISK;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.ADDREPLICA;
import static org.apache.solr.common.params.CollectionParams.CollectionAction.MOVEREPLICA;
@@ -1662,4 +1665,84 @@ public class TestPolicy extends SolrTestCaseJ4 {
assertEquals("127.0.0.1:51147_solr" , op.getNode());
}
+ public void testDiskSpaceReqd() {
+ String autoScaleJson = "{" +
+ " cluster-preferences: [" +
+ " { minimize : cores, precision: 2}" +
+ " ]," +
+ " cluster-policy: [" +
+ " { replica : '0' , nodeRole: overseer}" +
+
+ " ]" +
+ "}";
+
+
+ Map<String, Map> nodeValues = (Map<String, Map>) Utils.fromJSONString("{" +
+ "node1:{cores:12, freedisk: 334, heap:10480, sysprop.rack:rack3}," +
+ "node2:{cores:4, freedisk: 262, heap:6873, sysprop.fs : ssd, sysprop.rack:rack1}," +
+ "node3:{cores:7, freedisk: 749, heap:7834, sysprop.rack:rack4}," +
+ "node4:{cores:0, freedisk: 900, heap:16900, nodeRole:overseer, sysprop.rack:rack2}" +
+ "}");
+
+ SolrCloudManager cloudManager = new DelegatingCloudManager(null) {
+ @Override
+ public NodeStateProvider getNodeStateProvider() {
+ return new DelegatingNodeStateProvider(null) {
+ @Override
+ public Map<String, Object> getNodeValues(String node, Collection<String> keys) {
+ Map<String, Object> result = new LinkedHashMap<>();
+ keys.stream().forEach(s -> result.put(s, nodeValues.get(node).get(s)));
+ return result;
+ }
+
+ @Override
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ if (node.equals("node1")) {
+ Map m = Utils.makeMap("newColl",
+ Utils.makeMap("shard1", Collections.singletonList(new ReplicaInfo("r1", "shard1",
+ new Replica("r1", Utils.makeMap(ZkStateReader.NODE_NAME_PROP, "node1")),
+ Utils.makeMap(FREEDISK.perReplicaValue, 200)))));
+ return m;
+ } else if (node.equals("node2")) {
+ Map m = Utils.makeMap("newColl",
+ Utils.makeMap("shard2", Collections.singletonList(new ReplicaInfo("r1", "shard2",
+ new Replica("r1", Utils.makeMap(ZkStateReader.NODE_NAME_PROP, "node2")),
+ Utils.makeMap(FREEDISK.perReplicaValue, 200)))));
+ return m;
+ }
+ return Collections.emptyMap();
+ }
+ };
+ }
+
+ @Override
+ public ClusterStateProvider getClusterStateProvider() {
+ return new DelegatingClusterStateProvider(null) {
+ @Override
+ public Set<String> getLiveNodes() {
+ return new HashSet<>(Arrays.asList("node1", "node2", "node3", "node4"));
+ }
+
+ @Override
+ public DocCollection getCollection(String name) throws IOException {
+ return new DocCollection(name, Collections.emptyMap(), Collections.emptyMap(), DocRouter.DEFAULT) {
+ @Override
+ public Replica getLeader(String sliceName) {
+ if (sliceName.equals("shard1"))
+ return new Replica("r1", Utils.makeMap(ZkStateReader.NODE_NAME_PROP, "node1"));
+ if (sliceName.equals("shard2"))
+ return new Replica("r2", Utils.makeMap(ZkStateReader.NODE_NAME_PROP, "node2"));
+ return null;
+ }
+ };
+ }
+ };
+ }
+ };
+ List<ReplicaPosition> locations = PolicyHelper.getReplicaLocations(
+ "newColl", new AutoScalingConfig((Map<String, Object>) Utils.fromJSONString(autoScaleJson)),
+ cloudManager, null, Arrays.asList("shard1", "shard2"), 1, 0, 0, null);
+ assertTrue(locations.stream().allMatch(it -> "node3".equals(it.node)));
+ }
+
}