You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2017/06/27 09:59:42 UTC
lucene-solr:feature/autoscaling: Refactored to pull out ReplicaInfo
as a top level class
Repository: lucene-solr
Updated Branches:
refs/heads/feature/autoscaling 6a1ddda02 -> 257ba055d
Refactored to pull out ReplicaInfo as a top level class
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/257ba055
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/257ba055
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/257ba055
Branch: refs/heads/feature/autoscaling
Commit: 257ba055d815a1d05141ddbfceb6a4bff55a4dbd
Parents: 6a1ddda
Author: Noble Paul <no...@apache.org>
Authored: Tue Jun 27 19:29:32 2017 +0930
Committer: Noble Paul <no...@apache.org>
Committed: Tue Jun 27 19:29:32 2017 +0930
----------------------------------------------------------------------
.../solrj/impl/SolrClientDataProvider.java | 2 +-
.../apache/solr/cloud/autoscaling/Clause.java | 1 -
.../cloud/autoscaling/ClusterDataProvider.java | 2 +-
.../cloud/autoscaling/MoveReplicaSuggester.java | 1 -
.../apache/solr/cloud/autoscaling/Policy.java | 50 ++-------------
.../solr/cloud/autoscaling/PolicyHelper.java | 2 +-
.../solr/cloud/autoscaling/ReplicaCount.java | 4 +-
.../solr/cloud/autoscaling/ReplicaInfo.java | 64 ++++++++++++++++++++
.../org/apache/solr/cloud/autoscaling/Row.java | 1 -
.../solr/cloud/autoscaling/TestPolicy.java | 38 ++++++------
10 files changed, 92 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/257ba055/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
index 6f6d309..104fcf0 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/impl/SolrClientDataProvider.java
@@ -35,7 +35,7 @@ import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.request.GenericSolrRequest;
import org.apache.solr.client.solrj.response.SimpleSolrResponse;
import org.apache.solr.cloud.autoscaling.ClusterDataProvider;
-import org.apache.solr.cloud.autoscaling.Policy.ReplicaInfo;
+import org.apache.solr.cloud.autoscaling.ReplicaInfo;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.SolrException;
import org.apache.solr.common.cloud.ClusterState;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/257ba055/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
index 09053a4..76caeaa 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Clause.java
@@ -31,7 +31,6 @@ import java.util.Objects;
import java.util.Optional;
import java.util.Set;
-import org.apache.solr.cloud.autoscaling.Policy.ReplicaInfo;
import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.cloud.rule.ImplicitSnitch;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/257ba055/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ClusterDataProvider.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ClusterDataProvider.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ClusterDataProvider.java
index 710db87..4127ea6 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ClusterDataProvider.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ClusterDataProvider.java
@@ -38,7 +38,7 @@ public interface ClusterDataProvider extends Closeable {
* <p>
* the format is {collection:shard :[{replicadetails}]}
*/
- Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys);
+ Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys);
Collection<String> getNodes();
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/257ba055/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/MoveReplicaSuggester.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/MoveReplicaSuggester.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/MoveReplicaSuggester.java
index 4c0d6a0..6576a6a 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/MoveReplicaSuggester.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/MoveReplicaSuggester.java
@@ -22,7 +22,6 @@ import java.util.List;
import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.cloud.autoscaling.Clause.Violation;
-import org.apache.solr.cloud.autoscaling.Policy.ReplicaInfo;
import org.apache.solr.cloud.autoscaling.Policy.Suggester;
import org.apache.solr.common.util.Pair;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/257ba055/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
index 6fd3a5b..2c3ad9a 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Policy.java
@@ -32,7 +32,6 @@ import java.util.Objects;
import java.util.Set;
import java.util.SortedSet;
import java.util.TreeSet;
-import java.util.function.Predicate;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@@ -40,7 +39,6 @@ import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.cloud.autoscaling.Clause.Violation;
import org.apache.solr.common.IteratorWriter;
import org.apache.solr.common.MapWriter;
-import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.params.CollectionParams.CollectionAction;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.StrUtils;
@@ -300,46 +298,6 @@ public class Policy implements MapWriter {
}
- public static class ReplicaInfo implements MapWriter {
- final String name;
- String core, collection, shard;
- Replica.Type type;
- Map<String, Object> variables;
-
- public ReplicaInfo(String name, String coll, String shard, Replica.Type type, Map<String, Object> vals) {
- this.name = name;
- this.variables = vals;
- this.collection = coll;
- this.shard = shard;
- this.type = type;
- }
-
- @Override
- public void writeMap(EntryWriter ew) throws IOException {
- ew.put(name, (MapWriter) ew1 -> {
- if(variables !=null){
- for (Map.Entry<String, Object> e : variables.entrySet()) {
- ew1.put(e.getKey(), e.getValue());
- }
- }
- if(type != null) ew1.put("type", type.toString());
- });
- }
-
- public String getCore() {
- return core;
- }
-
- public String getCollection() {
- return collection;
- }
-
- public String getShard() {
- return shard;
- }
- }
-
-
/* A suggester is capable of suggesting a collection operation
* given a particular session. Before it suggests a new operation,
* it ensures that ,
@@ -439,7 +397,7 @@ public class Policy implements MapWriter {
}
List<Pair<ReplicaInfo, Row>> getValidReplicas(boolean sortDesc, boolean isSource, int until) {
- List<Pair<Policy.ReplicaInfo, Row>> allPossibleReplicas = new ArrayList<>();
+ List<Pair<ReplicaInfo, Row>> allPossibleReplicas = new ArrayList<>();
if (sortDesc) {
if (until == -1) until = getMatrix().size();
@@ -452,11 +410,11 @@ public class Policy implements MapWriter {
return allPossibleReplicas;
}
- void addReplicaToList(Row r, boolean isSource, List<Pair<Policy.ReplicaInfo, Row>> replicaList) {
+ void addReplicaToList(Row r, boolean isSource, List<Pair<ReplicaInfo, Row>> replicaList) {
if (!isAllowed(r.node, isSource ? Hint.SRC_NODE : Hint.TARGET_NODE)) return;
- for (Map.Entry<String, Map<String, List<Policy.ReplicaInfo>>> e : r.collectionVsShardVsReplicas.entrySet()) {
+ for (Map.Entry<String, Map<String, List<ReplicaInfo>>> e : r.collectionVsShardVsReplicas.entrySet()) {
if (!isAllowed(e.getKey(), Hint.COLL)) continue;
- for (Map.Entry<String, List<Policy.ReplicaInfo>> shard : e.getValue().entrySet()) {
+ for (Map.Entry<String, List<ReplicaInfo>> shard : e.getValue().entrySet()) {
if (!isAllowed(e.getKey(), Hint.SHARD)) continue;//todo fix
if(shard.getValue() == null || shard.getValue().isEmpty()) continue;
replicaList.add(new Pair<>(shard.getValue().get(0), r));
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/257ba055/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
index 33d4b97..a67484d 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/PolicyHelper.java
@@ -50,7 +50,7 @@ public class PolicyHelper {
}
@Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
return delegate.getReplicaInfo(node, keys);
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/257ba055/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaCount.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaCount.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaCount.java
index 7d47071..fed899e 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaCount.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaCount.java
@@ -71,9 +71,9 @@ class ReplicaCount extends Number implements MapWriter {
return total();
}
- public void increment(List<Policy.ReplicaInfo> infos) {
+ public void increment(List<ReplicaInfo> infos) {
if (infos == null) return;
- for (Policy.ReplicaInfo info : infos) {
+ for (ReplicaInfo info : infos) {
switch (info.type) {
case NRT:
nrt++;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/257ba055/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaInfo.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaInfo.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaInfo.java
new file mode 100644
index 0000000..8d09997
--- /dev/null
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/ReplicaInfo.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cloud.autoscaling;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.solr.common.MapWriter;
+import org.apache.solr.common.cloud.Replica;
+
+
+public class ReplicaInfo implements MapWriter {
+ final String name;
+ String core, collection, shard;
+ Replica.Type type;
+ Map<String, Object> variables;
+
+ public ReplicaInfo(String name, String coll, String shard, Replica.Type type, Map<String, Object> vals) {
+ this.name = name;
+ this.variables = vals;
+ this.collection = coll;
+ this.shard = shard;
+ this.type = type;
+ }
+
+ @Override
+ public void writeMap(EntryWriter ew) throws IOException {
+ ew.put(name, (MapWriter) ew1 -> {
+ if (variables != null) {
+ for (Map.Entry<String, Object> e : variables.entrySet()) {
+ ew1.put(e.getKey(), e.getValue());
+ }
+ }
+ if (type != null) ew1.put("type", type.toString());
+ });
+ }
+
+ public String getCore() {
+ return core;
+ }
+
+ public String getCollection() {
+ return collection;
+ }
+
+ public String getShard() {
+ return shard;
+ }
+}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/257ba055/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Row.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Row.java b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Row.java
index bc86a2d..ca8059c 100644
--- a/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Row.java
+++ b/solr/solrj/src/java/org/apache/solr/cloud/autoscaling/Row.java
@@ -30,7 +30,6 @@ import org.apache.solr.common.MapWriter;
import org.apache.solr.common.cloud.Replica;
import org.apache.solr.common.util.Pair;
import org.apache.solr.common.util.Utils;
-import org.apache.solr.cloud.autoscaling.Policy.ReplicaInfo;
import static org.apache.solr.common.params.CoreAdminParams.NODE;
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/257ba055/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
----------------------------------------------------------------------
diff --git a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
index 440cd29..8247524 100644
--- a/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
+++ b/solr/solrj/src/test/org/apache/solr/cloud/autoscaling/TestPolicy.java
@@ -88,10 +88,10 @@ public class TestPolicy extends SolrTestCaseJ4 {
" 'node_name':'node1'," +
" 'state':'active'}}}}}}";
- public static Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaDetails(String node, String clusterState) {
+ public static Map<String, Map<String, List<ReplicaInfo>>> getReplicaDetails(String node, String clusterState) {
ValidatingJsonMap m = ValidatingJsonMap
.getDeepCopy((Map) Utils.fromJSONString(clusterState), 6, true);
- Map<String, Map<String, List<Policy.ReplicaInfo>>> result = new LinkedHashMap<>();
+ Map<String, Map<String, List<ReplicaInfo>>> result = new LinkedHashMap<>();
m.forEach((collName, o) -> {
ValidatingJsonMap coll = (ValidatingJsonMap) o;
@@ -101,9 +101,9 @@ public class TestPolicy extends SolrTestCaseJ4 {
ValidatingJsonMap r = (ValidatingJsonMap) o2;
String node_name = (String) r.get("node_name");
if (!node_name.equals(node)) return;
- Map<String, List<Policy.ReplicaInfo>> shardVsReplicaStats = result.computeIfAbsent(collName, k -> new HashMap<>());
- List<Policy.ReplicaInfo> replicaInfos = shardVsReplicaStats.computeIfAbsent(shard, k -> new ArrayList<>());
- replicaInfos.add(new Policy.ReplicaInfo(replicaName, collName, shard, Replica.Type.get((String) r.get(ZkStateReader.REPLICA_TYPE)), new HashMap<>()));
+ Map<String, List<ReplicaInfo>> shardVsReplicaStats = result.computeIfAbsent(collName, k -> new HashMap<>());
+ List<ReplicaInfo> replicaInfos = shardVsReplicaStats.computeIfAbsent(shard, k -> new ArrayList<>());
+ replicaInfos.add(new ReplicaInfo(replicaName, collName, shard, Replica.Type.get((String) r.get(ZkStateReader.REPLICA_TYPE)), new HashMap<>()));
});
});
});
@@ -404,7 +404,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
for (int i = 0; i < l3.size(); i++) {
Object o = l3.get(i);
Map m3 = (Map) o;
- l3.set(i, new Policy.ReplicaInfo(m3.keySet().iterator().next().toString()
+ l3.set(i, new ReplicaInfo(m3.keySet().iterator().next().toString()
,coll.toString(), shard.toString(), Replica.Type.get((String)m3.get("type")), new HashMap<>()));
}
});
@@ -419,8 +419,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
@Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return (Map<String, Map<String, List<Policy.ReplicaInfo>>>) Utils.getObjectByPath(m,false, Arrays.asList("replicaInfo", node));
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return (Map<String, Map<String, List<ReplicaInfo>>>) Utils.getObjectByPath(m,false, Arrays.asList("replicaInfo", node));
}
@Override
@@ -513,8 +513,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
Row r2 = r1.addReplica("c1", "s1",null);
assertEquals(1, r1.collectionVsShardVsReplicas.get("c1").get("s1").size());
assertEquals(2, r2.collectionVsShardVsReplicas.get("c1").get("s1").size());
- assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(0) instanceof Policy.ReplicaInfo);
- assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(1) instanceof Policy.ReplicaInfo);
+ assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(0) instanceof ReplicaInfo);
+ assertTrue(r2.collectionVsShardVsReplicas.get("c1").get("s1").get(1) instanceof ReplicaInfo);
}
public void testMerge() {
@@ -677,7 +677,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
@Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
return clusterDataProvider.getReplicaInfo(node, keys);
}
@@ -735,8 +735,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
" {'core_node2':{}}]}}}");
Map m = (Map) Utils.getObjectByPath(replicaInfoMap, false, "127.0.0.1:60089_solr/compute_plan_action_test");
m.put("shard1", Arrays.asList(
- new Policy.ReplicaInfo("core_node1", "compute_plan_action_test", "shard1", Replica.Type.NRT, Collections.emptyMap()),
- new Policy.ReplicaInfo("core_node2", "compute_plan_action_test", "shard1", Replica.Type.NRT, Collections.emptyMap())
+ new ReplicaInfo("core_node1", "compute_plan_action_test", "shard1", Replica.Type.NRT, Collections.emptyMap()),
+ new ReplicaInfo("core_node2", "compute_plan_action_test", "shard1", Replica.Type.NRT, Collections.emptyMap())
));
Map<String, Map<String, Object>> tagsMap = (Map) Utils.fromJSONString("{" +
@@ -755,8 +755,8 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
@Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
- return (Map<String, Map<String, List<Policy.ReplicaInfo>>>) replicaInfoMap.get(node);
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ return (Map<String, Map<String, List<ReplicaInfo>>>) replicaInfoMap.get(node);
}
@Override
@@ -817,7 +817,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
@Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
return clusterDataProvider.getReplicaInfo(node, keys);
}
@@ -861,7 +861,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
@Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
return getReplicaDetails(node, clusterState);
}
@@ -888,7 +888,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
@Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
return getReplicaDetails(node, clusterState);
}
@@ -945,7 +945,7 @@ public class TestPolicy extends SolrTestCaseJ4 {
}
@Override
- public Map<String, Map<String, List<Policy.ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
+ public Map<String, Map<String, List<ReplicaInfo>>> getReplicaInfo(String node, Collection<String> keys) {
return getReplicaDetails(node, clusterState);
}