You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@lucene.apache.org by no...@apache.org on 2016/03/09 19:17:51 UTC
[1/2] lucene-solr git commit: SOLR-8728
Repository: lucene-solr
Updated Branches:
refs/heads/branch_5_5 a1562c526 -> 4faadb625
SOLR-8728
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/c17f80a7
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/c17f80a7
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/c17f80a7
Branch: refs/heads/branch_5_5
Commit: c17f80a7a22a4215ebcfe08939b6c0acc30df7c2
Parents: a1562c5
Author: Noble Paul <no...@apache.org>
Authored: Fri Mar 4 16:22:37 2016 +0530
Committer: Noble Paul <no...@apache.org>
Committed: Wed Mar 9 23:47:04 2016 +0530
----------------------------------------------------------------------
.../cloud/OverseerCollectionMessageHandler.java | 30 ++++++++++++--------
.../java/org/apache/solr/cloud/rule/Rule.java | 4 ++-
.../org/apache/solr/cloud/ShardSplitTest.java | 27 ++++++++++++++++++
3 files changed, 48 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c17f80a7/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
index 0c7752b..9169d58 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
@@ -1308,13 +1308,18 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
// TODO: change this to handle sharding a slice into > 2 sub-shards.
+
+ Map<Position, String> nodeMap = identifyNodes(clusterState,
+ new ArrayList<>(clusterState.getLiveNodes()),
+ new ZkNodeProps(collection.getProperties()),
+ subSlices, repFactor - 1);
+
List<Map<String, Object>> replicas = new ArrayList<>((repFactor - 1) * 2);
- for (int i = 1; i <= subSlices.size(); i++) {
- Collections.shuffle(nodeList, RANDOM);
- String sliceName = subSlices.get(i - 1);
- for (int j = 2; j <= repFactor; j++) {
- String subShardNodeName = nodeList.get((repFactor * (i - 1) + (j - 2)) % nodeList.size());
- String shardName = collectionName + "_" + sliceName + "_replica" + (j);
+
+ for (Map.Entry<Position, String> entry : nodeMap.entrySet()) {
+ String sliceName = entry.getKey().shard;
+ String subShardNodeName = entry.getValue();
+ String shardName = collectionName + "_" + sliceName + "_replica" + (entry.getKey().index);
log.info("Creating replica shard " + shardName + " as part of slice " + sliceName + " of collection "
+ collectionName + " on " + subShardNodeName);
@@ -1349,7 +1354,6 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
replicas.add(propMap);
}
- }
// we must set the slice state into recovery before actually creating the replica cores
// this ensures that the logic inside Overseer to update sub-shard state to 'active'
@@ -2076,8 +2080,8 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
ZkNodeProps message,
List<String> shardNames,
int repFactor) throws IOException {
- List<Map> maps = (List) message.get("rule");
- if (maps == null) {
+ List<Map> rulesMap = (List) message.get("rule");
+ if (rulesMap == null) {
int i = 0;
Map<Position, String> result = new HashMap<>();
for (String aShard : shardNames) {
@@ -2090,7 +2094,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
}
List<Rule> rules = new ArrayList<>();
- for (Object map : maps) rules.add(new Rule((Map) map));
+ for (Object map : rulesMap) rules.add(new Rule((Map) map));
Map<String, Integer> sharVsReplicaCount = new HashMap<>();
@@ -2157,8 +2161,10 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
// Kind of unnecessary, but it does put the logic of whether to override maxShardsPerNode in one place.
- node = getNodesForNewReplicas(clusterState, collection, shard, 1, node,
- overseer.getZkController().getCoreContainer()).get(0).nodeName;
+ if(node == null) {
+ node = getNodesForNewReplicas(clusterState, collection, shard, 1, node,
+ overseer.getZkController().getCoreContainer()).get(0).nodeName;
+ }
log.info("Node not provided, Identified {} for creating new replica", node);
if (!clusterState.liveNodesContain(node)) {
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c17f80a7/solr/core/src/java/org/apache/solr/cloud/rule/Rule.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/rule/Rule.java b/solr/core/src/java/org/apache/solr/cloud/rule/Rule.java
index d73a9bf..209b755 100644
--- a/solr/core/src/java/org/apache/solr/cloud/rule/Rule.java
+++ b/solr/core/src/java/org/apache/solr/cloud/rule/Rule.java
@@ -163,7 +163,9 @@ public class Rule {
Map<String,Integer> nodesInThisShard = shardVsNodeSet.get(shardCondition.val.equals(WILD_WILD_CARD) ? entry.getKey() : shardName);
if (nodesInThisShard != null) {
for (Map.Entry<String,Integer> aNode : nodesInThisShard.entrySet()) {
- Object obj = nodeVsTags.get(aNode.getKey()).get(tag.name);
+ Map<String, Object> tagValues = nodeVsTags.get(aNode.getKey());
+ if(tagValues == null) continue;
+ Object obj = tagValues.get(tag.name);
if (tagCondition.canMatch(obj, phase)) countMatchingThisTagValue += aNode.getValue();
}
}
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/c17f80a7/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
----------------------------------------------------------------------
diff --git a/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java b/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
index bbcae33..22735ab 100644
--- a/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
+++ b/solr/core/src/test/org/apache/solr/cloud/ShardSplitTest.java
@@ -24,7 +24,9 @@ import org.apache.solr.client.solrj.SolrRequest;
import org.apache.solr.client.solrj.SolrServerException;
import org.apache.solr.client.solrj.impl.CloudSolrClient;
import org.apache.solr.client.solrj.impl.HttpSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.client.solrj.response.CollectionAdminResponse;
import org.apache.solr.client.solrj.response.QueryResponse;
import org.apache.solr.common.SolrDocument;
import org.apache.solr.common.cloud.ClusterState;
@@ -86,7 +88,32 @@ public class ShardSplitTest extends BasicDistributedZkTest {
// and the new sub-shards don't have any.
waitForRecoveriesToFinish(true);
//waitForThingsToLevelOut(15);
+ }
+
+ @Test
+ public void testSplitShardWithRule() throws Exception {
+ waitForThingsToLevelOut(15);
+
+ if (usually()) {
+ log.info("Using legacyCloud=false for cluster");
+ CollectionsAPIDistributedZkTest.setClusterProp(cloudClient, "legacyCloud", "false");
+ }
+ log.info("Starting testSplitShardWithRule");
+ String collectionName = "shardSplitWithRule";
+ CollectionAdminRequest.Create createRequest = new CollectionAdminRequest.Create()
+ .setCollectionName(collectionName)
+ .setNumShards(1)
+ .setReplicationFactor(2)
+ .setRule("shard:*,replica:<2,node:*");
+ CollectionAdminResponse response = createRequest.process(cloudClient);
+ assertEquals(0, response.getStatus());
+
+ CollectionAdminRequest.SplitShard splitShardRequest = new CollectionAdminRequest.SplitShard()
+ .setCollectionName(collectionName)
+ .setShardName("shard1");
+ response = splitShardRequest.process(cloudClient);
+ assertEquals(String.valueOf(response.getErrorMessages()), 0, response.getStatus());
}
private void incompleteOrOverlappingCustomRangeTest() throws Exception {
[2/2] lucene-solr git commit: SOLR-8728: ReplicaAssigner throws NPE
when a partial list of nodes are only participating in replica placement.
splitshard should preassign nodes using rules, if rules are present
Posted by no...@apache.org.
SOLR-8728: ReplicaAssigner throws NPE when a partial list of nodes are only participating in replica
placement. splitshard should preassign nodes using rules, if rules are present
Project: http://git-wip-us.apache.org/repos/asf/lucene-solr/repo
Commit: http://git-wip-us.apache.org/repos/asf/lucene-solr/commit/4faadb62
Tree: http://git-wip-us.apache.org/repos/asf/lucene-solr/tree/4faadb62
Diff: http://git-wip-us.apache.org/repos/asf/lucene-solr/diff/4faadb62
Branch: refs/heads/branch_5_5
Commit: 4faadb625e91b8ef323724855748258570137084
Parents: c17f80a
Author: Noble Paul <no...@apache.org>
Authored: Wed Mar 9 23:47:35 2016 +0530
Committer: Noble Paul <no...@apache.org>
Committed: Wed Mar 9 23:47:35 2016 +0530
----------------------------------------------------------------------
solr/CHANGES.txt | 4 ++++
.../apache/solr/cloud/OverseerCollectionMessageHandler.java | 7 ++++---
2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4faadb62/solr/CHANGES.txt
----------------------------------------------------------------------
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 9fd37ae..860ec2e 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -28,6 +28,10 @@ Bug Fixes
hasn't refreshed yet). In this case the reported size of the file is -1.
(Shai Erera, Alexey Serba, Richard Coggins)
+* SOLR-8728: ReplicaAssigner throws NPE when a partial list of nodes are only participating in replica
+ placement. splitshard should preassign nodes using rules, if rules are present (noble, Shai Erera)
+
+
======================= 5.5.0 =======================
Consult the LUCENE_CHANGES.txt file for additional, low level, changes in this release
http://git-wip-us.apache.org/repos/asf/lucene-solr/blob/4faadb62/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
----------------------------------------------------------------------
diff --git a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
index 9169d58..dfaff3c 100644
--- a/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
+++ b/solr/core/src/java/org/apache/solr/cloud/OverseerCollectionMessageHandler.java
@@ -2159,9 +2159,10 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
"Collection: " + collection + " shard: " + shard + " does not exist");
}
ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
+ boolean skipCreateReplicaInClusterState = message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false);
// Kind of unnecessary, but it does put the logic of whether to override maxShardsPerNode in one place.
- if(node == null) {
+ if (!skipCreateReplicaInClusterState) {
node = getNodesForNewReplicas(clusterState, collection, shard, 1, node,
overseer.getZkController().getCoreContainer()).get(0).nodeName;
}
@@ -2172,7 +2173,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
}
if (coreName == null) {
coreName = Assign.buildCoreName(coll, shard);
- } else if (!message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false)) {
+ } else if (!skipCreateReplicaInClusterState) {
//Validate that the core name is unique in that collection
for (Slice slice : coll.getSlices()) {
for (Replica replica : slice.getReplicas()) {
@@ -2187,7 +2188,7 @@ public class OverseerCollectionMessageHandler implements OverseerMessageHandler
ModifiableSolrParams params = new ModifiableSolrParams();
if (!Overseer.isLegacy(zkStateReader.getClusterProps())) {
- if (!message.getBool(SKIP_CREATE_REPLICA_IN_CLUSTER_STATE, false)) {
+ if (!skipCreateReplicaInClusterState) {
ZkNodeProps props = new ZkNodeProps(Overseer.QUEUE_OPERATION, ADDREPLICA.toLower(), ZkStateReader.COLLECTION_PROP,
collection, ZkStateReader.SHARD_ID_PROP, shard, ZkStateReader.CORE_NAME_PROP, coreName,
ZkStateReader.STATE_PROP, Replica.State.DOWN.toString(), ZkStateReader.BASE_URL_PROP,